From c854b5be40fb7a1225eb23cdd5014b47940e6306 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Wed, 8 Dec 2021 15:57:50 -0500 Subject: [PATCH 01/46] Update Ansible - Updates deployment playbooks for prod and staging to handle the possibility of a `secrets.py` file. - Updates vars and vault files with new variable pointers and encryption. - Adds a new template file that will be rendered as `secrets.py` in the root of the indicators directory. --- ansible/ansible-deploy-staging.yaml | 12 + ansible/ansible-deploy.yaml | 12 + .../templates/claims_hosp-secrets-prod.py.j2 | 11 + ansible/vars.yaml | 5 + ansible/vault.yaml | 457 +++++++++--------- 5 files changed, 273 insertions(+), 224 deletions(-) create mode 100755 ansible/templates/claims_hosp-secrets-prod.py.j2 diff --git a/ansible/ansible-deploy-staging.yaml b/ansible/ansible-deploy-staging.yaml index 3056d79f2..f08839800 100644 --- a/ansible/ansible-deploy-staging.yaml +++ b/ansible/ansible-deploy-staging.yaml @@ -27,6 +27,10 @@ local_action: stat path="templates/{{ indicator }}-params-prod.json.j2" register: template + - name: Check to see if we have a secrets template to send. + local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2" + register: template + - name: Set production params file. copy: src: files/{{ indicator }}-params-prod.json @@ -42,3 +46,11 @@ owner: "{{ runtime_user }}" group: "{{ runtime_user }}" when: template.stat.exists + + - name: Set production secrets template. + template: + src: templates/{{ indicator }}-secrets-prod.py.j2 + dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py" + owner: "{{ runtime_user }}" + group: "{{ runtime_user }}" + when: template.stat.exists diff --git a/ansible/ansible-deploy.yaml b/ansible/ansible-deploy.yaml index f35aa40f8..9863639ed 100644 --- a/ansible/ansible-deploy.yaml +++ b/ansible/ansible-deploy.yaml @@ -27,6 +27,10 @@ local_action: stat path="templates/{{ indicator }}-params-prod.json.j2" register: template + - name: Check to see if we have a secrets template to send. + local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2" + register: template + - name: Set production params file. copy: src: files/{{ indicator }}-params-prod.json @@ -42,3 +46,11 @@ owner: "{{ runtime_user }}" group: "{{ runtime_user }}" when: template.stat.exists + + - name: Set production secrets template. + template: + src: templates/{{ indicator }}-secrets-prod.py.j2 + dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py" + owner: "{{ runtime_user }}" + group: "{{ runtime_user }}" + when: template.stat.exists diff --git a/ansible/templates/claims_hosp-secrets-prod.py.j2 b/ansible/templates/claims_hosp-secrets-prod.py.j2 new file mode 100755 index 000000000..b00b147d7 --- /dev/null +++ b/ansible/templates/claims_hosp-secrets-prod.py.j2 @@ -0,0 +1,11 @@ +class claims: + HOST = 'ftp.delphi.cmu.edu' + USER = {{ claims_hosp_ftp_user }} + PASS = {{ claims_hosp_ftp_password }} + PORT = 2222 + + +class covidcast: + HOST = "delphi.midas.cs.cmu.edu" + USER = {{ claims_hosp_midas_user }} + PASS = {{ claims_hosp_midas_password }} diff --git a/ansible/vars.yaml b/ansible/vars.yaml index eaeff437d..c44844e8e 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -26,6 +26,11 @@ changehc_sftp_host: "{{ vault_changehc_sftp_host }}" changehc_sftp_port: "{{ vault_changehc_sftp_port }}" changehc_sftp_user: "{{ vault_changehc_sftp_user }}" changehc_sftp_password: "{{ vault_changehc_sftp_password }}" +# claims_hosp +claims_hosp_ftp_user: "{{ vault_claims_hosp_ftp_user }}" +claims_hosp_ftp_password: "{{ vault_claims_hosp_ftp_password }}" +claims_hosp_midas_user: "{{ vault_claims_hosp_midas_user }}" +claims_hosp_midas_password: "{{ vault_claims_hosp_midas_password }}" # NCHS nchs_mortality_token: "{{ vault_nchs_mortality_token }}" # SirCAL diff --git a/ansible/vault.yaml b/ansible/vault.yaml index 950d0ed51..ed2916927 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,225 +1,234 @@ $ANSIBLE_VAULT;1.1;AES256 -63316430313534303165663533363834353939653931383036363236343034666463366362303263 -3732646465653464336239613162343439323734636430660a393334323166376563653037303336 -63303663346462653963326361313238633263663133383965393431613139333432323139636361 -3130343634663234390a663433383962663438643434316433653762663330383437383432363730 -65313561343261613833343063626334333934613666333164306664363861376561343162616337 -31316266646439346533343934363262396538623836333465656536666465346564646563353338 -35326563303134616336393931313334616439383366343064643833363532353236393138393562 -62333034663932626438633836316130323964636439316265303230393735363637306438613662 -30333666663562663931346662353930386566316236303937333136353563303938393533643166 -63623262396661373137323839316530396365626334653262386539336162306335313035643364 -65343233643930653632303734663234643363323732653766383934373539393063643132623336 -36323433656234636130376238336132353364653664383238343337653436623838663036616639 -37343061373964366636313834613439343035306630353563396533313737643663623665333663 -63323834633661306336633062636666663536396435373234323032323234336632343266363536 -64316132366335356233393038346436643338356263666464316530653438656434323938633833 -65333461663636306133313662346466323431376639343438663135626265323965303462623965 -63383761376438623566636139303536383233356137626464393362646339393331653131313539 -30306235343938363038346361393735623033346531366330653939613437633136353163346234 -39363763353537353961366164346161646261653166653363393062343866336535306131323364 -65656535663337376138653236373762366463323962653064306436616439646132666633653438 -31383663386635643837383332643665643134656637306565346630386135366137383334323533 -36366130656238326532616165643664666464616331363038623764386232333733373630396434 -38626366346464616565343963313664616135643266346239316166393734613232666430306465 -38643762653039646630666362643264643636356662396230643539323965623863643435303161 -38303037653436643739623166363062356638333733653961386138616539653265306636316564 -32366231313366313932653933393636393438656639653837363431383931623139616432613737 -66343966646532613633663536363639393966356335343836343536393132636663613738653763 -63666139356537633735393262663239313766366533633432383864313565393739313631313661 -66346136363233323861633061663934633264613763383635653763656435346365313261323765 -66643837636138623436653430643734396534346238626337326337363932393137613764333938 -39646439626435623433303561656165663037313136636532663166643734623937313834643464 -61613931323738636263653033343832326437666233313434376262323433353535653235613766 -64336431363862616230363135333732636436393033363564626338373237313333323531373064 -66303939326530393638633631346666316263316437363463646537333363323965616134386162 -34653734356165613262386432653965613833353830383531613061623031343535373638383233 -66663463383230663561373237333435613661653662376139356132643133373333313163636436 -66663666653638373431633930346164333962323235313432306330363663303064383837633664 -35366164656663306165323130376333333064383864656263323934616637313031336330326136 -66333664626437386537353266623631633462386562623738633834313364663961353963653366 -38646238373663383638626131346363656530653638653466616364373565646639383232653363 -36633837363161626338326364383433356630626435383063623133393132323363303939393565 -30333863663563313464613534383035666435663231323135376338663035373231343035623962 -34613365626266313237346232636562386136353636313263626538343864346564336532636365 -62623262326633656261383136393066636432353764623337616564313933313639313735646138 -34386539643230393836623739643130366339663762386436616365303164313831303033633564 -61656230613235636466346134336665343062346333393230353435666361643264343434333231 -35646361653831643866316335633934356534346232656462336233373865333139386637333764 -65373630366235363332373832303630323938316162656239653033623162356531333161323936 -34316335613462613062613763366234636234663864313237643137396461313365653835346265 -32663036353734366661323434653865336437313338313335353566373462363266656661643466 -66666335313439303730656565373034643062333762666538643535623764376465353731336534 -39306365383331323338356266633230643336643439363865366630626339656236366635613735 -36626530356562613564303034616634633635383239646166663036373736626334383239646464 -36636233386631383134386665613036616534306330366436653063376362343462363932303931 -62633065613332303062333366636539323337663865306431303866393737346564336265356331 -38316136653932653634383530346465393337643732653039333732343933663238636534613535 -30353563383136656264636237613164663134613639616563323933646262303731333134346266 -32383166663934303030313062666232303064393862356337313466343964346564363531616662 -62616235353733373839626362613932663932666636363134313761336633373736313362333636 -65663533343736323966376436373930653061363030656464333163303238633932306438346433 -39373265613836343132626335353133386566656161393931313865643839613163643563376434 -62633666356261616666353463393663616431396438376462313865356161333264333530313931 -65646439323861653235346134326366393535636430346438656666633438316364316661313939 -63666464663431656531336638633531306365663266643838636339346330303533613138653665 -32326135326232336236653531363237383465303637393433386133323036333162353266306335 -31316531656634323835383533366137353937646266623837326465316266326464633638366164 -64663066653435626632633962313765613765373835623263613666313039663762646139343666 -39646131373337303434613363616330393634313765316634383238313337623863373339353863 -61313532396434656139636564306462346535386236616535353365643437633139333436616639 -33643331643937323562633034383263646438393731323366313765633439326463633934636137 -39613735373763386136323634343536383432333463393136383830303863376432633137623137 -38313335623735643132326430613661623164373364343666386638363562333438663031613135 -30313239613137383133623963656461336439376339656330663234653361306538323762646637 -65653436393635373139363466616336633337396539346237663564616130656135393032386239 -62336338326261346235376238613435653930656338376464616461393463613639393263366634 -32376237353765393936313430653636323365336539333732393637363734343238376234643835 -61383262336163626233333130663639396661313432666331333366623662646637333162623463 -30616638633239373462393463653165393735643435343637643735333561373333636438386633 -30323230366662353565383662363464616237333339373331613734623061353636313231306333 -32336339663536393462333865613035393135343836303935623434306334396366383863643732 -39396134353138376538623738353163336633323533376534393139666261346362626332393833 -32663335336161646230326630343163646333626130646562396632626638643565613630336437 -65316132393764303333653831393366326137323163643563376433316534373330626266373839 -64393966643538323133306662653861623839626466336566303832356530616438366338336335 -63393230643737306436363062666664386535383865373365363065343165383132623563666437 -62653933653637613263363165666163396261313531303938323361616461633236353663366466 -64623866653839303036623737616539363933376431646532663761646630396163393537626433 -37613334316336313236666266316364666333623432636466396334646230343730386635363161 -37396238353131373734393335383366633261626230633566326332376264393534346233396265 -36663465366666333135343338346438373163343934376563306137666334643535616133393261 -30373934343666356562653634623433626465666562666262356535643230366132396634363363 -63346635663237636231313032666531353230636363666437363037636530343064623231393361 -66346634383935653564313134306333363063313762383131336263333763656532393235383432 -65363234356136363039383333313530383633353761616162643331356335633130333161623163 -31333437623837653230313736366565663238663064646332353236303434383966323135343563 -36663463353432353464323332303031646231656535303764396134656632386265363263353366 -33643131623664306330633962363337613264623762323765643162623734356336356536396363 -63303036333366323165613238623362306536383933383763623835653537366534313361376233 -33336432306266626465656661326531326362346137336639323165323936346539396331326133 -38653333643462623461633966313531343833303434613366663736396562646536323433616630 -64323765333132346239663538663263336131376261353039373061343532666465313537313734 -38636233316335363137366635616363623839626665646630623930303031393665393165616439 -31613566383831336631383239646638643665383937343364313539393932363036323134383934 -38653533326233383430376233326132616133626365653237363965306563366634633334336434 -66653939363531323937623833343366363066323131333838636561363930626266613230373966 -30373336343266333931633031636330323030613532623563346539386264663035336433363765 -38306539373530653536666162643238623265326663653238663765613562366334643435616233 -37333333653866383939303064383265346561636430623661613861376134623864356132323636 -37613633343035656262653731623633353737633133613133363164313835373964376138313637 -38366465333536656633346139386237336166333966636661373034313731626333313136646266 -35313136626664633436366439306130633165633064366461653132373831613265323830303230 -31383235303263343636393562313733666266333939613834643630646164653162323930383361 -61666564663563336234383136343363346631366531356439643963643631396336616162626138 -63653430653761383361316238353465363036633562373839363337333236383735333636653138 -64303536323931303833386433653864653361383763323565643335353863613866313361356234 -34376363303436323033313038356462623034396363353466636639623836623266373337346538 -65663365646534613232303162356361353636316364366338353033353930643930343734616433 -36303634383133643164323565353137386632613039343238303061356638663734383135346437 -32326639363738643338366330343034343462636564346535303537346234656335343766333732 -38356237363361646334646336343964333731656635346139353464346238353562633366363837 -36356331323561643638316265626561306139643738653832323761613666343738336462353432 -33666235306562386134643330636663633832663831613864643839306161336265353566393438 -32393166353266383630303462363037663336663932623235393536653466343264376631343465 -63313862363666643666666138346364303462633033366363626266313462633539663035666432 -37643636333761336239626663346434383264666162323964323230663061613834393430633339 -63386264366437356636316662396234393734356564653564366234663030303262333366663666 -32623633373461356265663132643564366466303435366234626361373030316333323062623639 -31666565343338613863616364323564336565356139626565363665363761376230383130386235 -38343134326136366236386162346262663936633336633363316234333066343462363466393436 -31633938626433303662366532376562303737623735353365656233666637316263623031306131 -36346663336333663731353333313465396565346237613433653734313961623539653566343561 -30346335333064613236343932666532616364383763356636343232386235303465643465616530 -31333333393636393764643434646536373563303735353238626637303632353233613630666533 -66613265323632336237326137363330336631353265316161613337356232326235356163303431 -31633231356564346437353063616330663937333964356638376432316563303161316230373534 -31353938386130393734663939656664663830316537393764643163303964326335336533313566 -31616530336465333538366535343938323865663932333735653135393533363164633762376566 -39636134366333383538316130363835323937343764623033376362316438373436316630333361 -31363834343466363339333935386264656239653562316261363737356361303461333238316233 -39343731623730333236616464356464633431613835366364663436666363393130613339613965 -33343166616236616464393036303939613863396661633735633263306232356131636662323037 -30363863656266303933346633393066386235313838643835386363313532656530626164303730 -31333434386631633234323064383233363935383162666134326339333966333633343764346262 -65383462663131623933353432363566363038366438613366313730613130643438656236363136 -65643538346335316334666434356532643239346163626563336561373836613038613834383465 -37353930383038353865303766306232636536333738336130313831616262656362353734343634 -35363233396161356665663031376266336231633462373034336639343563333938383133646435 -62666438386638333130336333393837626531306266356566613866303032646263326634366333 -30343863306237333133393636616533633661643663636232316537633964363532366132383734 -38373166303662343963303132353831663736393165316335333232356535386433666366666362 -62353030333763356162626361633962626264313761613466346537643137326164323265346339 -38333931653130323965333734373762393561643630363261366536353137383361623734386165 -64343139613364383361303639303236336561336233396530393331353638343031353136623863 -35343663343965646537643835336330653636313133616538353165323366333037636438353234 -35613864313962386536323238663461303139333533363836613237643561366236653231616166 -36353533306334356533393564303839646661313634656633623737393337663866303366313331 -61376536353163306135646465373565326539633030653535393565306437386661353564623137 -39383534653663653262623331316139353265363063346530303131613965643566313161613335 -31373434386266383331626339346463356566313263316330376230363531303536303937303335 -61323333623930663837643837373136393365653839323036323335616330393164356662656137 -32323933303239383934643362376637376466613365373765666434313361626533653834306164 -35326432323137363734326339623732353964666232366465663934353939643363663435643461 -61616637313430326438633036646335346366366561333031643532613131393163613261313233 -62313264353764356136303737316266363338316164376235633761613236343333616431366432 -62643131656539376163653161326164336439666332653433336130326263663364306165613931 -33396136363739353132646666343537366231636135313238613461336563346339646530353631 -31653866633939663063636362613430383462343362653064363538343434346336346438373639 -32666430323836646336363961393065303838306461393064663563383262333631363334653637 -35656233633438613535336339333535306431646263376563306266346633373130343230386530 -66373330653138336534333330366537613836616238373335656465313739663236393364323633 -64383164613961636635633963363630626336393165396162313736643532376537306262616233 -36663530386162663264326533303736623063303335336665356161626232303039626338303539 -64366336663966653239663130336436373735316130613366396263363534333330383966363663 -62386165333330616236666332653237346637336262393838636130333738353733363730316432 -38663036363066613130393730656164333463666130346534663538643833616234386539333633 -62623635336635363938633534633032366266393336613564333131323865636532313238383464 -33336666323065323633333566313634366663383932623864336635643061346663643738333537 -35336530373330323033363333663036623237306632316661336430376434323737346630666234 -32646630643463323636323666373762323638386631353964386337656563363239623334363033 -35313738666264663663343064653863346161396239646233343964383066333432306434323861 -35386233303734646264383638376633366263636239653466316531396666616239363430383261 -37626264653130653639346237333530383766643238396232313635653930656266343034313766 -34653530343363643235663562616364343962393036663637363833383665666662393630383666 -35376135363837613538303039386438613632653464643361316265373837623932663535363236 -33333038356664323133303930333836386437656265383631343565663563646130366265383137 -36366262373935356331636662356232623837383361656636313234666361386662393533306238 -64396464323263313731313731386161636266366231306639303536616165613831626231303834 -65346361383237643433656135333739396130356631326539343662633032616264326330373730 -63326432356139663637306431393839303465363634326663346461646663643537356338623939 -62663731323730346635396338313735653932633536363864313164336436323936343164376461 -63393230306234616561653834396537313166633465336461623462303135303063643438363631 -36336164626134643462353738383163326236383237353730373430396134623662386663666639 -62303634646631656330646436626461376664636535653631356566616232376664343335373439 -64663465306336663661363061336531636534613736373737303462383534663661316166323762 -33616234373236363661326534663161636162333765636462383962373833346337653262303261 -30333662323563356665303761393936393832306430336437663066643834353039663639373964 -64653365346631303939666130666461353032663462613936343266356365626437656338616166 -30326563636138643665373063326439396566306533363364353837306530643138366135663564 -65303233643930663530643135623332666336656166643033323463353863386636643635626562 -30323664373137313130363266366466326437613630613363336431366433366132343433626130 -32613663303261633164363533636164653932313336343763633264353633656132333333663061 -61373130663737386231383866653066626564396364323131653134376638323237316234666164 -32623432326537636361323939373332646562316535333562643330363133396234633962653762 -36623138356131333331643963303635656431396463396231343365393830653432333235316464 -36393434643732656362376138383635343636636463646335363763346666393830323435666335 -65643335373362646263646634313032333435323334343062326165653233373264316435653764 -37346564323435386335663632633035326366323861663233316166643662306136303366376337 -65313164323461656264646163356563313162396434383639326431353530373335326164333964 -64373831626539393362313733626135633133353332613036383937666663663739643336613733 -66353661356165633733646135313636316161363264633030663231386432343532633037623833 -36616266613261376432353164343162663063353232343763336466323138343731313639373039 -64643533326631393465333439363966616163336137633236336636653130346333633631333361 -64396333396235643335623831666232376232636131656162323532386562666563303764613137 -39386531303938623033383836356639306533376130383963393331363762643433366461336236 -33323836323966636437663939626132343537663237636137323939363639323133326533366162 -34653539373938303237636332313535653437616537663231613466376538323364643334633435 -39613665653335666431303163653662376263623234613639313132613236373332353839346335 -64633233326632653634383831316266386438623939386433346563366461653565393763616239 -31613931653865313566313036623736353333643763316166343131383566373239396435353136 -32306237313363376364383463373238353739623331356465393236646232623561323962666537 -65633437313962636337626530653536386264383537643266663863633532353234386236636537 -62663632633564376438646232363962376665393566393834376264363531363438306566653234 -34333332653162346339353762386135363039333463353435303239396239636561643839313566 -35356433616239616234 +65363733383463363636386638303566363861646436363666656665663263653838373739383036 +6432353762653138336663653530633764323565383238370a666666633932306336353766396630 +64363066303830616536373936313666643435656639656538646334366433613363313165303832 +3335313031323032330a656464343664656235353536386630633331343633353263623637356264 +63636139653566366263333539656632316433333266666263356661663233613166616661336232 +62366137326264656531396434663766396233623562333562326633646666356465336435643338 +37316431343033316662663438376630343837616137313564376638313966363838666434373538 +63613834316439353033613235346634383834343132393934636365386339353039653663323161 +62613237393232643738333534396263633963653938336638376436663439366263373063303861 +61316132663638656436623862643834366236623130653864643766663631336361373236323264 +34383362393764643165343832643835336534653433313436363631363161313436363463616435 +30323534333265393030343032346531643330303362383562636566623964633266666261343533 +38643037333166316663303363633464643531633930313834646134303362643463653564333062 +39356431363937646637613439616136626231626366643364633637383338613232656165643832 +61373563653734313035633263616665373630623130396537616366636134613731376637363966 +63323430303466663433333861666435303435353135396238343538363136353035646435623136 +38636436623638613135313736366435323230343035383765636463666533303738633063656661 +36303831386662366365616434373833306465383562323930613236663031313165326633356434 +39313562623338376635326134636438323963646437353933336238346364623830333364326230 +61316661636163366361613464376165373666356236623135396631326637643735616461346264 +37333433343332633136663530616237356639396535653334386138643135643032383337333366 +33393335656139393538386266386432306532346638623530336239636264316236336637303636 +35363938643730313539323163366462373764613536346262666133383131626663653965343366 +31646162303832376362663432313965333433323766343162383936646430366664313437396534 +32373164343533613239393062636338643064636432323036666366653137633538363434376563 +63353139666462366235666436613865323632306663323339636664386437383561336435616533 +33353337666231663739316435663133616137653039363637346435336234373234313266303765 +64323030353735353764666466313362626637643064386138326130363064653764633137646538 +38343566643831383361336337616535393134366333306431373864633930383037343238326532 +38626238626465663561313661366362623238323934363639643033646133396661366533346433 +36613931376234646464333865376339396362616463313637353565383061626431633665396139 +66646333313438353432613435363231663265356535663634613066636161356466336334383835 +32383738313036643739623838656662366635396237393834373062383664336537373638313965 +38343836306635626632626266666334396364653331363932316562373632353434323337363034 +31306539383838663737363639316137386234343538633561386633366632393236373662636238 +66323261643535376533666539626266663132386435366230646263396138623065386137646638 +39616539363165613630336664373966643064303834646437616131633937653335383565376462 +30653162313730366431356533633166353631393732386532666632623666646230666166303464 +66326663353037373164316332386139313135363962623030643635336464356562383530313537 +63363964343738356632653531373930656431343763343435623364303430323866303730366263 +38346236616362356333326536366330653032323733313230363137656465653537396539613939 +30646463303438316433306266633738383837376233653833313565336633386236306363613739 +65616162666662636531396266396561373832366233623362356330353831393866366662656235 +38346533396334326632666663326161653832373366313633373965636663616334336466383465 +35363661333562393331366166663036383562666532623136333462336663353836333464316230 +32336638336132663936656361646431646434323634336535636661643164343737343861333430 +66393565623761643930323132643232653265346130303832613730373862636134616261623139 +63363062383032393738363538393664333436376436353764346133666163656431346331656266 +32323530316438396338633938306162383661353638373566386230323230343837333930383837 +37343265383734313966613765646165316133373665656631636139613331346530643466346166 +62613739326563396662393835353630653538343235363863326466353065353936343262303435 +65336133616130373366653336613832326363366532326465366238383061653832393739313235 +34663431316261666634356234623963373636313132663739333838373634663331633337653039 +36396539386636323936373631303339356339393435623731303763653236323766616464303938 +37633034343734633063653364393765396466366131613435343030393638306563373263646263 +35396333636464366364313435643630663638633333333537373930323830373831323432383061 +62313932653737323134663661643531333862646638303639393835646536653830353735636130 +62653631333530633833323832353831316366646162373235363232333231643631646562316163 +66613835666339643237646330343262366235336435343130313566656531356233393836613035 +38366639323334663731623334616234356562373761613535386230336263313337353662656561 +62366635333061643561653530633034366631333230613633656632633532326233663366636263 +66613632666164353830326361373163663666643835336164653135623563306439643530353737 +34323736396361336466353233366461663437616265396361323237663861396136353864383133 +66343937363164396364326665656261343662386631353532636562613332653636396539376161 +61393430336662323563366531396233393635626432353037373865643039323635663062666236 +39333265363363393231303939386338386532393766303330303562393230393865633263623832 +33643531383633346362626533646130383262653032386563396238643761363130313065633231 +39643039643264636262346366663566343966333963343932633034363861383830343731623230 +34396362626364306636666138623861653161646564393963316263663866636338396530636237 +62643561323236383863623432326563306533363566623963353537346430313835313338383165 +31626636353765356361383639643337303233343064346430346639303663306565353565323162 +65306533656466373863333637663039353537326536623962303866363233323939386561343137 +37373634323263333863333339366434623436643666316232353136613230613438313662666266 +61643432656332323661323732363537383537623661626530353364663237643463356266323261 +61333865336461363133633032343930313031383934336465363036303636393332336265373539 +66303434333232316562363064646330636565333737663238373938326135623837616664643330 +65323539653464346334613462323162666534353530383532623565336234643434303337326233 +66343932623036633466393637623333383263616434313437663231623036343466663232333262 +64396333346364663636643831323538373633376239396165353166626233656234343632623337 +33393663653139663937383832333032316130393666663261663631653435346566363363646164 +37353064633031383536626163383834366637643561653832633466346466663732383961626130 +39653164643732336362326531626130346432623932656535353964393837363961656230343064 +38613335303838306164663932396236376366343162306437323030626132373464363464376363 +31346663623563366535353165343736316336323437623430313233323435396130386662666138 +39363732373266666261643830346237316330633438323430333437393537626264343839366637 +61636537323663663130323334633364633738353161373239393434336461383062346231613932 +33313461646233356464313836626230613731626536653137366465613532623762313533343062 +61306137613336653162383134306464616462623331303836306339326330653362393732356538 +35353731373733353634356666353362336632333166383262663565383636613231643131323666 +62663134316462613665386263386461346535663839633066653132653264373963353339376662 +63363834343266663066343163366563373766623037616566663533303035666665653162316536 +32346437633964666464653239373933636363366334646634313563386633643336333362356464 +39613732376439393333643066326166623861653962656630396336306262663766323061663664 +33393234666663336533626331396161646337383032653532386534333762613335333035353237 +37333864396365633734623963663539653565613838643633343032386239613936633631646238 +37396262313834376166626237656530353232333635633135306638363434653866386463333630 +64376336383730663365656365613235373661363962646361333463613836393566326331623438 +35633662343937393739623031393437616434386462323033626563383164613838373066346338 +34616262363563373361353136613761633637366465663339373863316539623838343139303363 +31616232383038363539346465353734323465303835353465326533386437333034643466373338 +36343362326237383432616634363338383963303530613538303731613132633637633530366565 +63393466613665353435643734323932316361373032653933393230666137633130653637343236 +35393864633662346266383935393066633137363766343431616238613766613132366638653638 +61353833613834666132613834393237333663396666326639353235353563323034613831306330 +35376661393164333461646362313562316638383930353439333865353930373237373230393830 +33373134336238653863663532656632303638343134613566336332616537333334396236313135 +37633266383762646637303533653336303666326431656230366630616261343365303238663762 +31383264616134346432313033623461376163353534346439346639356637353861373762663838 +36386535623162353232616164326264653531343633343766306135363364623164363531356131 +30666565316337663162343032623662663939613265346330363431323938326532376161373034 +64353731653734636438393465663963393137353864643366326630666630643635626637333536 +66666139343134326137633664663830333465326137326438663833353731623831613864626136 +31626363396662333832663931383031616637393639616434383364393930383566323336663234 +30643162393130343134653736313830616234353064663736333738623339313761303036316639 +61326130343664633732656166383337353234373531346561363130663962313638383231353437 +31316239643263653230613538316337383265343735313531376237333330386362613539313665 +37333431383062336365636231373638313336363135626538393937613865353433393639666663 +38653431363666303632636131643961373437366565636363323330323239656530316138316666 +34333232636534643535346131633539646663636261663738333138396363336332666262353830 +34393534653439663239343162396236643033316561343939663239366233643830363463656163 +33363662386266396462386537316233356136396662616536383263643863663133623564656261 +32393437393438663563613530656439663231346563363063646163366335633637383637653631 +36613663393865373337313735623230613939396539623462663938633432376334306166386562 +31396333653862336336643662323266333566363764336363383335303731313930613237646436 +35336231653263363630663136613332356530393866333538633231363033643738373664623232 +32366261333634386636353336303537353362376463616630306235313436646264653733343933 +37316130326232653335323863326133383664663166653362613661346262626436646634633566 +64633530666434326439303633643030313435633831623364643566343366396262313063656366 +64316531663538336636636533376332323763663638643239356263306662613132633233623138 +64623461623865343736333661303934656630353535643465616635303264633465353438613532 +30313938333266383233376431623761373934373765363739336135346232313032373239313734 +33313838393136656131333465653833346164373735633762643363663337386537326665653736 +30386439316266636334303261363734323965363864313030613539363764663935656233323966 +33373731666537326266623264636536633433373663313233613330623337336331353833336565 +34356438666135633138333538386266323166393737366230313136306263383935636565623037 +64333936356362666665623263636535666235356233616664663335333833613139356337383337 +35386137383461313963333763376339646138646161383735616563306134633863313363633730 +30656538663837666338646636333261366566353764623439333765396365633361303865303439 +39353836326664386664393662376138303161653765663637643566633239396265633730323830 +36616639396232346561393533623664313437643635353531373232613231346130303966303433 +37356437353166643461663265313334343334623038343139383066343362653230343331663864 +62356335366563356536343165616362326536653233353537303739373837393132333036633863 +65363732313933666237333933323763616634396366613766303931373161313132363135393663 +31383232646562323832323531303639383838633734336563666464653833666466313532346362 +64656463326462623562613139666532313866343834663239653037306331346431613135363431 +37623063366163646435313939663562656436653565383961333362626663373533343836626436 +61343461343235383861366163303764313437646263636335643165303064643663633739346131 +33383536306234356439646133653634653336633138626132366630323732613462306166313863 +31633931363035653461623965303964626339666633636137393566343339323833653764386535 +33623834616164663261343763373062623335383966323932323833393165363630316131613930 +36336132663537333266323865313439656435306137303339363964633462356363613539643061 +34323764306363366536353738303665303063333939363165666365303561383739303566313633 +38633833633362313165396132646337383663333363393335333234303362646232616534303130 +37633638356636646231656335333532353138653630633831643231303564383166626237323337 +37663639643233303539303735366362663362663465356636623439336130356234353931363336 +37653735633034316539643433643836393031343162633764643738326234396535643063366430 +65333966333961303162376531616633333933336165313732303464646637623066343737303034 +65663533643232376164333563333464346136363237313731663030613263653631383836363238 +61613037623765383265643934336363646564313937336532626231343964393764386561376236 +61303832323336663962313030353838386338343738636431363438373166303830353638663761 +33626261363263386163636336336133636565303532303466623538616335623564636630313835 +36323336326437626431643964646265303134376661396164643739663435356161306239346265 +32616334626363653064316232326339663030653065373634626436626665306338343663366136 +64343634616536353965626666323232396137643462626366363962373130313638303961323334 +65376139636638636333653337333465333463666463366661316539363131373364653734613065 +34633861653239393864363031343835396161633231303666656533376462393063353839346363 +32373633353962663938623930356162313032643830363436656366326334396364336261626637 +34303835363662356563633764326439373837346337646131303930646638646531313537373666 +30643536383532623231646165303330393330383335616633646666386463326130663432363236 +65346664326165383264383362373333666338316438396439623063613636663836633330353132 +65316233366166633733663765366566373437326536386461376338616438653335303961336430 +35336230653962363462323863353835633033313235313561356261613263353433336535366230 +36326335646536663965343764643438326133313463333336656636373132643637653263303037 +65303031366138663863393464656436643964656462383462313562306131616139623738393636 +63363935363931616661626662653863613866306537633533663465313961336137373437336363 +63316132336230316661316233343538383661396338393735383136316639663033393538623131 +35643063303239623339313639623962376364363730613963363631336135383066313537633865 +66643838636231363832646532303065313863316134343066613161646336336237646530343432 +39346139373030646631393635623736343739613362666433303765366431653436393965643739 +38363865313830663736383866653530383739363063373930383264633763353430653434356664 +35643162616466343837653130373566636530393137653165346630643865336634333662666430 +38626333353232386438653331613737376365326535346563306164386434336132363865346263 +36633030373235353732363939316238663935393537666535393661323062396635353435303636 +37393631633864316262306630356566366465363836376361616237393062393065663735653966 +30363534646331386330613030376238356631373466333931363030643730326238356265663162 +62363563613564303731353231633637393133326162653638633430396561653236326362376238 +65633633626262323033386262396433323332303230373838363933336564386165373534643961 +38346130646466633265346262623639653135636332396332353562373233373138656433646663 +32613832393334623232663336313538656163303738366632656638376134646565626636343536 +33366563313263343936353634303238316437646664316563386130306639376363326362303330 +31633234623638663566336132383738393730306436666134333637336434316332353962663837 +38656432623063333863626136316261636365656462323239613564643163326336306564306166 +64336135366634643064373234613336623438666539646565383835396166306337626136623566 +32663032633739653435313539353434663865613636316437393034333830623739633162313864 +30353031636536383239376432646530633538623563646234363462386635323765656337646430 +38323337363632343236303834623535613432363366306564656539653937646262343132626564 +32366136373164303639373862666161373532643934366165643362326534373331393537376339 +66386561626531373438616662343861396362643431633462306562343964636638643338363963 +66623231343766303364323632613434373864353566643962643564323465303566313235313164 +38613438643461343631316265386432323762313132353930396234633132306433353064333733 +36373430383930646632356331306134303533643865363961336335323565376330313666393731 +64383430626536353932343337663930333735356634666534316438303334356131313461366239 +61353539343138303332353530396137356265356130656332366337653139396434393533633762 +31616139613139656631616332376339623630636235333434373930633938323361386334383730 +32666139653663643938333431656336633132313035616232613037313265646261316464336162 +33633731303539306530616561353533643132366165336465326430313565386465313935383036 +33393835323139313732396339313336623736363664666561353764633266336566356230643263 +62636232623233333630343364356361666564396236396562356631353737663437386634353138 +35666134373133656339303634646539663131326631666232386164613034646336353266323839 +32343361353932333431303132366536356230323566623135626662393664633435613834363433 +33313963656230653137323464343536313835393938343830656639663932363763653432386661 +65323164656663643438666539636366643136653433313238653835336166366336646236663262 +62653532306534313966313935656336303837323838656134323035626164373533333065343566 +38666439613837636135323636373762363432336134323761383930643735376562333565353862 +34343666333838663133306265383338363333313431613564663033353436663262633536373465 +34633265613035326434626563373164663365626561346334643534376565333463393534643638 +64653962383233303266333566326237616234346565636366373264356337623832383034316333 +62613135343438306235653863326532626466373364616435383466653032633038323435316566 +62636431616361333337623739653631653831333135353266383931663836663764343063613863 +66323338303433613437643338653863633862313133303263326663343737346638373333663436 +61383865313232633864306332323165653366623337323162316431386161636535623862356561 +38393438376164353163313334613965663635376563313661373530383334646465333236613035 +38653265363364663365653535306661646136343332393039383130636361643537363264633663 +33333535623637373732303738316438323461376366316238613131666332393265333031336231 +32323030343764636565666539616339643435303164623535336239663337633735323461366466 +66636663396266333965643935636662623233623861616439663564613430333235343834313965 +30356538376665363232666634393331393636363064613363333366336664303434373331336133 +36366337366534326634303238643334626462663336343834316632303333636230333961643132 +61363032616563346530323763346561383330373137343638363334313331623563363336366635 +65323234373034393931636665343831636639363038643732366432626434373539313538363733 +65653734356131313066633436363331376165316234373632613830633931346235633736656335 +37633963376262336231636566336266366163316265633631396163343762323866656639663430 +33303032383332313231313364633535633861353531633433356363613239623864 From c3227c4849eb90d3712d2e9074623ca4adb3156e Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Wed, 8 Dec 2021 16:03:13 -0500 Subject: [PATCH 02/46] Add `hosp_claims` automation rig - Adds new Python and shell scripts to automate `hosp_claims` indicator. - Adds a symlink for `secrets.py`. --- ansible/vault.yaml | 467 +++++++++--------- claims_hosp/HospClaims/automate/README.md | 12 + .../HospClaims/automate/agg_claims_drops.py | 116 +++++ .../automate/download_claims_ftp_files.py | 110 +++++ .../HospClaims/automate/ftp_to_covidcast.py | 62 +++ .../automate/get_latest_claims_name.py | 42 ++ .../automate/hosp_claims_master_script.sh | 72 +++ .../automate/hosp_claims_regen_script.sh | 68 +++ .../HospClaims/automate/regen_old_issue.py | 47 ++ .../HospClaims/automate/sanity_checks.py | 289 +++++++++++ claims_hosp/HospClaims/automate/secrets.py | 1 + .../HospClaims/automate/update_json.py | 60 +++ 12 files changed, 1113 insertions(+), 233 deletions(-) create mode 100644 claims_hosp/HospClaims/automate/README.md create mode 100644 claims_hosp/HospClaims/automate/agg_claims_drops.py create mode 100644 claims_hosp/HospClaims/automate/download_claims_ftp_files.py create mode 100644 claims_hosp/HospClaims/automate/ftp_to_covidcast.py create mode 100644 claims_hosp/HospClaims/automate/get_latest_claims_name.py create mode 100755 claims_hosp/HospClaims/automate/hosp_claims_master_script.sh create mode 100755 claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh create mode 100755 claims_hosp/HospClaims/automate/regen_old_issue.py create mode 100644 claims_hosp/HospClaims/automate/sanity_checks.py create mode 120000 claims_hosp/HospClaims/automate/secrets.py create mode 100644 claims_hosp/HospClaims/automate/update_json.py diff --git a/ansible/vault.yaml b/ansible/vault.yaml index ed2916927..914b39f87 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,234 +1,235 @@ $ANSIBLE_VAULT;1.1;AES256 -65363733383463363636386638303566363861646436363666656665663263653838373739383036 -6432353762653138336663653530633764323565383238370a666666633932306336353766396630 -64363066303830616536373936313666643435656639656538646334366433613363313165303832 -3335313031323032330a656464343664656235353536386630633331343633353263623637356264 -63636139653566366263333539656632316433333266666263356661663233613166616661336232 -62366137326264656531396434663766396233623562333562326633646666356465336435643338 -37316431343033316662663438376630343837616137313564376638313966363838666434373538 -63613834316439353033613235346634383834343132393934636365386339353039653663323161 -62613237393232643738333534396263633963653938336638376436663439366263373063303861 -61316132663638656436623862643834366236623130653864643766663631336361373236323264 -34383362393764643165343832643835336534653433313436363631363161313436363463616435 -30323534333265393030343032346531643330303362383562636566623964633266666261343533 -38643037333166316663303363633464643531633930313834646134303362643463653564333062 -39356431363937646637613439616136626231626366643364633637383338613232656165643832 -61373563653734313035633263616665373630623130396537616366636134613731376637363966 -63323430303466663433333861666435303435353135396238343538363136353035646435623136 -38636436623638613135313736366435323230343035383765636463666533303738633063656661 -36303831386662366365616434373833306465383562323930613236663031313165326633356434 -39313562623338376635326134636438323963646437353933336238346364623830333364326230 -61316661636163366361613464376165373666356236623135396631326637643735616461346264 -37333433343332633136663530616237356639396535653334386138643135643032383337333366 -33393335656139393538386266386432306532346638623530336239636264316236336637303636 -35363938643730313539323163366462373764613536346262666133383131626663653965343366 -31646162303832376362663432313965333433323766343162383936646430366664313437396534 -32373164343533613239393062636338643064636432323036666366653137633538363434376563 -63353139666462366235666436613865323632306663323339636664386437383561336435616533 -33353337666231663739316435663133616137653039363637346435336234373234313266303765 -64323030353735353764666466313362626637643064386138326130363064653764633137646538 -38343566643831383361336337616535393134366333306431373864633930383037343238326532 -38626238626465663561313661366362623238323934363639643033646133396661366533346433 -36613931376234646464333865376339396362616463313637353565383061626431633665396139 -66646333313438353432613435363231663265356535663634613066636161356466336334383835 -32383738313036643739623838656662366635396237393834373062383664336537373638313965 -38343836306635626632626266666334396364653331363932316562373632353434323337363034 -31306539383838663737363639316137386234343538633561386633366632393236373662636238 -66323261643535376533666539626266663132386435366230646263396138623065386137646638 -39616539363165613630336664373966643064303834646437616131633937653335383565376462 -30653162313730366431356533633166353631393732386532666632623666646230666166303464 -66326663353037373164316332386139313135363962623030643635336464356562383530313537 -63363964343738356632653531373930656431343763343435623364303430323866303730366263 -38346236616362356333326536366330653032323733313230363137656465653537396539613939 -30646463303438316433306266633738383837376233653833313565336633386236306363613739 -65616162666662636531396266396561373832366233623362356330353831393866366662656235 -38346533396334326632666663326161653832373366313633373965636663616334336466383465 -35363661333562393331366166663036383562666532623136333462336663353836333464316230 -32336638336132663936656361646431646434323634336535636661643164343737343861333430 -66393565623761643930323132643232653265346130303832613730373862636134616261623139 -63363062383032393738363538393664333436376436353764346133666163656431346331656266 -32323530316438396338633938306162383661353638373566386230323230343837333930383837 -37343265383734313966613765646165316133373665656631636139613331346530643466346166 -62613739326563396662393835353630653538343235363863326466353065353936343262303435 -65336133616130373366653336613832326363366532326465366238383061653832393739313235 -34663431316261666634356234623963373636313132663739333838373634663331633337653039 -36396539386636323936373631303339356339393435623731303763653236323766616464303938 -37633034343734633063653364393765396466366131613435343030393638306563373263646263 -35396333636464366364313435643630663638633333333537373930323830373831323432383061 -62313932653737323134663661643531333862646638303639393835646536653830353735636130 -62653631333530633833323832353831316366646162373235363232333231643631646562316163 -66613835666339643237646330343262366235336435343130313566656531356233393836613035 -38366639323334663731623334616234356562373761613535386230336263313337353662656561 -62366635333061643561653530633034366631333230613633656632633532326233663366636263 -66613632666164353830326361373163663666643835336164653135623563306439643530353737 -34323736396361336466353233366461663437616265396361323237663861396136353864383133 -66343937363164396364326665656261343662386631353532636562613332653636396539376161 -61393430336662323563366531396233393635626432353037373865643039323635663062666236 -39333265363363393231303939386338386532393766303330303562393230393865633263623832 -33643531383633346362626533646130383262653032386563396238643761363130313065633231 -39643039643264636262346366663566343966333963343932633034363861383830343731623230 -34396362626364306636666138623861653161646564393963316263663866636338396530636237 -62643561323236383863623432326563306533363566623963353537346430313835313338383165 -31626636353765356361383639643337303233343064346430346639303663306565353565323162 -65306533656466373863333637663039353537326536623962303866363233323939386561343137 -37373634323263333863333339366434623436643666316232353136613230613438313662666266 -61643432656332323661323732363537383537623661626530353364663237643463356266323261 -61333865336461363133633032343930313031383934336465363036303636393332336265373539 -66303434333232316562363064646330636565333737663238373938326135623837616664643330 -65323539653464346334613462323162666534353530383532623565336234643434303337326233 -66343932623036633466393637623333383263616434313437663231623036343466663232333262 -64396333346364663636643831323538373633376239396165353166626233656234343632623337 -33393663653139663937383832333032316130393666663261663631653435346566363363646164 -37353064633031383536626163383834366637643561653832633466346466663732383961626130 -39653164643732336362326531626130346432623932656535353964393837363961656230343064 -38613335303838306164663932396236376366343162306437323030626132373464363464376363 -31346663623563366535353165343736316336323437623430313233323435396130386662666138 -39363732373266666261643830346237316330633438323430333437393537626264343839366637 -61636537323663663130323334633364633738353161373239393434336461383062346231613932 -33313461646233356464313836626230613731626536653137366465613532623762313533343062 -61306137613336653162383134306464616462623331303836306339326330653362393732356538 -35353731373733353634356666353362336632333166383262663565383636613231643131323666 -62663134316462613665386263386461346535663839633066653132653264373963353339376662 -63363834343266663066343163366563373766623037616566663533303035666665653162316536 -32346437633964666464653239373933636363366334646634313563386633643336333362356464 -39613732376439393333643066326166623861653962656630396336306262663766323061663664 -33393234666663336533626331396161646337383032653532386534333762613335333035353237 -37333864396365633734623963663539653565613838643633343032386239613936633631646238 -37396262313834376166626237656530353232333635633135306638363434653866386463333630 -64376336383730663365656365613235373661363962646361333463613836393566326331623438 -35633662343937393739623031393437616434386462323033626563383164613838373066346338 -34616262363563373361353136613761633637366465663339373863316539623838343139303363 -31616232383038363539346465353734323465303835353465326533386437333034643466373338 -36343362326237383432616634363338383963303530613538303731613132633637633530366565 -63393466613665353435643734323932316361373032653933393230666137633130653637343236 -35393864633662346266383935393066633137363766343431616238613766613132366638653638 -61353833613834666132613834393237333663396666326639353235353563323034613831306330 -35376661393164333461646362313562316638383930353439333865353930373237373230393830 -33373134336238653863663532656632303638343134613566336332616537333334396236313135 -37633266383762646637303533653336303666326431656230366630616261343365303238663762 -31383264616134346432313033623461376163353534346439346639356637353861373762663838 -36386535623162353232616164326264653531343633343766306135363364623164363531356131 -30666565316337663162343032623662663939613265346330363431323938326532376161373034 -64353731653734636438393465663963393137353864643366326630666630643635626637333536 -66666139343134326137633664663830333465326137326438663833353731623831613864626136 -31626363396662333832663931383031616637393639616434383364393930383566323336663234 -30643162393130343134653736313830616234353064663736333738623339313761303036316639 -61326130343664633732656166383337353234373531346561363130663962313638383231353437 -31316239643263653230613538316337383265343735313531376237333330386362613539313665 -37333431383062336365636231373638313336363135626538393937613865353433393639666663 -38653431363666303632636131643961373437366565636363323330323239656530316138316666 -34333232636534643535346131633539646663636261663738333138396363336332666262353830 -34393534653439663239343162396236643033316561343939663239366233643830363463656163 -33363662386266396462386537316233356136396662616536383263643863663133623564656261 -32393437393438663563613530656439663231346563363063646163366335633637383637653631 -36613663393865373337313735623230613939396539623462663938633432376334306166386562 -31396333653862336336643662323266333566363764336363383335303731313930613237646436 -35336231653263363630663136613332356530393866333538633231363033643738373664623232 -32366261333634386636353336303537353362376463616630306235313436646264653733343933 -37316130326232653335323863326133383664663166653362613661346262626436646634633566 -64633530666434326439303633643030313435633831623364643566343366396262313063656366 -64316531663538336636636533376332323763663638643239356263306662613132633233623138 -64623461623865343736333661303934656630353535643465616635303264633465353438613532 -30313938333266383233376431623761373934373765363739336135346232313032373239313734 -33313838393136656131333465653833346164373735633762643363663337386537326665653736 -30386439316266636334303261363734323965363864313030613539363764663935656233323966 -33373731666537326266623264636536633433373663313233613330623337336331353833336565 -34356438666135633138333538386266323166393737366230313136306263383935636565623037 -64333936356362666665623263636535666235356233616664663335333833613139356337383337 -35386137383461313963333763376339646138646161383735616563306134633863313363633730 -30656538663837666338646636333261366566353764623439333765396365633361303865303439 -39353836326664386664393662376138303161653765663637643566633239396265633730323830 -36616639396232346561393533623664313437643635353531373232613231346130303966303433 -37356437353166643461663265313334343334623038343139383066343362653230343331663864 -62356335366563356536343165616362326536653233353537303739373837393132333036633863 -65363732313933666237333933323763616634396366613766303931373161313132363135393663 -31383232646562323832323531303639383838633734336563666464653833666466313532346362 -64656463326462623562613139666532313866343834663239653037306331346431613135363431 -37623063366163646435313939663562656436653565383961333362626663373533343836626436 -61343461343235383861366163303764313437646263636335643165303064643663633739346131 -33383536306234356439646133653634653336633138626132366630323732613462306166313863 -31633931363035653461623965303964626339666633636137393566343339323833653764386535 -33623834616164663261343763373062623335383966323932323833393165363630316131613930 -36336132663537333266323865313439656435306137303339363964633462356363613539643061 -34323764306363366536353738303665303063333939363165666365303561383739303566313633 -38633833633362313165396132646337383663333363393335333234303362646232616534303130 -37633638356636646231656335333532353138653630633831643231303564383166626237323337 -37663639643233303539303735366362663362663465356636623439336130356234353931363336 -37653735633034316539643433643836393031343162633764643738326234396535643063366430 -65333966333961303162376531616633333933336165313732303464646637623066343737303034 -65663533643232376164333563333464346136363237313731663030613263653631383836363238 -61613037623765383265643934336363646564313937336532626231343964393764386561376236 -61303832323336663962313030353838386338343738636431363438373166303830353638663761 -33626261363263386163636336336133636565303532303466623538616335623564636630313835 -36323336326437626431643964646265303134376661396164643739663435356161306239346265 -32616334626363653064316232326339663030653065373634626436626665306338343663366136 -64343634616536353965626666323232396137643462626366363962373130313638303961323334 -65376139636638636333653337333465333463666463366661316539363131373364653734613065 -34633861653239393864363031343835396161633231303666656533376462393063353839346363 -32373633353962663938623930356162313032643830363436656366326334396364336261626637 -34303835363662356563633764326439373837346337646131303930646638646531313537373666 -30643536383532623231646165303330393330383335616633646666386463326130663432363236 -65346664326165383264383362373333666338316438396439623063613636663836633330353132 -65316233366166633733663765366566373437326536386461376338616438653335303961336430 -35336230653962363462323863353835633033313235313561356261613263353433336535366230 -36326335646536663965343764643438326133313463333336656636373132643637653263303037 -65303031366138663863393464656436643964656462383462313562306131616139623738393636 -63363935363931616661626662653863613866306537633533663465313961336137373437336363 -63316132336230316661316233343538383661396338393735383136316639663033393538623131 -35643063303239623339313639623962376364363730613963363631336135383066313537633865 -66643838636231363832646532303065313863316134343066613161646336336237646530343432 -39346139373030646631393635623736343739613362666433303765366431653436393965643739 -38363865313830663736383866653530383739363063373930383264633763353430653434356664 -35643162616466343837653130373566636530393137653165346630643865336634333662666430 -38626333353232386438653331613737376365326535346563306164386434336132363865346263 -36633030373235353732363939316238663935393537666535393661323062396635353435303636 -37393631633864316262306630356566366465363836376361616237393062393065663735653966 -30363534646331386330613030376238356631373466333931363030643730326238356265663162 -62363563613564303731353231633637393133326162653638633430396561653236326362376238 -65633633626262323033386262396433323332303230373838363933336564386165373534643961 -38346130646466633265346262623639653135636332396332353562373233373138656433646663 -32613832393334623232663336313538656163303738366632656638376134646565626636343536 -33366563313263343936353634303238316437646664316563386130306639376363326362303330 -31633234623638663566336132383738393730306436666134333637336434316332353962663837 -38656432623063333863626136316261636365656462323239613564643163326336306564306166 -64336135366634643064373234613336623438666539646565383835396166306337626136623566 -32663032633739653435313539353434663865613636316437393034333830623739633162313864 -30353031636536383239376432646530633538623563646234363462386635323765656337646430 -38323337363632343236303834623535613432363366306564656539653937646262343132626564 -32366136373164303639373862666161373532643934366165643362326534373331393537376339 -66386561626531373438616662343861396362643431633462306562343964636638643338363963 -66623231343766303364323632613434373864353566643962643564323465303566313235313164 -38613438643461343631316265386432323762313132353930396234633132306433353064333733 -36373430383930646632356331306134303533643865363961336335323565376330313666393731 -64383430626536353932343337663930333735356634666534316438303334356131313461366239 -61353539343138303332353530396137356265356130656332366337653139396434393533633762 -31616139613139656631616332376339623630636235333434373930633938323361386334383730 -32666139653663643938333431656336633132313035616232613037313265646261316464336162 -33633731303539306530616561353533643132366165336465326430313565386465313935383036 -33393835323139313732396339313336623736363664666561353764633266336566356230643263 -62636232623233333630343364356361666564396236396562356631353737663437386634353138 -35666134373133656339303634646539663131326631666232386164613034646336353266323839 -32343361353932333431303132366536356230323566623135626662393664633435613834363433 -33313963656230653137323464343536313835393938343830656639663932363763653432386661 -65323164656663643438666539636366643136653433313238653835336166366336646236663262 -62653532306534313966313935656336303837323838656134323035626164373533333065343566 -38666439613837636135323636373762363432336134323761383930643735376562333565353862 -34343666333838663133306265383338363333313431613564663033353436663262633536373465 -34633265613035326434626563373164663365626561346334643534376565333463393534643638 -64653962383233303266333566326237616234346565636366373264356337623832383034316333 -62613135343438306235653863326532626466373364616435383466653032633038323435316566 -62636431616361333337623739653631653831333135353266383931663836663764343063613863 -66323338303433613437643338653863633862313133303263326663343737346638373333663436 -61383865313232633864306332323165653366623337323162316431386161636535623862356561 -38393438376164353163313334613965663635376563313661373530383334646465333236613035 -38653265363364663365653535306661646136343332393039383130636361643537363264633663 -33333535623637373732303738316438323461376366316238613131666332393265333031336231 -32323030343764636565666539616339643435303164623535336239663337633735323461366466 -66636663396266333965643935636662623233623861616439663564613430333235343834313965 -30356538376665363232666634393331393636363064613363333366336664303434373331336133 -36366337366534326634303238643334626462663336343834316632303333636230333961643132 -61363032616563346530323763346561383330373137343638363334313331623563363336366635 -65323234373034393931636665343831636639363038643732366432626434373539313538363733 -65653734356131313066633436363331376165316234373632613830633931346235633736656335 -37633963376262336231636566336266366163316265633631396163343762323866656639663430 -33303032383332313231313364633535633861353531633433356363613239623864 +61366430313636613638303831613735613235663034316436313361326233653736303261646162 +3039323465316136313336623130323063343439356463640a646535383133343035613931363736 +62663830633537376231336633346538346564653461613964346637353937306536383565313466 +3231633437303435640a306539313532653037653461343136396266386263303939373462326635 +34656132313633636230303666353630303666353261306262336330623133636534653466646436 +65333465343761636239626161643031313663303232303532343237646139386439643333343234 +34636261386339356135343936336362353966633330346166373763343364343061383231316138 +37653461303161383032323330663363343234383363383638363665653030346434613966353833 +63393735663632616463343463646563633131343765363333306164363062303332353838353465 +65623663363731616161366334313662626165633264343631373737396431663963336635633132 +35633366393037326662643366323261633935366137633834393632326633626533626435323666 +61623761616536386237326436623733646466653761316365353336356262363461616334643334 +35353636343033656366623762396237386131623164656362663730656237306430613362656630 +61393335393435346366613432616533393539333266333232366264386263303763336662646238 +33643064353161366161353035643035336436353335316261333531363735326339326262353961 +31643966306161303037323238323538343434666637623530303732356463303764373034626237 +32643934613161353431623761616339616366313264333534393234376630646433306134333563 +38663366353530303764636662616233663262313230613364393632386562646533646331383361 +30396136336461313134626138333034383235393835393462313030613233386338346436386538 +62363530383366366333343738653061323233333730616262336630363533323431303339623330 +33346533613336663362386363626438336365343231363235366138646235626235613231373739 +38613236323237626632363465633433313037376533303832626462353932626661666365643866 +65316234363838636263303432303366363264653537383137366332623034653736626636346463 +38383462306530336665656364336666373065636430646534333761656365653934613838633933 +36616330353339356335343161343463616438373663386366663262373866326333633964346135 +37333463626232306636366262626261616332653763653266386232666433303764623637633731 +35666236346565313666373736386236376136303334313163623862313336316663393161323635 +31343031393736633537363234383163383936613134653837393134633466313661653761333133 +61383538623933363630393861313738326461326663633235646335333132373238313166646534 +63383233303362373765303162363861363866386362626532303334633339333730356261353063 +66323166306632313333343862356465613437633662343734373465333337346536373937646438 +31363038373634376239666462313736336231663836616531363534653035623738303764616533 +36313965303862623563353537366566356235376437316561633931343963393631613631643764 +37313331306634343366316234316336643965623739643864633866386364313064393061373635 +35343832383363663231616432626130616664333564656661326662316339663761663564363131 +37663931343833643866633835376538653235336433633364356534666432623730636332366364 +65313532383830666563313761663030353663376139386138646663616232623139623332356264 +37343363626464346534386534326261316337363730666162313061393434656263356438616633 +38393039316134616137376436623439633565376366386237353463656366383033336365653964 +61636331346264396339393437663632643634323133666566313864663036303662333064303963 +62376533663932653939663338376161333039383633623931306431656461376238306339633230 +38323566303464386165626332333037386265303361323134366236346637633236303234303438 +37313632396437626363666338383965393438653063393034316432633932623137646539633832 +31303038623161323233616365373265623739306238316433343230303137613965393539306666 +62666466303039666632313139376536653938656430356163613137383163666563646138336439 +66353232613235623130323631653131323562383430323832643431653939626563396332613734 +33316665666638613030313235373937383965336138366565643533613737303064656439616665 +34623863623538626462386338316336393739643166366531303266626661623162393639653235 +63613466393133653564333033336333343534333063633536306430366532323338313762353831 +38626233376331303334646130393230333331333330323435303962346666616165343234666562 +61636530336339326230316638373965636232613937646465303239356438616336343137383063 +64653538326235373262656432613865613163656163306138316531396166366138613937623339 +39333965313337366338666538353134326437353966373931613437373362626561303336343239 +30323932383265366161336566343738313866356630643161656266636332353261346461353035 +31383333636661343262636165383336653262333336656666366664363030353034616535633832 +65386436636535393562386135653138313466643963663166666264663932623064333264396563 +34303761623663623834386330303435306431333261373337353532663364306634396537353061 +32376535396465626562346665313038346133653561336633306561393638656236336137616236 +30333961363363643833343334616434643039653438666238373231616366663339663437656331 +30303765316434633562636632353066356665373464663438653931653762356564653932656435 +66653363333032303931653233303262656530656432666564323665316332383239373430303536 +30633061396134653666633966363438356334353033346461366438343139363537313661393764 +34633531626365623361353238306430666661666164653864393237636335313561663732383065 +35353965613264666266306566313264333339626436653938393665343162303739623562306661 +32313035333132366463616234326137353834326637306263393132393333633938376531623633 +65663463383165633334376562323936643934633835376362336130643438626134633765336664 +34663366343161336234333737363238666132316230653461346166313961316263373034643263 +33393337633033386562653862653362363135343937633362616130366364333634363561326666 +39366435323931313238646662393934303236613839336438373233393033656630323166663039 +62643435666432393733623135316239343961373665366436383035643833343663383337333633 +35616461353566356430646164613166346539643137373062653534303538326135366638376464 +62653836616239353064356530666338313137383639643739656665353535666465343961386330 +31303263653833633965313161323330376134636534646631393663653562373637613037323065 +39326230343934646662353434333934373538353437366564323937363434616464663730303834 +34643063323535323462333761626531623464373762363761336634663634353630366462643537 +37316537333465636330333634363363656435336265373564653663333935373661643236646632 +64633763373961636237393931663862646565373036653332373236643938353635346436313862 +31613630356365646239386233636234623133383533303461333061306266373839613832613934 +61383234383862383738326561353231383039383766333236376233636137373830316236316436 +39616262383038346463633136646638653263333335333530363134663336383431643765366230 +33393631376134306434316161653038303864636365303731616536376164646262646437373431 +30613631383565346637343735303165366662383331326639666530353238373266356565333031 +37626530633762346338363632646461353838346463643431326463613331636133656562616162 +30373931393136366137366435333133633031316234613336323334643964653965343934323736 +34336565653133646563646334316164653139363564663765386438303030386565633036656136 +30623033613037346166346666336461313439653930386362653631346463313066663232336332 +65613565343162373865313166646131393839616662643666363030313434663532326538643537 +38386135633831343065643331623366626365393663336538623136616538333363383338353534 +30353735383433306563646639636233656164323238643261666435306536626132663531373464 +62343933643631383930313965643666396465316536376465623834323663383064646535623265 +30656666383339383161326233313564646664326162393337343933613334386638373638663763 +32306530376539386365313461326461643663316264386435303062633735336633303531323432 +65383564363634353963663733623361333934323334633335343432373335396134633133313065 +62336337313739653330386534613934396234633262623836363230323936333463613532613037 +36663063623137313764336435346366626435613535303236343165313934306437373766333936 +66333135666362353832353439663630663265623638356634313765663461313031393030656636 +65366138346632623832623234636331386632313066643236643465643539333538373734333535 +37303264366332643130323035363934373161363535363533643161346133633333613665383834 +34393033346438323165313639336439663437613935373563643338633439646463653064303364 +66333132646639356561303465653534323633613134666531636530326437373139623130303766 +35316339363264383231303932383634353038343538363337366637666561653363656630643639 +63323936363532393131346164613237636434623639633563633330646465386233636565336463 +64646263326239393737353236643034306538326139303139316133643062643739313839356438 +30323632633833636465663834363539323761376466643763373661323537636436626263646239 +35393364363134353137323230346338663962353139333333643462663933343466626236633563 +33623630643536356635623865306239623336363238396235666464393438376566353631363838 +30366165376561313263373430376132616534363933336236626535383439333865346238333335 +64393733396234303738656434393333613534353635363536643363663234653230323532386631 +34643463363363383034613634356531333733393063616338383463636632633037346239633731 +32366563346463353363346436663163363663643466393336346633643633336534623464623938 +63393465336366316237653538663466363033346439373664396330303236303265303866653431 +32386163313961353762336365663634383931363365386236613263656266346434663965313938 +37353766313566366330656463376232616462326138386565373535376661663963336234393433 +63306136656533316562333061326432663032363336346137383064363438656435373834353230 +63323736653734323638346464376635313435616631346439313630616134313662313764613035 +32346162383332663038386639363839653961613730306361643962623466636335626164623263 +33333061396435363834363537383863653738383964323037343034616164396464333834353961 +33316632343862626337646134636361623162306166363032343630313131346234613366616362 +61313463656537663033353534376234373037363561393163383430643735636664633666326239 +32616237646530623935373238623435386536323038346235656565343664396638386536326633 +33633063626362333030623063353634643433666163653665313266383265323734316631316330 +66383339643630633533643631333733633563333335313935366262353861396437643833356530 +62383739633632383162663166313839383239323265383737666634346535366562626164333038 +63633263383464643763336235333965383964343566636633643330326466613532613936343730 +36653734383434633637393431653730323434313362326334336332396335376263353935333064 +62666461383238633336306465633862653465323662306434656230633064313032653230613262 +37353533613631336638656535633834303963316435623634343837333862346330383336316233 +38646237636336633635366237363864353338353830316162613866646238396433666133306461 +61393736363039303438353734643163623138613162393233626332366534623733326236373437 +30323438386530303766353839343532303964643534363665326437623964616264383133643133 +32363963633833353962373165643866643031363561353137396361646537393365306363313136 +62643134323263613736613962363734646635646333386563303238393166383130373165316337 +65333266633834393735323237653330326662343965663266363866613538396230616632663932 +32663835653463343633313062333832623133643537333264636139663231326436316263396231 +36346565376162323034376538396365643261303261336139383963636236346330353432616137 +66373438386430666664326636663336626533316438373265353639656535356634626630383635 +33656136616130306531623761383737383666393134663965396634626130303035363664376162 +65313665653032393662636162353636613662656534333862313662383365353963383432616465 +61613134316364363138326336663139363739646565613738623763396531386532376537373432 +38366438363038343033336639393734373438393630313161616237653139623932353363616531 +36353134376334623766646338346462366131356438373265383534323430623732373631626634 +66383137616131383839623562303737656239366364363036333531313762393863613631323239 +36356263303136656264333464313466336330386533633464363463386465653433316437656438 +30386339653766313030636438616336343865326232356161623261643435363530643337643535 +37376132616639376563613337326334313938396436653565306661396363366263373266663833 +30303834336334653133616666373661343530303865353761613231653938393636366565313439 +64383830323763323665623664626135663164643163636135396338303538393062323061613632 +62393636313935643837616538623031363832303936383832656135646263366266386633633930 +66326634613665313739386362653764626163626133633030323765383731643830333165313165 +38633233366237613161336239393363393432346664613632376631306165656532363566383736 +36653733383034383039623436356562353732366433396463356139656538643265333164373638 +62333332393566343061616137303666313534366337646563376238326466303735643466623861 +64333537303862363762623165386264623834663164306338646639616263376336646535323834 +39643864653566613931323038333361326139363466346132616665613732356139363138666634 +37656132343663623132623365356635316530626265396230646339633038376236306237653662 +30326563393534643162333466383832646134316432613965666639393437366536336638653236 +63366234666363303466386231333763363636663432393666303364336139643664383930313339 +31636436393461333062623231623131393164356233373066613439306538323364663233376334 +32653234626436383639316265356266333135663536383266633662633730656163326230313030 +63316538393638666432383832616435643936373962646165356534666434356235343034336530 +39336334383736383437623434326634363963383662663464633537323438613430346338353262 +31653333383538646265343761383838633665316564653033363936376239303166383264613937 +62613366613162653662653566333762633631303661633766353238656661653063656638303061 +37656366666237353339393434336263663431386663303664656662366531353330616532303566 +66303337663236643536626638646336653132363933396462373966646138393733663966653631 +65613864353738323736323766623261613134333864363065623831393439653233663965393463 +32646136656365386236356338306131316234346537656532313365643836383965373662623137 +35376434386564323834376361633361666330396566386333393630656465623632313266633136 +35383230633133666534663836646563386465643863343933656163326566313232336630613262 +32376239376331306163333331393037346531323134316338336562363364613466346330396432 +62626663373862386133613265336133653134353031303064663639653931366330323332336338 +34313665633632643133643439656633636138396662633835336264656539353862313433373562 +64366535663739356536363765313435343235366664616263383261666635346464616161633135 +62366564643038303034393664363861316134626537303934373066636133313830316432633465 +34313338383836313561306637396333383761383032386438613231633366643765636530636461 +34383434373431336136626361363463353364626632343231343064646139643163333863616430 +64656336393832343864663730663463386637303933343566306634326662613530386434323538 +37303163663261316430633533656138333137623839376435653463383031343036363262303361 +31613535346335376163356530643065306564333563353061366538623865623937373263376565 +33353532303135333061326532313965633562316437366261343437323338376334626239323738 +62313861393063373530623263336535353466646232336237636230353137633438666130363062 +39333137316335623735616162633837363361616135333966386239303339336162616231663365 +37306262306239313134666533323263616438646464313063636562666566633137616539373331 +37303966383964653132623737326163616439643131326166323737643533626333646339613238 +61376432316666393562636237393836626361376438396662643033616636366664373161343836 +33356362643530343861326165663464363161393236323365306261396434636137373962376532 +65386336326637383464346239326134653365633731633963396234363934616635666364323562 +34356331376631346562303631316535303933633935616461653438383761313763633934363733 +36343838653632393438343933666635363163313262373538363833353665663062316132393038 +33373230323230643464333631313539643837376137396530363866666666666639376434396639 +30653536363366393739636431623563323133663934336631636265306338303330353166373536 +64643439373238333666656131353564613036613230363037653834383464383662636262666434 +38356161343933343363376264613636353565666561646631636436666465643963656662343861 +62323563623038373162376135666337326263356536613662663535306263356566613838363861 +37343733303735636538653033636130373062613130383562653966383963633862626462336566 +31383864373230336635333038366662376135373064353530643366393537393339393662346138 +61636664343133343731346538623539643337323761346564343033356239366638633731616336 +30656235343039386462323335356662653466633765643165363730363831643738616531643238 +31666163316266666565306335326534623962323538626338396537326631393332393166333837 +34353632366663373830633030646637653663376330343265646263666363656634613735366536 +38393435373366643963656231643235666135623562343630306565663933613961623638633238 +34336561326464333134613935383433343534383730383731333733616430323661306334313566 +32616432303633653738663666643161383035636661373431303431366637303430383266336538 +31343731396231616261326461613633356432353331373633646232393530653661623162383739 +65356364323433353262366233333835393266376530316563636464316334313833333030376663 +65666533366364343239316639353939613261333966353939343830383063626232306164396231 +37373637323235313666316564303163363834346162633462336636663463646134653330376335 +31303339646466663562373138626635323939633435363834376264306164636638353262333265 +34373438633763356339366664613238303135346561613735663733666466323633653833383031 +37633066363165376332633339343835386638303131626636326362653565643931373863656239 +31303063363936343732383336343761393963316432646430663163383332616539643031313266 +38323134626561336230303166333864643865626535316561333035616463376562663261663339 +62646638353139363333353339653535343365303435313830613632633938633733386337323534 +63323961623461383134386164333335346239666133333562653039323963633233363833333565 +65393239353733663463316633613031626433303235346534656262343937313161323862383066 +32316333333234393431636663643738643431343432386435313734306332366237656464313430 +63623739646635373537656236306464656534633335373030613735306532393639336166323936 +37326436303530376630386130626536663730633734656161666536613966643065343031663963 +30386165663763613432643831343136363164353966313533653436636362303639623932333531 +31373730373235666662303838633466376338653365376331306162653363353861663935626630 +32313832656266653234323335626432333663663661376631363331626337396365646237353837 +32343934663234663566313531353938616132316531646262363366613133623330306163636365 +33646666383832616265363336666335326462386532343534326465643839333165303833326165 +31336166633639393539656630323264616462633263356430353637356437316534393832323463 +33653766376364343532376231353339366533336466396161356331343865393864616430313835 +32356135363065643033643265363732316563623166646632643062633264343761653965363332 +64656165383661373834343336653338336538393630393635643839383731626137373132343330 +63653433613832626263323063623464386132666135653139353631643930666332313333323966 +31346637643930383861363062626436663938303232373265613637396230356334623534383330 +39323165303061323863643233323238336362643838373632663537333331383433653665643064 +31386239646666313166323736336235643535333566336264613335303339353063656438346562 +63663637636337356436653637656164383462633137343833343134656334383266326236333535 +32303134376338646436393630303635623830663337343537333166396362363736386232623632 +3737313339643935376561666432363666663631386434666365 diff --git a/claims_hosp/HospClaims/automate/README.md b/claims_hosp/HospClaims/automate/README.md new file mode 100644 index 000000000..58ebf15b6 --- /dev/null +++ b/claims_hosp/HospClaims/automate/README.md @@ -0,0 +1,12 @@ +## Dependencies +- paramiko +- click +- numpy +- pandas +- pathlib +- imap_tools +- matplotlib +- requests + +Maria todo: + - remove all absolute path references \ No newline at end of file diff --git a/claims_hosp/HospClaims/automate/agg_claims_drops.py b/claims_hosp/HospClaims/automate/agg_claims_drops.py new file mode 100644 index 000000000..28ba97c1a --- /dev/null +++ b/claims_hosp/HospClaims/automate/agg_claims_drops.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +"""Aggregates chunks of drops. + +Drops are expected to be numbered as: + +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz +... etc. +""" + +# standard +from collections import defaultdict +from pathlib import Path + +# third party +import click +import numpy as np +import pandas as pd + + +def agg_and_write(data_path, force=True): + """ + Aggregate drops given a folder path. Will output an aggregated version in the + same folder. Example below. + + Input files in folder: + ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz + ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz + + Will create: + ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_07052020_1456.csv.gz + + + Args: + data_path: path to the folder with duplicated drops. + force: if aggregated file exists, whether to overwrite or not + """ + + files = np.array(list(Path(data_path).glob("*"))) + + for f in files: + out_path = f.parents[0] / f.name + dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, + "patCountyFIPS": str}) + if "servicedate" in dfs.columns: + dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) + if "patCountyFIPS" in dfs.columns: + dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) + if "patHRRname" in dfs.columns: + dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) + if "patAgeGroup" in dfs.columns: + dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) + if "patHRRid" in dfs.columns: + dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) + + assert np.sum( + dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", + "Pat HRR Name", "PatAgeGroup"])) == 0, \ + "Duplication across drops!" + assert dfs.shape[1] == 10, "Wrong number of columns" + + dfs.to_csv(out_path, index=False) + print(f"Wrote {out_path}") + + # matches = defaultdict(list) + # for i, f in enumerate(files): + # drop_datetime = f.name.split("_")[3:] + # if len(drop_datetime) > 2: # there is a group number + # group, date, time = drop_datetime + # dateid = date + time + # matches[dateid].append(i) + # + # for match, file_idxs in matches.items(): + # # check if file exists before writing + # out_name = files[file_idxs][0].name.split("_") + # out_name = '_'.join(out_name[:3] + out_name[4:]) + # out_path = files[file_idxs][0].parents[0] / out_name + # if out_path.exists() and not force: + # # print(f"{out_path} exists, skipping") + # continue + # + # dfs = [pd.read_csv(files[i], dtype={"PatCountyFIPS": str, + # "patCountyFIPS": str}) for i in file_idxs] + # n_rows = [a.shape[0] for a in dfs] + # dfs = pd.concat(dfs) + # if "servicedate" in dfs.columns: + # dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) + # if "patCountyFIPS" in dfs.columns: + # dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) + # if "patHRRname" in dfs.columns: + # dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) + # if "patAgeGroup" in dfs.columns: + # dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) + # if "patHRRid" in dfs.columns: + # dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) + # + # assert np.sum( + # dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", + # "Pat HRR Name", "PatAgeGroup"])) == 0, \ + # "Duplication across drops!" + # assert dfs.shape[1] == 10, "Wrong number of columns" + # assert sum(n_rows) == dfs.shape[0], "Sum of rows is incorrect" + # + # safe_out(out_path, dfs, force) + + +@click.command() +@click.argument('data_path') +@click.option('--force', '-f', is_flag=True, default=False) +def run_cli(data_path, force): + agg_and_write(data_path, force=force) + + +if __name__ == "__main__": + run_cli() diff --git a/claims_hosp/HospClaims/automate/download_claims_ftp_files.py b/claims_hosp/HospClaims/automate/download_claims_ftp_files.py new file mode 100644 index 000000000..28baff0e8 --- /dev/null +++ b/claims_hosp/HospClaims/automate/download_claims_ftp_files.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Downloads files modified in the last 24 hours from the delphi ftp server.""" + +# standard +import datetime +import functools +import sys +from os import path + +# third party +import click +import paramiko + +# first party +from secrets import claims + + +class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): + def missing_host_key(self, client, hostname, key): + return + + +def print_callback(filename, bytes_so_far, bytes_total): + rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) + if (rough_percent_transferred % 25) == 0: + print(f'{filename} transfer: {rough_percent_transferred}%') + + +def get_timestamp(name): + try: + split_name = name.split("_") + yyyymmdd = split_name[3] + hhmm = ''.join(filter(str.isdigit, split_name[4])) + timestamp = datetime.datetime.strptime(''.join([yyyymmdd, hhmm]), + "%Y%m%d%H%M") + except Exception: + timestamp = datetime.datetime(1900, 1, 1) + + return timestamp + + +def flip_MMDDYYYY_to_DDMMYYYY(name): + # flip date from MMDDYYYY to DDMMYYYY + split_name = name.split("_") + date = split_name[4] + flip_date = date[2:4] + date[:2] + date[4:] + split_name[4] = flip_date + name = '_'.join(split_name) + return name + + +def flip_YYYYMMDD_to_DDMMYYYY(name): + split_name = name.split("_") + date = split_name[3] + flip_date = date[6:] + date[4:6] + date[:4] + split_name[3] = flip_date + name = '_'.join(split_name) + return name + + +@click.command() +@click.argument("out_path") +def download(out_path): + current_time = datetime.datetime.now() + seconds_in_day = 24 * 60 * 60 + print(f"current time is {current_time}") + + # open client + client = paramiko.SSHClient() + client.set_missing_host_key_policy(AllowAnythingPolicy()) + + client.connect(claims.HOST, + username=claims.USER, password=claims.PASS, port=claims.PORT) + sftp = client.open_sftp() + sftp.chdir('/hosp/receiving') + + + # go through files in recieving dir + files_to_download = [] + for fileattr in sftp.listdir_attr(): + # file_time = datetime.datetime.fromtimestamp(fileattr.st_mtime) + file_time = get_timestamp(fileattr.filename) + time_diff_to_current_time = current_time - file_time + if 0 < time_diff_to_current_time.total_seconds() <= seconds_in_day: + files_to_download.append(fileattr.filename) + + # make sure we don't download more that the 3 chunked drops (2x a day) for OP + # and the 1 chunk (2x a day) for IP - 01/07/21, *2 for multiple day drops + assert len(files_to_download) <= 2 * ((3 * 2) + 2), "more files dropped than expected" + + filepaths_to_download = {} + for file in files_to_download: + flipped_file = flip_YYYYMMDD_to_DDMMYYYY(file) + if "INPATIENT" in file: + full_path = path.join(out_path, flipped_file) + if path.exists(full_path): + print(f"{flipped_file} exists, skipping") + else: + filepaths_to_download[file] = full_path + + # download! + for infile, outfile in filepaths_to_download.items(): + callback_for_filename = functools.partial(print_callback, infile) + sftp.get(infile, outfile, callback=callback_for_filename) + + client.close() + + +if __name__ == "__main__": + download() diff --git a/claims_hosp/HospClaims/automate/ftp_to_covidcast.py b/claims_hosp/HospClaims/automate/ftp_to_covidcast.py new file mode 100644 index 000000000..0333f027c --- /dev/null +++ b/claims_hosp/HospClaims/automate/ftp_to_covidcast.py @@ -0,0 +1,62 @@ +"""FTP created files over to Delphi Covidcast ingestion.""" +# standard +import datetime +import os +from pathlib import Path + +# third party +import click +import paramiko + +# first party +from secrets import covidcast + +NUM_FILES = 71*6*2 # expect (71 dates x 6 geos x 2 signals) +NUM_SE_FILES = 71*6*1 # expect (71 dates x 6 geos x 1 signals) + + +class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): + def missing_host_key(self, client, hostname, key): + return + + +@click.command() +@click.argument("local_receiving_dir") +def upload(local_receiving_dir): + """Upload files to the delphi covidcast ingestion folders + + Args: + local_receiving_dir: local dir containing the non-se signal files + + """ + today = datetime.datetime.now().date() + + # open client + client = paramiko.SSHClient() + client.set_missing_host_key_policy(AllowAnythingPolicy()) + client.connect(covidcast.HOST, username=covidcast.USER, password=covidcast.PASS) + sftp = client.open_sftp() + + files_to_upload = [] + for file in Path(local_receiving_dir).glob("*.csv"): + files_to_upload.append(file) + + assert len(files_to_upload) == NUM_FILES, "more files to upload than expected!" + + # upload signal without se + sftp.chdir("/common/covidcast/receiving/hospital-admissions") + for i, file in enumerate(files_to_upload): + assert ( + datetime.datetime.fromtimestamp(os.path.getmtime(file)).date() == today + ), f"uploading old file {file}" + + sftp.put(file, file.name) + if (i % 61) == 0: + print(f"Finished {i} out of {len(files_to_upload)}") + + print(f"Successfully uploaded the hospital-admissions claims signal") + client.close() + + +if __name__ == "__main__": + upload() diff --git a/claims_hosp/HospClaims/automate/get_latest_claims_name.py b/claims_hosp/HospClaims/automate/get_latest_claims_name.py new file mode 100644 index 000000000..9dadbe5dc --- /dev/null +++ b/claims_hosp/HospClaims/automate/get_latest_claims_name.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Return the latest drop.""" + +# standard +import datetime +from pathlib import Path + +# third party +import click + + +@click.command() +@click.argument("dir_path") +def get_latest_filename(dir_path): + current_date = datetime.datetime.now() + files = list(Path(dir_path).glob("*")) + + latest_timestamp = datetime.datetime(1900, 1, 1) + latest_filename = None + for file in files: + split_name = file.name.split("_") + if len(split_name) == 5: + ddmmyyyy = split_name[3] + hhmm = ''.join(filter(str.isdigit, split_name[4])) + timestamp = datetime.datetime.strptime(''.join([ddmmyyyy, hhmm]), + "%d%m%Y%H%M") + if timestamp > latest_timestamp: + if timestamp <= current_date: + latest_timestamp = timestamp + latest_filename = file + + assert current_date.date() == latest_timestamp.date(), "no drop for today" + + # write to stdout for shell script to use + print(latest_filename) + + # return for other uses + return latest_filename + + +if __name__ == "__main__": + get_latest_filename() diff --git a/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh b/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh new file mode 100755 index 000000000..e3230a5c2 --- /dev/null +++ b/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -o errexit +#set -o nounset +set -o pipefail + +BASE="/home/indicators/runtime/claims_hosp/HospClaims" + +AUTO_DIR="$BASE/automate" +HOSP_CLAIMS_PKG_DIR="/home/indicators/runtime/claims_hosp" +CLAIMS_DIR="$BASE/claims_data" +GEO_DIR="/common/covidcast/covid-19/geographical_scope" +CURRENT_dmY=$(date '+%d%m%Y') +CURRENT_Ymd=$(date '+%Y%m%d') +CURRENT_YmdHM=$(date '+%Y%m%d_%H%M') +RECEIVING_DIR="$BASE/receiving/results_$CURRENT_YmdHM" +RECEIVING_SE_DIR="$BASE/receiving/results_se_$CURRENT_YmdHM" + +# pull latest data +echo "downloading drops" +cd "$AUTO_DIR" || exit +python3 download_claims_ftp_files.py "$CLAIMS_DIR" + +# aggregate data +echo "aggregating drops" +python3 agg_claims_drops.py "$CLAIMS_DIR" + +# find the latest files (these have timestamps) +echo "finding today's latest claims drop" +claims_file=$(python3 get_latest_claims_name.py "$CLAIMS_DIR") + +# make receiving directories +mkdir "$RECEIVING_DIR" + +# generate the sensor +cd "$HOSP_CLAIMS_PKG_DIR" || exit + +source env/bin/activate + +python $AUTO_DIR/update_json.py \ + "$claims_file" \ + "$GEO_DIR" \ + "$HOSP_PKG_DIR" \ + "$RECEIVING_DIR" + +python -m delphi_claims_hosp + +deactivate + +sanity_check() { + geo=$1 + cd "$AUTO_DIR" || exit + python3 sanity_checks.py "$RECEIVING_DIR" "$geo" +} + +echo "running sanity checks" +sanity_check state +sanity_check msa +sanity_check hrr +sanity_check county + +# plot states without se +cd "$AUTO_DIR" || exit +python3 sanity_checks.py "$RECEIVING_DIR" state -p + +# upload files to covidcast +python3 ftp_to_covidcast.py "$RECEIVING_DIR" + +# delete raw data +rm "$CLAIMS_DIR"/*.csv.gz + +# delete signal files +rm -r "$RECEIVING_DIR" diff --git a/claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh b/claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh new file mode 100755 index 000000000..2111f7b7e --- /dev/null +++ b/claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh @@ -0,0 +1,68 @@ +#!/bin/sh +set -o errexit +#set -o nounset +set -o pipefail + +BASE="/home/indicators/runtime/claims_hosp/HospClaims" +AUTO_DIR="$BASE/automate" +HOSP_CLAIMS_PKG_DIR="/home/indicators/runtime/claims_hosp" +CLAIMS_DIR="$BASE/claims_data" +GEO_DIR="/common/covidcast/covid-19/geographical_scope" +RECEIVING_DIR="$1" + +# pull latest data +echo "downloading drops" +cd "$AUTO_DIR" || exit +python3 download_claims_ftp_files.py "$CLAIMS_DIR" + +# find the latest files (these have timestamps) +echo "finding today's latest claims drop" +claims_file=$(python3 get_latest_claims_name.py "$CLAIMS_DIR") + +# only keep latest file +cd "$CLAIMS_DIR" || exit +claims_filename=$(basename "$claims_file") +echo "$claims_filename" +mv "$claims_filename" .. +cd .. +rm -f "$CLAIMS_DIR"/*.csv.gz +mv "$claims_filename" "$CLAIMS_DIR" + +# aggregate data +cd "$AUTO_DIR" || exit +echo "aggregating drops" +python3 agg_claims_drops.py "$CLAIMS_DIR" + +# generate the sensor +cd "$HOSP_CLAIMS_PKG_DIR" || exit + +source env/bin/activate + +python $AUTO_DIR/update_json.py \ + "$claims_file" \ + "$GEO_DIR" \ + "$HOSP_PKG_DIR" \ + "$RECEIVING_DIR" + +python -m delphi_claims_hosp + +deactivate + +sanity_check() { + geo=$1 + cd "$AUTO_DIR" || exit + python3 sanity_checks.py "$RECEIVING_DIR" "$geo" +} + +echo "running sanity checks" +sanity_check state +sanity_check msa +sanity_check hrr +sanity_check county + +# plot states without se +#cd "$AUTO_DIR" || exit +#python3 sanity_checks.py "$RECEIVING_DIR" state -p + +# delete raw data +rm "$CLAIMS_DIR"/*.csv.gz diff --git a/claims_hosp/HospClaims/automate/regen_old_issue.py b/claims_hosp/HospClaims/automate/regen_old_issue.py new file mode 100755 index 000000000..838d4b24d --- /dev/null +++ b/claims_hosp/HospClaims/automate/regen_old_issue.py @@ -0,0 +1,47 @@ +from datetime import datetime, timedelta +import os +import logging + + +def regen(issue_date: datetime): + fake_date = datetime.strftime(issue_date, '%Y%m%d') + fake_datetime = datetime.strftime(issue_date, '%Y-%m-%d %H:%M:%S') + + out_dir = f"/home/maria/Delphi/HospClaims/regen/issue_{fake_date}" + out_dir_no_se = out_dir + "/hospital-admissions" + #if os.path.isdir(out_dir_no_se) and len(os.listdir(out_dir_no_se)) > 0: + # logging.info(f"files in output dir, skipping {issue_date}") + # return False + + os.makedirs(out_dir_no_se, exist_ok=True) + os.system( + f"faketime '{fake_datetime}' /home/maria/Delphi/HospClaims/automate/hosp_claims_regen_script.sh {out_dir_no_se}") + + logging.info(str(issue_date.date())) + + +def main(): + hour = 23 + + start_date = datetime(2021, 6, 12, hour) + end_date = datetime(2021, 6, 13, hour) + #start_date = datetime(2020, 6, 2, hour) + #end_date = datetime(2020, 8, 4, hour) + n_dates = (end_date - start_date).days + 1 + date_range = [start_date + timedelta(days=a) for a in range(n_dates)] + + logging.basicConfig(level=logging.DEBUG, filename="out.log", + filemode="a+", + format="%(asctime)-15s %(levelname)-8s %(message)s") + + #date_range = [datetime(2020, 6, 21, hour)] + for date in date_range: + try: + regen(date) + except Exception as e: + logging.info(e) + continue + + +if __name__ == "__main__": + main() diff --git a/claims_hosp/HospClaims/automate/sanity_checks.py b/claims_hosp/HospClaims/automate/sanity_checks.py new file mode 100644 index 000000000..f2b03fe65 --- /dev/null +++ b/claims_hosp/HospClaims/automate/sanity_checks.py @@ -0,0 +1,289 @@ +"""Sanity check results from generating DV estimates. + +Author: Maria Jahja +Created: 2020-05-12 + +Plotting code modified from: http://blog.marmakoide.org/?p=94 +""" + +# standard packages +import logging +import sys +from collections import defaultdict +from datetime import datetime, timedelta +from pathlib import Path + +# third party +import click +import matplotlib.dates as mpld +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from matplotlib.backends.backend_pdf import PdfPages + +# first party +EPIDATA_DIR = Path.home() / "Delphi/delphi-epidata/src/client" +FIPS_DIR = Path.home() / "Delphi/covid-19/doctor-visits/maria/data/fips_full.csv" +sys.path.append(str(EPIDATA_DIR)) +from delphi_epidata import Epidata + + +class EMRHospChecks: + DATE_FORMAT = mpld.DateFormatter('%m-%d') + + def __init__(self, data_path, level, se): + self.level = level + self.data = self.get_data(data_path, level, se) + self.locs = list(sorted(set(self.data["adj"]["val"].keys()) | \ + set(self.data["nadj"]["val"].keys()))) + self.se = se + + # read in geo file for fips + self.geo = pd.read_csv(FIPS_DIR, dtype={"FIPS": int}) + self.geo.drop_duplicates('FIPS', inplace=True) + + @staticmethod + def get_data(data_path, level, se): + """ + Compile data values and dates for given data_path and geographic level + + Args: + data_path: path to the data files + level: geographic level to pull + se: bool if se's are included in the file + + Returns: + dictionary with data + """ + + def extract(all_files, all_dates): + """Extract data from the files.""" + res = {"val": defaultdict(list), + "se": defaultdict(list), + "dates": defaultdict(list)} + for f, d in zip(all_files, all_dates): + df = pd.read_csv(open(f, "rb"), dtype={"geo_id": str}).to_numpy() + for row in df: + geo = row[0] + res["val"][geo].append(row[1]) + res["se"][geo].append(row[2]) + res["dates"][geo].append(d) + return res + + data = {"adj_files": [], "nadj_files": [], "dates": []} + for f in sorted(data_path.glob("*")): + name = f.name.split("_") + if f.suffix == ".csv" and name[1] == level: + name_idx = -2 if se else 3 + if name[name_idx] == "adj": + data["adj_files"].append(f) + else: + data["nadj_files"].append(f) + data["dates"].append(name[0]) + + # extract data + data["dates"] = sorted(list(set(data["dates"]))) + data["adj"] = extract(data["adj_files"], data["dates"]) + data["nadj"] = extract(data["nadj_files"], data["dates"]) + + # convert dates + data["dates"] = pd.to_datetime(data["dates"]) + data["first_date"] = data["dates"].min() + data["last_date"] = data["dates"].max() + data["first_plot_date"] = data["last_date"] - timedelta(days=30) + data["epidata_date_range"] = Epidata.range( + str(data["first_plot_date"].date()).replace('-', ''), + str(data["last_date"].date()).replace('-', '')) + + return data + + def check_se_na(self): + """ + Checks that all SE are reported as 'NA' due to + privacy concerns from the company. + + Returns: + true if pass, false otherwise + """ + + for kind in ["adj", "nadj"]: + for geo, ses in self.data[kind]["se"].items(): + for se in ses: + if not np.isnan(se): + logging.error(f"{geo}, {se} not nan") + return False + return True + + def check_range(self): + """ + Checks that all percentages are within [0, 100]. + + Returns: + true if pass, false otherwise + """ + for kind in ["adj", "nadj"]: + for geo, vals in self.data[kind]["val"].items(): + for val in vals: + if not (0 <= val <= 100): + logging.error(f"{geo}, {val} not in [0, 100]") + return False + return True + + def check_quantity(self): + """Checks how many geographies were generated.""" + n_geos = {} + logging.info(f"geographies generated for {self.level}") + for kind in ["adj", "nadj"]: + for geo, vals in self.data[kind]['val'].items(): + n_geos[geo] = len(vals) + + min_geo = np.min([v for k, v in n_geos.items()]) + max_geo = np.max([v for k, v in n_geos.items()]) + avg_geo = np.mean([v for k, v in n_geos.items()]) + std_geo = np.std([v for k, v in n_geos.items()]) + logging.info(f"\t{kind}" + f"\nmin:\t{min_geo}\nmax:\t{max_geo}" + f"\navg:\t{avg_geo:.2f}\nstd:\t{std_geo:.2f}") + + def get_filled_df(self, loc, kind): + df = pd.DataFrame({"val": self.data[kind]["val"][loc]}, + index=pd.to_datetime(self.data[kind]["dates"][loc])) + + if self.data["first_plot_date"] not in df.index: + df = df.append( + pd.DataFrame({"val": np.nan}, index=[self.data["first_plot_date"]])) + if self.data["last_date"] not in df.index: + df = df.append(pd.DataFrame({"val": np.nan}, index=[self.data["last_date"]])) + df.sort_index(inplace=True) + df = df.asfreq('D', fill_value=np.nan) + return df[df.index > self.data["first_plot_date"]] + + def get_epidata_df(self, loc, kind): + epi_kind = "smoothed_adj_covid19_from_claims" if kind == "adj" else "smoothed_covid19_from_claims" + if self.level == "msa": + loc = int(float(loc)) + + rows = Epidata.covidcast("hospital-admissions", epi_kind, "day", + self.level, self.data["epidata_date_range"], loc) + vals = [] + obs_dates = [] + for row in rows['epidata']: + vals.append(row['value']) + obs_dates.append(row['time_value']) + + obs_dates = [datetime.strptime(str(d), "%Y%m%d") for d in obs_dates] + df = pd.DataFrame({'date': obs_dates, 'val': vals}) + df = df.set_index('date') + return df + + def get_county_name(self, fips_code): + """Return name of a county given it's fips code.""" + loc = self.geo[self.geo["FIPS"] == fips_code] + if len(loc) == 0: + return fips_code + return f'{loc["Name"].iloc[0]} County, {loc["State"].iloc[0]}' + + def plot(self, outname): + """ Create PDF plots of the generated values by location. + + Args: + outname: name for the output pdf file + """ + + # start pdf document + pdf_pages = PdfPages(f'{outname}-{self.level}-hosp-claims-plots.pdf') + n_plot = len(self.locs) + n_plots_per_page = 25 + + # init plotting axis and counter + fig, axs = None, None + j = 0 + + for i, loc in enumerate(self.locs): + + # start new page if needed + if i % n_plots_per_page == 0: + fig, axs = plt.subplots(5, 5, figsize=(10, 10), sharex=True) + axs = axs.ravel() + j = 0 + + # plot + adj_ts = self.get_filled_df(loc, "adj") + axs[j].plot(adj_ts.index, adj_ts["val"], label="New (Adj)", color="blue") + + if not self.se: + nadj_ts = self.get_filled_df(loc, "nadj") + axs[j].plot(nadj_ts.index, nadj_ts["val"], label="New", color="green") + + # current data. left unlabeled to clear clutter, but colors correspond to + # the "new" lines. only plot first 52 cases (it's rather slow to run otherwise) + if self.level == "state" or \ + ((self.level == "county") and (loc in ["53033", "36061"])): + try: + epi_adj_ts = self.get_epidata_df(loc, "adj") + axs[j].plot(epi_adj_ts.index, epi_adj_ts["val"], + color="lightskyblue", linestyle="--") + if not self.se: + epd_nadj_ts = self.get_epidata_df(loc, "nadj") + axs[j].plot(epd_nadj_ts.index, epd_nadj_ts["val"], + color="lightgreen", linestyle="--") + except: + logging.warning(f"could not retrieve {loc} in epidata, skipping") + + # set title + if self.level == "county": + axs[j].set_title(self.get_county_name(int(loc)), fontsize=10) + else: + axs[j].set_title(loc) + + # set legend and format + if i == 0 or j == 0: + axs[j].legend() + + axs[j].xaxis.set_major_formatter(self.DATE_FORMAT) + axs[j].tick_params(axis='both', which='major', labelsize=5, labelrotation=90) + + # close the page if needed + if (i + 1) % n_plots_per_page == 0 or (i + 1) == n_plot: + plt.tight_layout() + pdf_pages.savefig(fig) + plt.close() + j += 1 + + pdf_pages.close() + logging.info(f"plotted to '{outname}-{self.level}-hosp-claims-plots.pdf'") + + +def run(respath, geo, se, plot): + """Run sanity checks and produce plots. + + Args: + respath: path to result csvs + geo: geo level, one of state, msa, hrr, county + se: boolean whether data includes se or not + plot: boolean whether to plot or not + """ + assert geo in ["state", "msa", "hrr", "county"], f"{geo} is invalid" + + ehc = EMRHospChecks(Path(respath), geo, se) + assert ehc.check_range(), "range failed" + if not se: + assert ehc.check_se_na(), "se is all na failed" + ehc.check_quantity() + if plot: + ehc.plot(str(datetime.today().date())) + logging.info("finished checks") + + +@click.command() +@click.argument('respath') +@click.argument('geo') +@click.option('--se', is_flag=True, default=False) +@click.option('--plot', '-p', is_flag=True, default=False) +def run_cli(respath, geo, se, plot): + logging.basicConfig(level=logging.INFO) + run(respath, geo, se, plot) + + +if __name__ == "__main__": + run_cli() diff --git a/claims_hosp/HospClaims/automate/secrets.py b/claims_hosp/HospClaims/automate/secrets.py new file mode 120000 index 000000000..4ae593b93 --- /dev/null +++ b/claims_hosp/HospClaims/automate/secrets.py @@ -0,0 +1 @@ +../../secrets.py \ No newline at end of file diff --git a/claims_hosp/HospClaims/automate/update_json.py b/claims_hosp/HospClaims/automate/update_json.py new file mode 100644 index 000000000..9ae028570 --- /dev/null +++ b/claims_hosp/HospClaims/automate/update_json.py @@ -0,0 +1,60 @@ +"""Change the json template to run the package.""" + +import argparse +import json +from pathlib import Path + + +def get_hosp(edi_file, geo_dir, out_dir, receiving_dir, se): + """Output the json.params file needed to run the emr_hosp package. + + Args: + edi_file: Path to EDI file (claims) + geo_dir: Dir containing the geo_map files + out_dir: Output dir to put the json file + receiving_dir: Output dir to put the hosp estimates + se: Boolean to output SEs or not + """ + + if se: + weekday = [True] + else: + weekday = [True, False] + + data = { + "common": { + "export_dir": receiving_dir, + "log_exceptions": False + }, + "indicator": { + "input_file": edi_file, + "start_date": None, + "end_date": None, + "drop_date": None, + "n_backfill_days": 70, + "n_waiting_days": 3, + "write_se": se, + "obfuscated_prefix": "wip_henear", + "parallel": True, + "geos": ["state", "msa", "hrr", "county", "hhs", "nation"], + #"geos": ['hhs', 'nation'], + "weekday": weekday + } + } + print(f"Using {edi_file}") + + with open(Path(out_dir) / 'params.json', 'w') as outfile: + json.dump(data, outfile, indent=4) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('edi_file') + parser.add_argument('geo_dir') + parser.add_argument('out_dir') + parser.add_argument('receiving_dir') + parser.add_argument('--se', action="store_true") + + args = parser.parse_args() + get_hosp(args.edi_file, args.geo_dir, + args.out_dir, args.receiving_dir, args.se) From 095f792cba1fec75cb630584e79d699c79a0f645 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Thu, 9 Dec 2021 15:25:48 -0500 Subject: [PATCH 03/46] Re-add new secrets after fixing merge conflict with main --- ansible/vault.yaml | 460 +++++++++++++++++++++++---------------------- 1 file changed, 235 insertions(+), 225 deletions(-) diff --git a/ansible/vault.yaml b/ansible/vault.yaml index 6e26c86e7..cede7133f 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,226 +1,236 @@ $ANSIBLE_VAULT;1.1;AES256 -37363765653062643761336537303863333332346238656661646635613262616135333839346662 -3533306333623537356332366433636363643261656133620a663637636661663766376665313639 -61353639386239633430383864646533613364633765636664616134333138376266346462383463 -3232336263326536350a663637386366636434303465623937633237323638303430346162643261 -38343738333737353637306161373031366563646333666536663535663930323032323465316232 -64663738616334343866386536356136393732656334383737326661613235313937666637383966 -34313364316563613764383063373463663664353966636665663766323937656237326664363037 -39653333613732356535656366306161643266323938393362393261653733383435383335356531 -32663333616438393635633464336431616366363066646463663937383834303165646361646530 -35653839303939623932373532663332353662643134303830383765336132343831393939353538 -32626335613362303063326466343061353532333832333264383538336362383737633738323762 -61633237616238643765363630366433396134626535343463313431383037313764643535353835 -65306336666432656331333839303136373535323265373531363537353733303062373636323033 -63303333306437326431616635646263666166666339353632313134636363646362633434623639 -39346433376436306433373939626239363732376232636337373234373534646464313535333837 -32666538626538316534383531316531333865323262373232643761643035633961386530376235 -30633361333239376463313933343939353630323539666131373761636230623064306531376562 -36393036386137626166363433626330653431636131323836656433636533323937363265616337 -33306163626438636564353838366132623730353931376537323264323564313666623964306335 -65383537303563623732613862366431326530313364386562336136666435663130636136626435 -62336531626537393932376436633664363261316639386237323461343236396633363933643637 -31303534653138336533303566633963333534303964363962626264363062623562386164343564 -64313733666336383733613738666537623238303935353830393530353637363266656465363731 -34616437343262333861396361313730346533646364356461626232376362396639353135643332 -34656532393038656362616665343864393033613637646633343763613564636535643461396661 -36353636313639636661613131366433333866663263303362333665353630656462646663306335 -39653431393864373934393132323333623736663231323233323366313932656161363930353062 -34313138316134313533643934353237333236623438653734366439616230383264343834343462 -36356538653763656235316463653638666635616638303739316463343262646337313661326330 -32373439386133646134343039303161643863323736613466623538626234323866306265343034 -38656339313136393037386364663263393264376436626639393935386363663862633330613238 -30306563376133613762623365316337363966356633383832666134663034616636633866346264 -65613539613331386337633731373261336161346532633264373632376163386333653161666432 -34386533343238303431363236303062393564303438633633313330323564353733323163353935 -64656238663735343839666430366334336363363262353732333035663832353262653065613632 -39316261646366626138333639343263363966653965323265373266363438323733363461333433 -62396533313139333834323431323162353336643733373662373335616138333665383536326232 -34363235643731613663386361353533333563323934396362323738663534376632356635303065 -65366366616137656430626233653538656135623166623333623638303936313231393461393139 -36623330386332653932613761353738636531643437353663633463356537383431373764653930 -64636232613163343665653037663538326163313665383632633930323937383133653232366361 -37356331373762626638343439343136303738353062386435663636346165653963333761396533 -63353565366561653839353136306333313366613738353837616237623162653363393066623061 -63613536333061613638376333313463636666396236353337323363323263383133386166316362 -64343538646334356637663936373261633733303337353533393364326465343133663064336366 -34656463326333643531333064313763343738666466333535616232613539623438363961626332 -33313164613433346566633861666462343939663834366636326536613234326339646330376534 -62306637623233313739393630623033383366616239333036316132643730326234383135396332 -30303133303561623534363339303762663566366664616433313966336532373135633536613333 -38343934626230646334343062356431656466663665663466326331383237343439313333303561 -64306330383363306537336466623934363563363535326161653630303461343461663365343063 -33386263306338323862326336376234363061366530303436633366366466663464623235366265 -63383438383933656264323162373262323532303431363535663965356366323663396566396234 -33313535313133303866636266326535316466383839653662396531616361613561393864383239 -33636264383964616331363866633862666630346331393630356464643833333938653865393832 -35393334373737626233333466333835386263343039613362633333653538303633646162306163 -37663962646133326438396630626132613266663731363436616536616162373439663537336431 -30376636323637616437376431356135363562376530303333343465366533623331663139616538 -61356233633431663464336265323964313138393335633063636565383066616339313936393534 -30643032633836376362616362333536316139323238366230653636663539653666306337313833 -66313164346332333539383034333235323335323062633465353237613161643338356562373634 -31353665363336323462666335306435623938356539613231356535323138653431646364393861 -32626233653932383831313035336438643635613264346266393334303530396562383961336530 -64663430376561663232616663343531656538643166363261333337303836623264623838303065 -66393765383462623661633862346261323464346665613961363833306666656263633733303230 -30353532613965316538643435313234323264353963656566653761323830336231363037646132 -35356130653038346664306266643261666163313964353035393466646536646165666361346231 -65613239396235353064366231353939323062623065306163643530626262326566356331333465 -36386134666135396165366132636263383537313535613663663833396538613039343064376334 -64323364613431613962396334653165363937313265663864643630343538666237623062393633 -35306265313962616362616561643633653239356338653030653362623932666264626166366136 -37616661643464373837656237366430343561653533393530366238616537333061353366623833 -34346335646436633430316563636466323466663433613230376636326232306633393462653162 -37383638646234363764666364616635333039636133326561323765353939363732366434626639 -66666565623464653561363534316565633939666261376137623966326338643937383732613931 -35623131643937346461373835366436386264636333333165666639646338346632653035313238 -39666632616262663232306637646239303436366662333264383231323565363164383561303039 -63313964663564313335623639656563613136323666663930646332356538313839323463633235 -64613966633237323638366236636338633236396565303063363864666639303737336663646233 -64363737646564613331353534396432383433653263643730323337306566336631333736326466 -30366339373161316438636434626137653130363932303134663162373466666362373132303331 -61643933663531663335316137636436613361623233356430333430343833626162383266626239 -62383836313438343733623762303166333836623736346233356565353136396138376366343663 -35663134666136633639333031353066666135343661656536393536363866393562653061343439 -34633231396666663234303431323532336262363637376436363430636436656266363266616137 -31353635383833623862373333306562623966376235643935323538353939376633313437663665 -36363134313931633730626431393664643936333532346166333530613637626336333466643233 -33316564393162383536366531623538343961376166653630393062323134646136393339393937 -36383432663362623932623865386330663239663563333965373238353763343036636236346335 -61346330383639353961613161666364616234656638616465343462363138383338613034363938 -38383966313132663931613838323339363565323264336431333263653765323132636237373764 -38303434353862333465393334623635323334333132386464386666306630376537373762623133 -39346531326331353838666434323735313434333239643938333763313166343131656336316439 -32333463343338643562623634653737393333346432376530356333316338636436643236616663 -31383336373262353062623431323061643861396333656464623635336334663836353732376639 -62636536336265363631636262333461366366393137666538346239353364616236326366636435 -63323336333735326338303263333935386261353735643662303064616564393233326635303939 -33636530386438316466313138323439373632303439633065393261393865396265653338623963 -39353935663861333165343930643865383166373264373534353166633633653338303830333464 -32623838353838386238303461376435333237653963343539356434646532323032353733633464 -39336132323666323533316433376137656463616436383035366630396637333662326535623265 -34663961323530656135616233626138646665373539356266346565363932616133343066316338 -33663663373732666431333963313464336635303138346364303462376366353066323938333331 -35373863353536656462643238373936306365653361326539366462616230626165323331333262 -37356365623934303062666361636563306266356239666264326463343038393636303166353636 -31383835326337343635313330303962666636363166666237663565393235643436316464616464 -32363738666630333232326462313234663734333335653933636236613932633938353265326561 -34636632373336363061316662646132373462636337303233373733353030623637663561366637 -31366632623563323337626265373832386466386230343039356234363136383533373437343866 -32323234356139393935616334623061613431336266353937636636373138613335346265373261 -34393137323232326332373535343930643764666331373937333765353738303964666131643663 -61333136623834383030343431613863353634626631396637643630373737666635643435316131 -36306130316434356534613433626334323831666136383065633632633962313238373935613336 -33323933336235333562333264626531666436313331393362393532363936323362613633626464 -38326561623333316432663466376336323764663336336563383232666434636332646634363463 -61393432326638316664613638323466656235363137303562376436653439623931363766313631 -63663032313436386666303463363265313262623366616332373165646631333539313933663533 -64346533383661346563666132363163663762343664656232336436646434663236613436626461 -37643937626366643235323062353632376137343334303235383337346565333039346132633462 -66303733386336376666353735396238653930363833306133313736653233663436623462636161 -36613136626365663936336338626139323631646462316335613163633030336165613730383765 -61666134346234366633353264353338333734353239643632636434333863356131616230303862 -39373931353165393135343164613561393430343763333330353736326266313439656338366464 -63633234643635383262613533306434633064393039393431326464353638623235663537633665 -38663165393765303464306662363532363231383265393134303136626632346434623562363438 -61646165383537353337373739623866306133646461376261323565306530613136376461343134 -35363862376534383764616665373061393436616165666531373930616130326166396335646136 -34303833336236613537343033636563386139393432633234303838343230613638633336623435 -63653735353365346438663933663039653232356332363836653864383634616664396362613036 -62656532356363366234383735313464633432666437623866333231373838353861386336663035 -32303930396433323432366535656436346538346363363839373532396462393864393731643132 -63633036623461616431303132623264346330613962656161323465643961346461393861646433 -34323030643763653239656131353666633334633134376230656533396564666131613038336464 -62356264383661376133336566636431636635353063353561333266373437383164336366393762 -36656532363335316266393265643961336135666434373364653938326634656631613535383938 -38646135303136646639386230386135636634656135373832393736623363653234313932623034 -66633430366561386165396433306566663566613333333865646435383363323334373663663364 -64633865646365666233326264363236623330633762376632633965653935653533663333356134 -34646264303966613837666435343661383935386662386433343661653866313535613138613166 -62386633373561303437333162613830343539623034356563376362323134313635633033313034 -64373463326464313438656364383262653465343238636164326337373664653865616363656230 -35343630313732636433636336346332356366626234303163366238353164313563616162663430 -30353736646563646332393632333939613335393761323763316335346166623665623334363561 -31333639396337383133336431376366333535353566333665353238303765313763393930393930 -65633435366564616161343632356231353033326365313232363636393530356132636163353731 -36383563323938633063313931313636366433383735326538383533633033323133386135623939 -37363530343437353035626464326136313863633335303735333839616161633236303662633930 -31356363656433303431643531643230646634303363336230313838383931373365383539376331 -39316630373662633533303936623035663635343963393134636630633165636164343265333531 -66633265633133303463616637373832353463646233356636303736346436373934346536353765 -35613564663332663333343235643462653137653232633565363338356630666161393765306132 -39353035396361356266613535653531613565343932306262666531613731353434343861343632 -34623439353839626531326161303230336661616335656365646366623763386235393062303333 -61646261396430376134613631663566623832396437363331653062333566653236383862616631 -31356332356434393834366637366539316466363466333362643830616439303034646538616134 -64356638633531663537386164663030386435646639626664326137383663346666316536376434 -64313063633864396533323831333866336162333963666639653161633762616234363334663539 -66343262316135353261306161323062343166386662633635633936663337356238643933306530 -34643335396564363530366330346466633364613365333230613161343233333331393963656537 -30636538303563666339356362303962316466323937373130373835623961656539396364666435 -61373933363438323333636666646262393935663563323734636535313935393531363364343763 -61313132376236616666353364383636666461323038383332643131326536333466633365363039 -39386537643466636333653862373637666337366363323962633334373434323263616430393530 -34633363353639306664396461373739343331356132643861306636326631376132336430336236 -38646339363163623336326164363133636539613134303635303833316632376162656632373163 -65346231663538316534393238633364343631383438663539323633396632376465643630323232 -33386266353065653766393937366430306635363734323634383931346330366533383234646434 -63376232653434643164616366623230653637623434646130663432636630663364396161623362 -37313564383232383365616539653936383636653439343239303731666136306530626664306466 -39343761666464356435643437666338653037653332666463653336303136396632396235356538 -63623236626236613734633633396137313264376139656163306238653830306665393065366536 -38393238633262656638636661666537343638643538633538663730316565393336353562366135 -32336331303765313033643439363034613633313831643832303438346238316239653337653663 -63613038306332323334613062303361363865346134653765323837313135346561383931303530 -32346438633134393563363730313931373431346639633437313038303632663234383334303734 -38386532393831653763356534336135306339653265386532346530623236323933396138313066 -31343162616233333639656665336133343439356161336166333731626437613961663233313237 -37383433633936623733383565633131633433366533303739613862313765396634663832313830 -38333831643730653135326432353531663936613061313337376365386437383838663662346232 -31363563373433373862353439633330626330633838663131653633663933333066623366373465 -39323331303638376461326331643663663833646263383238313664303431613831623637653835 -62326461633762356365383665326162323536386631623738643061353066343461393530393661 -64333261383834353734633031363635366532333338376337643263343531353530343233356531 -39663439326638633361306463393430336666386334616239363933656563333565383963636464 -37656266366539663934396330666464666430376139303235643938663865386130383063623466 -34383564366230376534613232363231333438386534616634633639323035386639383635636535 -33326532326638333833353761626334646663373666396566653337323764353662316235323834 -61643434393039313731323334353631333261343339343130633438333430313231363163336265 -30356539663131353831343631353931663064373064383735383138626165323563326339663534 -61306330343163356532373062643232663631376238313939643062363332613934636131373832 -61613936333366363936623631623432636563326638653264323164643366396361326631623736 -37636665336633653161393362353565383037326130626366653265656561386266643035383865 -35386662376466626466633762316238306234646336336630323562353437623164373031663732 -39323430326663336137643532613962666537363832333034323635386462626635323664333265 -37393833353937366364653066333330386162646664363833336162643237363165323432326664 -36336236306437356234356535663632353531346533613430326430616134313963353139393936 -66353636656135323635356663666465343834363037386132376432373639333564316336353766 -31653766326332663236643932613933393562363437613530613236666364393335336131336163 -65636234326131363264666236656562306539306336626366333463386465383064343265646432 -39353538653563303662396461383630383262636132353036363630393666326637346239363435 -37396132663963633232393163326437363331643539353061383232323238326462316132363033 -62323563363665336334383066373866636238663337316534363061333965653834633166383334 -37376564633939383733303636366639626239613230306238393064643136633966343735386561 -37393633626633653935396531623735666564633761326437383532386366303662636366386337 -39333764326661343035373039646336633539643434626166656232393433633365373332373562 -61303766646165313262626163373134363034356236633764616562366566333836663266616336 -65333030303861643961386430323965356464336536666163643034326664353566613861393338 -66383335326130643963336232636230376436386238643964636630613063376538393963363461 -31613430383363353037366363393666646635306662626362373865323036383636623531623166 -66626135313232376562666335653662376136313362653133623731663537636335353130313230 -63343665346564663530363930373066626437313131336631373033666334346637646165623964 -39623164613039393533656430343331396262633431323038323966616437666265666239326433 -33666366393833373964326530626137633763663337363665306133373930366362343633316162 -61383538666266613835623364306133333037353134633133346264333439376463636436383330 -65396231393438363731613732663631386162373439326634663133396533363432356334616533 -34353166306632613063646339666636323366653262363230323561366431393730663439326232 -61336130363533366431333437653736326236323461353530363237613166316436656262356339 -33613134323937323732346461306563343762333936613131353364353132623237303433393636 -65656565323737366265383964613730613132663562383534396633303739346462633761303437 -37656536663537353738326663623862623161356536643733343361313737613635656633626536 -37633964326334356139643431303933306634383139663863613635353462616335653330326132 -32313161613835333039306465363436393433346334333339303661383431313631343162643462 -32396163313535653930346236396337393531313061656432623439613637323366393939343332 -33376639343333343133396533353034643733366564643738376263653435623739333066306133 -6434 \ No newline at end of file +34613064323664616266326436376330366432336666656438346663383165363865363966343266 +6636656139316131663836633137613730393836666437330a343930613438623366393130653964 +61366435633630353363333631326566663865376462326231643930336435336132346233663934 +3135643864366232380a313333663766383838373565663365333361636438306563346466386662 +37646331643730343964326532373834646366383063306539633566363031653837653438373364 +33383566333135346431383432373564306131316137373636656133653935663732393535323433 +33663463646434303663353864326563653463353532366235663236613664616432623736306635 +32306262303431326338333334356263326665363763643865356237643831616638353432373134 +66336436616464393239663563633061336361626462316434333438373534646432366233343565 +63323732326538663262383363653963333736616161303336303166653432653038613732663265 +39336230363038666133643964373666366239393463646563383431613964376161653335386165 +38646466663363613261316633636133616163396138663062656338613866366236383636623566 +30663434393463316632656663373932343562356366356337353463626234373037663633636366 +32383931343232633036656664616534653565373834356436376130626533356239333661643663 +38393132666661383161616535353165336330646164613631356436323636306632316665383665 +61643534366430383930663563363364663465306638363431656135333630626338653632306363 +38396162383061643936356338343061653766643332613139306536663033323339313366376131 +37336661643735613235383439663137336533653034663838646135386566343836316664633563 +65353061633863396164396538663534623861313861343530616131343237326265316335316438 +35396264306634643736383436396266656265396337653630336162306238303738373935623335 +61623162646162313437306134613336386136613863623365373839656232613431626165366439 +36323336626664663561356538656432366132353131343263313036356335363861623365376431 +30383262363533383732346437373461353061316333316336336438616636626337393937323530 +65666162323334646336313431313736323330613834303466323662336432393866303235633532 +37623163656264353537373736323765353338333162363632393462633237376531333263666665 +38663531396463623933363563376134653133643738616663663736643161393737653738323032 +38643863666264643737393536383936633262636465356339653463323136366230326366323863 +61656536626430336337353430373834643737336630663066613136343538336238366237353862 +35643034363932613433616562313433353964623134393733393039393363343932353137313562 +31323837393665356563306432393234663866623739653336313339393731626364643366306462 +61373365613035623334656539333565346333646235653161646365396466326231646361333466 +39393663373266666231366637343864663232303033633763386132633563333238386261373537 +30373732383737666338663066393138356165303965313536633466633334356465343130363763 +36333262366366323964656239366361346136316633353137353533343333633162616435336539 +34613533643138363663386631303464363535643638653432303830333338613063643530396536 +37633737386139646664333132643731663133383331366237666561623038626230363262623066 +63663030366430373736613530643362316433666566653663333566386439323030323832663162 +31383364626362613530366130383439613233323937653632373831393466363462353765333935 +61363334313962306366376266633430383464656636626338303639613831656632623865333831 +34346136366664323331613330343036336464303838323831333937353930303261346566313066 +62326264343231313664633035313033633862373932663639323966303131656663336334313061 +35613662323931323533626666643561313435326234656435333432376663353931366534313264 +64653630396231663732613663383230353130303562303266336662343635633238383062376662 +66383431373838653338326236366238366165396666656664346636626436616138393161396438 +38376465633031623265373733313333393933643265396432613435633138626663393362656631 +66356638636232316531383332303361656535316332636135376531366536386164363738623339 +66363233383436666431323665623937613533396663356664306137626666646461323334636339 +33303065326164613131366361656531373136373530663461666430313636336633373830303764 +35336636643266333861623265356638626665376163626434623739343734643434363937373261 +64633034623035623566313161613265353736626235376663353364656235363464356364666537 +62353836343733323730343234306662643835323664393634316336633937386135313866666262 +34313339626336343366363464623336633739313533396265336631623034396536303362306130 +37383037323237373562343133643432366464393562613966363630343563643466306633376437 +34623633646662326639623262346435386364306537656330376435613138393337333766323331 +37363137316239363936623264623630656333366165323036343366393039616331373831383539 +61656637633637613739383933623664633039363739336531326163353332363836333966646264 +34323436666334346363326562353363333030333432653239366337626235333831306663633765 +32323661313462663861663931356461656334363163393031396235383331663831353637633662 +33343339616266626634643737346432656161633830346335633734393338666537666165336463 +63356565306563626363366435306335636466653339323965636661653139626632343865393566 +33623335386332653534316462366363343535383634623732353635623733346139386333383335 +39376438636638653639303431306162373764386232636461643964343434636134333039636238 +64303566363765353133636665623662663833343439363564626461333938613731393435663863 +38356635316637366330313739633062333162396236386637373135386339653932366633373439 +61363334366333663338383039343935643836313361326139343530363866396333333962643838 +65316134306532306635643232313538633130383661306432356236343664643438333534356536 +66326638393139363565663131393561396536383965303262373338636163653364626462653663 +34623037373762646562376464353464363334663031336262376639313736313930343838333831 +31366634633937626562623838376266393562323764393661356362316635386261333834633462 +38646638663062306663616435346561366264306635303464313963353933386437613635646566 +65313339393632373664646135333933383838393464363835623964663561386365636530616666 +64323937393062313331613935373536333238303164613462396165343165326261323639313334 +38396365363031303537346136666236646262366635633333333736336632353262666333386534 +34663732316561633736316566353230393461326565393630303762623139656563353162356363 +36366361373466326131653561366534313863313539633734636566303335323963653831303031 +33616238373938303137393637383430383436613935343463643762306531626138643933646233 +61306235393566663362383165313866353661646461613030343761333432333539653965613238 +38366631623536386431313735326337303162613435343537393636383434303334303339323933 +34666462353965336433633932636435393532383733343639363538616163663933363239376638 +39353433333665643466303964646365313966626630646333343661313732613033633436356239 +32313766616535656431366661663663653530633766613139616363666332663331363035623634 +39616664336665346136306238633661393665326430636236303966646264353364616235353461 +37646661623130393033383335356631633637313466346536343539336130363066366235636561 +30323663643232616665373266383833323733356436303936633465303763333431626362336261 +63316562353130636163343265376533633730393831663838346331633261643030636166346239 +35663164303634343864373830333030353062343536393661623737323438363039656161353337 +35633662313435343365323630346238333566623261663563303763633864626632383538386538 +65306362353336366132383232316231313838626333613465616431653639626433323833633532 +31333165396137343530393637396631376439613638666462363964633432633233333239616361 +61356562663533356532363735313365303730616461616239373735633237393732633438646235 +30656137613536623635303337343834366238396235613733313433346164636462666338356636 +63633065393033663733616566323134303865393339363838383430663862323364646431383365 +34626233636136626362383234656464663837306638393334343330663265356337313436373739 +32353862373530386438323366303834386237653861393630303361303138636263653430393339 +66323461636331663063323865386430323866623663333835643034363763623136323638643065 +32336566313532356239623631373861616134636538353530363032336162633638386435646133 +65323538653937343536656633636332353234336666343962613236363066336661353165663731 +35353338336538306665643839303530383138393636663361646431666534633136333561333330 +66326137623237343237323264373031633566633632353438343362663162373461613961616534 +34643362326134376466326261323936313366623931376534343763353237636662313935393530 +32643037376238656465613765623532643035616431366662356365373263643933323439393662 +61363430343066646232323839373232313163636431303964646466386439366565383533333864 +64343063383934393938653861333365623237356334643737306138373638616562626438626330 +34333130363566313439396630633531366561323665626135363664373136383033393365346437 +32396631613562643230376161306433623433323462663431373365383537386233656433373236 +36643539336538623265643531343636646164366237363736343630353735363864333964353434 +35363363623561343239613461303365396634343031633861383963306562343364343062643232 +36393832386366343063306336633837373564376637346235336566616637623934396130336362 +36396561303639353063316538316330373136346636626266396639643430663830636561393639 +36343964646439303263353731623532303564363662626264636239393833383166343939636336 +36326166346637306236356463623836346461316331636439383738363530323562653237633533 +33336166323234666539396332353065613362653035663135313238386537343237346563663037 +61643732633761656438363137336366636163393431656362653661363566333139316138383737 +34346563363033383433306263363662313334333234383362623766356133386639396165656533 +33646230333365383231626533376536396239343661306537353065373036393666623133633135 +63303665653266653731636163383461356236353166393531303765616335616365316434656335 +62646430623361376266626138303239613439663236646238363030326239646365323664396331 +34346364353935323965346330376239653934346336306531616434636130393764376462626433 +61373735623038613461353966653733363166396636386631376538303334613932333233613065 +66616162363937376336613363376333303863343038653661343232653237663539356637613163 +36376132396637343735306566623264363633646532356131616435363038643837623433373633 +65653630346161323964353561326330633262383931623962616632393635363331393132646364 +33633838373438326531663566333738323466646234636466613066383662346461363766333366 +33376235353731326264376634326638316165366261373837343063383964633135343038306532 +66363430616362363666633737333230366338666366653932316639393133396137636164643031 +32373562643936396165353135636433393664366337303034313432636161346330386336323864 +34613939656435623863626466303433666164373438343234653739373939316636383533333962 +66333665373334623963353061616335656666336161353965336233303064633963666565643835 +34646534623932636134623866323531643463336136383633666432623162303034343865613562 +31313961643933633235383235636632333036613137356263356430656162666631333137643638 +33386261313635613937393539666332393561646266353535396161323533346333323164333834 +37643366313839393561316136363665356562326539303733333633613664393566373966363930 +63346164363836373361656261363938343235666537623130653232356132346162316535383533 +36383263666135356162333038383036633538383333633536626531643463346164623866393934 +64366264326634373332613237393735353963336132363331623537386431343837383863323232 +39313165656465306537633661616530383165343564343863393732396237313032333731336362 +64656131323939666237643462333431623336653339383631663734626638653634353834653239 +31333830383134626365656562383236323837643465386464346530333162333832313730663930 +30323536356130333166646135643939393861626133623531343236386638313763326666323532 +37376232656232633237633064363233613265393738613162626461333762343364633963346537 +38616430323931376166353036353031656665323065383538653137306536373561376536666134 +65373438666331333963333437333139373830366632376638376437613930373539643766613639 +64613930376631346534326132313136656334383930373839626337303532373366373330663035 +63356663373036653930333064323038373238623537626563346463613965653139373461323765 +65393964666532313530373037623237623132373766326539323235633062653239343663383738 +39643230393761666637396136633037376566323761313431363339376534643836326661316461 +37326561626335323762613034663764386530643937663233383036383336333839383634393062 +30643233383161316139346331326364643930343461336437373137336434646330663234623933 +33333331613335313365393636646161346138646437663365613966633165323065393732303366 +37393532633832346365363464316231306631373931306137366334643135343265626138373536 +30656262313764396636356132613061363433363461643561646636663031353162373961626663 +38303538383961396662313638313730316137666534646134393030663762373938313138623066 +63333962353734343035336264653937383130326263363633653932346566643638663538333536 +34386330613930353737653461623638303562336163626539333131306630613433616135373263 +64393463393439303363333937616630646335356536643031326434663337633137306331343165 +66653431396663346431663130303961336664313562396435393364613136323934336161366333 +66323761363165333766633032643237353236613739653436613566663263306665313137663263 +38363237643333633964633430313231646535343038386139653731333761323365323935336639 +35376635323466656138336638376236363431623039343462613633306166343863373135646462 +62313239383762613564326533646234303737353930333661363462303031653739666265653737 +37303462636230643139373730303935616133663232373537653032396361383861303135346431 +66393438353038333338616333396537326261376432336431343465633461626464623332373731 +64313832643064633963373334666135303337646166623366366563663136336435323461613531 +35376335366362623039663362353963326238356562386663303139613533343561363330393432 +37336430653264623133663464633439323364396533336366396461393735353862643732333438 +63336436343932666132313064616537653062353139306464353238623930323635396239613030 +63636363383831613136366634633765386336346239383234633937373161353334323836393963 +37643131333361376163663731646662356334616334663363643635393939393361313636326566 +39396332366566383432333232303562636138626337316333346532346265633132336238383731 +61646332363261646234346534303663643836343537393133633664363663393164343137323363 +31323337643135313835613938666566396133386466393938643565623336306636353363346338 +31393665316336626333323964333232653864366331616464623765356537356562343132666332 +36666137356637333766613637643761653666393264386161663135393961313739363963643131 +62633736363839393038386666376533643761316330343666323637386330643366303636303835 +38653366316532383938613032653830383839326638323865356162326132353838346335663037 +39373563626263653864666366383764323366346432356666353862326561386536626439353261 +31663137373630303831396265613439316133643834653933316332333462326265643730623765 +61616262323438316336353934383235313366316264383831313563373438663432663364396635 +34356236613635393137366334383162623933626265613336616335313162303262323264313932 +33343231313336313138626531613138313761353333353561303665373835366133313338656134 +33326230366231663366303166343138326564613336636533356235623936313864623862393562 +38326531396661356363383261626136383036633235633837313035386164333763656262666132 +34313931656466613561333362653933353535393939353538653139346366376537623965333962 +65643232386265633266383464353961363732323137373834343432303364323938323832323063 +31303233323530366563336635303061663738633535373365623037613131626338663565626236 +33313961653961366132666436393634326338643937656239346232666134646138336530333965 +34353062333962346630356630363333626431613266666233393538333133663833613832666236 +31326631636539653539303432363335316136653663316538323261313563366430333234303839 +34366164633534323034386462363738313365313963393064386363613533636161313137366662 +37663332333561363363303931643433666639353136313536383435323634316239613132336235 +64333532393436383135376435393862343939386330306530346163623939653635343635643566 +64376465343530303339383466666335303637626537333233616466346265373439346462303437 +33336665303664636236373566313636366538613563643736343839333964343331373738663038 +63663533666230386364663762333934666665333533363633356335376334646238336434666537 +35663736336364323066373239663630633164303536663830363164663834666331636263303361 +64313738393264653731653239323864313064616337653266646565636661336633353735663261 +62623039303961376430363764313434373034393235373835633733656633316266626230303961 +30653636323237643931316134636362363661343531663962663739393331303262333630396363 +62626462373761343461363166663665646138363132613236306266316262346662373364663764 +31303161616336643637336632396164303865393934613162363831643931316232393930323237 +39313365653537613132613238656663323166646632366462616162336638316237396663333230 +32303736626537353961313936633965346232396639303837353161633462613834663431333834 +35376365376437323835636537336339646432303632363863663831353864303336383038336438 +38613334343161346239373731663863626135613436373661336432343361356138353632386639 +62636234656265613265656538636662636136396164383731386463636562623866386466353465 +66333531373163646631393733613630396665653334306437343235386431666261326161383833 +38653634363830383532626232336566643865323162343833633066646562326631663965323337 +36653332396531376131303037356430353038633431343731633964383436373437316237306633 +65653335656166313630633430303632346163393232343865373263343537656334393939383163 +65306132306439353235313235353566653532646237323535313035353231366465353631663236 +37376661623730616137336266623138313537316236363865313263613862653633666264323661 +61393938393738383736396335643538346134663738613265623633613265346136363137623235 +61336662656430313132326639393337636263373061376665643030636639613065613637386566 +36346538623836343462343465303565303235643936646433623836333162346130383837346338 +62333636653166323065356664353061343339373162623530343163373232396438393735343239 +62393834343236616432316138333438633636393464653335306361636132333538656539333263 +66366334366139613431393435363964626638363636346135383566663635373339346133633833 +34323563353163326664623739613736313731646432303438363530613331643537663736343232 +62313565333961346230313831396538353666386566626666313861363366366561653461383764 +36356431386161633365363130323435346530326162363031653266643139636534313838326234 +32396664346631663730666438643866383363313137636562336134313665316265643662616462 +66666161623261356432633065626533313863636232333430653462373733383332386465613934 +31646231383730336162396264376663353362306530336636643466356461663331326166376630 +61333638346164626332643435333937383464303437366539323965366265323938613935383639 +33393366386532393634336638373338303837653862343433373734356462663931656632376331 +32326466353430303565656237343463383033303662336539633964373936346233326563313732 +66326637656262663762373533393536623565393633366531376565333339363262656432393761 +39626631646537396662303430313136666336326437656462653830636130626363333633626663 +35356131346537303833313936336433346638616632396133613936613938656331366231626333 +61313661353361616537616362633663363935663736643439636338313863343765346338316662 +33656235336239316539336464383861353137353162623334623437366232303335623764646438 +34383436666261333164306331363930643133363335343035646331613931306532653139313466 +33393231336136623732353365346439333039616262633866323066383939633165636264626630 +38376238356235316265373535306531343565363039363364353237376534616631323130633430 +303662306139303061306330333339333635 From b70e0bb8b72824c3fa3681c27dacd01eba73c54f Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Thu, 9 Dec 2021 16:29:53 -0500 Subject: [PATCH 04/46] Register template and template-secrets in case we have both --- ansible/ansible-deploy-staging.yaml | 4 ++-- ansible/ansible-deploy.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/ansible-deploy-staging.yaml b/ansible/ansible-deploy-staging.yaml index f08839800..d26ff10cf 100644 --- a/ansible/ansible-deploy-staging.yaml +++ b/ansible/ansible-deploy-staging.yaml @@ -29,7 +29,7 @@ - name: Check to see if we have a secrets template to send. local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2" - register: template + register: template-secrets - name: Set production params file. copy: @@ -53,4 +53,4 @@ dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py" owner: "{{ runtime_user }}" group: "{{ runtime_user }}" - when: template.stat.exists + when: template-secrets.stat.exists diff --git a/ansible/ansible-deploy.yaml b/ansible/ansible-deploy.yaml index 9863639ed..952347021 100644 --- a/ansible/ansible-deploy.yaml +++ b/ansible/ansible-deploy.yaml @@ -29,7 +29,7 @@ - name: Check to see if we have a secrets template to send. local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2" - register: template + register: template-secrets - name: Set production params file. copy: @@ -53,4 +53,4 @@ dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py" owner: "{{ runtime_user }}" group: "{{ runtime_user }}" - when: template.stat.exists + when: template-secrets.stat.exists From ab1c2b851e6fd9a3e5631d506d47de6200e244f2 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Fri, 10 Dec 2021 07:32:29 -0500 Subject: [PATCH 05/46] Update claims_hosp/HospClaims/automate/update_json.py Co-authored-by: Katie Mazaitis --- claims_hosp/HospClaims/automate/update_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/HospClaims/automate/update_json.py b/claims_hosp/HospClaims/automate/update_json.py index 9ae028570..093cee1c6 100644 --- a/claims_hosp/HospClaims/automate/update_json.py +++ b/claims_hosp/HospClaims/automate/update_json.py @@ -6,7 +6,7 @@ def get_hosp(edi_file, geo_dir, out_dir, receiving_dir, se): - """Output the json.params file needed to run the emr_hosp package. + """Output the json.params file needed to run the claims_hosp package. Args: edi_file: Path to EDI file (claims) From 419303848954c36897c660f7a38889b36e56f99b Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 3 May 2022 09:39:33 -0400 Subject: [PATCH 06/46] Remove unused lines --- .../HospClaims/automate/agg_claims_drops.py | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/claims_hosp/HospClaims/automate/agg_claims_drops.py b/claims_hosp/HospClaims/automate/agg_claims_drops.py index 28ba97c1a..6fe56dfb3 100644 --- a/claims_hosp/HospClaims/automate/agg_claims_drops.py +++ b/claims_hosp/HospClaims/automate/agg_claims_drops.py @@ -63,47 +63,6 @@ def agg_and_write(data_path, force=True): dfs.to_csv(out_path, index=False) print(f"Wrote {out_path}") - # matches = defaultdict(list) - # for i, f in enumerate(files): - # drop_datetime = f.name.split("_")[3:] - # if len(drop_datetime) > 2: # there is a group number - # group, date, time = drop_datetime - # dateid = date + time - # matches[dateid].append(i) - # - # for match, file_idxs in matches.items(): - # # check if file exists before writing - # out_name = files[file_idxs][0].name.split("_") - # out_name = '_'.join(out_name[:3] + out_name[4:]) - # out_path = files[file_idxs][0].parents[0] / out_name - # if out_path.exists() and not force: - # # print(f"{out_path} exists, skipping") - # continue - # - # dfs = [pd.read_csv(files[i], dtype={"PatCountyFIPS": str, - # "patCountyFIPS": str}) for i in file_idxs] - # n_rows = [a.shape[0] for a in dfs] - # dfs = pd.concat(dfs) - # if "servicedate" in dfs.columns: - # dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) - # if "patCountyFIPS" in dfs.columns: - # dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) - # if "patHRRname" in dfs.columns: - # dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) - # if "patAgeGroup" in dfs.columns: - # dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) - # if "patHRRid" in dfs.columns: - # dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) - # - # assert np.sum( - # dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", - # "Pat HRR Name", "PatAgeGroup"])) == 0, \ - # "Duplication across drops!" - # assert dfs.shape[1] == 10, "Wrong number of columns" - # assert sum(n_rows) == dfs.shape[0], "Sum of rows is incorrect" - # - # safe_out(out_path, dfs, force) - @click.command() @click.argument('data_path') From 17b85d5825805f16be33f9a76dd06d1640165e34 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 3 May 2022 12:29:15 -0400 Subject: [PATCH 07/46] Update claims_hosp/HospClaims/automate/update_json.py Co-authored-by: Katie Mazaitis --- claims_hosp/HospClaims/automate/update_json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/claims_hosp/HospClaims/automate/update_json.py b/claims_hosp/HospClaims/automate/update_json.py index 093cee1c6..3bdcf5b3b 100644 --- a/claims_hosp/HospClaims/automate/update_json.py +++ b/claims_hosp/HospClaims/automate/update_json.py @@ -37,7 +37,6 @@ def get_hosp(edi_file, geo_dir, out_dir, receiving_dir, se): "obfuscated_prefix": "wip_henear", "parallel": True, "geos": ["state", "msa", "hrr", "county", "hhs", "nation"], - #"geos": ['hhs', 'nation'], "weekday": weekday } } From a631d77f0264742b0ccb09aab185082ce0dc736f Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 3 May 2022 12:34:51 -0400 Subject: [PATCH 08/46] In light of #1419 being merged, we can change to 70 dates --- claims_hosp/HospClaims/automate/ftp_to_covidcast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/claims_hosp/HospClaims/automate/ftp_to_covidcast.py b/claims_hosp/HospClaims/automate/ftp_to_covidcast.py index 0333f027c..2356783c3 100644 --- a/claims_hosp/HospClaims/automate/ftp_to_covidcast.py +++ b/claims_hosp/HospClaims/automate/ftp_to_covidcast.py @@ -11,8 +11,8 @@ # first party from secrets import covidcast -NUM_FILES = 71*6*2 # expect (71 dates x 6 geos x 2 signals) -NUM_SE_FILES = 71*6*1 # expect (71 dates x 6 geos x 1 signals) +NUM_FILES = 70*6*2 # expect (70 dates x 6 geos x 2 signals) +NUM_SE_FILES = 70*6*1 # expect (70 dates x 6 geos x 1 signals) class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): From 126f6e1ebd7dbf72b41d2b72c14084b00f039137 Mon Sep 17 00:00:00 2001 From: Brian Clark Date: Tue, 3 May 2022 13:29:34 -0400 Subject: [PATCH 09/46] Remove unused line --- claims_hosp/HospClaims/automate/hosp_claims_master_script.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh b/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh index e3230a5c2..836709bf4 100755 --- a/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh +++ b/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh @@ -1,6 +1,5 @@ #!/bin/sh set -o errexit -#set -o nounset set -o pipefail BASE="/home/indicators/runtime/claims_hosp/HospClaims" From d9b087f593030a5791a5a57fd694c66cfe5e96a6 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 07:52:38 -0400 Subject: [PATCH 10/46] add scripts and modification required for pulling data --- .../delphi_claims_hosp/agg_claims_drops.py | 61 +++++++++ .../download_claims_ftp_files.py | 121 ++++++++++++++++++ .../get_latest_claims_name.py | 35 +++++ claims_hosp/delphi_claims_hosp/run.py | 19 ++- claims_hosp/params.json.template | 1 + claims_hosp/setup.py | 1 + 6 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 claims_hosp/delphi_claims_hosp/agg_claims_drops.py create mode 100644 claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py create mode 100644 claims_hosp/delphi_claims_hosp/get_latest_claims_name.py diff --git a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py new file mode 100644 index 000000000..5b3a2967e --- /dev/null +++ b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +"""Aggregates chunks of drops. + +Drops are expected to be numbered as: + +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz +... etc. +""" + +# standard +from pathlib import Path + +# third party +import numpy as np +import pandas as pd + + +def agg_and_write(data_path, logger): + """ + Aggregate drops given a folder path. Will output an aggregated version in the + same folder. Example below. + + Input files in folder: + ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz + ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz + + Will create: + ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_07052020_1456.csv.gz + + Args: + data_path: path to the folder with duplicated drops. + force: if aggregated file exists, whether to overwrite or not + """ + + files = np.array(list(Path(data_path).glob("*"))) + + for f in files: + out_path = f.parents[0] / f.name + dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, + "patCountyFIPS": str}) + if "servicedate" in dfs.columns: + dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) + if "patCountyFIPS" in dfs.columns: + dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) + if "patHRRname" in dfs.columns: + dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) + if "patAgeGroup" in dfs.columns: + dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) + if "patHRRid" in dfs.columns: + dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) + + assert np.sum( + dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", + "Pat HRR Name", "PatAgeGroup"])) == 0, \ + "Duplication across drops!" + assert dfs.shape[1] == 10, "Wrong number of columns" + + dfs.to_csv(out_path, index=False) + logger.info(f"Wrote {out_path}") diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py new file mode 100644 index 000000000..73132ce53 --- /dev/null +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Downloads files modified in the last 24 hours from the delphi ftp server.""" + +# standard +import datetime +import functools +from os import path + +# third party +from secrets import claims +import paramiko + + +class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): + """ + Class for missing host key policy. + """ + def missing_host_key(self, client, hostname, key): + """ + Function for the missing host key. + """ + return + + +def print_callback(filename, bytes_so_far, bytes_total, logger): + """ + Print the callback information. + """ + rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) + if (rough_percent_transferred % 25) == 0: + logger.info(f'{filename} transfer: {rough_percent_transferred}%') + + +def get_timestamp(name): + """ + Get the reference date in datetime format. + """ + try: + split_name = name.split("_") + yyyymmdd = split_name[3] + hhmm = ''.join(filter(str.isdigit, split_name[4])) + timestamp = datetime.datetime.strptime(''.join([yyyymmdd, hhmm]), + "%Y%m%d%H%M") + except Exception: + timestamp = datetime.datetime(1900, 1, 1) + + return timestamp + + +def flip_MMDDYYYY_to_DDMMYYYY(name): + """ + Flip date from MMDDYYYY to DDMMYYYY. + """ + # flip date from MMDDYYYY to DDMMYYYY + split_name = name.split("_") + date = split_name[4] + flip_date = date[2:4] + date[:2] + date[4:] + split_name[4] = flip_date + name = '_'.join(split_name) + return name + + +def flip_YYYYMMDD_to_DDMMYYYY(name): + """ + Flip date from DDMMYYYY to MMDDYYYY. + """ + split_name = name.split("_") + date = split_name[3] + flip_date = date[6:] + date[4:6] + date[:4] + split_name[3] = flip_date + name = '_'.join(split_name) + return name + + +def download(out_path, logger): + """ + The main function to pull the latest raw files. + """ + current_time = datetime.datetime.now() + seconds_in_day = 24 * 60 * 60 + logger.info(f"current time is {current_time}") + + # open client + client = paramiko.SSHClient() + client.set_missing_host_key_policy(AllowAnythingPolicy()) + + client.connect(claims.HOST, + username=claims.USER, password=claims.PASS, port=claims.PORT) + sftp = client.open_sftp() + sftp.chdir('/hosp/receiving') + + + # go through files in recieving dir + files_to_download = [] + for fileattr in sftp.listdir_attr(): + # file_time = datetime.datetime.fromtimestamp(fileattr.st_mtime) + file_time = get_timestamp(fileattr.filename) + time_diff_to_current_time = current_time - file_time + if 0 < time_diff_to_current_time.total_seconds() <= seconds_in_day: + files_to_download.append(fileattr.filename) + + # make sure we don't download more that the 3 chunked drops (2x a day) for OP + # and the 1 chunk (2x a day) for IP - 01/07/21, *2 for multiple day drops + assert len(files_to_download) <= 2 * ((3 * 2) + 2), "more files dropped than expected" + + filepaths_to_download = {} + for file in files_to_download: + flipped_file = flip_YYYYMMDD_to_DDMMYYYY(file) + if "INPATIENT" in file: + full_path = path.join(out_path, flipped_file) + if path.exists(full_path): + logger.info(f"{flipped_file} exists, skipping") + else: + filepaths_to_download[file] = full_path + + # download! + for infile, outfile in filepaths_to_download.items(): + callback_for_filename = functools.partial(print_callback, infile, logger) + sftp.get(infile, outfile, callback=callback_for_filename) + + client.close() diff --git a/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py new file mode 100644 index 000000000..c7f0dcad5 --- /dev/null +++ b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Return the latest drop.""" + +# standard +import datetime +from pathlib import Path + +def get_latest_filename(dir_path, logger): + """ + Get the latest filename from the list of downloaded raw files. + """ + current_date = datetime.datetime.now() + files = list(Path(dir_path).glob("*")) + + latest_timestamp = datetime.datetime(1900, 1, 1) + latest_filename = None + for file in files: + split_name = file.name.split("_") + if len(split_name) == 5: + ddmmyyyy = split_name[3] + hhmm = ''.join(filter(str.isdigit, split_name[4])) + timestamp = datetime.datetime.strptime(''.join([ddmmyyyy, hhmm]), + "%d%m%Y%H%M") + if timestamp > latest_timestamp: + if timestamp <= current_date: + latest_timestamp = timestamp + latest_filename = file + + assert current_date.date() == latest_timestamp.date(), "no drop for today" + + # write to stdout for shell script to use + logger.info(latest_filename) + + # return for other uses + return latest_filename diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index 58cba1c56..4be8b71d9 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -15,6 +15,9 @@ # first party from .config import Config +from .download_claims_ftp_files import download +from .agg_claims_drops import agg_and_write +from .get_latest_claims_name import get_latest_filename from .update_indicator import ClaimsHospIndicatorUpdater @@ -31,7 +34,7 @@ def run_module(params): - "log_exceptions" (optional): bool, whether to log exceptions to file. - "log_filename" (optional): str, name of file to write logs - "indicator": - - "input_file": str, optional filenames to download. If null, + - "input_dir": str, directory to downloaded raw files. If null, defaults are set in retrieve_files(). - "start_date": str, YYYY-MM-DD format, first day to generate data for. - "end_date": str or null, YYYY-MM-DD format, last day to generate data for. @@ -53,11 +56,21 @@ def run_module(params): __name__, filename=params["common"].get("log_filename"), log_exceptions=params["common"].get("log_exceptions", True)) + # pull latest data + download(params["indicator"]["input_dir"], logger) + + # aggregate data + agg_and_write(params["indicator"]["input_dir"], logger) + + # find the latest files (these have timestamps) + claims_file = get_latest_filename(params["indicator"]["input_dir"], + params["indicator"]["write_se"], logger) + # handle range of estimates to produce # filename expected to have format: EDI_AGG_INPATIENT_DDMMYYYY_HHMM{timezone}.csv.gz if params["indicator"]["drop_date"] is None: dropdate_dt = datetime.strptime( - Path(params["indicator"]["input_file"]).name.split("_")[3], "%d%m%Y") + Path(claims_file).name.split("_")[3], "%d%m%Y") else: dropdate_dt = datetime.strptime(params["indicator"]["drop_date"], "%Y-%m-%d") @@ -114,7 +127,7 @@ def run_module(params): signal_name ) updater.update_indicator( - params["indicator"]["input_file"], + claims_file, params["common"]["export_dir"], logger, ) diff --git a/claims_hosp/params.json.template b/claims_hosp/params.json.template index d6df27ed3..34e13a63d 100644 --- a/claims_hosp/params.json.template +++ b/claims_hosp/params.json.template @@ -5,6 +5,7 @@ }, "indicator": { "input_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz", + "input_dir": "./retrieve_files", "start_date": "2020-02-01", "end_date": null, "drop_date": null, diff --git a/claims_hosp/setup.py b/claims_hosp/setup.py index 940e1d165..d7e46a13d 100644 --- a/claims_hosp/setup.py +++ b/claims_hosp/setup.py @@ -4,6 +4,7 @@ required = [ "numpy", "pandas", + "paramiko", "pydocstyle", "pytest", "pytest-cov", From 0e59cd1ba4627e74a7ba065fbafed786487d514e Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 08:01:40 -0400 Subject: [PATCH 11/46] rm downloaded files at the end --- claims_hosp/delphi_claims_hosp/run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index 4be8b71d9..d0ec4fc71 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -7,6 +7,7 @@ # standard packages import time +import os from datetime import datetime, timedelta from pathlib import Path @@ -133,6 +134,8 @@ def run_module(params): ) logger.info("finished updating", geo = geo) + os.system(f'rm -rf {params["indicator"]["input_dir"]}') + elapsed_time_in_seconds = round(time.time() - start_time, 2) logger.info("Completed indicator run", elapsed_time_in_seconds = elapsed_time_in_seconds) From c927e00b55e592b58450b49ac50b1268bdda271b Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 08:03:12 -0400 Subject: [PATCH 12/46] add logger info for it --- claims_hosp/delphi_claims_hosp/run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index d0ec4fc71..43621937d 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -133,8 +133,10 @@ def run_module(params): logger, ) logger.info("finished updating", geo = geo) - + + # Remove all the raw files os.system(f'rm -rf {params["indicator"]["input_dir"]}') + logger.info('Remove all the raw files.') elapsed_time_in_seconds = round(time.time() - start_time, 2) logger.info("Completed indicator run", From c69cc2f37c067cbe551d3de6719bc4cb10dcd8c5 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 18:56:11 -0400 Subject: [PATCH 13/46] fixed errors in pulling --- .../delphi_claims_hosp/agg_claims_drops.py | 2 ++ .../download_claims_ftp_files.py | 27 +++++-------------- claims_hosp/delphi_claims_hosp/run.py | 8 +++--- claims_hosp/params.json.template | 9 +++++-- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py index 5b3a2967e..accee88e3 100644 --- a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py +++ b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py @@ -37,6 +37,8 @@ def agg_and_write(data_path, logger): files = np.array(list(Path(data_path).glob("*"))) for f in files: + if ".csv.gz" not in str(f): + continue out_path = f.parents[0] / f.name dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, "patCountyFIPS": str}) diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 73132ce53..b173d53a6 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -7,7 +7,6 @@ from os import path # third party -from secrets import claims import paramiko @@ -22,7 +21,7 @@ def missing_host_key(self, client, hostname, key): return -def print_callback(filename, bytes_so_far, bytes_total, logger): +def print_callback(filename, logger, bytes_so_far, bytes_total): """ Print the callback information. """ @@ -46,23 +45,9 @@ def get_timestamp(name): return timestamp - -def flip_MMDDYYYY_to_DDMMYYYY(name): - """ - Flip date from MMDDYYYY to DDMMYYYY. - """ - # flip date from MMDDYYYY to DDMMYYYY - split_name = name.split("_") - date = split_name[4] - flip_date = date[2:4] + date[:2] + date[4:] - split_name[4] = flip_date - name = '_'.join(split_name) - return name - - def flip_YYYYMMDD_to_DDMMYYYY(name): """ - Flip date from DDMMYYYY to MMDDYYYY. + Flip date from YYYYMMDD to MMDDYYYY. """ split_name = name.split("_") date = split_name[3] @@ -72,7 +57,7 @@ def flip_YYYYMMDD_to_DDMMYYYY(name): return name -def download(out_path, logger): +def download(ftp_credentials, out_path, logger): """ The main function to pull the latest raw files. """ @@ -84,8 +69,10 @@ def download(out_path, logger): client = paramiko.SSHClient() client.set_missing_host_key_policy(AllowAnythingPolicy()) - client.connect(claims.HOST, - username=claims.USER, password=claims.PASS, port=claims.PORT) + client.connect(ftp_credentials["host"], + username=ftp_credentials["user"], + password=ftp_credentials["pass"], + port=ftp_credentials["port"]) sftp = client.open_sftp() sftp.chdir('/hosp/receiving') diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index 43621937d..cd03311aa 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -58,14 +58,14 @@ def run_module(params): log_exceptions=params["common"].get("log_exceptions", True)) # pull latest data - download(params["indicator"]["input_dir"], logger) + download(params["indicator"]["ftp_credentials"], + params["indicator"]["input_dir"], logger) # aggregate data agg_and_write(params["indicator"]["input_dir"], logger) # find the latest files (these have timestamps) - claims_file = get_latest_filename(params["indicator"]["input_dir"], - params["indicator"]["write_se"], logger) + claims_file = get_latest_filename(params["indicator"]["input_dir"], logger) # handle range of estimates to produce # filename expected to have format: EDI_AGG_INPATIENT_DDMMYYYY_HHMM{timezone}.csv.gz @@ -133,7 +133,7 @@ def run_module(params): logger, ) logger.info("finished updating", geo = geo) - + # Remove all the raw files os.system(f'rm -rf {params["indicator"]["input_dir"]}') logger.info('Remove all the raw files.') diff --git a/claims_hosp/params.json.template b/claims_hosp/params.json.template index 34e13a63d..e200fa8fc 100644 --- a/claims_hosp/params.json.template +++ b/claims_hosp/params.json.template @@ -4,7 +4,6 @@ "log_exceptions": false }, "indicator": { - "input_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz", "input_dir": "./retrieve_files", "start_date": "2020-02-01", "end_date": null, @@ -15,7 +14,13 @@ "obfuscated_prefix": "foo_obfuscated", "parallel": false, "geos": ["state", "msa", "hrr", "county"], - "weekday": [true, false] + "weekday": [true, false], + "ftp_credentials": { + "host": "", + "user": "", + "pass": "", + "port": 2222 + } }, "validation": { "common": { From 40411f84bb66c009535e808e586d90373ccda2b2 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 19:21:42 -0400 Subject: [PATCH 14/46] add unit tests --- .../tests/test_download_claims_ftp_files.py | 21 ++++++++++++++++++ .../tests/test_get_latest_claims_name.py | 22 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 claims_hosp/tests/test_download_claims_ftp_files.py create mode 100644 claims_hosp/tests/test_get_latest_claims_name.py diff --git a/claims_hosp/tests/test_download_claims_ftp_files.py b/claims_hosp/tests/test_download_claims_ftp_files.py new file mode 100644 index 000000000..a1f4df192 --- /dev/null +++ b/claims_hosp/tests/test_download_claims_ftp_files.py @@ -0,0 +1,21 @@ +# standard +import datetime + +# third party +import numpy as np + +# first party +from delphi_claims_hosp.download_claims_ftp_files import (flip_YYYYMMDD_to_DDMMYYYY, + get_timestamp) + + +class TestDownloadClaimsFtpFiles: + + def test_flip_YYYYMMDD_to_DDMMYYYY(self): + name = "SYNEDI_AGG_INPATIENT_20200611_1451CDT" + expected = "SYNEDI_AGG_INPATIENT_11062020_1451CDT" + assert(flip_YYYYMMDD_to_DDMMYYYY(name)==expected) + + def test_get_timestamp(self): + name = "SYNEDI_AGG_INPATIENT_20200611_1451CDT" + assert(get_timestamp(name).date() == datetime.date(2020, 6, 11)) diff --git a/claims_hosp/tests/test_get_latest_claims_name.py b/claims_hosp/tests/test_get_latest_claims_name.py new file mode 100644 index 000000000..8a5ddfb54 --- /dev/null +++ b/claims_hosp/tests/test_get_latest_claims_name.py @@ -0,0 +1,22 @@ +# standard +import time + +# third party +import pytest + +from delphi_utils import get_structured_logger +from delphi_claims_hosp.get_latest_claims_name import get_latest_filename + + +class TestGetLatestFileName: + + start_time = time.time() + logger = get_structured_logger( + __name__, filename="test.log", + log_exceptions=True) + + def test_get_latest_claims_name(self): + dir_path = "./test_data/" + + with pytest.raises(AssertionError): + get_latest_filename(dir_path, self.logger) From b753b7ef4fa7c71023da4b720e379df6e6671af5 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 20:21:21 -0400 Subject: [PATCH 15/46] change the path to test.log --- claims_hosp/tests/test_get_latest_claims_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/tests/test_get_latest_claims_name.py b/claims_hosp/tests/test_get_latest_claims_name.py index 8a5ddfb54..627ebadc7 100644 --- a/claims_hosp/tests/test_get_latest_claims_name.py +++ b/claims_hosp/tests/test_get_latest_claims_name.py @@ -12,7 +12,7 @@ class TestGetLatestFileName: start_time = time.time() logger = get_structured_logger( - __name__, filename="test.log", + __name__, filename="./tests/test.log", log_exceptions=True) def test_get_latest_claims_name(self): From edf8e3f91bd9c26aae980aa8ba0b5cb4159169c4 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 20:21:41 -0400 Subject: [PATCH 16/46] fixed an error --- claims_hosp/delphi_claims_hosp/run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index cd03311aa..c7ed0e31a 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -135,7 +135,9 @@ def run_module(params): logger.info("finished updating", geo = geo) # Remove all the raw files - os.system(f'rm -rf {params["indicator"]["input_dir"]}') + for fn in os.listdir(params["indicator"]["input_dir"]): + if ".csv.gz" in fn: + os.system(f'rm {params["indicator"]["input_dir"]}/{fn}') logger.info('Remove all the raw files.') elapsed_time_in_seconds = round(time.time() - start_time, 2) From 2d4400bf57ba713f84e206630c205ffa49caab4d Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Mon, 6 Jun 2022 20:37:36 -0400 Subject: [PATCH 17/46] change the format of geo_id at hrr level from float to str --- claims_hosp/delphi_claims_hosp/update_indicator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/claims_hosp/delphi_claims_hosp/update_indicator.py b/claims_hosp/delphi_claims_hosp/update_indicator.py index b4169370d..2e4aaa8ca 100644 --- a/claims_hosp/delphi_claims_hosp/update_indicator.py +++ b/claims_hosp/delphi_claims_hosp/update_indicator.py @@ -113,6 +113,7 @@ def geo_reindex(self, data): new_code=self.geo) elif self.geo == "hrr": data_frame = data # data is already adjusted in aggregation step above + data_frame[self.geo] = data_frame[self.geo].astype(str).str.zfill(3) else: logging.error( "%s is invalid, pick one of 'county', 'state', 'msa', 'hrr', 'hhs', nation'", From a7f9146b4ddb2e4be1411175b716b79583aa67a6 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 7 Jun 2022 11:14:15 -0400 Subject: [PATCH 18/46] fix hrr geo id type --- claims_hosp/delphi_claims_hosp/update_indicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/update_indicator.py b/claims_hosp/delphi_claims_hosp/update_indicator.py index 2e4aaa8ca..cab92b551 100644 --- a/claims_hosp/delphi_claims_hosp/update_indicator.py +++ b/claims_hosp/delphi_claims_hosp/update_indicator.py @@ -150,7 +150,7 @@ def update_indicator(self, input_filepath, outpath, logger): # load data base_geo = Config.HRR_COL if self.geo == Config.HRR_COL else Config.FIPS_COL data = load_data(input_filepath, self.dropdate, base_geo) - data_frame = self.geo_reindex(data) + data_frame = self.geo_reindex(data) # handle if we need to adjust by weekday wd_params = Weekday.get_params( From a2fcfcb554ded60d0e863773202f7198a6b849dd Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 7 Jun 2022 11:14:30 -0400 Subject: [PATCH 19/46] add unit tests --- claims_hosp/tests/test_get_latest_claims_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/tests/test_get_latest_claims_name.py b/claims_hosp/tests/test_get_latest_claims_name.py index 627ebadc7..a64c39710 100644 --- a/claims_hosp/tests/test_get_latest_claims_name.py +++ b/claims_hosp/tests/test_get_latest_claims_name.py @@ -12,7 +12,7 @@ class TestGetLatestFileName: start_time = time.time() logger = get_structured_logger( - __name__, filename="./tests/test.log", + __name__, filename="./test.log", log_exceptions=True) def test_get_latest_claims_name(self): From 40fa62d4b3bfd5145abcdd4603baf0c9e4868335 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 7 Jun 2022 13:02:27 -0400 Subject: [PATCH 20/46] change back. Geo ids for hrr are still float numbers --- claims_hosp/delphi_claims_hosp/update_indicator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/update_indicator.py b/claims_hosp/delphi_claims_hosp/update_indicator.py index cab92b551..5522917fa 100644 --- a/claims_hosp/delphi_claims_hosp/update_indicator.py +++ b/claims_hosp/delphi_claims_hosp/update_indicator.py @@ -113,7 +113,6 @@ def geo_reindex(self, data): new_code=self.geo) elif self.geo == "hrr": data_frame = data # data is already adjusted in aggregation step above - data_frame[self.geo] = data_frame[self.geo].astype(str).str.zfill(3) else: logging.error( "%s is invalid, pick one of 'county', 'state', 'msa', 'hrr', 'hhs', nation'", From 78e3c4fa66f2ef394b8d909badb89a4971db7e0a Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 7 Jun 2022 13:25:02 -0400 Subject: [PATCH 21/46] fix linting --- claims_hosp/.pylintrc | 4 ++- .../delphi_claims_hosp/agg_claims_drops.py | 7 +++-- .../download_claims_ftp_files.py | 29 ++++++------------- .../get_latest_claims_name.py | 4 +-- .../delphi_claims_hosp/update_indicator.py | 2 +- claims_hosp/retrieve_files/.gitignore | 0 .../tests/test_download_claims_ftp_files.py | 6 ++-- 7 files changed, 21 insertions(+), 31 deletions(-) create mode 100644 claims_hosp/retrieve_files/.gitignore diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc index f30837c7e..8ba5e540a 100644 --- a/claims_hosp/.pylintrc +++ b/claims_hosp/.pylintrc @@ -7,7 +7,9 @@ disable=logging-format-interpolation, # Allow pytest functions to be part of a class. no-self-use, # Allow pytest classes to have one test. - too-few-public-methods + too-few-public-methods, + broad-except + [BASIC] diff --git a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py index accee88e3..45b57d03a 100644 --- a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py +++ b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py @@ -19,8 +19,9 @@ def agg_and_write(data_path, logger): """ - Aggregate drops given a folder path. Will output an aggregated version in the - same folder. Example below. + Aggregate drops given a folder path. + + Will output an aggregated version in the same folder. Example below. Input files in folder: ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz @@ -32,8 +33,8 @@ def agg_and_write(data_path, logger): Args: data_path: path to the folder with duplicated drops. force: if aggregated file exists, whether to overwrite or not - """ + """ files = np.array(list(Path(data_path).glob("*"))) for f in files: diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index b173d53a6..a97d7718a 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -11,29 +11,22 @@ class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): - """ - Class for missing host key policy. - """ + """Class for missing host key policy.""" + def missing_host_key(self, client, hostname, key): - """ - Function for the missing host key. - """ + """Check missing host key.""" return def print_callback(filename, logger, bytes_so_far, bytes_total): - """ - Print the callback information. - """ + """Print the callback information.""" rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) if (rough_percent_transferred % 25) == 0: logger.info(f'{filename} transfer: {rough_percent_transferred}%') def get_timestamp(name): - """ - Get the reference date in datetime format. - """ + """Get the reference date in datetime format.""" try: split_name = name.split("_") yyyymmdd = split_name[3] @@ -45,10 +38,8 @@ def get_timestamp(name): return timestamp -def flip_YYYYMMDD_to_DDMMYYYY(name): - """ - Flip date from YYYYMMDD to MMDDYYYY. - """ +def change_date_format(name): + """Flip date from YYYYMMDD to MMDDYYYY.""" split_name = name.split("_") date = split_name[3] flip_date = date[6:] + date[4:6] + date[:4] @@ -58,9 +49,7 @@ def flip_YYYYMMDD_to_DDMMYYYY(name): def download(ftp_credentials, out_path, logger): - """ - The main function to pull the latest raw files. - """ + """Pull the latest raw files.""" current_time = datetime.datetime.now() seconds_in_day = 24 * 60 * 60 logger.info(f"current time is {current_time}") @@ -92,7 +81,7 @@ def download(ftp_credentials, out_path, logger): filepaths_to_download = {} for file in files_to_download: - flipped_file = flip_YYYYMMDD_to_DDMMYYYY(file) + flipped_file = change_date_format(file) if "INPATIENT" in file: full_path = path.join(out_path, flipped_file) if path.exists(full_path): diff --git a/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py index c7f0dcad5..39b4808d0 100644 --- a/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py +++ b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py @@ -6,9 +6,7 @@ from pathlib import Path def get_latest_filename(dir_path, logger): - """ - Get the latest filename from the list of downloaded raw files. - """ + """Get the latest filename from the list of downloaded raw files.""" current_date = datetime.datetime.now() files = list(Path(dir_path).glob("*")) diff --git a/claims_hosp/delphi_claims_hosp/update_indicator.py b/claims_hosp/delphi_claims_hosp/update_indicator.py index 5522917fa..b4169370d 100644 --- a/claims_hosp/delphi_claims_hosp/update_indicator.py +++ b/claims_hosp/delphi_claims_hosp/update_indicator.py @@ -149,7 +149,7 @@ def update_indicator(self, input_filepath, outpath, logger): # load data base_geo = Config.HRR_COL if self.geo == Config.HRR_COL else Config.FIPS_COL data = load_data(input_filepath, self.dropdate, base_geo) - data_frame = self.geo_reindex(data) + data_frame = self.geo_reindex(data) # handle if we need to adjust by weekday wd_params = Weekday.get_params( diff --git a/claims_hosp/retrieve_files/.gitignore b/claims_hosp/retrieve_files/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/claims_hosp/tests/test_download_claims_ftp_files.py b/claims_hosp/tests/test_download_claims_ftp_files.py index a1f4df192..95f967ca3 100644 --- a/claims_hosp/tests/test_download_claims_ftp_files.py +++ b/claims_hosp/tests/test_download_claims_ftp_files.py @@ -5,16 +5,16 @@ import numpy as np # first party -from delphi_claims_hosp.download_claims_ftp_files import (flip_YYYYMMDD_to_DDMMYYYY, +from delphi_claims_hosp.download_claims_ftp_files import (change_date_format, get_timestamp) class TestDownloadClaimsFtpFiles: - def test_flip_YYYYMMDD_to_DDMMYYYY(self): + def test_change_date_format(self): name = "SYNEDI_AGG_INPATIENT_20200611_1451CDT" expected = "SYNEDI_AGG_INPATIENT_11062020_1451CDT" - assert(flip_YYYYMMDD_to_DDMMYYYY(name)==expected) + assert(change_date_format(name)==expected) def test_get_timestamp(self): name = "SYNEDI_AGG_INPATIENT_20200611_1451CDT" From 93fdafa555d01afc635916104f822e83fa9532e3 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Wed, 8 Jun 2022 21:01:40 -0400 Subject: [PATCH 22/46] remove unused code --- claims_hosp/HospClaims/automate/README.md | 12 - .../HospClaims/automate/agg_claims_drops.py | 75 ----- .../automate/download_claims_ftp_files.py | 110 ------- .../HospClaims/automate/ftp_to_covidcast.py | 62 ---- .../automate/get_latest_claims_name.py | 42 --- .../automate/hosp_claims_master_script.sh | 71 ----- .../automate/hosp_claims_regen_script.sh | 68 ----- .../HospClaims/automate/regen_old_issue.py | 47 --- .../HospClaims/automate/sanity_checks.py | 289 ------------------ claims_hosp/HospClaims/automate/secrets.py | 1 - .../HospClaims/automate/update_json.py | 59 ---- 11 files changed, 836 deletions(-) delete mode 100644 claims_hosp/HospClaims/automate/README.md delete mode 100644 claims_hosp/HospClaims/automate/agg_claims_drops.py delete mode 100644 claims_hosp/HospClaims/automate/download_claims_ftp_files.py delete mode 100644 claims_hosp/HospClaims/automate/ftp_to_covidcast.py delete mode 100644 claims_hosp/HospClaims/automate/get_latest_claims_name.py delete mode 100755 claims_hosp/HospClaims/automate/hosp_claims_master_script.sh delete mode 100755 claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh delete mode 100755 claims_hosp/HospClaims/automate/regen_old_issue.py delete mode 100644 claims_hosp/HospClaims/automate/sanity_checks.py delete mode 120000 claims_hosp/HospClaims/automate/secrets.py delete mode 100644 claims_hosp/HospClaims/automate/update_json.py diff --git a/claims_hosp/HospClaims/automate/README.md b/claims_hosp/HospClaims/automate/README.md deleted file mode 100644 index 58ebf15b6..000000000 --- a/claims_hosp/HospClaims/automate/README.md +++ /dev/null @@ -1,12 +0,0 @@ -## Dependencies -- paramiko -- click -- numpy -- pandas -- pathlib -- imap_tools -- matplotlib -- requests - -Maria todo: - - remove all absolute path references \ No newline at end of file diff --git a/claims_hosp/HospClaims/automate/agg_claims_drops.py b/claims_hosp/HospClaims/automate/agg_claims_drops.py deleted file mode 100644 index 6fe56dfb3..000000000 --- a/claims_hosp/HospClaims/automate/agg_claims_drops.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 - -"""Aggregates chunks of drops. - -Drops are expected to be numbered as: - -../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz -../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz -... etc. -""" - -# standard -from collections import defaultdict -from pathlib import Path - -# third party -import click -import numpy as np -import pandas as pd - - -def agg_and_write(data_path, force=True): - """ - Aggregate drops given a folder path. Will output an aggregated version in the - same folder. Example below. - - Input files in folder: - ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz - ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz - - Will create: - ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_07052020_1456.csv.gz - - - Args: - data_path: path to the folder with duplicated drops. - force: if aggregated file exists, whether to overwrite or not - """ - - files = np.array(list(Path(data_path).glob("*"))) - - for f in files: - out_path = f.parents[0] / f.name - dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, - "patCountyFIPS": str}) - if "servicedate" in dfs.columns: - dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) - if "patCountyFIPS" in dfs.columns: - dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) - if "patHRRname" in dfs.columns: - dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) - if "patAgeGroup" in dfs.columns: - dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) - if "patHRRid" in dfs.columns: - dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) - - assert np.sum( - dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", - "Pat HRR Name", "PatAgeGroup"])) == 0, \ - "Duplication across drops!" - assert dfs.shape[1] == 10, "Wrong number of columns" - - dfs.to_csv(out_path, index=False) - print(f"Wrote {out_path}") - - -@click.command() -@click.argument('data_path') -@click.option('--force', '-f', is_flag=True, default=False) -def run_cli(data_path, force): - agg_and_write(data_path, force=force) - - -if __name__ == "__main__": - run_cli() diff --git a/claims_hosp/HospClaims/automate/download_claims_ftp_files.py b/claims_hosp/HospClaims/automate/download_claims_ftp_files.py deleted file mode 100644 index 28baff0e8..000000000 --- a/claims_hosp/HospClaims/automate/download_claims_ftp_files.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python3 -"""Downloads files modified in the last 24 hours from the delphi ftp server.""" - -# standard -import datetime -import functools -import sys -from os import path - -# third party -import click -import paramiko - -# first party -from secrets import claims - - -class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): - def missing_host_key(self, client, hostname, key): - return - - -def print_callback(filename, bytes_so_far, bytes_total): - rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) - if (rough_percent_transferred % 25) == 0: - print(f'{filename} transfer: {rough_percent_transferred}%') - - -def get_timestamp(name): - try: - split_name = name.split("_") - yyyymmdd = split_name[3] - hhmm = ''.join(filter(str.isdigit, split_name[4])) - timestamp = datetime.datetime.strptime(''.join([yyyymmdd, hhmm]), - "%Y%m%d%H%M") - except Exception: - timestamp = datetime.datetime(1900, 1, 1) - - return timestamp - - -def flip_MMDDYYYY_to_DDMMYYYY(name): - # flip date from MMDDYYYY to DDMMYYYY - split_name = name.split("_") - date = split_name[4] - flip_date = date[2:4] + date[:2] + date[4:] - split_name[4] = flip_date - name = '_'.join(split_name) - return name - - -def flip_YYYYMMDD_to_DDMMYYYY(name): - split_name = name.split("_") - date = split_name[3] - flip_date = date[6:] + date[4:6] + date[:4] - split_name[3] = flip_date - name = '_'.join(split_name) - return name - - -@click.command() -@click.argument("out_path") -def download(out_path): - current_time = datetime.datetime.now() - seconds_in_day = 24 * 60 * 60 - print(f"current time is {current_time}") - - # open client - client = paramiko.SSHClient() - client.set_missing_host_key_policy(AllowAnythingPolicy()) - - client.connect(claims.HOST, - username=claims.USER, password=claims.PASS, port=claims.PORT) - sftp = client.open_sftp() - sftp.chdir('/hosp/receiving') - - - # go through files in recieving dir - files_to_download = [] - for fileattr in sftp.listdir_attr(): - # file_time = datetime.datetime.fromtimestamp(fileattr.st_mtime) - file_time = get_timestamp(fileattr.filename) - time_diff_to_current_time = current_time - file_time - if 0 < time_diff_to_current_time.total_seconds() <= seconds_in_day: - files_to_download.append(fileattr.filename) - - # make sure we don't download more that the 3 chunked drops (2x a day) for OP - # and the 1 chunk (2x a day) for IP - 01/07/21, *2 for multiple day drops - assert len(files_to_download) <= 2 * ((3 * 2) + 2), "more files dropped than expected" - - filepaths_to_download = {} - for file in files_to_download: - flipped_file = flip_YYYYMMDD_to_DDMMYYYY(file) - if "INPATIENT" in file: - full_path = path.join(out_path, flipped_file) - if path.exists(full_path): - print(f"{flipped_file} exists, skipping") - else: - filepaths_to_download[file] = full_path - - # download! - for infile, outfile in filepaths_to_download.items(): - callback_for_filename = functools.partial(print_callback, infile) - sftp.get(infile, outfile, callback=callback_for_filename) - - client.close() - - -if __name__ == "__main__": - download() diff --git a/claims_hosp/HospClaims/automate/ftp_to_covidcast.py b/claims_hosp/HospClaims/automate/ftp_to_covidcast.py deleted file mode 100644 index 2356783c3..000000000 --- a/claims_hosp/HospClaims/automate/ftp_to_covidcast.py +++ /dev/null @@ -1,62 +0,0 @@ -"""FTP created files over to Delphi Covidcast ingestion.""" -# standard -import datetime -import os -from pathlib import Path - -# third party -import click -import paramiko - -# first party -from secrets import covidcast - -NUM_FILES = 70*6*2 # expect (70 dates x 6 geos x 2 signals) -NUM_SE_FILES = 70*6*1 # expect (70 dates x 6 geos x 1 signals) - - -class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): - def missing_host_key(self, client, hostname, key): - return - - -@click.command() -@click.argument("local_receiving_dir") -def upload(local_receiving_dir): - """Upload files to the delphi covidcast ingestion folders - - Args: - local_receiving_dir: local dir containing the non-se signal files - - """ - today = datetime.datetime.now().date() - - # open client - client = paramiko.SSHClient() - client.set_missing_host_key_policy(AllowAnythingPolicy()) - client.connect(covidcast.HOST, username=covidcast.USER, password=covidcast.PASS) - sftp = client.open_sftp() - - files_to_upload = [] - for file in Path(local_receiving_dir).glob("*.csv"): - files_to_upload.append(file) - - assert len(files_to_upload) == NUM_FILES, "more files to upload than expected!" - - # upload signal without se - sftp.chdir("/common/covidcast/receiving/hospital-admissions") - for i, file in enumerate(files_to_upload): - assert ( - datetime.datetime.fromtimestamp(os.path.getmtime(file)).date() == today - ), f"uploading old file {file}" - - sftp.put(file, file.name) - if (i % 61) == 0: - print(f"Finished {i} out of {len(files_to_upload)}") - - print(f"Successfully uploaded the hospital-admissions claims signal") - client.close() - - -if __name__ == "__main__": - upload() diff --git a/claims_hosp/HospClaims/automate/get_latest_claims_name.py b/claims_hosp/HospClaims/automate/get_latest_claims_name.py deleted file mode 100644 index 9dadbe5dc..000000000 --- a/claims_hosp/HospClaims/automate/get_latest_claims_name.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -"""Return the latest drop.""" - -# standard -import datetime -from pathlib import Path - -# third party -import click - - -@click.command() -@click.argument("dir_path") -def get_latest_filename(dir_path): - current_date = datetime.datetime.now() - files = list(Path(dir_path).glob("*")) - - latest_timestamp = datetime.datetime(1900, 1, 1) - latest_filename = None - for file in files: - split_name = file.name.split("_") - if len(split_name) == 5: - ddmmyyyy = split_name[3] - hhmm = ''.join(filter(str.isdigit, split_name[4])) - timestamp = datetime.datetime.strptime(''.join([ddmmyyyy, hhmm]), - "%d%m%Y%H%M") - if timestamp > latest_timestamp: - if timestamp <= current_date: - latest_timestamp = timestamp - latest_filename = file - - assert current_date.date() == latest_timestamp.date(), "no drop for today" - - # write to stdout for shell script to use - print(latest_filename) - - # return for other uses - return latest_filename - - -if __name__ == "__main__": - get_latest_filename() diff --git a/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh b/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh deleted file mode 100755 index 836709bf4..000000000 --- a/claims_hosp/HospClaims/automate/hosp_claims_master_script.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh -set -o errexit -set -o pipefail - -BASE="/home/indicators/runtime/claims_hosp/HospClaims" - -AUTO_DIR="$BASE/automate" -HOSP_CLAIMS_PKG_DIR="/home/indicators/runtime/claims_hosp" -CLAIMS_DIR="$BASE/claims_data" -GEO_DIR="/common/covidcast/covid-19/geographical_scope" -CURRENT_dmY=$(date '+%d%m%Y') -CURRENT_Ymd=$(date '+%Y%m%d') -CURRENT_YmdHM=$(date '+%Y%m%d_%H%M') -RECEIVING_DIR="$BASE/receiving/results_$CURRENT_YmdHM" -RECEIVING_SE_DIR="$BASE/receiving/results_se_$CURRENT_YmdHM" - -# pull latest data -echo "downloading drops" -cd "$AUTO_DIR" || exit -python3 download_claims_ftp_files.py "$CLAIMS_DIR" - -# aggregate data -echo "aggregating drops" -python3 agg_claims_drops.py "$CLAIMS_DIR" - -# find the latest files (these have timestamps) -echo "finding today's latest claims drop" -claims_file=$(python3 get_latest_claims_name.py "$CLAIMS_DIR") - -# make receiving directories -mkdir "$RECEIVING_DIR" - -# generate the sensor -cd "$HOSP_CLAIMS_PKG_DIR" || exit - -source env/bin/activate - -python $AUTO_DIR/update_json.py \ - "$claims_file" \ - "$GEO_DIR" \ - "$HOSP_PKG_DIR" \ - "$RECEIVING_DIR" - -python -m delphi_claims_hosp - -deactivate - -sanity_check() { - geo=$1 - cd "$AUTO_DIR" || exit - python3 sanity_checks.py "$RECEIVING_DIR" "$geo" -} - -echo "running sanity checks" -sanity_check state -sanity_check msa -sanity_check hrr -sanity_check county - -# plot states without se -cd "$AUTO_DIR" || exit -python3 sanity_checks.py "$RECEIVING_DIR" state -p - -# upload files to covidcast -python3 ftp_to_covidcast.py "$RECEIVING_DIR" - -# delete raw data -rm "$CLAIMS_DIR"/*.csv.gz - -# delete signal files -rm -r "$RECEIVING_DIR" diff --git a/claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh b/claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh deleted file mode 100755 index 2111f7b7e..000000000 --- a/claims_hosp/HospClaims/automate/hosp_claims_regen_script.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh -set -o errexit -#set -o nounset -set -o pipefail - -BASE="/home/indicators/runtime/claims_hosp/HospClaims" -AUTO_DIR="$BASE/automate" -HOSP_CLAIMS_PKG_DIR="/home/indicators/runtime/claims_hosp" -CLAIMS_DIR="$BASE/claims_data" -GEO_DIR="/common/covidcast/covid-19/geographical_scope" -RECEIVING_DIR="$1" - -# pull latest data -echo "downloading drops" -cd "$AUTO_DIR" || exit -python3 download_claims_ftp_files.py "$CLAIMS_DIR" - -# find the latest files (these have timestamps) -echo "finding today's latest claims drop" -claims_file=$(python3 get_latest_claims_name.py "$CLAIMS_DIR") - -# only keep latest file -cd "$CLAIMS_DIR" || exit -claims_filename=$(basename "$claims_file") -echo "$claims_filename" -mv "$claims_filename" .. -cd .. -rm -f "$CLAIMS_DIR"/*.csv.gz -mv "$claims_filename" "$CLAIMS_DIR" - -# aggregate data -cd "$AUTO_DIR" || exit -echo "aggregating drops" -python3 agg_claims_drops.py "$CLAIMS_DIR" - -# generate the sensor -cd "$HOSP_CLAIMS_PKG_DIR" || exit - -source env/bin/activate - -python $AUTO_DIR/update_json.py \ - "$claims_file" \ - "$GEO_DIR" \ - "$HOSP_PKG_DIR" \ - "$RECEIVING_DIR" - -python -m delphi_claims_hosp - -deactivate - -sanity_check() { - geo=$1 - cd "$AUTO_DIR" || exit - python3 sanity_checks.py "$RECEIVING_DIR" "$geo" -} - -echo "running sanity checks" -sanity_check state -sanity_check msa -sanity_check hrr -sanity_check county - -# plot states without se -#cd "$AUTO_DIR" || exit -#python3 sanity_checks.py "$RECEIVING_DIR" state -p - -# delete raw data -rm "$CLAIMS_DIR"/*.csv.gz diff --git a/claims_hosp/HospClaims/automate/regen_old_issue.py b/claims_hosp/HospClaims/automate/regen_old_issue.py deleted file mode 100755 index 838d4b24d..000000000 --- a/claims_hosp/HospClaims/automate/regen_old_issue.py +++ /dev/null @@ -1,47 +0,0 @@ -from datetime import datetime, timedelta -import os -import logging - - -def regen(issue_date: datetime): - fake_date = datetime.strftime(issue_date, '%Y%m%d') - fake_datetime = datetime.strftime(issue_date, '%Y-%m-%d %H:%M:%S') - - out_dir = f"/home/maria/Delphi/HospClaims/regen/issue_{fake_date}" - out_dir_no_se = out_dir + "/hospital-admissions" - #if os.path.isdir(out_dir_no_se) and len(os.listdir(out_dir_no_se)) > 0: - # logging.info(f"files in output dir, skipping {issue_date}") - # return False - - os.makedirs(out_dir_no_se, exist_ok=True) - os.system( - f"faketime '{fake_datetime}' /home/maria/Delphi/HospClaims/automate/hosp_claims_regen_script.sh {out_dir_no_se}") - - logging.info(str(issue_date.date())) - - -def main(): - hour = 23 - - start_date = datetime(2021, 6, 12, hour) - end_date = datetime(2021, 6, 13, hour) - #start_date = datetime(2020, 6, 2, hour) - #end_date = datetime(2020, 8, 4, hour) - n_dates = (end_date - start_date).days + 1 - date_range = [start_date + timedelta(days=a) for a in range(n_dates)] - - logging.basicConfig(level=logging.DEBUG, filename="out.log", - filemode="a+", - format="%(asctime)-15s %(levelname)-8s %(message)s") - - #date_range = [datetime(2020, 6, 21, hour)] - for date in date_range: - try: - regen(date) - except Exception as e: - logging.info(e) - continue - - -if __name__ == "__main__": - main() diff --git a/claims_hosp/HospClaims/automate/sanity_checks.py b/claims_hosp/HospClaims/automate/sanity_checks.py deleted file mode 100644 index f2b03fe65..000000000 --- a/claims_hosp/HospClaims/automate/sanity_checks.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Sanity check results from generating DV estimates. - -Author: Maria Jahja -Created: 2020-05-12 - -Plotting code modified from: http://blog.marmakoide.org/?p=94 -""" - -# standard packages -import logging -import sys -from collections import defaultdict -from datetime import datetime, timedelta -from pathlib import Path - -# third party -import click -import matplotlib.dates as mpld -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from matplotlib.backends.backend_pdf import PdfPages - -# first party -EPIDATA_DIR = Path.home() / "Delphi/delphi-epidata/src/client" -FIPS_DIR = Path.home() / "Delphi/covid-19/doctor-visits/maria/data/fips_full.csv" -sys.path.append(str(EPIDATA_DIR)) -from delphi_epidata import Epidata - - -class EMRHospChecks: - DATE_FORMAT = mpld.DateFormatter('%m-%d') - - def __init__(self, data_path, level, se): - self.level = level - self.data = self.get_data(data_path, level, se) - self.locs = list(sorted(set(self.data["adj"]["val"].keys()) | \ - set(self.data["nadj"]["val"].keys()))) - self.se = se - - # read in geo file for fips - self.geo = pd.read_csv(FIPS_DIR, dtype={"FIPS": int}) - self.geo.drop_duplicates('FIPS', inplace=True) - - @staticmethod - def get_data(data_path, level, se): - """ - Compile data values and dates for given data_path and geographic level - - Args: - data_path: path to the data files - level: geographic level to pull - se: bool if se's are included in the file - - Returns: - dictionary with data - """ - - def extract(all_files, all_dates): - """Extract data from the files.""" - res = {"val": defaultdict(list), - "se": defaultdict(list), - "dates": defaultdict(list)} - for f, d in zip(all_files, all_dates): - df = pd.read_csv(open(f, "rb"), dtype={"geo_id": str}).to_numpy() - for row in df: - geo = row[0] - res["val"][geo].append(row[1]) - res["se"][geo].append(row[2]) - res["dates"][geo].append(d) - return res - - data = {"adj_files": [], "nadj_files": [], "dates": []} - for f in sorted(data_path.glob("*")): - name = f.name.split("_") - if f.suffix == ".csv" and name[1] == level: - name_idx = -2 if se else 3 - if name[name_idx] == "adj": - data["adj_files"].append(f) - else: - data["nadj_files"].append(f) - data["dates"].append(name[0]) - - # extract data - data["dates"] = sorted(list(set(data["dates"]))) - data["adj"] = extract(data["adj_files"], data["dates"]) - data["nadj"] = extract(data["nadj_files"], data["dates"]) - - # convert dates - data["dates"] = pd.to_datetime(data["dates"]) - data["first_date"] = data["dates"].min() - data["last_date"] = data["dates"].max() - data["first_plot_date"] = data["last_date"] - timedelta(days=30) - data["epidata_date_range"] = Epidata.range( - str(data["first_plot_date"].date()).replace('-', ''), - str(data["last_date"].date()).replace('-', '')) - - return data - - def check_se_na(self): - """ - Checks that all SE are reported as 'NA' due to - privacy concerns from the company. - - Returns: - true if pass, false otherwise - """ - - for kind in ["adj", "nadj"]: - for geo, ses in self.data[kind]["se"].items(): - for se in ses: - if not np.isnan(se): - logging.error(f"{geo}, {se} not nan") - return False - return True - - def check_range(self): - """ - Checks that all percentages are within [0, 100]. - - Returns: - true if pass, false otherwise - """ - for kind in ["adj", "nadj"]: - for geo, vals in self.data[kind]["val"].items(): - for val in vals: - if not (0 <= val <= 100): - logging.error(f"{geo}, {val} not in [0, 100]") - return False - return True - - def check_quantity(self): - """Checks how many geographies were generated.""" - n_geos = {} - logging.info(f"geographies generated for {self.level}") - for kind in ["adj", "nadj"]: - for geo, vals in self.data[kind]['val'].items(): - n_geos[geo] = len(vals) - - min_geo = np.min([v for k, v in n_geos.items()]) - max_geo = np.max([v for k, v in n_geos.items()]) - avg_geo = np.mean([v for k, v in n_geos.items()]) - std_geo = np.std([v for k, v in n_geos.items()]) - logging.info(f"\t{kind}" - f"\nmin:\t{min_geo}\nmax:\t{max_geo}" - f"\navg:\t{avg_geo:.2f}\nstd:\t{std_geo:.2f}") - - def get_filled_df(self, loc, kind): - df = pd.DataFrame({"val": self.data[kind]["val"][loc]}, - index=pd.to_datetime(self.data[kind]["dates"][loc])) - - if self.data["first_plot_date"] not in df.index: - df = df.append( - pd.DataFrame({"val": np.nan}, index=[self.data["first_plot_date"]])) - if self.data["last_date"] not in df.index: - df = df.append(pd.DataFrame({"val": np.nan}, index=[self.data["last_date"]])) - df.sort_index(inplace=True) - df = df.asfreq('D', fill_value=np.nan) - return df[df.index > self.data["first_plot_date"]] - - def get_epidata_df(self, loc, kind): - epi_kind = "smoothed_adj_covid19_from_claims" if kind == "adj" else "smoothed_covid19_from_claims" - if self.level == "msa": - loc = int(float(loc)) - - rows = Epidata.covidcast("hospital-admissions", epi_kind, "day", - self.level, self.data["epidata_date_range"], loc) - vals = [] - obs_dates = [] - for row in rows['epidata']: - vals.append(row['value']) - obs_dates.append(row['time_value']) - - obs_dates = [datetime.strptime(str(d), "%Y%m%d") for d in obs_dates] - df = pd.DataFrame({'date': obs_dates, 'val': vals}) - df = df.set_index('date') - return df - - def get_county_name(self, fips_code): - """Return name of a county given it's fips code.""" - loc = self.geo[self.geo["FIPS"] == fips_code] - if len(loc) == 0: - return fips_code - return f'{loc["Name"].iloc[0]} County, {loc["State"].iloc[0]}' - - def plot(self, outname): - """ Create PDF plots of the generated values by location. - - Args: - outname: name for the output pdf file - """ - - # start pdf document - pdf_pages = PdfPages(f'{outname}-{self.level}-hosp-claims-plots.pdf') - n_plot = len(self.locs) - n_plots_per_page = 25 - - # init plotting axis and counter - fig, axs = None, None - j = 0 - - for i, loc in enumerate(self.locs): - - # start new page if needed - if i % n_plots_per_page == 0: - fig, axs = plt.subplots(5, 5, figsize=(10, 10), sharex=True) - axs = axs.ravel() - j = 0 - - # plot - adj_ts = self.get_filled_df(loc, "adj") - axs[j].plot(adj_ts.index, adj_ts["val"], label="New (Adj)", color="blue") - - if not self.se: - nadj_ts = self.get_filled_df(loc, "nadj") - axs[j].plot(nadj_ts.index, nadj_ts["val"], label="New", color="green") - - # current data. left unlabeled to clear clutter, but colors correspond to - # the "new" lines. only plot first 52 cases (it's rather slow to run otherwise) - if self.level == "state" or \ - ((self.level == "county") and (loc in ["53033", "36061"])): - try: - epi_adj_ts = self.get_epidata_df(loc, "adj") - axs[j].plot(epi_adj_ts.index, epi_adj_ts["val"], - color="lightskyblue", linestyle="--") - if not self.se: - epd_nadj_ts = self.get_epidata_df(loc, "nadj") - axs[j].plot(epd_nadj_ts.index, epd_nadj_ts["val"], - color="lightgreen", linestyle="--") - except: - logging.warning(f"could not retrieve {loc} in epidata, skipping") - - # set title - if self.level == "county": - axs[j].set_title(self.get_county_name(int(loc)), fontsize=10) - else: - axs[j].set_title(loc) - - # set legend and format - if i == 0 or j == 0: - axs[j].legend() - - axs[j].xaxis.set_major_formatter(self.DATE_FORMAT) - axs[j].tick_params(axis='both', which='major', labelsize=5, labelrotation=90) - - # close the page if needed - if (i + 1) % n_plots_per_page == 0 or (i + 1) == n_plot: - plt.tight_layout() - pdf_pages.savefig(fig) - plt.close() - j += 1 - - pdf_pages.close() - logging.info(f"plotted to '{outname}-{self.level}-hosp-claims-plots.pdf'") - - -def run(respath, geo, se, plot): - """Run sanity checks and produce plots. - - Args: - respath: path to result csvs - geo: geo level, one of state, msa, hrr, county - se: boolean whether data includes se or not - plot: boolean whether to plot or not - """ - assert geo in ["state", "msa", "hrr", "county"], f"{geo} is invalid" - - ehc = EMRHospChecks(Path(respath), geo, se) - assert ehc.check_range(), "range failed" - if not se: - assert ehc.check_se_na(), "se is all na failed" - ehc.check_quantity() - if plot: - ehc.plot(str(datetime.today().date())) - logging.info("finished checks") - - -@click.command() -@click.argument('respath') -@click.argument('geo') -@click.option('--se', is_flag=True, default=False) -@click.option('--plot', '-p', is_flag=True, default=False) -def run_cli(respath, geo, se, plot): - logging.basicConfig(level=logging.INFO) - run(respath, geo, se, plot) - - -if __name__ == "__main__": - run_cli() diff --git a/claims_hosp/HospClaims/automate/secrets.py b/claims_hosp/HospClaims/automate/secrets.py deleted file mode 120000 index 4ae593b93..000000000 --- a/claims_hosp/HospClaims/automate/secrets.py +++ /dev/null @@ -1 +0,0 @@ -../../secrets.py \ No newline at end of file diff --git a/claims_hosp/HospClaims/automate/update_json.py b/claims_hosp/HospClaims/automate/update_json.py deleted file mode 100644 index 3bdcf5b3b..000000000 --- a/claims_hosp/HospClaims/automate/update_json.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Change the json template to run the package.""" - -import argparse -import json -from pathlib import Path - - -def get_hosp(edi_file, geo_dir, out_dir, receiving_dir, se): - """Output the json.params file needed to run the claims_hosp package. - - Args: - edi_file: Path to EDI file (claims) - geo_dir: Dir containing the geo_map files - out_dir: Output dir to put the json file - receiving_dir: Output dir to put the hosp estimates - se: Boolean to output SEs or not - """ - - if se: - weekday = [True] - else: - weekday = [True, False] - - data = { - "common": { - "export_dir": receiving_dir, - "log_exceptions": False - }, - "indicator": { - "input_file": edi_file, - "start_date": None, - "end_date": None, - "drop_date": None, - "n_backfill_days": 70, - "n_waiting_days": 3, - "write_se": se, - "obfuscated_prefix": "wip_henear", - "parallel": True, - "geos": ["state", "msa", "hrr", "county", "hhs", "nation"], - "weekday": weekday - } - } - print(f"Using {edi_file}") - - with open(Path(out_dir) / 'params.json', 'w') as outfile: - json.dump(data, outfile, indent=4) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('edi_file') - parser.add_argument('geo_dir') - parser.add_argument('out_dir') - parser.add_argument('receiving_dir') - parser.add_argument('--se', action="store_true") - - args = parser.parse_args() - get_hosp(args.edi_file, args.geo_dir, - args.out_dir, args.receiving_dir, args.se) From 42d72173a1060748abf00f57105b8d61b36d8be5 Mon Sep 17 00:00:00 2001 From: Jingjing Tang <31444565+jingjtang@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:05:42 -0400 Subject: [PATCH 23/46] Use os.remove to clean raw files Co-authored-by: Katie Mazaitis --- claims_hosp/delphi_claims_hosp/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index c7ed0e31a..7817ef4f1 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -137,7 +137,7 @@ def run_module(params): # Remove all the raw files for fn in os.listdir(params["indicator"]["input_dir"]): if ".csv.gz" in fn: - os.system(f'rm {params["indicator"]["input_dir"]}/{fn}') + os.remove(f'{params["indicator"]["input_dir"]}/{fn}') logger.info('Remove all the raw files.') elapsed_time_in_seconds = round(time.time() - start_time, 2) From a011ffa11029eb5e3be239df5826661634fc08af Mon Sep 17 00:00:00 2001 From: Jingjing Tang <31444565+jingjtang@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:06:13 -0400 Subject: [PATCH 24/46] Use mock for the logger in unit tests Co-authored-by: Katie Mazaitis --- claims_hosp/tests/test_get_latest_claims_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/tests/test_get_latest_claims_name.py b/claims_hosp/tests/test_get_latest_claims_name.py index a64c39710..6ecf06e6c 100644 --- a/claims_hosp/tests/test_get_latest_claims_name.py +++ b/claims_hosp/tests/test_get_latest_claims_name.py @@ -11,7 +11,7 @@ class TestGetLatestFileName: start_time = time.time() - logger = get_structured_logger( + logger = unittest.mock.Mock() __name__, filename="./test.log", log_exceptions=True) From 06875d7c90f56ba54e1d43fec35756305348af2a Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 14 Jun 2022 12:19:36 -0400 Subject: [PATCH 25/46] include filename with errors --- claims_hosp/delphi_claims_hosp/agg_claims_drops.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py index 45b57d03a..bb9618204 100644 --- a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py +++ b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py @@ -38,7 +38,8 @@ def agg_and_write(data_path, logger): files = np.array(list(Path(data_path).glob("*"))) for f in files: - if ".csv.gz" not in str(f): + filename = str(f) + if ".csv.gz" not in filename: continue out_path = f.parents[0] / f.name dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, @@ -57,8 +58,8 @@ def agg_and_write(data_path, logger): assert np.sum( dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", "Pat HRR Name", "PatAgeGroup"])) == 0, \ - "Duplication across drops!" - assert dfs.shape[1] == 10, "Wrong number of columns" + f'Duplication across drops in {filename}!' + assert dfs.shape[1] == 10, f'Wrong number of columns in {filename}' dfs.to_csv(out_path, index=False) logger.info(f"Wrote {out_path}") From cdd0583bcd5b4b7a2095669eaa88492e6a28fa6a Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 14 Jun 2022 12:22:01 -0400 Subject: [PATCH 26/46] delete commented-out code --- claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index a97d7718a..50eb6eee1 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -69,7 +69,6 @@ def download(ftp_credentials, out_path, logger): # go through files in recieving dir files_to_download = [] for fileattr in sftp.listdir_attr(): - # file_time = datetime.datetime.fromtimestamp(fileattr.st_mtime) file_time = get_timestamp(fileattr.filename) time_diff_to_current_time = current_time - file_time if 0 < time_diff_to_current_time.total_seconds() <= seconds_in_day: From 4f86e1eb1a3ddd380aefa21b1c0c996aa98b7e40 Mon Sep 17 00:00:00 2001 From: Jingjing Tang <31444565+jingjtang@users.noreply.github.com> Date: Tue, 14 Jun 2022 12:23:49 -0400 Subject: [PATCH 27/46] Update logger info with variables Co-authored-by: Katie Mazaitis --- claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 50eb6eee1..619fa8a1e 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -22,7 +22,7 @@ def print_callback(filename, logger, bytes_so_far, bytes_total): """Print the callback information.""" rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) if (rough_percent_transferred % 25) == 0: - logger.info(f'{filename} transfer: {rough_percent_transferred}%') + logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred) def get_timestamp(name): From 747d58029a834fe6f82604860067eef84d62b730 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 14 Jun 2022 12:25:16 -0400 Subject: [PATCH 28/46] Remove continue but check file links with .csv.gz --- claims_hosp/delphi_claims_hosp/agg_claims_drops.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py index bb9618204..f5206f334 100644 --- a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py +++ b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py @@ -35,12 +35,10 @@ def agg_and_write(data_path, logger): force: if aggregated file exists, whether to overwrite or not """ - files = np.array(list(Path(data_path).glob("*"))) + files = np.array(list(Path(data_path).glob("*.csv.gz"))) for f in files: filename = str(f) - if ".csv.gz" not in filename: - continue out_path = f.parents[0] / f.name dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, "patCountyFIPS": str}) From c0482c3d01b5e9433353f1d1cd26d86eb62e9103 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 14 Jun 2022 12:34:00 -0400 Subject: [PATCH 29/46] fix the error in switching to Mock in unittest.mock --- claims_hosp/tests/test_get_latest_claims_name.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/claims_hosp/tests/test_get_latest_claims_name.py b/claims_hosp/tests/test_get_latest_claims_name.py index 6ecf06e6c..2f3021af0 100644 --- a/claims_hosp/tests/test_get_latest_claims_name.py +++ b/claims_hosp/tests/test_get_latest_claims_name.py @@ -1,19 +1,18 @@ # standard import time +from unittest.mock import Mock # third party import pytest -from delphi_utils import get_structured_logger + from delphi_claims_hosp.get_latest_claims_name import get_latest_filename class TestGetLatestFileName: start_time = time.time() - logger = unittest.mock.Mock() - __name__, filename="./test.log", - log_exceptions=True) + logger = Mock() def test_get_latest_claims_name(self): dir_path = "./test_data/" From ce1f45e1e411afa4ae3b1e5b62b4b7419e317f57 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 14 Jun 2022 12:34:46 -0400 Subject: [PATCH 30/46] remove unused variables --- claims_hosp/tests/test_get_latest_claims_name.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/claims_hosp/tests/test_get_latest_claims_name.py b/claims_hosp/tests/test_get_latest_claims_name.py index 2f3021af0..ded5c9718 100644 --- a/claims_hosp/tests/test_get_latest_claims_name.py +++ b/claims_hosp/tests/test_get_latest_claims_name.py @@ -10,8 +10,6 @@ class TestGetLatestFileName: - - start_time = time.time() logger = Mock() def test_get_latest_claims_name(self): From 2546ceae9fb457cea04977eb39238f55b2871032 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 14 Jun 2022 12:35:18 -0400 Subject: [PATCH 31/46] remove unused messages --- claims_hosp/.pylintrc | 1 - 1 file changed, 1 deletion(-) diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc index 8ba5e540a..ef44f2925 100644 --- a/claims_hosp/.pylintrc +++ b/claims_hosp/.pylintrc @@ -7,7 +7,6 @@ disable=logging-format-interpolation, # Allow pytest functions to be part of a class. no-self-use, # Allow pytest classes to have one test. - too-few-public-methods, broad-except From 7adf66f69a468c86538c8bb4f013ac821775ec9e Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 14 Jun 2022 13:17:20 -0400 Subject: [PATCH 32/46] add too-few-public-methods back to message control for linting --- claims_hosp/.pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc index ef44f2925..8ba5e540a 100644 --- a/claims_hosp/.pylintrc +++ b/claims_hosp/.pylintrc @@ -7,6 +7,7 @@ disable=logging-format-interpolation, # Allow pytest functions to be part of a class. no-self-use, # Allow pytest classes to have one test. + too-few-public-methods, broad-except From 0c39be0df634c483314fb6daf493a041b419f156 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Wed, 15 Jun 2022 10:07:42 -0400 Subject: [PATCH 33/46] add logger info for files to download --- claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 619fa8a1e..f195ce713 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -73,6 +73,7 @@ def download(ftp_credentials, out_path, logger): time_diff_to_current_time = current_time - file_time if 0 < time_diff_to_current_time.total_seconds() <= seconds_in_day: files_to_download.append(fileattr.filename) + logger.info("File to download", filename=fileattr.filename) # make sure we don't download more that the 3 chunked drops (2x a day) for OP # and the 1 chunk (2x a day) for IP - 01/07/21, *2 for multiple day drops @@ -84,7 +85,7 @@ def download(ftp_credentials, out_path, logger): if "INPATIENT" in file: full_path = path.join(out_path, flipped_file) if path.exists(full_path): - logger.info(f"{flipped_file} exists, skipping") + logger.info("Skip the existing file", filename=flipped_file) else: filepaths_to_download[file] = full_path From c77c8117f2e38cc892f4019a7c19ff9331ff9c93 Mon Sep 17 00:00:00 2001 From: Jingjing Tang <31444565+jingjtang@users.noreply.github.com> Date: Wed, 15 Jun 2022 10:08:30 -0400 Subject: [PATCH 34/46] Update logger info for the latest claims file Co-authored-by: Katie Mazaitis --- claims_hosp/delphi_claims_hosp/get_latest_claims_name.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py index 39b4808d0..e417183c7 100644 --- a/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py +++ b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py @@ -26,8 +26,7 @@ def get_latest_filename(dir_path, logger): assert current_date.date() == latest_timestamp.date(), "no drop for today" - # write to stdout for shell script to use - logger.info(latest_filename) + logger.info("Latest claims file", filename=latest_filename) # return for other uses return latest_filename From 0be164cea585a8afe3fb12538546026591778ff8 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Wed, 15 Jun 2022 10:19:45 -0400 Subject: [PATCH 35/46] update the function for renmaing the raw drops --- .../delphi_claims_hosp/modify_claims_drops.py | 58 +++++++++++++++++++ claims_hosp/tests/test_modify_claims_drops.py | 15 +++++ 2 files changed, 73 insertions(+) create mode 100644 claims_hosp/delphi_claims_hosp/modify_claims_drops.py create mode 100644 claims_hosp/tests/test_modify_claims_drops.py diff --git a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py new file mode 100644 index 000000000..6fcd88b85 --- /dev/null +++ b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +"""Modify the drops. + +Drops are expected to be numbered as: + +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz +... etc. +""" + +# standard +from pathlib import Path + +# third party +import numpy as np +import pandas as pd + + +def modify_and_write(data_path, logger, force=True): + """ + Modify drops given a folder path. + + Will rename necessary columns in the input files, and check the number of + columns and duplications. + + Args: + data_path: path to the folder with duplicated drops. + force: if aggregated file exists, whether to overwrite or not + + """ + files = np.array(list(Path(data_path).glob("*.csv.gz"))) + + for f in files: + filename = str(f) + out_path = f.parents[0] / f.name + dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, + "patCountyFIPS": str}) + if "servicedate" in dfs.columns: + dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) + if "patCountyFIPS" in dfs.columns: + dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) + if "patHRRname" in dfs.columns: + dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) + if "patAgeGroup" in dfs.columns: + dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) + if "patHRRid" in dfs.columns: + dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) + + assert np.sum( + dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", + "Pat HRR Name", "PatAgeGroup"])) == 0, \ + f'Duplication across drops in {filename}!' + assert dfs.shape[1] == 10, f'Wrong number of columns in {filename}' + + if force: + dfs.to_csv(out_path, index=False) + logger.info(f"Wrote {out_path}") diff --git a/claims_hosp/tests/test_modify_claims_drops.py b/claims_hosp/tests/test_modify_claims_drops.py new file mode 100644 index 000000000..f2fdfa4d2 --- /dev/null +++ b/claims_hosp/tests/test_modify_claims_drops.py @@ -0,0 +1,15 @@ +# standard +from unittest.mock import Mock + +# third party + +# first party +from delphi_claims_hosp.modify_claims_drops import (modify_and_write) + + +class TestDropsModification: + + def test_modify_and_write(self): + data_path = "./test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz" + logger = Mock() + modify_and_write(data_path, logger, force=False) From 2c9b21dafde761ccf6189befdb0756e6675e6086 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Wed, 15 Jun 2022 10:25:55 -0400 Subject: [PATCH 36/46] remove agg function --- .../delphi_claims_hosp/agg_claims_drops.py | 63 ------------------- 1 file changed, 63 deletions(-) delete mode 100644 claims_hosp/delphi_claims_hosp/agg_claims_drops.py diff --git a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py b/claims_hosp/delphi_claims_hosp/agg_claims_drops.py deleted file mode 100644 index f5206f334..000000000 --- a/claims_hosp/delphi_claims_hosp/agg_claims_drops.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -"""Aggregates chunks of drops. - -Drops are expected to be numbered as: - -../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz -../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz -... etc. -""" - -# standard -from pathlib import Path - -# third party -import numpy as np -import pandas as pd - - -def agg_and_write(data_path, logger): - """ - Aggregate drops given a folder path. - - Will output an aggregated version in the same folder. Example below. - - Input files in folder: - ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz - ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz - - Will create: - ../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_07052020_1456.csv.gz - - Args: - data_path: path to the folder with duplicated drops. - force: if aggregated file exists, whether to overwrite or not - - """ - files = np.array(list(Path(data_path).glob("*.csv.gz"))) - - for f in files: - filename = str(f) - out_path = f.parents[0] / f.name - dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, - "patCountyFIPS": str}) - if "servicedate" in dfs.columns: - dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) - if "patCountyFIPS" in dfs.columns: - dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) - if "patHRRname" in dfs.columns: - dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) - if "patAgeGroup" in dfs.columns: - dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) - if "patHRRid" in dfs.columns: - dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) - - assert np.sum( - dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", - "Pat HRR Name", "PatAgeGroup"])) == 0, \ - f'Duplication across drops in {filename}!' - assert dfs.shape[1] == 10, f'Wrong number of columns in {filename}' - - dfs.to_csv(out_path, index=False) - logger.info(f"Wrote {out_path}") From 080ad4515a19e5af22dec5812af11a1acb74cfaf Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Wed, 15 Jun 2022 10:26:25 -0400 Subject: [PATCH 37/46] update import info for the modification function --- claims_hosp/delphi_claims_hosp/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index 7817ef4f1..9092f46dd 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -17,7 +17,7 @@ # first party from .config import Config from .download_claims_ftp_files import download -from .agg_claims_drops import agg_and_write +from .modify_claims_drops import modify_and_write from .get_latest_claims_name import get_latest_filename from .update_indicator import ClaimsHospIndicatorUpdater @@ -62,7 +62,7 @@ def run_module(params): params["indicator"]["input_dir"], logger) # aggregate data - agg_and_write(params["indicator"]["input_dir"], logger) + modify_and_write(params["indicator"]["input_dir"], logger) # find the latest files (these have timestamps) claims_file = get_latest_filename(params["indicator"]["input_dir"], logger) From 819b043698a7f86a51d27e417acf4f2e7579240f Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Wed, 15 Jun 2022 10:27:03 -0400 Subject: [PATCH 38/46] add j2 file for claims_hosp params --- .../templates/claims_hosp-params-prod.py.j2 | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 ansible/templates/claims_hosp-params-prod.py.j2 diff --git a/ansible/templates/claims_hosp-params-prod.py.j2 b/ansible/templates/claims_hosp-params-prod.py.j2 new file mode 100644 index 000000000..5b536ada1 --- /dev/null +++ b/ansible/templates/claims_hosp-params-prod.py.j2 @@ -0,0 +1,43 @@ +{ + "common": { + "export_dir": "./receiving", + "log_exceptions": false + }, + "indicator": { + "input_dir": "./retrieve_files", + "start_date": "2020-02-01", + "end_date": null, + "drop_date": null, + "n_backfill_days": 70, + "n_waiting_days": 3, + "write_se": false, + "obfuscated_prefix": "foo_obfuscated", + "parallel": false, + "geos": ["state", "msa", "hrr", "county"], + "weekday": [true, false], + "ftp_credentials": { + "host": "{{ claims_hosp_ftp_host }}", + "user": "{{ claims_hosp_ftp_user }}", + "pass": "{{ claims_hosp_ftp_password }}", + "port": 2222 + } + }, + "validation": { + "common": { + "data_source": "hospital-admissions", + "span_length": 14, + "min_expected_lag": {"all": "3"}, + "max_expected_lag": {"all": "4"}, + "dry_run": true, + "suppressed_errors": [] + }, + "static": { + "minimum_sample_size": 5, + "missing_se_allowed": true, + "missing_sample_size_allowed": true + }, + "dynamic": { + "ref_window_size": 7 + } + } +} From cfb682c5da01bb08b77d57640330b9a1e5edfdd6 Mon Sep 17 00:00:00 2001 From: Kathryn M Mazaitis Date: Wed, 15 Jun 2022 14:47:29 -0400 Subject: [PATCH 39/46] Manually merge vault changes --- ansible/vault.yaml | 472 +++++++++++++++++++++++---------------------- 1 file changed, 237 insertions(+), 235 deletions(-) diff --git a/ansible/vault.yaml b/ansible/vault.yaml index cede7133f..8339482ba 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,236 +1,238 @@ $ANSIBLE_VAULT;1.1;AES256 -34613064323664616266326436376330366432336666656438346663383165363865363966343266 -6636656139316131663836633137613730393836666437330a343930613438623366393130653964 -61366435633630353363333631326566663865376462326231643930336435336132346233663934 -3135643864366232380a313333663766383838373565663365333361636438306563346466386662 -37646331643730343964326532373834646366383063306539633566363031653837653438373364 -33383566333135346431383432373564306131316137373636656133653935663732393535323433 -33663463646434303663353864326563653463353532366235663236613664616432623736306635 -32306262303431326338333334356263326665363763643865356237643831616638353432373134 -66336436616464393239663563633061336361626462316434333438373534646432366233343565 -63323732326538663262383363653963333736616161303336303166653432653038613732663265 -39336230363038666133643964373666366239393463646563383431613964376161653335386165 -38646466663363613261316633636133616163396138663062656338613866366236383636623566 -30663434393463316632656663373932343562356366356337353463626234373037663633636366 -32383931343232633036656664616534653565373834356436376130626533356239333661643663 -38393132666661383161616535353165336330646164613631356436323636306632316665383665 -61643534366430383930663563363364663465306638363431656135333630626338653632306363 -38396162383061643936356338343061653766643332613139306536663033323339313366376131 -37336661643735613235383439663137336533653034663838646135386566343836316664633563 -65353061633863396164396538663534623861313861343530616131343237326265316335316438 -35396264306634643736383436396266656265396337653630336162306238303738373935623335 -61623162646162313437306134613336386136613863623365373839656232613431626165366439 -36323336626664663561356538656432366132353131343263313036356335363861623365376431 -30383262363533383732346437373461353061316333316336336438616636626337393937323530 -65666162323334646336313431313736323330613834303466323662336432393866303235633532 -37623163656264353537373736323765353338333162363632393462633237376531333263666665 -38663531396463623933363563376134653133643738616663663736643161393737653738323032 -38643863666264643737393536383936633262636465356339653463323136366230326366323863 -61656536626430336337353430373834643737336630663066613136343538336238366237353862 -35643034363932613433616562313433353964623134393733393039393363343932353137313562 -31323837393665356563306432393234663866623739653336313339393731626364643366306462 -61373365613035623334656539333565346333646235653161646365396466326231646361333466 -39393663373266666231366637343864663232303033633763386132633563333238386261373537 -30373732383737666338663066393138356165303965313536633466633334356465343130363763 -36333262366366323964656239366361346136316633353137353533343333633162616435336539 -34613533643138363663386631303464363535643638653432303830333338613063643530396536 -37633737386139646664333132643731663133383331366237666561623038626230363262623066 -63663030366430373736613530643362316433666566653663333566386439323030323832663162 -31383364626362613530366130383439613233323937653632373831393466363462353765333935 -61363334313962306366376266633430383464656636626338303639613831656632623865333831 -34346136366664323331613330343036336464303838323831333937353930303261346566313066 -62326264343231313664633035313033633862373932663639323966303131656663336334313061 -35613662323931323533626666643561313435326234656435333432376663353931366534313264 -64653630396231663732613663383230353130303562303266336662343635633238383062376662 -66383431373838653338326236366238366165396666656664346636626436616138393161396438 -38376465633031623265373733313333393933643265396432613435633138626663393362656631 -66356638636232316531383332303361656535316332636135376531366536386164363738623339 -66363233383436666431323665623937613533396663356664306137626666646461323334636339 -33303065326164613131366361656531373136373530663461666430313636336633373830303764 -35336636643266333861623265356638626665376163626434623739343734643434363937373261 -64633034623035623566313161613265353736626235376663353364656235363464356364666537 -62353836343733323730343234306662643835323664393634316336633937386135313866666262 -34313339626336343366363464623336633739313533396265336631623034396536303362306130 -37383037323237373562343133643432366464393562613966363630343563643466306633376437 -34623633646662326639623262346435386364306537656330376435613138393337333766323331 -37363137316239363936623264623630656333366165323036343366393039616331373831383539 -61656637633637613739383933623664633039363739336531326163353332363836333966646264 -34323436666334346363326562353363333030333432653239366337626235333831306663633765 -32323661313462663861663931356461656334363163393031396235383331663831353637633662 -33343339616266626634643737346432656161633830346335633734393338666537666165336463 -63356565306563626363366435306335636466653339323965636661653139626632343865393566 -33623335386332653534316462366363343535383634623732353635623733346139386333383335 -39376438636638653639303431306162373764386232636461643964343434636134333039636238 -64303566363765353133636665623662663833343439363564626461333938613731393435663863 -38356635316637366330313739633062333162396236386637373135386339653932366633373439 -61363334366333663338383039343935643836313361326139343530363866396333333962643838 -65316134306532306635643232313538633130383661306432356236343664643438333534356536 -66326638393139363565663131393561396536383965303262373338636163653364626462653663 -34623037373762646562376464353464363334663031336262376639313736313930343838333831 -31366634633937626562623838376266393562323764393661356362316635386261333834633462 -38646638663062306663616435346561366264306635303464313963353933386437613635646566 -65313339393632373664646135333933383838393464363835623964663561386365636530616666 -64323937393062313331613935373536333238303164613462396165343165326261323639313334 -38396365363031303537346136666236646262366635633333333736336632353262666333386534 -34663732316561633736316566353230393461326565393630303762623139656563353162356363 -36366361373466326131653561366534313863313539633734636566303335323963653831303031 -33616238373938303137393637383430383436613935343463643762306531626138643933646233 -61306235393566663362383165313866353661646461613030343761333432333539653965613238 -38366631623536386431313735326337303162613435343537393636383434303334303339323933 -34666462353965336433633932636435393532383733343639363538616163663933363239376638 -39353433333665643466303964646365313966626630646333343661313732613033633436356239 -32313766616535656431366661663663653530633766613139616363666332663331363035623634 -39616664336665346136306238633661393665326430636236303966646264353364616235353461 -37646661623130393033383335356631633637313466346536343539336130363066366235636561 -30323663643232616665373266383833323733356436303936633465303763333431626362336261 -63316562353130636163343265376533633730393831663838346331633261643030636166346239 -35663164303634343864373830333030353062343536393661623737323438363039656161353337 -35633662313435343365323630346238333566623261663563303763633864626632383538386538 -65306362353336366132383232316231313838626333613465616431653639626433323833633532 -31333165396137343530393637396631376439613638666462363964633432633233333239616361 -61356562663533356532363735313365303730616461616239373735633237393732633438646235 -30656137613536623635303337343834366238396235613733313433346164636462666338356636 -63633065393033663733616566323134303865393339363838383430663862323364646431383365 -34626233636136626362383234656464663837306638393334343330663265356337313436373739 -32353862373530386438323366303834386237653861393630303361303138636263653430393339 -66323461636331663063323865386430323866623663333835643034363763623136323638643065 -32336566313532356239623631373861616134636538353530363032336162633638386435646133 -65323538653937343536656633636332353234336666343962613236363066336661353165663731 -35353338336538306665643839303530383138393636663361646431666534633136333561333330 -66326137623237343237323264373031633566633632353438343362663162373461613961616534 -34643362326134376466326261323936313366623931376534343763353237636662313935393530 -32643037376238656465613765623532643035616431366662356365373263643933323439393662 -61363430343066646232323839373232313163636431303964646466386439366565383533333864 -64343063383934393938653861333365623237356334643737306138373638616562626438626330 -34333130363566313439396630633531366561323665626135363664373136383033393365346437 -32396631613562643230376161306433623433323462663431373365383537386233656433373236 -36643539336538623265643531343636646164366237363736343630353735363864333964353434 -35363363623561343239613461303365396634343031633861383963306562343364343062643232 -36393832386366343063306336633837373564376637346235336566616637623934396130336362 -36396561303639353063316538316330373136346636626266396639643430663830636561393639 -36343964646439303263353731623532303564363662626264636239393833383166343939636336 -36326166346637306236356463623836346461316331636439383738363530323562653237633533 -33336166323234666539396332353065613362653035663135313238386537343237346563663037 -61643732633761656438363137336366636163393431656362653661363566333139316138383737 -34346563363033383433306263363662313334333234383362623766356133386639396165656533 -33646230333365383231626533376536396239343661306537353065373036393666623133633135 -63303665653266653731636163383461356236353166393531303765616335616365316434656335 -62646430623361376266626138303239613439663236646238363030326239646365323664396331 -34346364353935323965346330376239653934346336306531616434636130393764376462626433 -61373735623038613461353966653733363166396636386631376538303334613932333233613065 -66616162363937376336613363376333303863343038653661343232653237663539356637613163 -36376132396637343735306566623264363633646532356131616435363038643837623433373633 -65653630346161323964353561326330633262383931623962616632393635363331393132646364 -33633838373438326531663566333738323466646234636466613066383662346461363766333366 -33376235353731326264376634326638316165366261373837343063383964633135343038306532 -66363430616362363666633737333230366338666366653932316639393133396137636164643031 -32373562643936396165353135636433393664366337303034313432636161346330386336323864 -34613939656435623863626466303433666164373438343234653739373939316636383533333962 -66333665373334623963353061616335656666336161353965336233303064633963666565643835 -34646534623932636134623866323531643463336136383633666432623162303034343865613562 -31313961643933633235383235636632333036613137356263356430656162666631333137643638 -33386261313635613937393539666332393561646266353535396161323533346333323164333834 -37643366313839393561316136363665356562326539303733333633613664393566373966363930 -63346164363836373361656261363938343235666537623130653232356132346162316535383533 -36383263666135356162333038383036633538383333633536626531643463346164623866393934 -64366264326634373332613237393735353963336132363331623537386431343837383863323232 -39313165656465306537633661616530383165343564343863393732396237313032333731336362 -64656131323939666237643462333431623336653339383631663734626638653634353834653239 -31333830383134626365656562383236323837643465386464346530333162333832313730663930 -30323536356130333166646135643939393861626133623531343236386638313763326666323532 -37376232656232633237633064363233613265393738613162626461333762343364633963346537 -38616430323931376166353036353031656665323065383538653137306536373561376536666134 -65373438666331333963333437333139373830366632376638376437613930373539643766613639 -64613930376631346534326132313136656334383930373839626337303532373366373330663035 -63356663373036653930333064323038373238623537626563346463613965653139373461323765 -65393964666532313530373037623237623132373766326539323235633062653239343663383738 -39643230393761666637396136633037376566323761313431363339376534643836326661316461 -37326561626335323762613034663764386530643937663233383036383336333839383634393062 -30643233383161316139346331326364643930343461336437373137336434646330663234623933 -33333331613335313365393636646161346138646437663365613966633165323065393732303366 -37393532633832346365363464316231306631373931306137366334643135343265626138373536 -30656262313764396636356132613061363433363461643561646636663031353162373961626663 -38303538383961396662313638313730316137666534646134393030663762373938313138623066 -63333962353734343035336264653937383130326263363633653932346566643638663538333536 -34386330613930353737653461623638303562336163626539333131306630613433616135373263 -64393463393439303363333937616630646335356536643031326434663337633137306331343165 -66653431396663346431663130303961336664313562396435393364613136323934336161366333 -66323761363165333766633032643237353236613739653436613566663263306665313137663263 -38363237643333633964633430313231646535343038386139653731333761323365323935336639 -35376635323466656138336638376236363431623039343462613633306166343863373135646462 -62313239383762613564326533646234303737353930333661363462303031653739666265653737 -37303462636230643139373730303935616133663232373537653032396361383861303135346431 -66393438353038333338616333396537326261376432336431343465633461626464623332373731 -64313832643064633963373334666135303337646166623366366563663136336435323461613531 -35376335366362623039663362353963326238356562386663303139613533343561363330393432 -37336430653264623133663464633439323364396533336366396461393735353862643732333438 -63336436343932666132313064616537653062353139306464353238623930323635396239613030 -63636363383831613136366634633765386336346239383234633937373161353334323836393963 -37643131333361376163663731646662356334616334663363643635393939393361313636326566 -39396332366566383432333232303562636138626337316333346532346265633132336238383731 -61646332363261646234346534303663643836343537393133633664363663393164343137323363 -31323337643135313835613938666566396133386466393938643565623336306636353363346338 -31393665316336626333323964333232653864366331616464623765356537356562343132666332 -36666137356637333766613637643761653666393264386161663135393961313739363963643131 -62633736363839393038386666376533643761316330343666323637386330643366303636303835 -38653366316532383938613032653830383839326638323865356162326132353838346335663037 -39373563626263653864666366383764323366346432356666353862326561386536626439353261 -31663137373630303831396265613439316133643834653933316332333462326265643730623765 -61616262323438316336353934383235313366316264383831313563373438663432663364396635 -34356236613635393137366334383162623933626265613336616335313162303262323264313932 -33343231313336313138626531613138313761353333353561303665373835366133313338656134 -33326230366231663366303166343138326564613336636533356235623936313864623862393562 -38326531396661356363383261626136383036633235633837313035386164333763656262666132 -34313931656466613561333362653933353535393939353538653139346366376537623965333962 -65643232386265633266383464353961363732323137373834343432303364323938323832323063 -31303233323530366563336635303061663738633535373365623037613131626338663565626236 -33313961653961366132666436393634326338643937656239346232666134646138336530333965 -34353062333962346630356630363333626431613266666233393538333133663833613832666236 -31326631636539653539303432363335316136653663316538323261313563366430333234303839 -34366164633534323034386462363738313365313963393064386363613533636161313137366662 -37663332333561363363303931643433666639353136313536383435323634316239613132336235 -64333532393436383135376435393862343939386330306530346163623939653635343635643566 -64376465343530303339383466666335303637626537333233616466346265373439346462303437 -33336665303664636236373566313636366538613563643736343839333964343331373738663038 -63663533666230386364663762333934666665333533363633356335376334646238336434666537 -35663736336364323066373239663630633164303536663830363164663834666331636263303361 -64313738393264653731653239323864313064616337653266646565636661336633353735663261 -62623039303961376430363764313434373034393235373835633733656633316266626230303961 -30653636323237643931316134636362363661343531663962663739393331303262333630396363 -62626462373761343461363166663665646138363132613236306266316262346662373364663764 -31303161616336643637336632396164303865393934613162363831643931316232393930323237 -39313365653537613132613238656663323166646632366462616162336638316237396663333230 -32303736626537353961313936633965346232396639303837353161633462613834663431333834 -35376365376437323835636537336339646432303632363863663831353864303336383038336438 -38613334343161346239373731663863626135613436373661336432343361356138353632386639 -62636234656265613265656538636662636136396164383731386463636562623866386466353465 -66333531373163646631393733613630396665653334306437343235386431666261326161383833 -38653634363830383532626232336566643865323162343833633066646562326631663965323337 -36653332396531376131303037356430353038633431343731633964383436373437316237306633 -65653335656166313630633430303632346163393232343865373263343537656334393939383163 -65306132306439353235313235353566653532646237323535313035353231366465353631663236 -37376661623730616137336266623138313537316236363865313263613862653633666264323661 -61393938393738383736396335643538346134663738613265623633613265346136363137623235 -61336662656430313132326639393337636263373061376665643030636639613065613637386566 -36346538623836343462343465303565303235643936646433623836333162346130383837346338 -62333636653166323065356664353061343339373162623530343163373232396438393735343239 -62393834343236616432316138333438633636393464653335306361636132333538656539333263 -66366334366139613431393435363964626638363636346135383566663635373339346133633833 -34323563353163326664623739613736313731646432303438363530613331643537663736343232 -62313565333961346230313831396538353666386566626666313861363366366561653461383764 -36356431386161633365363130323435346530326162363031653266643139636534313838326234 -32396664346631663730666438643866383363313137636562336134313665316265643662616462 -66666161623261356432633065626533313863636232333430653462373733383332386465613934 -31646231383730336162396264376663353362306530336636643466356461663331326166376630 -61333638346164626332643435333937383464303437366539323965366265323938613935383639 -33393366386532393634336638373338303837653862343433373734356462663931656632376331 -32326466353430303565656237343463383033303662336539633964373936346233326563313732 -66326637656262663762373533393536623565393633366531376565333339363262656432393761 -39626631646537396662303430313136666336326437656462653830636130626363333633626663 -35356131346537303833313936336433346638616632396133613936613938656331366231626333 -61313661353361616537616362633663363935663736643439636338313863343765346338316662 -33656235336239316539336464383861353137353162623334623437366232303335623764646438 -34383436666261333164306331363930643133363335343035646331613931306532653139313466 -33393231336136623732353365346439333039616262633866323066383939633165636264626630 -38376238356235316265373535306531343565363039363364353237376534616631323130633430 -303662306139303061306330333339333635 +64343730333633363731313532616431356566353763613932666337653837623466346434626161 +6230333434636231386538396538343465646133646438630a643833396531656634346465666333 +31366434386538373830363161656636666435373663326263326366303932376132653265376139 +3632333431326631650a383737623235336466386635653965663261396362643539323236343464 +31366135323562356466383732643762663438343130366264326265646536313061353061663732 +36653363666235653166393036623063396634616339653433376534636361303233323761626439 +39623537623932613465663731633231323438646661396535663635323033356538336539373233 +62363531353662386635336634333138386134643230396666336237633739366333363265633561 +35646438663436643061303261363635316630636236393931323833336462396631653363393264 +64383765386438333531323161343232613534396465363361316431393062353965613933623434 +62633761376338346538363866306638336361356137346538313539353837633962653964613766 +63626365386566313365346235623234373234356561373239333130303937636237316233396237 +30613062393666326238333065376331383730653538343261656363653437616130383433646436 +63363038383230333863663662303461623236393562663638323732663237373866653131393464 +36616637326461633161636266313631343362663764666639373539393437343361366264616361 +39626533336335363533313964383931636438633338376339353137386362373131643136373635 +61363033663132376530313164353236313632613736643139346465386130313339623965633565 +64353935643465313766363764353930333165346236653364333638663564333066313830633637 +32623435656364383730393733363665653434623062333362646463656561656261656164653839 +32363537386638393234373765636237386636366463666134336530373836323666653232373834 +38356331383162363637303164643062663263666434663563333236326637643366633165626566 +62353437336331633365393461623464313062666463643162323638646537316434653661323662 +38386262343466646663636664356364323834326239396566363262613865386233346264636433 +62633334333262393637613365663464653362653665353538346163396333653364363663646336 +34373330663230656637336366626130613138633834663635326561306566313832326161346364 +66363535366166353964646434313865376439623434353437353639613765613163343361303735 +33306463336462306634386666393335373062353163636332666530373432353133613865346164 +34353463376466306162386239373334353130663738303736653366383033626539613866656236 +37636465633063653537616162643961623163646434386332393864353531633363333436333739 +63643062363263326336383238323139666263383130306431363761313230386434616138653361 +30626664366234643330663736346239326136306264633030646161383562343439333139616431 +65663532356336303334326633633930373834396462633861336563396630666135386137393465 +62653332386233623237333261393365353937356530323363633265616330613632386534343836 +38303932356264356234373635343431623032393461376262376563386439333537663332656462 +39623232333931353236313733356262666535636534363037356131613463356139663963363430 +33373033663665333538303037303937303537633365656262633765306435323434643236363832 +30323263303630656163623638623630396562393166336464316432343931666262663937356465 +66653331303232313535613531653861386537643262323065396661646332356133636230386332 +38336539663363316330646230656636613961386562373562353934666364386662616136353034 +61633063623161313164386661346635383832623161386165343836376633353863633465316539 +31666562363339383838666235353638326331396636343863393164333231343533343034363836 +63343035666632663464616562643433376565613237346331373466363862616364333430383339 +30306232303835353362633734333031366161396662666238626338623238316137663363343037 +65623239666539363339323266376666366231643763343334313062316462306662373334633232 +34633932363039623665656665343635643137323439313266373032616430653463363561343366 +34346338346363306266373665303066643463373234616163643331613364323933363539346365 +35363861656562663136616335383437326665646361373864363666656565346531363630613465 +32363465363132343366386266396137616563373537383864663438383336613336333838613438 +35316265313934343166653233343932323565343831303163323661663265306434366336343166 +31363165386163333432643535333634653235353432323162313533613266376566613662623433 +31323334666134393138303132306437623331663264666361373939616630333765363563346461 +61656161636135363637393534623030613433373064633166653061646130636463316563343735 +39623336316632636135623136386166646661386464623139633866313964663833383862663062 +62366264393666323836663936306561373864383030373634383362363536323264366439646434 +66653030383164383434376537656235323635373332386661643764393433346162376333323831 +37343638663964356336373436383461646566343538656331633536626637333931303065616266 +35383738313138646131343966326430643132363035633537633035616533376134383630376336 +66633661303932393565366632626564393261363539336262336263633539303438613338386636 +35393863623332383537326230313332616635663735633534353535386335383766646539656539 +64353833353333323663663632636135623037303363656366643433663264316630336461663436 +61623636623535356638313735326237666535653536373131313630336633626664333033316561 +34636262393833623861663761366339653136313462646666666565333537633738346531363630 +64323931363036646630333134613032636362313838623430376234303239313633396331643361 +31616333306439353735643435623437326430316563303434653134353236653965326436323766 +34376634363966323034613331336434323437313964383331663866386237396563656636643034 +63616330323363373833363731663164383362623639353532626265313263346637313263386138 +30393132316332643461613538326632303463633536663036306666656663663033363839333735 +37316131306331356133303965343639323761363934613636306639363066373563653961623830 +31303139653631343766363261306464343930306639633838663164323030636362626338313233 +64316331373539643038343832656637393231373937386633613662313863653732343566633562 +64393636373439333363343431383536366137623464336235643365333139383965386434666230 +65663461323365626535626362313166646336366262363866653533316235663437306337343664 +31366339363965626237363731313464363434303437316233663931366533333765353830373764 +37323466363265356338383363393563626232656630306139666561323039636563643336336234 +32376366636661643232373632366130613062356134656563633665626639366634336133366266 +38633563323564333233623438376630613161383939353537386336623465346236396565663933 +35353237663866316632613866663063363165353239633166653633323537306434386135333461 +30356532373635663461666436373636356336633238306665613638303132323239626633313238 +65666431363434626165346632326534353263363265343638653938636134306636613937626632 +62373062393162656434376463656263316266616532346635393432633531303132636565663463 +39663561613030653531393464366130393363363564366135366662613262366133313964323330 +35616361313638316434303132323963303439363930613365353562346666656434363732323339 +38353164383637383836343262343333333331393334313132313965333430636135343538386236 +34343266323934383831366137623535356131326136313837626633623335636436313464393236 +66663836393936303037336230663963316463646236333335383162666565643864313230343834 +63303633366462306139653930356134346236643435656637353062326132326562316261306539 +35306465363064663636313136623530653934323762626261623361303362613437613437643563 +33326333306433613235393165623138313563356235353165373030643365373032333236653135 +33393830393338323764623130383162656337623366623464333133366264396365643965336462 +62353236613263313832373033366666373166363931393964646265396138313234663162353763 +30623233356632663831613231366331633631643765653261393036326630316632383166353536 +32613234393366306265613738373736656564613362386333303531313065656439373532343735 +35396233663936646135613462616435323836333862303034353534306535623338336339363465 +33383832306136303363353966316365383134656236343363663363353963366537323538626135 +30336565393263623736366633623636366438646662656163343464313330306136303939313730 +35303065363639303233343230303966343265343465346162616633303539623735623030323439 +65393463316337343135663538333166306539343039383065346364343139333065623736363333 +62306536636535356361333431643765323566316562366366346662336263656335366435646330 +39373933323530653866343038393632393761353761633362373961376563636262643466656465 +38386531303663353862363532616237303336366539386165623330633765626436353330656265 +38386338643735633264386561616430656430376566363838306436353333346563383237306630 +63643638366132323639313031616262643530343635343365353665313765396666303733663332 +36643231353061383231313434343439383363643662653562356133643332653063353136313338 +33383735333734623237396364336334643038386166366364333435356465353631313263396531 +35366234663966373731333364346665373464316538656666306463666263396465663633353639 +65313865323832666331306164326562356530613563313766623937356464373136386234373865 +38626635633165393632366534396139373938306637323933373737333165353230633838663262 +34653336323334393365653565663037396561613365353136646432356337393833343934313037 +35353734373264333165666262353539666561663664653736343137376563303864363463646161 +62363062633838663965626164623634316665393234663463636538663561366339613664326563 +32356635633834306362633339636439656531613637653132386162346364373137643138633733 +33303730356238343234613265323835626662633638333266636662343864386138633463643736 +38343563366532646632363265623337333366623936373134336534316239326537663139303261 +30363264613533643532633430336564316432323533636366343334373564376661323535336535 +31336437666438393139303932366666343666616337306363353963303962343133373564346530 +66623038643636343033626333303662386464323037366565363039636366376539313633346232 +66326533323035663230383432303731386634383437663663633661396433333362333861336533 +37666231313935303663636564633830643465306666633062303039353962633131323934616663 +32316131633933656262303636643534656538386435356364646463643666333136653965636136 +31633837666565303262623733633934343562663464373763643238343862346363363065333336 +33316364383737653538353437633262623634633031386530643436336234663961343139643236 +38343539326638623034303434613934613066393664643236383530656264396635313163316363 +31383435623037636636613965353939333165333138666531643039623661663664626465336466 +64666438646261656561383635613834656536353166623738353638306663343234386238666230 +62313432383965303837663430393438626538663764383565343565303531663062623136663536 +65306333313734326462303937343132316663366463303466373230393231623430323865623361 +32323737613236633833653435663839396533333662616330393639333534343662636133336336 +35376634633038333839323933613331663966393231383630633665343033346134613939333233 +63313735626135346239346132323661303431323735666662313237373239663837346633653738 +65363565313032326666353261643833303431363838376364386264313838323162613764323063 +63333466383064393539373463626663656438663137636232323363303566633038303362613035 +33326331396665303863346337616433343265663666626538653738306366636465303037663638 +35323131646233666165353264643733373161663737373139633137353339666537363466336465 +34623234653636643362336361373638343034326166393665356231616535326566623366333234 +31323764623531613736656535313333643334376664643035326236363332346439623562393130 +63316235656565346333393966616161616233653161316565646666383263396364383561353163 +62613733653066653761393834326431346166656539383965356165323862346463353734386265 +38376331386234383861633631646539653536373631323334636663663536303438373964333037 +37313663656361633039383965346237343165373339663232366632323664353430373462643731 +39393062363661613938313966353835333763646231306264303561363031613938653433626565 +38656433373630323032393132323136333862386432633833373064373162663361613465356161 +65636361656136663866316634306561306436393631363036313034656235313631386538303363 +34656133373663623430313235306662393439396261386539626330373664356336373936326463 +34633335333438646537333932373836316162383638636165396635333364316561346635306538 +33313033313332303933313034666533346461663133626433316466623538326366376236663738 +65626336636633323865393663373737306365383336386630336635313966313064396563346266 +64643739393264343735623265653738383832326639363634616237346162356132643864356666 +33353037353235646333613938656334656130316336306266303164633762313564336539363462 +31343032663439333635343430623165316531353135386531353363366130353838663662663832 +37636266393962316362386632336232336234376237623335326163353865343562343435663834 +65393035383930643330313633313763616663636161343835643439393865663539303461366661 +34383638653636376535323364303866303434663866303337333532633134306335646231663830 +30633864363762653231633333383234343734313738366164356637303734343436363434393337 +38363861633439393965643839376337333866363862316566353666326333396531623562313064 +35396436356137623065633938643130303936623335346663376331666236383439383763313965 +38653332646634386336343766653666303166633631346339363638363131646363346237313830 +39663137396333303335343762336661636238383836303537396163383237643338313338316464 +36303765633264313061373038303238663738653662373132613865306433346435643235323535 +66373838616435303731313666316534336339623636336631323063343739363565383564393239 +30666538663731666166633435393030633631653362383263656365303339633533326665613138 +62303332363430663231643436363038613332373436326661313964656339633830663535646335 +35656136313765653330373538643638616163383761333266366631393133353736656132333631 +35626265613263303531373561613632633834383636353534383734383064356265363230383663 +63663639633430346361383332323033373132386430316466653463373865626165383462303066 +63303031616363333365306363346630313263356237363766396663333765333861353734643637 +66373835633830363739323530633861323465366637623036613637376232663761613633343533 +38663138643339376136363765313831306130313737616461376339373662333363383536303135 +65353834656237613033306635666132303265383334316237323439323662356136343038333836 +36336633643731336533633864633035346434663065373663316130326432353265636539666261 +32656333346331326265373537386431653562636462303863333139353563376362653166643665 +33643764303363383363346337373539633632323733613630316130643130663931356532663665 +30653739316234386435623633663562383539666136313335336262333934646564363765343434 +35653235386538346133303435353630336162316664306531653765633834356463303262303831 +64636631316330396235336138666664396465343736373561633239666135663863663864613863 +38313433323965316535633232316365303966396162666334393266366638336330313434306339 +39376162653238303834643264323830653831316636353563396163333037616237393434636466 +61646631653138303064306563333161623337646161656130656635626431656164323333323433 +37633434626639616631643432653137346462343038376663643638383364313161336466633939 +32633863396239373430366439653033613538646339346163633735373939326134306662666665 +66363531353630396632653466396136353134616635323264623837353432626333626661323632 +64663639343761623164373066356638613135626664353930636532666434303133656562313461 +34363037386138373564376264666630653836393462366234316662653232333932323038656437 +62393834666562393139303131623733663739303837383563366534656232323361663564623263 +35613534303438323165636462646635383165323430316635366466383036323932363665333733 +37643636643833626161636438333437383965303636316439366237323737323066373334343332 +35643833326238633065346662646130383265363839396131323430613830643566646237346263 +66333537623266366364303464366366633536393432336331663533613261383737366534613330 +38663861346332626438376435363733653436663161643332323433316666346331623433323964 +66373732376266356339666564663139313861303934326530333064373536366465636533646632 +32356238643134643935643433376430336438313664623832636537373534386635376337353930 +34373133633930336261333034626461643861653637353930613833306138663833323132316632 +37343831366463343464366239653332653932353730383034326431373730383730316437666133 +63633964353631363230363064643136336635663237636331613665333665343735646261326330 +61623834623633366131613361653530376666383230613561386463303963626432363166386239 +34323034303334383235613438373336646534353763376166646365396361626232643435613162 +64623736313466343964366532656335653431373635363265353936343836353635333965313231 +33313934353862363536653564616630336338303463663161376133383362666537316662356635 +30653966313663346464373937333066636266333231313130626132393561326334626236303336 +63363062626364376164313631663762383464663634393539366635366135643435393735393962 +35376463326635313632613335633065616264643235353535643537656364623531316461306335 +66306161393034643364333831303962363665366266396663343936373532633133346632393862 +63643330303564333533343432313830613263333761376534623062313263386538626162363838 +32336363393538303735653533326333333266363433613938626136313863393763303161343135 +35323366323661316638306633656437633766343338383662373432363134393261393239666135 +64363962643635383432383963386661643766346666353461643433386334323837353532613238 +61616538336665653536396439316634396537653330663834636531356136613338633462613364 +32646232346539393966393264313431373465663966336232313966383865346433353632623562 +66646437353036396338656163363937393866623235333735623535633261313565366563303065 +32316565363238643533353133333361666535316231316663623932616363633966383432363364 +35646466316435616339666335633266373363326661653834633062316163663662376363626235 +39363532663039633462346632386266393665376433653964626230633738353061323031633765 +66363234363331353334383932336261663765373630366437663764306239646362393437303032 +33343839306431613130663636346533623330346364613065393439363961356130653335313039 +35636133306162643261333830643639666162376162303630306631303465376461346131326436 +31306135623031323036613836353233383231363232643363663063363465303730646232326463 +34393863366332343536363064646164326666383234373235353064306134333632633533613063 +31626166376663326533376431666138323834356537633362363963316266316433366633633365 +39396566343366336662333336303066333636333566376436653464376536613730343734363961 +30346164303939663032353761306264646432373033343237346662323138386638303037653465 +66336331613565336337353336323738346565373330326432636239386561323639383932326133 +66326330653338666264623863313065303739303530326466383935646538333132646439343961 +37363139393034316165376633656662373662613030396338623032323636333633333461363766 +63343232386265353963383832653830383736313765313561326239373062646539386663373135 +66373666333033343963316439653036386430333935666534343632366336313033323166313931 +31336533636131313864323665303861303563613639343332363632633065303162363639366531 +62386434316230346163323330623262616562633431613066633864616230616263363034366339 +33613633306531396430323862646638333237396465353435306438386631363266366264386263 +66653066653964376534636363383364313864623737346266373166633433323863396662663433 +35633665626537633539666335383565633263316365653231626139653266636266616663346462 +65333038643233663833653562633235383232656532316330623430323632626337333666336563 +61333034626564623438356234373063663134316665376361323661326639616333373832326665 +64643062383766353166646662633836363765663764653339363063643736313733666564653265 +35653132613731386465323733623037386265626536623139333634376536383039653939353131 +33333864346237653264373533386431633036373764376334393465373335383734643638613237 +64616662316163613366643433323666373830643664393461316664303461376463616135656337 +32666539376436613633306238363033353337636535343235663539316535383561616433306563 +36303762343563333238666366383338356438653463373032653338343631303363 From 6f4976ce78984bddad02e56ba567f4c2dd3c2910 Mon Sep 17 00:00:00 2001 From: Jingjing Tang <31444565+jingjtang@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:51:15 -0400 Subject: [PATCH 40/46] Update logger info Co-authored-by: Katie Mazaitis --- claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index f195ce713..7aa748d4f 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -52,7 +52,7 @@ def download(ftp_credentials, out_path, logger): """Pull the latest raw files.""" current_time = datetime.datetime.now() seconds_in_day = 24 * 60 * 60 - logger.info(f"current time is {current_time}") + logger.info("starting download", time=current_time) # open client client = paramiko.SSHClient() From 1c83ffde32c2dbc0551a1550c0f5ffd992b27f2a Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Thu, 16 Jun 2022 15:24:24 -0400 Subject: [PATCH 41/46] add min_max_dates, csv_export_count, etc to logger info --- claims_hosp/delphi_claims_hosp/run.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index 9092f46dd..6c7405a36 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -102,6 +102,8 @@ def run_module(params): weekday = params["indicator"]["weekday"], write_se = params["indicator"]["write_se"]) + max_dates = [] + n_csv_export = [] # generate indicator csvs for geo in params["indicator"]["geos"]: for weekday in params["indicator"]["weekday"]: @@ -132,6 +134,8 @@ def run_module(params): params["common"]["export_dir"], logger, ) + max_dates.append(updater.output_dates[-1]) + n_csv_export.append(len(updater.output_dates)) logger.info("finished updating", geo = geo) # Remove all the raw files @@ -141,5 +145,12 @@ def run_module(params): logger.info('Remove all the raw files.') elapsed_time_in_seconds = round(time.time() - start_time, 2) + min_max_date = min(max_dates) + max_lag_in_days = (datetime.now() - min_max_date).days + csv_export_count = sum(n_csv_export) + formatted_min_max_date = min_max_date.strftime("%Y-%m-%d") logger.info("Completed indicator run", - elapsed_time_in_seconds = elapsed_time_in_seconds) + elapsed_time_in_seconds = elapsed_time_in_seconds, + csv_export_count = csv_export_count, + max_lag_in_days = max_lag_in_days, + oldest_final_export_date = formatted_min_max_date) From 50c93fb5ea4ecca89fc43690dd17b6d12173af35 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Fri, 17 Jun 2022 13:44:47 -0400 Subject: [PATCH 42/46] update the unit test for modification of the raw files --- .../delphi_claims_hosp/modify_claims_drops.py | 5 ++++- claims_hosp/tests/test_modify_claims_drops.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py index 6fcd88b85..f40236249 100644 --- a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py +++ b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py @@ -30,7 +30,7 @@ def modify_and_write(data_path, logger, force=True): """ files = np.array(list(Path(data_path).glob("*.csv.gz"))) - + dfs_list = [] for f in files: filename = str(f) out_path = f.parents[0] / f.name @@ -56,3 +56,6 @@ def modify_and_write(data_path, logger, force=True): if force: dfs.to_csv(out_path, index=False) logger.info(f"Wrote {out_path}") + else: + dfs_list.append(dfs) + return files, dfs_list diff --git a/claims_hosp/tests/test_modify_claims_drops.py b/claims_hosp/tests/test_modify_claims_drops.py index f2fdfa4d2..2905a3caa 100644 --- a/claims_hosp/tests/test_modify_claims_drops.py +++ b/claims_hosp/tests/test_modify_claims_drops.py @@ -1,15 +1,19 @@ # standard from unittest.mock import Mock +from pathlib import Path # third party - -# first party from delphi_claims_hosp.modify_claims_drops import (modify_and_write) class TestDropsModification: def test_modify_and_write(self): - data_path = "./test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz" + data_path = "./tests/test_data/" logger = Mock() - modify_and_write(data_path, logger, force=False) + files, dfs_list = modify_and_write(data_path, logger, force=False) + expected_colnames = ['PatCountyFIPS', 'Pat HRR Name', 'Pat HRR ID', 'PatAgeGroup'] + assert len(files) == 1 + assert len(dfs_list) == 1 + assert all(files == [Path('tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz')]) + assert set(expected_colnames).issubset(set(dfs_list[0].columns)) From 300397ff16fbcc797b05351df49f9de0469c15b1 Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Fri, 17 Jun 2022 13:48:50 -0400 Subject: [PATCH 43/46] fix the error in the unit test --- claims_hosp/tests/test_modify_claims_drops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/claims_hosp/tests/test_modify_claims_drops.py b/claims_hosp/tests/test_modify_claims_drops.py index 2905a3caa..e4a6ba83f 100644 --- a/claims_hosp/tests/test_modify_claims_drops.py +++ b/claims_hosp/tests/test_modify_claims_drops.py @@ -9,11 +9,11 @@ class TestDropsModification: def test_modify_and_write(self): - data_path = "./tests/test_data/" + data_path = "./test_data/" logger = Mock() files, dfs_list = modify_and_write(data_path, logger, force=False) expected_colnames = ['PatCountyFIPS', 'Pat HRR Name', 'Pat HRR ID', 'PatAgeGroup'] assert len(files) == 1 assert len(dfs_list) == 1 - assert all(files == [Path('tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz')]) + assert files[0] == Path('./test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz') assert set(expected_colnames).issubset(set(dfs_list[0].columns)) From 76b4db0fab385b2cba888dbc9b00c54e2f6dbd2e Mon Sep 17 00:00:00 2001 From: Jingjing Tang Date: Tue, 21 Jun 2022 19:51:39 -0400 Subject: [PATCH 44/46] rename the parameter for test_mode check --- claims_hosp/delphi_claims_hosp/modify_claims_drops.py | 10 +++++----- claims_hosp/tests/test_modify_claims_drops.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py index f40236249..0ab93ebcc 100644 --- a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py +++ b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py @@ -17,7 +17,7 @@ import pandas as pd -def modify_and_write(data_path, logger, force=True): +def modify_and_write(data_path, logger, test_mode=False): """ Modify drops given a folder path. @@ -26,7 +26,7 @@ def modify_and_write(data_path, logger, force=True): Args: data_path: path to the folder with duplicated drops. - force: if aggregated file exists, whether to overwrite or not + test_mode: Don't overwrite the drops if test_mode==True """ files = np.array(list(Path(data_path).glob("*.csv.gz"))) @@ -53,9 +53,9 @@ def modify_and_write(data_path, logger, force=True): f'Duplication across drops in {filename}!' assert dfs.shape[1] == 10, f'Wrong number of columns in {filename}' - if force: + if test_mode: + dfs_list.append(dfs) + else: dfs.to_csv(out_path, index=False) logger.info(f"Wrote {out_path}") - else: - dfs_list.append(dfs) return files, dfs_list diff --git a/claims_hosp/tests/test_modify_claims_drops.py b/claims_hosp/tests/test_modify_claims_drops.py index e4a6ba83f..91d3d048f 100644 --- a/claims_hosp/tests/test_modify_claims_drops.py +++ b/claims_hosp/tests/test_modify_claims_drops.py @@ -11,7 +11,7 @@ class TestDropsModification: def test_modify_and_write(self): data_path = "./test_data/" logger = Mock() - files, dfs_list = modify_and_write(data_path, logger, force=False) + files, dfs_list = modify_and_write(data_path, logger, test_mode=True) expected_colnames = ['PatCountyFIPS', 'Pat HRR Name', 'Pat HRR ID', 'PatAgeGroup'] assert len(files) == 1 assert len(dfs_list) == 1 From aef95115e303170dc6a805e8352ca8dee361685e Mon Sep 17 00:00:00 2001 From: Kathryn M Mazaitis Date: Wed, 22 Jun 2022 16:32:50 -0400 Subject: [PATCH 45/46] Remove try/except from get_timestamp by converting to re --- claims_hosp/.pylintrc | 1 - .../download_claims_ftp_files.py | 17 ++++++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc index 8ba5e540a..7fc2f5c30 100644 --- a/claims_hosp/.pylintrc +++ b/claims_hosp/.pylintrc @@ -8,7 +8,6 @@ disable=logging-format-interpolation, no-self-use, # Allow pytest classes to have one test. too-few-public-methods, - broad-except [BASIC] diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 7aa748d4f..7cb30c60e 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -5,6 +5,7 @@ import datetime import functools from os import path +import re # third party import paramiko @@ -24,19 +25,13 @@ def print_callback(filename, logger, bytes_so_far, bytes_total): if (rough_percent_transferred % 25) == 0: logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred) - +FILENAME_TIMESTAMP = re.compile(r".*EDI_AGG_INPATIENT_(?P[0-9]*)_(?P[0-9]*)[^0-9]*") def get_timestamp(name): """Get the reference date in datetime format.""" - try: - split_name = name.split("_") - yyyymmdd = split_name[3] - hhmm = ''.join(filter(str.isdigit, split_name[4])) - timestamp = datetime.datetime.strptime(''.join([yyyymmdd, hhmm]), - "%Y%m%d%H%M") - except Exception: - timestamp = datetime.datetime(1900, 1, 1) - - return timestamp + m = FILENAME_TIMESTAMP.match(name) + if not m: + return datetime.datetime(1900, 1, 1) + return datetime.datetime.strptime(''.join(m.groups()), "%Y%m%d%H%M") def change_date_format(name): """Flip date from YYYYMMDD to MMDDYYYY.""" From 170f2e349c91c4806f4bc583af44169a6de33d64 Mon Sep 17 00:00:00 2001 From: Kathryn M Mazaitis Date: Fri, 24 Jun 2022 17:26:05 -0400 Subject: [PATCH 46/46] Update expected file count for inpatient-only --- .../delphi_claims_hosp/download_claims_ftp_files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py index 7cb30c60e..6c2a3f184 100644 --- a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -70,9 +70,10 @@ def download(ftp_credentials, out_path, logger): files_to_download.append(fileattr.filename) logger.info("File to download", filename=fileattr.filename) - # make sure we don't download more that the 3 chunked drops (2x a day) for OP - # and the 1 chunk (2x a day) for IP - 01/07/21, *2 for multiple day drops - assert len(files_to_download) <= 2 * ((3 * 2) + 2), "more files dropped than expected" + # make sure we don't download more than the 1 chunk (2x a day) drops for IP - 01/07/21, + # *2 for multiple day drops + assert len(files_to_download) <= 2 * (2), \ + f"more files dropped ({len(files_to_download)}) than expected (4)" filepaths_to_download = {} for file in files_to_download: