From 4aeeb94f369a6fdef8eea39eabd7093c3fd0ff61 Mon Sep 17 00:00:00 2001 From: Rowanmh Date: Thu, 6 Feb 2025 19:42:48 +1100 Subject: [PATCH 1/8] Add code for a 'Databook default all' column in the framework If set for a parameter, then when creating a new databook it will just fill in an 'All' population instead of each population row. --- atomica/data.py | 8 ++++++-- atomica/excel.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/atomica/data.py b/atomica/data.py index 18535a8e..d8531d7d 100644 --- a/atomica/data.py +++ b/atomica/data.py @@ -261,13 +261,15 @@ class instance (e.g. if creating a new databook). pop_type = spec.get("population type") databook_order = spec.get("databook order") full_name = spec["display name"] + default_all = spec.get("databook default all") + allowed_units = [framework.get_databook_units(full_name)] if pd.isna(databook_order): order = np.inf else: order = databook_order pages[databook_page].append((spec.name, order)) - data.tdve[spec.name] = TimeDependentValuesEntry(full_name, data.tvec, allowed_units=[framework.get_databook_units(full_name)], comment=spec["guidance"], pop_type=pop_type) + data.tdve[spec.name] = TimeDependentValuesEntry(full_name, data.tvec, allowed_units=allowed_units, comment=spec["guidance"], pop_type=pop_type, default_all=default_all) data.tdve[spec.name].write_units = True data.tdve[spec.name].write_uncertainty = True if obj_type == "pars": @@ -276,6 +278,8 @@ class instance (e.g. if creating a new databook). data.tdve[spec.name].tvec = [] # If parameter is timed, don't show any years data.tdve[spec.name].write_uncertainty = False # Don't show uncertainty for timed parameters. In theory users could manually add the column and sample over it, but because the duration is rounded to the timestep, it's likely to have confusing stepped effects data.tdve[spec.name].pop_type = pop_type + if default_all: + data.tdve[spec.name].ts['All'] = TimeSeries(units=allowed_units[0]) # Now convert pages to full names and sort them into the correct order for _, spec in framework.sheets["databook pages"][0].iterrows(): @@ -654,7 +658,7 @@ def add_pop(self, code_name: str, full_name: str, pop_type: str = None) -> None: # Since TDVEs in databooks must have the unit set in the framework, all ts objects must share the same units # And, there is only supposed to be one type of unit allowed for TDVE tables (if the unit is empty, it will be 'N.A.') # so can just pick the first of the allowed units - if tdve.pop_type == pop_type: + if tdve.pop_type == pop_type and not (hasattr(tdve, 'default_all') and tdve.default_all): tdve.ts[code_name] = TimeSeries(units=tdve.allowed_units[0]) def rename_pop(self, existing_code_name: str, new_code_name: str, new_full_name: str) -> None: diff --git a/atomica/excel.py b/atomica/excel.py index e864fde5..9501b184 100644 --- a/atomica/excel.py +++ b/atomica/excel.py @@ -884,7 +884,7 @@ class TimeDependentValuesEntry: """ - def __init__(self, name, tvec: np.array = None, ts=None, allowed_units: list = None, comment: str = None, pop_type: str = None): + def __init__(self, name, tvec: np.array = None, ts=None, allowed_units: list = None, comment: str = None, pop_type: str = None, default_all: bool = False): if ts is None: ts = sc.odict() @@ -906,6 +906,8 @@ def __init__(self, name, tvec: np.array = None, ts=None, allowed_units: list = N self.write_units = None #: Write a column for units (if None, units will be written if any of the TimeSeries have units) self.write_uncertainty = None #: Write a column for uncertainty (if None, uncertainty will be written if any of the TimeSeries have uncertainty) self.write_assumption = None #: Write a column for assumption/constant (if None, assumption will be written if any of the TimeSeries have an assumption) + + self.default_all = default_all #If the TDVE should by default only have the 'All' population in the databook e.g. the same value for all populations unless exceptions are manually entered def __repr__(self): output = sc.prepr(self) From 4d4fa1713924f5bc5d767f0b73f8687d56af311a Mon Sep 17 00:00:00 2001 From: Romesh Abeysuriya Date: Tue, 18 Feb 2025 11:03:22 +1000 Subject: [PATCH 2/8] Add test --- tests/sir_framework_default_all.xlsx | Bin 0 -> 27090 bytes tests/test_tox_databooks.py | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/sir_framework_default_all.xlsx diff --git a/tests/sir_framework_default_all.xlsx b/tests/sir_framework_default_all.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..99b7b42940ba3e133959ba8709ee237e3ea7e964 GIT binary patch literal 27090 zcmeFZWpEtblCCReSXCiLI zozp*7S9er&m9p~5H*>9Al9vJoK?VQ=fB*mhAOL9Iz{6_*1OUi}0sue)fB@1GvbAwC zv2oH0Qx-t|GxeQTVOn8Otz06UgSaiRcM#SV(pq7g5@k= z2rO5DOK^r%OQ5uA$Ye0AFXsl-yw2jd`?pV_{ehw>Yj4z*?Tn;ZbHT{|;!opc zEUA;yuof9aBW8o=(}P14%kHG56wr_LpI*jB#t-E$(u+5C zk!vUJw3A??(QLO^K@O=%u|Izvy_3c8gQB~6;rbdEj^_YJ7~}(H(kdx9W zs6tP4`{}{bpDaUv!mj6FV&zCj`+NMqkpB-R`#=2Yl?gI(ee^IP7vis>!w>W8aqxmt zt^(q11j;^s5^Hdc(S^iV>s@5n@XDBhz+%1~KCh#z>)g@D!vy!gm@6X@k-3PPT&qG- zpY5H&DM=ktMC~iL`w?8`@8<8*#U$OyT{>bZzqOR*OO0$2iOyXK*TPTHs$oJR6=R2> z@}&D~4$5e37(P}5&I>A?RfW_vv*jHpPG$Norj#7P@`Z6opU$SE3_BW{E?0XETM^tn zVX7#ZahTT_<~VQ?y6YKRb>9eQb|Jla(a2^ED-tpzJ~B>;4Uy#C`D!*WUyWtE_p?Fv zeH*?U35dii+xXPIe}sf{3|_Y!7y!Tw5&!`1(=u*Wbgs4zmWH;rmcP}oNY&Ez7d!G> z=j^9}Hs|^YAPFd={QQ~}xdd@a)|2BX6VnD)i6>WN^D!rUb5DYbxK*i20=s+eJBA+Y zy|$ijXs4poD2aCl;J2x;FdD+#dm|1Ao}02V0U3)3MFbgjrwKd?5xsG}pLaaQ2F5aO z(0~D;IE2d;P0TbHWzA)GQRvJ8xv*#*3iK&+Xf&WC5z`ebuE!m$R+}Aqp?=~A{X&H4 zT&_tI<*s(nECUmsjv|N9E2&JP(ZtmQN=3R-O#V84gNVOCZFF1(CL)R($~FaxW}Gds z!gY$l1!#-MyA;x6E*DT$U+91eP5UIO1K2PR;+?|RL=zP&S;O_c$3(oEhr-9doOt0#E6x|m=;o~f6}|#Oi!AbS!_PPfZ!k-l za=lqs2reGln|JjGQJV|+s=+|Q1AXV}OKN#8nNZt<5Lu$~oEJTDt>Jeki3Rv_*`68uFEa{W z3P_vSx+Sm2N5dB`;SKoyfbO_#cE&2imnKq-o=p4|5^$OoLGKb)o|L6D=NNK>NX<$yS! zuC*&QgSzb@uYKX+U*(K_*2GgakTjM~D17zT&Gzo<1h=j-N*Kt)d{#(x^*a+)Jd|CR zWcd+7rmYUflkL{ie-4X1m%Y<^^vmB`mbBB-W2XJ!&Yn5j-BHH=EfVvb`PY7+AclG) z8%YHa!pyV6ln~G4!?*QcB^paNd*TSaFXCiv5AS?^-!Eec3ehm9@MxK{d|I_Obj&=t zw0LzgrG$W=>F(#b|Fd81$ubt7>0aHVe-mUlppD3$0eI{^ZD{+Q@|R6CyD z`ME?;%D04OO+X@G{AILewl+1hR_9!KRvytw5dxywmG^i>WJJ!G+H--{RLRU3Ewzk{ zd+jVYSZ1k&tc(xbqBM8W&;$fWa-ENz>9)zm4rdh>pq$3XL;huGnlfkYEtuBw9!AEL zE<;Q%c|kg+$`s?r+5I5I?i>zM$fL))>mDoMJ}Yl}4gIun@IR;<(zv8fGG{Ws0%ov|e|Hx!G z)5bpOx_5FBjllb--gbLs@Z|KFWNm{00HA-K@iz@|G&3=Aa-{q7#_(HAvJzKq3hCiP zE~xHss5ilZAGXL3m+aO}yS;7}hRE zSoOG@%Sa@Wmw-goBr|_Yc=zzdoglyR&y1k5%qs%FbYkf0pm15J@&<;7K#Za@M-t3r z+7|)Nco}Zer&rV}b?+|MV%X%|Qu#{miFBm+p^|3Yi*04%>SwC_+>wxQ+Ald(bH!#bhg7*rn`&#U zGV6P_$kfYwl<~L*QwzsQM;T7GR!;ckIDwb=DK@Fjedgt$IVk(XR+^KGwo1pNOAy^Wt=bCm3Hd&yNqID1JySLiU;M0DopX|H4xl z|9EPWOzbNDr>7=AWmbOEt`uR_ zC(3c(yxQ998!>YLI7qgJ8cj?>G%KP}si!%NO)VWElk|d@p=hwq%QL;aHMqO#yEa>& zTmgRq%d6E#Pf%+`^N8qaNxb~Y@$ip+YH7I$!PGjWpx4;Zj6x=ILc zPq1O)fa)_H?-3Z7^$;?-7$eNp(^$r`dX`;UM?sz2yY8TbekSv_M#6SKos}3hVl(qS zve2Nc>^O{IFN1ncKhR?@wQ4+-d?^hm#2c63roggvwcp0rDGIyH6Bx`O0SX1?4pFNA$$Ot$HsWl}B?Dpu6Z?-BFWY<=ooNDO^;ic6Zke zOm*{-mrj^4A~t6+tR)j2zynlGXGcMH^gfMW=59SWGW3W=Zhh#7pzUa0?T@~GJ5L#YXcLn~(_f)cU8hkvcSZU|BEG6k<9*MZ$wE9)S7I#3 zB0mJB_SW`(Z{RksZ)Wi$5;wCFusm4dj-mN3z<_mxiW#N6LcV{waB8@Usc-TqKH`=| zT(8bBGaVC`>52ZM0wlC?9~?CL1Rsppv21gpppoZHdt-nKmMeczcPhEx84bhj=vFir zan0--6`Fn3FWHrp zrhy0O-$#=5UAX4n5-{+@o2lHASQ^`NHb* zd=r|Dg1nD2jg$0KLKN^`q}0X~_0Qea`uSC+>CE4=X51QK(%+Q2M|7jZ>5{4zJn`Im z%n^yzFd5yBk=Z9p)6wxHs+7~E`UD5+d6>Q+|f~ma+sZ}+?jqu2I>1fdEXL!umr13QMc1K zQL0BVB`x>d>a=vnpTyUE;cjT^<177}9WHc}I}!(#$;D}C-$U*aT~xO2m=n=cV^X_s zTm(J^F^d2QF|0sS9C?6ooLRtbTm#6mfCJ&M59!+1EUgF@8XVfC6cVX1U<-*j*Pqwv z5Vx_+_65I$`H+5=-wD#cs|PF51EimFb#*#?>5~RJQxbu1kYf?BD`8i#FCb$SF?+)= zLxkZ^wvs^!RX}m#r3}csQ9uGw3r9jB^B{CY;e)6E0hTAxQyMT(2F-qmz&DGv4A`Os z(Wnl3Q7Qi=H-nV^?NLCU1pSfHtF_zqkCFZi+OZurpYswydLmnEKlua{y5kxfFc^&| zst4<+g%NNfLX1G%A4aWpe(u@rdTXB>7i!=&x|(yrC}i0|XbS;KR_g_nZv5QRF@~@0 zv&Vb(ha(lWMxyK7j)5=-{cgO1xiM<-0Q5_Aek%j>{AtQ+mezEH&7ET`*S!daX4d5L zhdg`^Y25Pn)Md_`XSjj{Z%LPFW1ZtYnM{klsqgs1%QmeTFLW7o-uO(qsEZiu=o#y% z!jZ#~BT2V|$8)!`9!@<_75kzMwou|-R>GV^ljy4dyiPHRr3U)=%pBW3L-fDUW9Gjy zN151_KOy6uPrVa~%wL|Buu`>6WKUJ)jUCHWkouIp5rf~uofT|Aor09q;gM@-)8d7P@MrkIOFQpKJWb&qZf7D5ShKT*S!R?WG6RXXc#*a+= zP+LUn;w6EBOmljTq$KnUQ*>>Hr&j~kz6W%B44(PGHFGxOgo&73(CkJZy$lg(J0^qK z;VYz}fPL&;o9g|!^xQAptbG@*TOtFe*sw`{o0UbyHG}DcWa3{-Gi+CVjbo!Cp^utA+US>qG?qqc{4dI0jYv!ef|I-Y2JwPq z=xQ$8tT={$6!^|HGYIB_CdSO#)2r>^>5N;3b7B7=M9qlcHQh@Ms2kuuK1dDfXp4}D z+gVAo7O%8RM~3`@9EK0mJ9D-LnpAN}N&YNnY#7GxLZCZj`pgLRm`TFI>?Qv`P_|ij zZ!KavQb|raw|o$glejtYPBkoZaz(yvg)F9lr`H3iOr@HrRUDTV8txK^v9-_IMF{S;xBEqs` zs!A|?{l03oRNK1i4rg>&N*8`gPEot}%o3Bhw>3rEH!7c5v?jJ#iB)qW@*E}xn42gY@RSs7 zN_vhv>`%RqDAf^!OIEmszCm<*irWxT8qmr7K$l3VZf=})QtFAwk`Tt=v}_S_8*~jg zGAi0mplD0LrQ(1PBGkJnw9`5Xf!b6RLDbXOhGq33vT_J+bvU}kzlQnv^1~7&rG*q7 zG~V*On$L}43iOmG6=k-4_!2y!+H^^8JTv=E=rXbOgh+}_&wU_$4?a+T_sgVcd3I(? zh#Vli3{wvRMg6bgTz3_d#~qN#jqjvjP6U3w(RP~`g;g1a_Zo#+Xt(mqrYdj9tu#zo zZW7pHiN=^aK9`Z4-oFgBG^auIGG&tXQ%TrhCQ-_+dzXTZD}Z<##GxJn{no{%tZ`{F zgLIJx=YqPu8ny?kgNzLOzYJ0?Q{dx#EAED3MXo7LL6q&Ffc$DyYKIS^TJ7)m2Dcl0 zsBi!=e})sNb&G~wnoYDciI4!7nAV@DMhwUIQ`oby=7!jHSKn9E z_V#YLeqf6b794%!le_u87#QZ0(2l3O>APx=Ie2qN&Pr!aP~kv-n(6;Rm;RBqf2&H8 z$=Io4!o3!sA%Gac45KJtWD-XsB1=#=MnRZP$rVusKcX061~x)I%P_p7`)m2OfJ>)v zwOcGc+-xS>zbr$5Um1yzzad3j#7nOq2@DZ<;1;xTg`nEv0EIwyAKc3a8WTP1)Bu`s z7_dn{EDuS_O;-C-JqkHT{TB9n;1F~6Gzf*{&%Pys(dI;5wU5(qJ5rW0pb*qHEeR}J)xI=0#vNTnpBo;kM zpOnReifr!)p0ngasl3FUOBWV^8~8w#2+iqoYGl0e+Csuuj$Dgj7`@39#ofgK4oY zECBnF>H7(`FLsZ{%XYdQboZrXd&kA0ly3vCrSfIMx#dgc@kK$6N=!Yz)SPirw9=Z4 zkHV0gwiP9GD0a?sNjmBQ=x4w}Fh=PwuQ#p$X<^sG6<~pzEl{;6F$TZh&H_N?&<^a3aF(zLRNRhxxwL?PMvZ zm6yZE5Y_8cO{l3;E0ABs6J=sP7e#)^eG8EB1`=gx#OIA7B$$G+2MSL59&aH?I_1pY zwAGLKt0luO1`{;X0;qi$8v#%{Mc5`0R4h&DJ)KRK-uCyI4Qv^8cTlLJLilKC69}dH zX%vTvn(*or>B0>yu<8>OAJS{Whg2g(r;>&xB>{OfornX7U|j@$FNls_6ic^zeLj*L zg!;M0i0-G6*Hd0HLH7eh<7i(kDC6jwdS4A8hnC={?yUC@mFN%q^Z;TY_!+~w7sTB4 zs#fgX(bUOnErk{kn1d5QJii6<2=UiNQItYRV1#`0AW=#z-Gifjt8e1kE2c| zqc%evLE$f(rs;AL@$RN03niI#pfiztlvf~TcWZxj)#qh&xiujHct8XA>1m8Q9_`Y`v= zNxDW(5J~PAH?!K1-jB3JL|6982z4{cXAX+4adNHk(B~%#gNy2O3G$(2hD`da00Tp0 zY;yQ~dWaz#aJf8W!L+Cso{Twwx=1%~--T2mK;QbJkPzEsU&iCqN^h+hLdrU;6trAM zkDOPVdp&__t=KNam@fkg3qz)MK;Ka6BW9_CkVW%jX=WH-UDT~#>2F_VWap^7Kqu); zK8q5qgJ@opxU5Qr9vm#>Q#r{5QS*2bjTnWI4;0p-+fPyYTLzyrvkujZkM6gv6@LNb z@J?+|HoX)b28mOZW;DP4fU(VbrJo(%`S@74Llfj~S6}b(gKxyx4=@~#O%&T1dpJ_! zpzXvHVkXp{bvnxpTJ`0puA&t`U{7#ZeWXfsN#y*`kA%*Ink;$Y`luCqjsdWKQPcy$ zYGP}hLFYW51fxcICDT=#QSK{yW9|e_LtvhOSUgB6KL z%+H7wlYSjq@zEF?5eI7dJTuK(#k^+Rh$cY9ywuyp7MYn6{&&I9T}e$%Co3|r=ejRV z{cr3S7?XD2b$rw4o%laH3amkszwH6UGw%vzh*oXcbh(<)@GEXRg<;4^P^BSaXm7fG zbXdCS3=k7Q+?P7wGzX}cTXiTW5KtSrCz}!nn-b9?R&JGsNecPH`P=kL&O5X!NYcvy zC;b8$-oS`96PMqN3o89~2vw0nwWn`1ST^RTRZ*l1k-oHO71s9h*ty+OFylbUC7lO6 zt_gUFciqS9UIx@(qkNf%k~Bd^ZmmtVedmRwM0QGZdXg^JNw-Oltp+QH0J|?u)uHL= zY$>}5kfu54+_o;;%vfp)zz!o|NhKBkwW?Xwe98=!|NeyWOtw|Gy7dw0Etw=e^US4f zW89=x7PSwe6Xk_6&?pfwY1i5*QU$i}NEr)>Wwub66$mQmuqU=H97hA^o zpQW%IzH+(lmKb-9J;cDZ@n9zM+!=EmUi@p{}A z7VlZD4Vo*jJsz)4y}v#i-44~;>he7G#-@tAe&_SL9-GR2@sE>%4%hX0x@|s?)APAI zy4d8o>ShDQMc($#fDgAx6lxeknw~%(OwNMoe{#7slFr6q2V23N)j{FvUR4_RYGJjH zc9r8NxB|45#$~TNA2KW`3%!|OxA)`9s%J&h1z!8v4uyIyH#rx zZ@)4I`Gst;tl4u(-30m1v#~lJt|8WK_#2X``&vV9#B)9Mg62!&b&@!gF|nT}!JU=U zJ-(fSxk+PadZ}o|=oPmG7pyvN4ke^s!WgS+N}@IrbD>h~Q6W6)%zXjrxCg+RsC|p= zZ?1On-CyAlwY`N^+VOT^JRT!3q=?cJV~9svf;eWWGXc}E3#|v$p{(1}1y4M9YgL2| z8~Z|C+>6;mCr8rCnHe1~qGlpU?Nq-~eY+T}G%V(vv8uK}A7~MSzi+n9BXf__l06_H;?jt;G|~P*tD6Bk3r%-~&6^TKO+!j1HDf?!w(#u^Aeh|}ln6a@ z+cLp_vDk*1A#%%yQn@$et$f6)*Li!QgpZoaj>_PgRo$ z_i#UXo!r$&MS4^(j&gwhJ-57I3aKSV9m4`H$S#wbe)eevszsh5lrm|6`4qB6n5hxC z0JPd?n*ju6%{=oX0Q>NR^?({Sae-c?f(Mg8V4GX97T*|cG$FiGDNN)O+I zrhCFh!5KPiP??KmQ;!OF4!6D@pUJ`n)k>1+Z+A?1jTqXL1aIJze3XKs(^jZ+VK zO@waW4OXc@6ubvmamCTZwQ#|w*_TDIq%)GN)vwR*dsoR@t|qzGIrN5Sn~$8P%un4N z_v;)Tt|rzGo}#j~mTH}A=kC%l6Ykw2j_=J_f1(eiu~`nF>3UrPV0VCihbNVmcsbr8 z&?e|jGSCy}zt%d)sKWC7v!?L(>rYzR=05D7>rXVF&E2qnsgp(n0*0N`YXC6g~aqvAn6d^f%G+le}@x=N#ilG7~o?}~b7Twk?SS~RZuVB=Qy+cF6 z0-k@Ea*)bswERSMIRZ*CyDULcogxXfzgwAZz2axmZt^;D3zpl)FiMcb8lkkvJtn-s z?fMTVMiNQGr2zY{tgxt}gqgau#k0meJ@YGg2{7~T;^1WqT13&e;W#~vRg@}eNZrmV z3N=x@&aS^?wG+TE>x06sX6NX~ zc~Uu5;Z?CoG}LH*3@>cf&b4m7QbGjUcS>34-+%`Y2OKjgk0FzmVQtz-4(9}kn%7Ss zz6p4ij|{@~Zh$p2K!huCh7PXHnx|WeNc5W<;8$hPsNLHtLv98$f<>@}JBX%8BHE-c zogae1%ZhSz)lR%rv3FNP6E6~?>@&tPU<0T0oxuYgh0y47)joPRptgZ?0j6$#i99w8 z{g}1Ed-dEXU$+q+O|Z3EUJbA@HO-%av(HeIq~xXBFM4ZaNs@=NHL51F!Lo0+x;{JrjVw4GYwR ze6(M4NpB*V?FKijPGP{Aok+K>&V%}Hmc4Rh+q=0wmfKc0?d#d(5XhN)*{j&sK7K$q zIlh0Ti?p|${jpgCWb@#i3qo{3eal0JB}i1VeH8JT5DdOzR=R+9UNc zYInqSnh&3b9gr$wG@cAS53iye?ygrg3OC`CqM1lz7oT{5CD546)}Azf(otrYGv|<7y$H09rme3d6+LS^ zueSA`D-qUbQYIBcNkSy%G_EmZ6PDu&wjwSEfVmNJ`yj7G)(WNt`1xadF|G|Y1&bKh zKi7@v5JQwwr>dWKWuP&TYi^0Udy})s=Q7sxrdiaKA!bo}cp(kE1)diGOXkg3#(>a;)+;Da(6wrJX_I`R)fgj>0Y_Jq(*?e+t*J>9tnl?UU;ms$ju&`P#^AZ^%TB#nHXXjY~SVbtOA z5@)s9MqAvU3swWFloFm>$UgA}9BU&C<3o$4AZ#^w>~K%$fPyC>wrY9EdhYQW5(KvD zpqk2APFuNr*irZ7l~~|}KgC||32N1wM}ey*0r)0MBm1l~If)k|eH>805Zyd5c!kb^ zu_pAP3YQzu0>ar!uZ$JRjZP1T7u^`Kxj95+I*P7Z#*Dhx_n^!yKe)Y&sY=pOs=_JE zJ?D%rM1;S~T^Mdioib6#Jms1{eO6!p*i=Ne)!pp+35Ct)BIakI^t=7d*ulW{HxG0! z)>guQ9W&5=9-pfG8yoNqLbMV_o+qgm;D?;IO|VN~r}W6dh7`4$8qCy#&d(=e8nSa= z)~8o#VYkEya&sPsO^Gio#}4gvoj^AmWFPBh8*l4Zl4#u2w=07*of`LSH>1QK(GXRs zlq?WZ-_qYfO`Y^C#?YMPzVxcq&8XTK1Xd#b7z(g7u@et%S7@}#F-}V$ht9cv;%D&f ztsQO?4__5!DunV`C%bJOq#j|X0@U1_=%95=!6g}Sdfbm<$Na&eUq-zkUJ0cuXFcAG zV7W7UQp^4WQmbYD6fEkdrLkWV>!)>Plmdl+Sd~_Jlnc9f@M?fjOAW|@Q+WPlKy$aYrvHddRjcTfY{M*=QOTg&J| zX2KruK(3+&B(u9g_sOg$7O@?<7^O~hv;I{~8`RM(Ehxzfl5LP$7?x5s)2M4LC|*f> zXFr0Yl};BW90oK_7sBkem{a9spUQYkwmoy?;ICWp?0nK0hsSE0g zpmoW^7g$WkRz*q|#p9DUj;91+dS5TZxP$2NS^T*udZtkzfr(^iOGFa>i-?`A$Ueb@2Gd90>b{9J30~7KB#I_#1*7XXJ{a{wt zF|+hW#_ru58J-H7($NsWf;JjS!pW8GOOVmvG{M`L6KeC?h#cZC;3DHm{dqo6+@K5R zr$ZcZI>6SVMNELi+O`GiUg^TXdoD!TeSiF`jnm`hdnfOEZ0Dq|_uKJ_sTWT>CEeX! z*1IuU*FVjZj~iqHm1mA24WboeD|unP8D?;*_Lby&=RoWw=i0SQh_!X*<&I~ zi$EeAquf(8TTOv#hHSLyfoVWZcXlUj=$$Fa5d-B`zG*>30-d8E@%hz8w)aE~MuKim zVwN%qTb7JsT3tq1m7*A%q^{1&!xL6g%yVLeDv65A-j}`0YHDB?u%kU}(iz=>Cov(? z*Q;8UB05=|;4JMDL({Z}Ct$nM9 zNx~>cGkWXh`oP}qFsWvRrDcT*YDf1k*nvU)+6$TdY;bQ5<>@DP`lzuptAhhO8P0X@ zu6?vgLOZ~OGQ3l^@^4zu{bV&bUx{s&#L5O-&2OOdldeFyZqz`YFm75W9m>26iLnzB z2_Qp=EjE$br9+6LhG0FhLoE)}CrMm9UD7FuR{Wv|7ME)1?AV~QhG1C-p@l1_H#VZP z$AYaRZ<;Ma?`>;yP7zMZhJFlrpq<>yx)UsfdiFop{PnqvC+BVCt*exUSMKbLznYt($X+XhDo;foGvj9j^oXWaJ39miN5&ymn*Y`BAS1 zcc-haKKGv{Gqe2wo_n7hl>et3F#W}W-e>gy-@OU`@~xx)V-sv%&P%N@uIgC_m|ElKy-P;faFRv=WfEl zzQ^uw`>>t&0bbriUTn(l>= zr`ENI!|I!|Qgqj@4#pJkWVCYdK$M1ZTo{Ex9_OnK(2CE9dfLD~z(6~(vh`$xr`%(? z!Lm)BHl!$`*GdUz-m4H(6&wX2PC`Jj0JYuO{i@R|TD<$R)>hQ7@~u%mkA3uQtYph= zQqnO|C}JRt4eVUXD*IP${^C~6Y6y^)CABS7^CWA5=Lbj%=W-IZhMA?8FC$-efKELh zS0FuY{$S17$h`w>pz=~+@8}mp^R(!CQw(BaX^}>zh7) z#YSbpDmTYdU8}_x^8PhU9~@^1XDPJXCMVf6fQO54sqo9`ms27A*bJ(b^$od>+R`bI zRk%T!gR3M}<2C;`JpbtRp-z+@8{+QH-AyOLM6@jr2jOZmoJE{PaL&^sXgeNW!o|6m zlb91=?I~TqlNp589A|A;ZM-dycQ$RftB>V@&zbLklGPmhSvpLA(lLqnpQ!}%-{|^Cs-;eN>YnV`JQ6ypR#oB@wBeYL8YuRhhbl^(E4!j=31_4 z&par?b|E{efxUEA$-~G;03>a8JH*Y9bHD_;o=QNxsqIZo0}bL$EN+5Oz4z{ln^kbO3HEl^yMehAchkeX&fpjbK3Tt_eKe z0=P!3CwujyZnTF*IvYFi#3w{1dIz;gqO&AO&NB8A9P>SmX_H6!1=8iq4=!0JY|uz5 znaMVXeyfCLRWKNhJgzPa=n$_Zkrg0zWPCL>chVBaKvVvrS=T2UYL?WWDq;Jn623r8 ziB*#R)D0{?hvRZOc9u;Y^$VCa`!^aN)yu^Wf1$Aq`xhEkkr?5>(YTUXc~vAg@#>ZRL}P=fb6Vp|Fs+__nlVwk@G#Wb z-q~7t)-o&4M@^I62jc0apT5{{GG6{70|y)abmuo2-uhw_2xoE^rP)h8#@Es{eP-AF zk;7$u&wRa@7r0z#@#LKEYH0CuXx$!O^p$d)m0Xn%*$bw0es{|~HN0mv=QSIlcaOF; zxW97M=l)t-`TLr;{Tn31!Y3jvpUD)_U+MktT~vSD-SXGAsy`q8>>T*Bsj4z=%mSDJ zM&u#*J#dD1K^iqb$*m4D*h=Ubuy=v`@X>#-WMp$Of7%6qTE;bjmXp)0t?f}Gn_~<) z&|j4;kA3N|gdW|Er=v25Um}0YEXE&JTSI@5hu12WTd7=%QC32^7=(d7B_Ri4Am&uU zNgCT?cIdSHCo5dqaOX^Si%kaX{wB;Jsbc@3QDTrfs0{%2+(_zz~O=Xue{X8&!LMjL+nni&}b`b!rtZ!a*C z;DAC7FjhMQf9qzCC2Y46#Z=?0gfdA;Lw#zJ4*Uq5<;24@_Mb(u%Sz?exb*SfrT(?n z8D!&K7}GRq78fwP9hL$(bpK|S$osy(nFaIj%wqJGi%G_J!RWn!M<={y{g+wP{+Q)6 zIR2wqDu!KW7ym2X@~1`bAI-wMK0*9mqBH4+dS-7%%4{Yd)J$% zKb;E;y&W$64LclxbfB8Ug3YDm=KU5n`=U|Rx?eEAef(q(HP<^0eRU)K)%7)s?@3h3 z+p%}KYnAGs%88~Xm(G95E`M5p{;6Gp{{LtfN6v8a_)kyr`CMlG?C1PFd0=E~ZThCSo&g408gGDdJ8_hqVYY97gge9iRSIXd!^CQ zjJ)A?Rwy{Wc;;|oJhN119XqR-XUI4}FH(dBtzS2E<#Aop%e_6b11+iUy0Nb3MiZMO zJafXSZnlTxo&y2EOEZ%7z=pzPmWC_2V!C!n0;xa=W3?r*JhJlmC!?w9-n^F&O*W%$ z6ogc%FL;0X<4;#6;R|I}Eh8D;FX{nVx*Hc>VHTG0hXgtPa91cbOq?Oe8QOs2AtekM zdV9@oWJDJ-2BKD~6;sw>N9{}8elf||Rhc(HLu9(V=}CI5$gLWYtHF6QADz0;@Y$O4 zSLWVfV;kMC);^d_uX?HGyqR>P%3!D_UShoIa1iiUFHscU#T|8KTY|%g3nAhd*#HQE zBpaZ)VyUsC8P?1L-xVg8cFZ~{L-IYjF|z&csm$Db>GiLpqYXAL4t3Xf5n8xfibXdIIAuvrEkDaH28CSY0PY}5PlE&WO-mx3W? zTP;9_OlZys*92T5!A3=y(-CA@luR+cyk62#rU#e_aHY4%MLPgnn<0wR{X=HQoX|xyuY(%zfj*QI_tXdnC zsw8K|7r1(QWlZI~X38rqkaqA=>qBdu>rgt29dXQ3Wq!f!cHplSs)*MKaVglzq;kFQ z5o0zB(36k_xeZ?7fmXyf+$7u3+#*9O$gP{!^^6{QyX)J|}}e+uHu#egDYe znud%m(GWrh(w48;WiLp)u=$P^kU&DM#>EPd6;W8WkZk0>G;C>A&bfqd+%K@F3m+98 zAy|{|h5TsNAOe_QIM(Oyt~S2?nooVmR?&i*adYixVdv5#cS!yoy{b+3ezo&5*F~)? zy)wVzA+{LFGcb_KcC)Oq*A}B40R`RS)?ub}OsDouz47Of+rFoRTj^E)93)-eSCkk2 zA2Bov$EU~hPfsT*rO{eRD70DxYkTj=Z4aF3jkTm_CRM@56=SQ>Hv->jK4-Ld zdLOpYpj}I#c%MnL4f2z1F_mWMPTL+0QOqotm?AT&vXxGRu<2|%L%FI|_s%EW{F}kM zyethl+vWW;P6jd~gDG>ns=lMyIaxW&_1$^5jDt0Qc=Nty{ra$ccl7c2qBk=CB;tO< zS3@?#!wT{EhV|YbYw?W+$gNiWs%Zf{86J{d@I2^Yqxd?$c_qV{{3d7J|xF+VJSIbr;Kgzp8#bv*>QWLYAcvCM2`I%Qs z-&j?%R$9}I#SyS!*OlyQaRr5H&n;xD9b>M{BqoAfuJy!$9@bDPAp@=0J@iopP7Ie- zaJ^ty z6HE1!=`oM(muAWxu`_w1Zh_7&EY(rpAVIYphlz<5%wKacB`+ zQBlf=Y_eFdQpYZ%hds#~?5M_ifDD)7g_E6lKVO+|!k6P;$sTG4l3SOF$uUGI++NDZ z#han-k+C3!uCV~37q-75i)W%vXOI+py(3GUL7;{&k#Ni$2(D^SNB2E)||PfgyXM~fGpI^fz+u~Gjia08J_pn!&5 z1IH?IiRUbAZMTrbLY*Na!(N3zN!v=kP#n%tlzsnFnpoV|xJMa)z{5`JOyx=)L4I$# z1$3Jk0VH$@9~vr0CcvSRz@Ad|Eo&-Hxv~|=q2?iGPFMwmkM=>T@@?=;!`enTH46qY zPHz+8u5V}W#`4?xdN`0G3SYglPysgzL#GF?q1Tl=VTipeoEt&hB}IE&lxf#HTRS`Q zJ%heDt&A3A4-C2Tp@bjkcbLlI^(AF`LINbHXK_S&cg7$bYQV{0Vkp*(D~@5S$|Qy8 zHFy1Z2i#Uj8%3Rv$v29{IP5aHHCBPAY){G`-FmsbsFOzyUv4?A7Jv^U24ukU!ZK0? z6I?7R{-5^F`YWonZ{tHZ4Bd@LDxEWQmw+@#gLH|M2t!H=BBiu+Nl1${(hP@|P5}`R z0SSqBksOWRWIfJvonNbC82lJRRS5`Tt z_8HKJ_wIV8A6=CWE`nPJuMQ29Pw!Da6%sqT)d9t8;&H#tgV&C(_lfZ4Gm#m`n$h&bIC* z>6Bh8tmq$*?G}c%z9R0Z;V$ea0uTUZ4>w=e#|T;o;jf<-u`#-masyU@ZVAqa!R>mF1Lii8b%c3PllPSuj$ zX13=DjkGOIP9_amoRE;!njX2N%ACnoZn(sv$e?ZT9`EYXrx?$fm;f7Mq`Lv46l`4m zuK^?2Nf4y~49j}0^&iBT!|}%!X2g!%<9%K20pEoR=0Al6Z=v zN+15ggC&NIDQ~mIQeEdE8?RjxN@!_3A(+xFW;GmY^uYhZf7yH~d&W%_+$8dt=HmN^dz3}lVjC_HH@LHryn+g&2=FmQx#BRERu&!lC4B~1z%)Zf zDm0j^x<*&vquK=(P|37hkpB5PEYpWI#@(gw4xO0?XAmk%Ki%Q1xnI?3Rk}auu1V&A zMmksWY(FF6skux5=(_?&#J(@1I02Id*iTG|;wgUWTb109PG!?KoTR}WsMYDC9Rp(N z4(}PuP+oU9QGT0Kqw)T#uW`UPZu^tFg-@CQS`#7U5vf3eBo916M&PknNjh%Sy3N zh?O$)Thix)k4~ReJi*2wVD=^4>p|m;i8P)Qruz!yu!&>NiC0~}%nLb>qRJ@uFMhFX zq*ZRD{P4Cqor`?f+%ny{9oG=Jx^g1kaz>%TD-{a19$H#tFT$0zgFOWj5_AxuYXMMb zDbTcwidd>txk@;O0SJ1ZX<>i_A>*z~!y;*S7CB-oR)(NA4y<1H??cI94;0g2i_-}6 zg6j$OR|989YDM7XosmF;!*!yO=!`sj-nx*q@#5@MsrzR+B=4HiUe+z7Z0}}T=O4Bn zznUM5wjox->!#<3DVrS|8kXQc7~~gw>g`x#kU-G?>0SH&i5)|4@>^}M0HKrhgZ0h# zL6}-IDh>~~oGJ@=e7qO-GLnZWs(CRfDwmxqBdSDSsmU}lFn3N1TDuo|E%ZjHN@j(A zd|B2J%ti(Fee8MYWtmqU;umRL>XA zVwa5rp_N+s-t80axU$6hQzdy%E@@j|`5191*t!L9%r>}5I7y_NAw3^cHU%&Gx9#Ak zWa&iA;w%?SP@B*Y$X#C6h*1MJE93DVXSrS|msXxnX&qB7Ul|98Q_+4{77n1@NDJ)D zQmUVBTiqjG=pe63q(TDGsreZ1G)F|BLXXyW6@FMgs1~ zQ9x2PP0yoUIo$}uMlLY6LX-G4pkj;XW+@ZU!PLiLSD{pi%=cjYz{Tt0)$hC8zR1&v`yRsjc1po6z&?}zZdk*-;7xBQPy3q)mn2f% zwz{I4s-bFJDZ~7Nles$bEMU*#wsq+8NA~RevWWf>&7uf?L&nE$mf^V`8jh(YT~2u7 zKNuyKKVy_6$xi1|!=t^Iun(v}n_Dt0fQRM0umd*$<^C*I7cFE197qCJ`Jn3oK^ zP6uYT;Uk3xD{(8&0Wk>Li!d7jeEa@@HEnKH8P?(k;9WZq{i8&gEnO5=WYt@8(}!y4 zeti!k*!ZU$y09B9&>$P>9S_cZist?Y;bPe2%EOV^>Y#y3 z;(j`(&hv(M0)hp#YD8B^#9tpSgji><$A~6uWutxY55wZ1Tpw&q8$^rs;swbeYsqJa zU3Q0}G@=ZnGJ=wkBtRKRrjq37QuRk|fbwdlLtdb85vcY7yO6mH=J;3^H`*b~CS(&- zKRx#-2d$Yu(oa+(gU`Imtx!5Q;Z754tQ@C8PZ*|iXAOM=xZ6oy2rp+S$Jygb{@Gl^ z{`5B!2{35H7#fnzDKiiPM3KO$Co+Bz|HV8s{DTguk(;&zwR}OgGk)R=xkuq$kj*Gu zr&$(rMvXTzxbNzeM!Aa#cVFub_9dkFO+OybJFD)PIT}M|)OlQJq2i`?=**NBP@x^~ zAzHzMrGwOuS{#P`Ilh2vt9cQt@5v{RLT_?K`@6#w10PKRHHZlewl?1;J}h&z*d|$ zh0?9PADe1CfH6xZ%I>Yw5IGmwaiRv%6H;!N-Qd&U-K!LxZXSC? zkinisN$v8AiPk=gygJ%EA9XOfCLQC1MvV1!k4H~W;iDd5XsCF&GQR5Umffab2}yW_ z4hJaE>F6$cPPmG0ZV7}*!Lx(G%e$?U!Q6NH>Pu+KNyY%f(-dwXiBK>^*}1}6v+!7< zeted&Ex=X58qKdmyHO~wF0mly4uNP|y?2;*seixo!toQTQ#>!F4=8laORoUjXjB-v z#0o_t!^p?HgWkT0cz1cbBUiw2%=(}%57SP?{zqfq-1@nAc!NG+P7K<#W|U|eOqq|0 zKARfL(`g)z{B&*UZxr zEPTsp+hWMyHF)Kded$$bn9t50m-CGOk)7yMo^4jFS0c?`<%_dqb03*G-ff2~71Ty> z_V5jeR2~ZPx0yA}5s-8suf2z`uzoag_Q~uXA;WA&9s@aV=9U6>BdC{0o7#&ksTckacdqXznm88$zakO?7< z5m#-O0Vc@GtT4V5V#vRYdQZt&V|mW8Z=jbUL0jaSbxX5Y%k;MH;a<+n`EYz7@0>~4 zLEw#@sG9h80R-jpO_;ZD1x7q789c*>2?}b1(P^A$> z|8B<6*nksDd7Ga5V)4MX1Vx9{cGUujl;Y3S4 zs53c5vcEOu>y}xJOa0kk`+VKnBV{U2ZPGAx;=8DV!}r;b^BwPVi?H)`L3MR}S_7XP zoXZMjm`6e+DJ4$!aHmxKsGUfAcoJtzv=dsRUC%gTSz=-h5OeBvmy|d$%w@QA9F?p;X;rQ0+gVFFRDsr{ zom212Nwfv^?0GkLWLYXwVWr#pdX$!v6TSif%ez|ZS-m)p^K_SWM@x(J&ZZ;HSf$gz zyQsCu$;zqqbSvL2a)TjT1wzQoc|wgqW*r1sqHf>$#?G4LgZ#yM+s ztYW->IDQn%My;VSYP4_He=;9y*(9q!TayMa>6VJD-PfYoqGu8@_mWg`^F4Oz! zYUg##IQMWBI^#;dAu*zF!liP1b9Z?AZG}Y2z2lUFtqsTJ`+Z4##rDx!-cuA?* z?)eUGrHwR&6YC=L513ca97kJ2H$CUPz1PR|4FYcTCEXUSAuESn#|*Hoe-C*roUCN+ zEbLu=S-4H0#@MiOPPCE0c5vuvz5O=Iivs!5w^HQ5(JHJptptXqCn4r9{D&Le8pk~E z=JpXk)g0z4iXy@^T1K}~L_eN=T#G@(RF5(>%oHMwzB#l0hR`J1)Wb&Ep)x2a#$8O4 zDpamhcA2$?-B-lbN3D?FJ{9zxQK`#LpnSkXQJn))N7%(L;u6G$PR$_3jdZq(YL;n{ zA)C>j-@lYDo7K5Ruz7C$^TtE76f^EIg>88ZTj?i+};karw5vZ)-^0gV$U0a1e*D4)z4#qRWaDIe^)j|n9{6M1X< z<6yAA+q3k`CQ8{$g?pNOOl-(QvGIPQ($ER6VXZaLSnz~GC+g#+ks6RBr;lu7)i-l> zlu1=LEe3lW@=SSR%L9kU77p@j$=fQwXCF z(#8i9^x9f&(=mvIQ<2#1x$&udOPLDM-Gzpkin?=by=V0l>jK;0w2)&QPxiG>*X$RvPL7|BfQ24>A9Z7?(=LLSotih)$xur5=BosjIAyXzebUCf)pXla3FoS6J-o`REESkJ zt1$&Bx4W3L_dd!tm(2cKq7pD%ksTYgUn6^2SCPKk(JA9=4?&)`PyXZqF zmjq+?@!2XyEq+UP8yPzhj-sInZ~XuZ<9!R1n)vFMDqFHBTs@Vl{)R>&aSJq5p6PRW zTh76X3i{^m6VJo`bqfp-75z;9*xx13bu{rUs|YtTT^}BLWeiW@ro>I z#Ce1!c>Se@E+4ogHi$~!vP`uIOZR)S<~_`dkx6qD`-%N(*-i;L>%f6&mlZplP_PD> zH;_w&tjtu1X4rk62;*Yl8CW+$jf1c~9`x+~jzC@3(Yzr;yU8>)E`xR|^XFv8ysUL9 zWHW}AG@>j@^aZ31i){i>H?=fdY5}p=iPis|EHI}+RmsrIG5Y5>f~r zyH=O9?Ns_qoEvYZ9NC<_I6L!t5ujp2a9cE~`2u`#9ryRk#esy(4XaYP`P~nHpWMHH zZ+H^~BH;Ht`M-s~es^G3&_B}U5rv2u+i)onY+wIMOZ_`>8&UjwlrLNg0BpkQoM7UA z3HC+gAcnNUNqVqislRsOKO$QZDTo1@aLTVP`v%JOQ$R!tVp1TS@&Hz5d;{fr$qSK! z7#;?v#FO7hxgPT)QV`RW;FNBv8!6Yu4MYlJwh5f_6y_?r;fStn4u}-Q-~l*=mE}gt zbqoO_1+i)yPPu#gnu1uvjR-(2q=N(Ixvv56vN}X5Vu27`D#iP!6tP?gk$_lD0Vn7R zToeASwgM4^Xmf{y(1iaDbZvS^1R?r_;UGM*e*;~+h7m!CmP$B?6Z~(WYl9^s2+@)T z2MI`BgZ@8*8X^$U%moLkO8*xi+}ec*M6^M{ftE7=1qe4qAp#NoLvWy_oz{LO}gs7CZXz|~rczo~D${~<2!!__64zo~D$?;);gz}0*Dzo~D$-yzPj z;OY$HKh^)7b0Hr0?-Log7N#@-0RCfIgDC%f$oWq>p4mU-e+@u2RncIsC;)&K_U(Z6 KlxG&d{{0`UMU7(s literal 0 HcmV?d00001 diff --git a/tests/test_tox_databooks.py b/tests/test_tox_databooks.py index 7958424f..0952a698 100644 --- a/tests/test_tox_databooks.py +++ b/tests/test_tox_databooks.py @@ -180,10 +180,26 @@ def test_databook_all(): d = at.PlotData(res, outputs=["sus", "ch_prev", "ch_all"], project=P) at.plot_series(d, axis="pops", data=P.data) +def test_databook_default_all(): + F = at.ProjectFramework(testdir/'sir_framework_default_all.xlsx') + D = at.ProjectData.new(F, np.arange(2000,2005), pops=4, transfers=1) + D.save(tmpdir / "databook_default_all_test.xlsx") # Test saving it back + + D.add_pop('new','new') + assert list(D.tdve['sus'].ts.keys()) == ['pop_0','pop_1','pop_2','pop_3','new'] + assert list(D.tdve['ch_prev'].ts.keys()) == ['All'] + + D.tdve['contacts'].ts['pop_0'] = D.tdve['contacts'].ts['All'] + del D.tdve['contacts'].ts['All'] + D.add_pop('new2','new2') + assert list(D.tdve['ch_prev'].ts.keys()) == ['All'] + assert list(D.tdve['contacts'].ts.keys()) == ['pop_0','new2'] + D.save(tmpdir / "databook_default_all_test_2.xlsx") # Test saving it back if __name__ == "__main__": # test_mixed_years_2() # test_mixed_years_1() # test_databooks() # test_databook_comments() - test_databook_all() + # test_databook_all() + test_databook_default_all() From e16cdfaee63b167c6894b91e2b017eb77cc4ba66 Mon Sep 17 00:00:00 2001 From: Romesh Abeysuriya Date: Tue, 18 Feb 2025 11:03:53 +1000 Subject: [PATCH 3/8] Fix migrations --- atomica/data.py | 11 +++++++++- atomica/migration.py | 41 +++++++++++++++++++++++++++++++++++++ tests/test_tox_migration.py | 5 +++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/atomica/data.py b/atomica/data.py index d8531d7d..6737645b 100644 --- a/atomica/data.py +++ b/atomica/data.py @@ -58,11 +58,20 @@ def __init__(self, framework): self.tdve_pages = sc.odict() #: This is an odict mapping worksheet name to an (ordered) list of TDVE code names appearing on that sheet # Internal storage used with methods while writing - self._pop_types = list(framework.pop_types.keys()) # : Store set of valid population types from framework + self._pop_types = list(framework.pop_types.keys()) #: Store set of valid population types from framework self._formats = None #: Temporary storage for the Excel formatting while writing a databook self._book = None #: Temporary storage for the workbook while writing a databook self._references = None #: Temporary storage for cell references while writing a databook + + def __setstate__(self, d): + from .migration import migrate + + self.__dict__ = d + projectdata = migrate(self) + self.__dict__ = projectdata.__dict__ + + def tables(self): """ Return iterator over all TDVE and TDC tables diff --git a/atomica/migration.py b/atomica/migration.py index 10abed30..fdc351f9 100644 --- a/atomica/migration.py +++ b/atomica/migration.py @@ -812,3 +812,44 @@ def _convert_framework_columns(framework): def _parset_add_initialization(parset): parset.initialization = None return parset + + +@migration("ProjectData", "1.30.0", "1.31.0", "Add pop types attribute and default_all") +def _projectdata_add_types_default(D): + + if not hasattr(D,'_pop_types'): + # We have to check for existence because + # Are any population types present? + pop_types = list(x['type'] for x in D.pops.values() if 'type' in x) + if not pop_types: + pop_types = [FS.DEFAULT_POP_TYPE] + + for pop, spec in D.pops.items(): + if 'type' not in spec: + spec['type'] = pop_types[0] + D._pop_types = list({x['type']:None for x in D.pops.values()}.keys()) # Using a dictionary here allows for order-preserving unique + + for tdve in D.tdve.values(): + + # Fix the '_add_pop_type' migration which added tdve.type instead of tdve.pop_type + if hasattr(tdve, 'type'): + tdve.pop_type = tdve.type + delattr(tdve, 'type') + elif not hasattr(tdve, 'pop_type'): + tdve.pop_type = D._pop_types[0] # The majority of the time, if the pop_type is missing, the default needs to be added + + # Also add the default_all attribute + tdve.default_all = False + + # Some old TDVE tables have a lowercase 'n.a.' instead of the correct default value + for ts in tdve.ts.values(): + if ts.units == FS.DEFAULT_SYMBOL_INAPPLICABLE.lower(): + ts.units = FS.DEFAULT_SYMBOL_INAPPLICABLE + + # A TDVE should always have some allowed units, if the allowed units have not been populated, then draw + # them from the ts entries. There are some older saved files that may have no allowed units even though the ts + # entries themselves have units. + if not hasattr(tdve, 'allowed_units') or not tdve.allowed_units: + tdve.allowed_units = list({x.units:None for x in tdve.ts.values()}.keys()) + + return D diff --git a/tests/test_tox_migration.py b/tests/test_tox_migration.py index 63393827..d3219bc4 100644 --- a/tests/test_tox_migration.py +++ b/tests/test_tox_migration.py @@ -33,6 +33,11 @@ def test_migration(): P.databook.save(tmpdir / "migration_test_databook_save") # Save original databook P.data.save(tmpdir / "migration_test_data_save") # Re-convert data to spreadsheet and save + # Test databook operations + P.data.add_pop('test_pop','test_pop') # Requires migration + P.data.add_transfer('test_transfer','test_transfer') + P.data.add_interaction('test_interaction','test_interaction') + if __name__ == "__main__": test_migration() From 5d26e8954451ea1d28a3abe9e2039b86829b5a9e Mon Sep 17 00:00:00 2001 From: Romesh Abeysuriya Date: Tue, 18 Feb 2025 11:06:09 +1000 Subject: [PATCH 4/8] Increment version, update logic for manually edited tables --- CHANGELOG.md | 4 ++++ atomica/data.py | 17 ++++++++++------- atomica/excel.py | 4 ++-- atomica/version.py | 2 +- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 471965e1..c1f60ebb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ This file records changes to the codebase grouped by version release. Unreleased changes are generally only present during development (relevant parts of the changelog can be written and saved in that section before a version number has been assigned) +## [1.31.0] - 2025-02-18 + +- Framework definitions of compartments, characteristics, and parameters support a new column 'databook default all'. If set to 'y', then when a databook is produced, the data entry table will only contain a record for 'All' instead of having population-specific rows. Further manual editing of the tables is supported as normal. + ## [1.30.0] - 2025-02-17 - Automatic calibration can now selectively weight parts of the time series to select or prioritise a subset of time points. diff --git a/atomica/data.py b/atomica/data.py index 6737645b..8f7b5eb9 100644 --- a/atomica/data.py +++ b/atomica/data.py @@ -63,7 +63,6 @@ def __init__(self, framework): self._book = None #: Temporary storage for the workbook while writing a databook self._references = None #: Temporary storage for cell references while writing a databook - def __setstate__(self, d): from .migration import migrate @@ -71,7 +70,6 @@ def __setstate__(self, d): projectdata = migrate(self) self.__dict__ = projectdata.__dict__ - def tables(self): """ Return iterator over all TDVE and TDC tables @@ -270,7 +268,7 @@ class instance (e.g. if creating a new databook). pop_type = spec.get("population type") databook_order = spec.get("databook order") full_name = spec["display name"] - default_all = spec.get("databook default all") + default_all = spec["databook default all"] == "y" allowed_units = [framework.get_databook_units(full_name)] if pd.isna(databook_order): @@ -287,8 +285,10 @@ class instance (e.g. if creating a new databook). data.tdve[spec.name].tvec = [] # If parameter is timed, don't show any years data.tdve[spec.name].write_uncertainty = False # Don't show uncertainty for timed parameters. In theory users could manually add the column and sample over it, but because the duration is rounded to the timestep, it's likely to have confusing stepped effects data.tdve[spec.name].pop_type = pop_type + if default_all: - data.tdve[spec.name].ts['All'] = TimeSeries(units=allowed_units[0]) + # add_pop normally adds TDVE rows, but it won't operate on any TDVEs that default to 'All' so we need to add the 'All' rows here + data.tdve[spec.name].ts["All"] = TimeSeries(units=allowed_units[0]) # Now convert pages to full names and sort them into the correct order for _, spec in framework.sheets["databook pages"][0].iterrows(): @@ -640,7 +640,7 @@ def add_pop(self, code_name: str, full_name: str, pop_type: str = None) -> None: :param code_name: The code name for the new population :param full_name: The full name/label for the new population - :param pop_type: String with the population type code name + :param pop_type: String with the population type code name (optional) - default is the type of the first population """ @@ -666,8 +666,11 @@ def add_pop(self, code_name: str, full_name: str, pop_type: str = None) -> None: for tdve in self.tdve.values(): # Since TDVEs in databooks must have the unit set in the framework, all ts objects must share the same units # And, there is only supposed to be one type of unit allowed for TDVE tables (if the unit is empty, it will be 'N.A.') - # so can just pick the first of the allowed units - if tdve.pop_type == pop_type and not (hasattr(tdve, 'default_all') and tdve.default_all): + # so can just pick the first of the allowed units. We will add the population row if the pop type matches and if + # the TDVE is either not a 'default_all' or if the user has removed the 'All' row from the TDVE despite it being default_all + if tdve.pop_type != pop_type or (tdve.default_all and ("All" in tdve.ts or "all" in tdve.ts)): + continue + else: tdve.ts[code_name] = TimeSeries(units=tdve.allowed_units[0]) def rename_pop(self, existing_code_name: str, new_code_name: str, new_full_name: str) -> None: diff --git a/atomica/excel.py b/atomica/excel.py index 9501b184..304490ab 100644 --- a/atomica/excel.py +++ b/atomica/excel.py @@ -906,8 +906,8 @@ def __init__(self, name, tvec: np.array = None, ts=None, allowed_units: list = N self.write_units = None #: Write a column for units (if None, units will be written if any of the TimeSeries have units) self.write_uncertainty = None #: Write a column for uncertainty (if None, uncertainty will be written if any of the TimeSeries have uncertainty) self.write_assumption = None #: Write a column for assumption/constant (if None, assumption will be written if any of the TimeSeries have an assumption) - - self.default_all = default_all #If the TDVE should by default only have the 'All' population in the databook e.g. the same value for all populations unless exceptions are manually entered + + self.default_all = default_all #: Record whether the framework specifies that this TDVE should default to having an 'All' row instead of population-specific rows (the user can manually modify further) def __repr__(self): output = sc.prepr(self) diff --git a/atomica/version.py b/atomica/version.py index 91aaedea..6d57766a 100644 --- a/atomica/version.py +++ b/atomica/version.py @@ -6,6 +6,6 @@ import sciris as sc -version = "1.30.0" +version = "1.31.0" versiondate = "2025-02-17" gitinfo = sc.gitinfo(__file__) From d8d59260b03f49f44366b45924068d467a13d630 Mon Sep 17 00:00:00 2001 From: Romesh Abeysuriya Date: Tue, 18 Feb 2025 11:06:25 +1000 Subject: [PATCH 5/8] Update formatting --- atomica/framework.py | 6 ++++++ atomica/migration.py | 22 +++++++++++----------- atomica/plotting.py | 18 +++++++++--------- tests/test_tox_databooks.py | 22 ++++++++++++---------- tests/test_tox_migration.py | 6 +++--- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/atomica/framework.py b/atomica/framework.py index 7028bcd0..76d13de1 100644 --- a/atomica/framework.py +++ b/atomica/framework.py @@ -632,6 +632,7 @@ def _sanitize_compartments(self) -> None: "is source": "n", "is junction": "n", "databook page": None, + "databook default all": "n", "default value": None, "population type": None, "databook order": None, # Default is for it to be randomly ordered if the databook page is not None @@ -643,6 +644,7 @@ def _sanitize_compartments(self) -> None: "is sink": {"y", "n"}, "is source": {"y", "n"}, "is junction": {"y", "n"}, + "databook default all": {"y", "n"}, } numeric_columns = ["databook order", "default value"] @@ -723,6 +725,7 @@ def _sanitize_characteristics(self) -> None: "default value": None, "databook page": None, "databook order": None, + "databook default all": "n", "guidance": None, "population type": None, "provenance": FS.DEFAULT_PROVENANCE, @@ -730,6 +733,7 @@ def _sanitize_characteristics(self) -> None: valid_content = { "display name": None, "components": None, + "databook default all": {"y", "n"}, } numeric_columns = ["databook order", "default value"] @@ -861,6 +865,7 @@ def _sanitize_parameters(self) -> None: "function": None, "databook page": None, "databook order": None, + "databook default all": "n", "targetable": "n", "guidance": None, "timescale": None, @@ -874,6 +879,7 @@ def _sanitize_parameters(self) -> None: "targetable": {"y", "n"}, "is derivative": {"y", "n"}, "timed": {"y", "n"}, + "databook default all": {"y", "n"}, } numeric_columns = ["databook order", "default value", "minimum value", "maximum value", "timescale"] diff --git a/atomica/migration.py b/atomica/migration.py index fdc351f9..724855a7 100644 --- a/atomica/migration.py +++ b/atomica/migration.py @@ -817,26 +817,26 @@ def _parset_add_initialization(parset): @migration("ProjectData", "1.30.0", "1.31.0", "Add pop types attribute and default_all") def _projectdata_add_types_default(D): - if not hasattr(D,'_pop_types'): + if not hasattr(D, "_pop_types"): # We have to check for existence because # Are any population types present? - pop_types = list(x['type'] for x in D.pops.values() if 'type' in x) + pop_types = list(x["type"] for x in D.pops.values() if "type" in x) if not pop_types: pop_types = [FS.DEFAULT_POP_TYPE] for pop, spec in D.pops.items(): - if 'type' not in spec: - spec['type'] = pop_types[0] - D._pop_types = list({x['type']:None for x in D.pops.values()}.keys()) # Using a dictionary here allows for order-preserving unique + if "type" not in spec: + spec["type"] = pop_types[0] + D._pop_types = list({x["type"]: None for x in D.pops.values()}.keys()) # Using a dictionary here allows for order-preserving unique for tdve in D.tdve.values(): # Fix the '_add_pop_type' migration which added tdve.type instead of tdve.pop_type - if hasattr(tdve, 'type'): + if hasattr(tdve, "type"): tdve.pop_type = tdve.type - delattr(tdve, 'type') - elif not hasattr(tdve, 'pop_type'): - tdve.pop_type = D._pop_types[0] # The majority of the time, if the pop_type is missing, the default needs to be added + delattr(tdve, "type") + elif not hasattr(tdve, "pop_type"): + tdve.pop_type = D._pop_types[0] # The majority of the time, if the pop_type is missing, the default needs to be added # Also add the default_all attribute tdve.default_all = False @@ -849,7 +849,7 @@ def _projectdata_add_types_default(D): # A TDVE should always have some allowed units, if the allowed units have not been populated, then draw # them from the ts entries. There are some older saved files that may have no allowed units even though the ts # entries themselves have units. - if not hasattr(tdve, 'allowed_units') or not tdve.allowed_units: - tdve.allowed_units = list({x.units:None for x in tdve.ts.values()}.keys()) + if not hasattr(tdve, "allowed_units") or not tdve.allowed_units: + tdve.allowed_units = list({x.units: None for x in tdve.ts.values()}.keys()) return D diff --git a/atomica/plotting.py b/atomica/plotting.py index 2f1567e8..1931b996 100644 --- a/atomica/plotting.py +++ b/atomica/plotting.py @@ -1250,23 +1250,24 @@ def _get_legend_handles(ax, handles, labels): if handles is None: if ax is None: ax = plt.gca() - elif isinstance(ax, plt.Figure): # Allows an argument of a figure instead of an axes # pragma: no cover + elif isinstance(ax, plt.Figure): # Allows an argument of a figure instead of an axes # pragma: no cover ax = ax.axes[-1] handles, labels = ax.get_legend_handles_labels() - else: # pragma: no cover + else: # pragma: no cover if labels is None: labels = [h.get_label() for h in handles] else: assert len(handles) == len(labels), f"Number of handles ({len(handles)}) and labels ({len(labels)}) must match" return ax, handles, labels + # Temporary copy of function from Sciris to remove after Sciris update def separatelegend(ax=None, handles=None, labels=None, reverse=False, figsettings=None, legendsettings=None): - """ Allows the legend of a figure to be rendered in a separate window instead """ + """Allows the legend of a figure to be rendered in a separate window instead""" # Handle settings - f_settings = sc.mergedicts({'figsize':(4.0,4.8)}, figsettings) # (6.4,4.8) is the default, so make it a bit narrower - l_settings = sc.mergedicts({'loc': 'center', 'bbox_to_anchor': None, 'frameon': False}, legendsettings) + f_settings = sc.mergedicts({"figsize": (4.0, 4.8)}, figsettings) # (6.4,4.8) is the default, so make it a bit narrower + l_settings = sc.mergedicts({"loc": "center", "bbox_to_anchor": None, "frameon": False}, legendsettings) # Get handles and labels _, handles, labels = _get_legend_handles(ax, handles, labels) @@ -1274,7 +1275,7 @@ def separatelegend(ax=None, handles=None, labels=None, reverse=False, figsetting # Set up new plot fig = plt.figure(**f_settings) ax = fig.add_subplot(111) - ax.set_position([-0.05,-0.05,1.1,1.1]) # This cuts off the axis labels, ha-ha + ax.set_position([-0.05, -0.05, 1.1, 1.1]) # This cuts off the axis labels, ha-ha ax.set_axis_off() # Hide axis lines # A legend renders the line/patch based on the object handle. However, an object @@ -1290,9 +1291,9 @@ def separatelegend(ax=None, handles=None, labels=None, reverse=False, figsetting handles2.append(h2) # Reverse order, e.g. for stacked plots - if reverse: # pragma: no cover + if reverse: # pragma: no cover handles2 = handles2[::-1] - labels = labels[::-1] + labels = labels[::-1] # Plot the new legend ax.legend(handles=handles2, labels=labels, **l_settings) @@ -1300,7 +1301,6 @@ def separatelegend(ax=None, handles=None, labels=None, reverse=False, figsetting return fig - def plot_bars(plotdata, stack_pops=None, stack_outputs=None, outer=None, legend_mode=None, show_all_labels=False, orientation="vertical") -> list: """ Produce a bar plot diff --git a/tests/test_tox_databooks.py b/tests/test_tox_databooks.py index 0952a698..03110d42 100644 --- a/tests/test_tox_databooks.py +++ b/tests/test_tox_databooks.py @@ -180,22 +180,24 @@ def test_databook_all(): d = at.PlotData(res, outputs=["sus", "ch_prev", "ch_all"], project=P) at.plot_series(d, axis="pops", data=P.data) + def test_databook_default_all(): - F = at.ProjectFramework(testdir/'sir_framework_default_all.xlsx') - D = at.ProjectData.new(F, np.arange(2000,2005), pops=4, transfers=1) + F = at.ProjectFramework(testdir / "sir_framework_default_all.xlsx") + D = at.ProjectData.new(F, np.arange(2000, 2005), pops=4, transfers=1) D.save(tmpdir / "databook_default_all_test.xlsx") # Test saving it back - D.add_pop('new','new') - assert list(D.tdve['sus'].ts.keys()) == ['pop_0','pop_1','pop_2','pop_3','new'] - assert list(D.tdve['ch_prev'].ts.keys()) == ['All'] + D.add_pop("new", "new") + assert list(D.tdve["sus"].ts.keys()) == ["pop_0", "pop_1", "pop_2", "pop_3", "new"] + assert list(D.tdve["ch_prev"].ts.keys()) == ["All"] - D.tdve['contacts'].ts['pop_0'] = D.tdve['contacts'].ts['All'] - del D.tdve['contacts'].ts['All'] - D.add_pop('new2','new2') - assert list(D.tdve['ch_prev'].ts.keys()) == ['All'] - assert list(D.tdve['contacts'].ts.keys()) == ['pop_0','new2'] + D.tdve["contacts"].ts["pop_0"] = D.tdve["contacts"].ts["All"] + del D.tdve["contacts"].ts["All"] + D.add_pop("new2", "new2") + assert list(D.tdve["ch_prev"].ts.keys()) == ["All"] + assert list(D.tdve["contacts"].ts.keys()) == ["pop_0", "new2"] D.save(tmpdir / "databook_default_all_test_2.xlsx") # Test saving it back + if __name__ == "__main__": # test_mixed_years_2() # test_mixed_years_1() diff --git a/tests/test_tox_migration.py b/tests/test_tox_migration.py index d3219bc4..76c63e4e 100644 --- a/tests/test_tox_migration.py +++ b/tests/test_tox_migration.py @@ -34,9 +34,9 @@ def test_migration(): P.data.save(tmpdir / "migration_test_data_save") # Re-convert data to spreadsheet and save # Test databook operations - P.data.add_pop('test_pop','test_pop') # Requires migration - P.data.add_transfer('test_transfer','test_transfer') - P.data.add_interaction('test_interaction','test_interaction') + P.data.add_pop("test_pop", "test_pop") # Requires migration + P.data.add_transfer("test_transfer", "test_transfer") + P.data.add_interaction("test_interaction", "test_interaction") if __name__ == "__main__": From 23ac4244a5224e6b808e07e543f815d3b8ef7c47 Mon Sep 17 00:00:00 2001 From: Romesh Abeysuriya Date: Tue, 18 Feb 2025 11:09:35 +1000 Subject: [PATCH 6/8] Update PyPI url --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2316c6cb..abea6312 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Atomica +1# Atomica [![Build Status](https://dev.azure.com/AtomicaTeam/Atomica/_apis/build/status/atomicateam.atomica?branchName=master)](https://dev.azure.com/AtomicaTeam/Atomica/_build/latest?definitionId=1&branchName=master) -[![PyPi version](https://badgen.net/pypi/v/atomica/)](https://pypi.com/project/atomica) +[![PyPi version](https://badgen.net/pypi/v/atomica/)](https://pypi.org/project/atomica) Atomica is a simulation engine for compartmental models. It can be used to simulate disease epidemics, health care cascades, and many other things. From aef5b474ed4c0719c0efd253c33cc5328878e970 Mon Sep 17 00:00:00 2001 From: Romesh Abeysuriya Date: Tue, 18 Feb 2025 11:33:33 +1000 Subject: [PATCH 7/8] Add version attribute --- atomica/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/atomica/data.py b/atomica/data.py index 8f7b5eb9..60763755 100644 --- a/atomica/data.py +++ b/atomica/data.py @@ -21,6 +21,7 @@ from collections import defaultdict import pandas as pd import itertools +from .version import version, gitinfo __all__ = ["InvalidDatabook", "ProjectData"] @@ -63,6 +64,9 @@ def __init__(self, framework): self._book = None #: Temporary storage for the workbook while writing a databook self._references = None #: Temporary storage for cell references while writing a databook + self.version = version #: Current Atomica version + self.gitinfo = sc.dcp(gitinfo) #: Atomica Git version information, if being run in a Git repository + def __setstate__(self, d): from .migration import migrate From 13b1117a8c2d49eb031d0ff4992a1eba1cb50327 Mon Sep 17 00:00:00 2001 From: Romesh Abeysuriya Date: Tue, 18 Feb 2025 12:51:32 +1000 Subject: [PATCH 8/8] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index abea6312..59573fd3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -1# Atomica +# Atomica [![Build Status](https://dev.azure.com/AtomicaTeam/Atomica/_apis/build/status/atomicateam.atomica?branchName=master)](https://dev.azure.com/AtomicaTeam/Atomica/_build/latest?definitionId=1&branchName=master)