From 1afe2c140411006c4fa0e8ac97ed7c1656ca5ad3 Mon Sep 17 00:00:00 2001 From: Alex Guinman Date: Wed, 13 May 2020 19:51:27 +1000 Subject: [PATCH 1/4] Move csv export to function --- nemreader/outputs.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/nemreader/outputs.py b/nemreader/outputs.py index 9e4e8a8..e0fbaf3 100644 --- a/nemreader/outputs.py +++ b/nemreader/outputs.py @@ -94,19 +94,24 @@ def output_as_csv(file_name, output_dir="."): last_date = rows[-1][1] output_file = "{}_{}_transposed.csv".format(nmi, last_date.strftime("%Y%m%d")) output_path = output_dir / output_file - with open(output_path, "w", newline="") as csvfile: - cwriter = csv.writer( - csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL - ) - cwriter.writerow(headings) - for row in rows: - cwriter.writerow(row) - - log.debug("Created %s", output_path) + save_to_csv(headings, rows, output_path) output_paths.append(output_path) return output_paths +def save_to_csv(headings: List[str], rows: List[list], output_path): + """ save data to csv file """ + with open(output_path, "w", newline="") as csvfile: + cwriter = csv.writer( + csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL + ) + cwriter.writerow(headings) + for row in rows: + cwriter.writerow(row) + log.debug("Created %s", output_path) + return output_path + + def flatten_and_group_rows( nmi: str, nmi_transactions: Dict[str, list], @@ -185,14 +190,6 @@ def output_as_daily_csv(file_name, output_dir="."): for row in rows: all_rows.append(row) - with open(output_path, "w", newline="") as csvfile: - cwriter = csv.writer( - csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL - ) - cwriter.writerow(headings) - for row in all_rows: - cwriter.writerow(row) - - log.debug("Created %s", output_path) + save_to_csv(headings, all_rows, output_path) return output_path From 8049da2cd53267b978754ffc27a20d22123fadbe Mon Sep 17 00:00:00 2001 From: Alex Guinman Date: Tue, 26 May 2020 19:38:21 +1000 Subject: [PATCH 2/4] Add more examples to readme --- README.md | 33 +++++++++++++++++++++++++++++++++ docs/plot_cal.png | Bin 0 -> 10717 bytes docs/plot_profile.png | Bin 0 -> 14205 bytes 3 files changed, 33 insertions(+) create mode 100644 docs/plot_cal.png create mode 100644 docs/plot_profile.png diff --git a/README.md b/README.md index fb8b42b..ffdc62a 100644 --- a/README.md +++ b/README.md @@ -51,3 +51,36 @@ period_start,period_end,E1,Q1,quality_method 2004-02-01 00:30:00,2004-02-01 01:00:00,1.111,2.222,A ... ``` + +## Charting + +You can easily chart the usage data using pandas: + +```python +import matplotlib.pyplot as plt +from nemreader import output_as_data_frames + +# Setup Pandas DataFrame +dfs = output_as_data_frames("examples/nem12/NEM12#000000000000002#CNRGYMDP#NEMMCO.zip") +nmi, df = dfs[0] # Return data for first NMI in file +df.set_index("period_start", inplace=True) + +# Chart time of day profile +hourly = df.groupby([(df.index.hour)]).sum() +plot = hourly.plot(title=nmi, kind="bar", y=["E1"]) +plt.show() +``` + +!["Time of day plot"](docs/plot_profile.png) + +Or even generate a calendar with daily usage totals: + +```python +# Chart daily usage calendar +import pandas as pd +import calmap +plot = calmap.calendarplot(pd.Series(df.E1), daylabels="MTWTFSS") +plt.show() +``` + +!["Calendar Plot"](docs/plot_cal.png) diff --git a/docs/plot_cal.png b/docs/plot_cal.png new file mode 100644 index 0000000000000000000000000000000000000000..71072554f1f3402ad55fd6cd9dcf856dfbf481c9 GIT binary patch literal 10717 zcmcI~cRbbq`#1TZk`W>$%9dBbyT{D-<%4Ep+Vcb?mH=l@+pOosPZT z*ZF*ZkKg^c@B4B8asP4G!*RUNd%mvgdOn}m^E#oL>I$S6uU*8$!y{EvlzoJUhhGEl z5$B2ERY#})GyM9;N=8iv53eMO|7rC(F|dW$O-W6Tc%JYYEk0g_hz1EB9{Zw_ ztjuH2@%3acw-3YBXHu9%!QfY~D4HqGXHjr0e~=-Rl};ZiL>L(sSC;G5r+q-x>lau2 zEN?Z;smZ4tL528`phvM6vnitMs%0X-k3Nc@}mnUnFvWqP=D2ty8a( z`s```6@$gjqm3EAUazV8y{$ruT|0(NDr0i@9#XJ4z+-_W|m-Z7tX6mRTr6 zWd-v8A2vM9cgjWGW{4t=Y7hF6!4(#$p7StgFrA~ztkM8?(o|-qi*6{?tea@jesaHc z-EC_xafVOo?2eUUzx~`qZx99H*CTidQe4q ze0gR<`&`q^b2GlH=ZKU;al1ZMQ~kNzs>n0oAZWrfawR|;cdcm8%Q1IFD<>k_=c;Et z!ZM8cpK9yugh?;ERR37rS#}d(l*jFc&72NU*BQ?HIe&?WfGB7tOnM3H`MlbAs^Q8= zr!+h6M9-^?c_JC5BHQO?KH3#FSqsfRzjx37_?~xtib1Jm`p7B)Vgkc4C-rD>b?0h? zywjV{``DBEMiD`6fig{$`aqHUj+y+TC$F9y4V-UPz3)sKN1%lRI^1 z(|=Z1Yga`12Rv#g^(?S(lBq-Ml0R-WFUK0gGR%aN?oJ%e`Sc$ynC;)Wbfe+YQYq7f zQMg0yfdyYu=d%Y|)II+9uTFnPdx|rBEiJXNv&*cj`_$b0psTB^qpORFi;HVM?F+2u z+tISO-=E(7NFpRK^v4aMW_mM;LrVV#sRfkL&J>qS@!FQX18W)Di4IP||1TTti>z#14*q0{DV*8tN6*9{Fa)<0V zDp|EwE|cl#B#OScO%wjQz^F1hHnx=({r7co5L?f*GEY27a7)0XV*^uF_2fq0n;s@? zt7+g(4`oOmN~Nq^l(MbuZP%qx)Tx?)A{-N=Pjfn#h%#N;-Ksh z1-DbNj-0%T^bN6_fiv8^2@*cSrB;2} z6%}zGKVD<-{EZ(L9v&L&-$~*dypzs^dfx2GnyTp4)R^tu&a7k9Mbez}A=oY_me$wZ zq1av7tLaA^tu^anAXfTSQu|q*AoT)aD(!zo358^}t$*WUMERKk`*iuUgfb36DG5Kq z{V8##$F8;fal?9M&5+4?H7&iCRl>$?2DTRIi(ki{!=b?uO@rQ4&CnQd%`jc58iT|vm2E3!a^vd*bZ(X{BWOk_g=aU(KM<|&AMbK8z2CpG7ycN5mwKc4n))Z&uhMd%UW=b`fcr+I zJ&JMZSG8-te#tpGt!Qh#en-3bxX9!cMtI7yKl*ayv-wi7Qb%X! zVu1i-g9IH6ib2VqrbhcVc{ z-7`Z{C-qJl`}X-8T*YJd#~U6E=8*IHcW9=3FcOjU_nY+^4t4I(kT^A*2)izRzeRDG zSTw%-iq>o2K{dhvQG_aT*W;S@vgNtZpFuWKuhJ&h-HC^K&y#AVWm-NMQ)Qz9+KKOe z@xh%fR!xig4o$da(vNTZQjGP8s4@GW_FJCf_(y%#LQnaw3XQ#$&-75KQgpzh31`jK zW&fnM{NZ>9A(~7U-j>m6DV&5abc87LA7f0tGLT~J=gxOBfb5nwHA?HN^F%bE$zjQx z8y*}W_)JpA?B$C_6xMyhGTz0;QV-Frh=#YFqj=uPbgYusGl#X_tbgwRM@38PGEq=K zSC_Jji%YTo9o1CHWJq2Cn*kO>L&F3~Khb~?jvs{MDykuO{FaJ?Oh{6b`FKQ?e+gfR5eQH`-SM02uMBQ95h+$AqYWD8<8s?`HT_D zisMu`dq1N|-zi7=DUGccbl?za;frU#iM6<4D>c zu@)k7aJmWXLonuh#G1gZ>2*@llrz}bw3|L?OLe`8XuRXf4||d!OuBsr52WujDbd5m zo^R#juS~)*HrVdxlGl5AnXzNiC*zDs%MAg4Ri*`qyR77@v42%=wG2EZ**>8P?b~gq z;ccMC5!CubaR9ROZfRC`NRMez>04ajI0;r&rhYcF7N)2T|AY zXE4Xc(Q$F4;qv2uWyUU#M!dc5*zmz80?C)J7*C`du@z z^yfGcC+}5buO{ijl1pymzFB{;Cthr)yRtiD`rRZ?XT%=rASjehf-Vmu;-7BlEze|( zv9@`f>N}pX;uKl{(~4Sh+btBDJ?sc=5rXs^*=WsJdw#!38Yl4{68E%k?_1f)eNDSZ z4}9N88!Eucyg(RgTBKhg*F|z}uBa_CHX)f$QXk(Tx059B16?l-k25>M=2yT9^Ge1r z8|=%@`(wo?bT%HHTB(0%&oX`G5rP8gh41_EpuxPIOg&YyC$9*ttgcQrz5h*wNm)5` zC4m7Ms6>43g`+sZlZ*}$BCehLDSL9HWn_8M=RTX0xBD~RMvFW?wBj4TAG`7~*Y9`t zLU+7Dtvh#HTbr@RP2}HR1?br;Va-KwP6PJ&2t5mf$1!~=h<#uwe|}5+pVbn>#FVMIT>Z;i#R1R z_SHi6;bRcKI>tQlA{}zx5sHTRbq4XF-nIcw^}(X=zhi^d3iRsFsyCgO>dwpr6X1-i zyoM8Wwu5i<`*ODxQUrd@6PBQJbiHKiSRRHl@oG}PE-d5`cAEOWyFNj8*Oq#3(u1+O zy81~`#LDksX(y+Crob%E@MzFPp)B+$6i!105|ZdwJ--DA_xA1Ekiv+2164Qy)pGKs z7?i)d)qnVtEVEw`a8S_ld&o*nE*WCyM#`bx+yk|wkS{7F*~F;@TcqXEdj~PJx%j72 z{MoF;fwE&wvRx!hd3qT??~flUd&!vjl4By@KtqA)d2KB|ev_5P%`eCVIYKUrDv)YX zLr`=PVU_0OwHmEwP>tClX%rKW*PE2*e%c<1cal){@ILG$X)f2!Q|lrjGDVn`=vSTW zP9)xQ&Jb~zt+}8iyLs;H?ME{@{`bFqE(Ris+MfxKifL8<+nR)kV^p^=ef z1A~JH<0I&A#|K;H9^Nu{xbj|0uLeC`5i(5jT)nLQ^)y5)lqM;I1Ah;n)FP-c=b`LV zs=b|5!(E3Q-K4Eeixu~^*13sb26BTDL`z7b>7{W@hGbKUQd-5UuWhr5t(nM|?@q6D z(X5~Yjs*~s*9)hq`gzN@qkdw-(&yu66JLD|J-pN^_($XHpRVaDw?CA;=zXR<-GbSg z)xu0PI;E^kh)HJf!ki(clB~kMs7h|R#F<^Px|L>qG))qsh#XZ<=^Z02%zuv5k}8r$ zRe`pH$nyrLq0ys9lz8enf@Xgy3~z7?Pkd&J6hA({F@-4VnR~w377-=*bdi=|H=rXM zjx)U$;O5q3pJt-KzODR#W-dH~gN2pV=4&XkM&LpGj}5}o1NE6B$t5#?x-Ozro9MFf zH_@gjl(&x2E7J2TK24I9Gxnj2@j$?4E@@8E7cHqysQycLqVFZEd;9p%-FIQRfB*hl zMk&eJ#I_KPtF(+njXvs$PQ_c(_-&?@mlb_PhA6dUkDf{lUt94vbl^u_4MRlD&Z&m6 z?|#gewY)hYn%gHT(?#F46ZXQ0vTcv@b$_OUYx@n4VvBAX|D%a%0q5D)kV?F8j)8v+ ze^WPg|8aw+*7+X&J$kY=cQNldJ}o15F; z*>GMW`pbt@*mqrKe64Hr_utzHQcp4c+jH-auDhf1^6DpEPI}Ho2#JU&WMLw2)5-nA zUSLRcRn+JuY4A<@n0U%$vNo$QgMPo7^exR>8AEfohEimo6zBY!E&qen?2jR33Eezc zUR#@wZ#ZWCYeWB~sA$RbbiuT4b>Y@T+QRm^>ANDPmpyYMS1{SV0q64cc?WGtUaSOK zTa{Iq?UCwbtLF^bll+uwE*xI07_*jgzBue5W2~b@ClVK1Ldd_qcM6$hMOow@6>LiO zrte!G@|8&*_oyH7VIqrd1AADX{Ge3Rb~Y|5Q{JoGQ=W6AhDJu5w{Oq(D=;oV|H4N9 zah?#(r>L~{MhZ>xO&e0iO0Bewj8s5G$tFc^8!6D{Nz+_>uTHP}AgzrpGA=|HaWhgp zw7l*nC#O=WnpSL8mh+%pn+clam*j%C__F19h2*#!0y<(p`dOHC#76lgn(Tj^2|4Br z(g{Tvg{!H?w+`^OX$dyX6T$}#LK~h3tgRY|-xyPLr$du3Z>*^;*5E3A19dwyVKl|7 zQv6$f8hL7Yrt+oOkikD$wY5pmD@7+KtI5d7*e?*G_c2C)kWw*qRC(}yfmu8;kazZmD35L)aa1 zQ8!i59=pP5tUeT(ts%%QWw#Jy3U~V zko^P(cw&bvBn8!A+-a@R(U}SIT=Ks~ZWv-4q5d1WoSMOrpqiamu|qkS;3E330ttGZ zJx35oD)L&sV`Kw~s`__-f>5?5&)hT%zN>ZGiGiNPV%$q}Z6vp^~nA zL-#rNC5xr4x(}G--O;aV<++5DB9>B-DFno1mt>KzTOBJ)o#*6r3k=)*_9olw_oh?~ z3=A3@B)qpd82$I3Gz2yb{+SPDPx;*|hRG@{Bzw8B;mF{Jl;j zd$rEoIqB&EwHR~~YqW(KnA}&vP>$eBeda3KSdd2KPoiuj8sGY=R;PF8plI&1~ z(Qo_m=g*&=*BSGMyZZch1%>TLDW2=S*EcU68?A7-PA;)cVK-8^P}0sYH}pl@nU^-L zq~v`(!aKCwZp7v6WN!;nTFc1j`d_CTWW>C6f6u|$_>0isnH zUV$Oh=*+{zW2=KX>Ti#ef6Ni4AvKuMKVbyn6A>2|R~1Xk%OvSJj1+yOql4*XJ9Xlo zRWqvdYNv{7fB53X-N9_NOcW~Ym?$gwsx^ECgR&byC1-iSK={{1CU((>(!A;?%6&6NbcdQCX6twy_?@)zCGlaJ_qy3IL z`ucLJsuI3^lLv>km%H-UxmGFe6INsY1cU55+*zfQ@)y5*_ik2pHn_f? zkVpsnb5V2>-a;zzJgls&ugN9%@q$}q%*~ncOb-X*Gj41A7#wUV?UM|jShs9>L%O`< z?SHarx;2=38k>4{yfkVrj0*&0;$V&jJ1ou(G56=YYX-NuxDE#5(X9a6<^xL(4i5BU z?r)`FJ$+|LOw%eW<58&F;tb?EIy!5UwY2fwZ6F~zIXTnw^DmW-eJ>wDkVxCuaCUZf z8oTz~z5HRxrP|ubX~}5%kVa(Ejf$4X*bLmq$2vMCP{*R8qNAl&RxyXhucrq+%0@jGO-R31RWGi&bTNAyC|uve$r2Ky1hi+g%y6%{2FmGBi3eH|V7NczrvgVNCkq!hS|pTxbQuRGOUj_H*3 z@ezaTyuHdFxVh>nDEwXk4~l04*q(!wddwbH(Up&{y+uX47#Vwk!X>|8rO4hV2JtEQ)?_bcdA`&>FW z>CmE=`X0DIegsXdF9OPfy-vUHCWL;mid#)g!x1K0#ZB2W44nYgcy%R#LJkjzAgO#O{KMR?8-+6!N|u&}V7sxycohiA~z(rQ{6 z&cnId;L|*TDa0o+k4QlLzc8P84zL1fWNmBvKu0G!CZ-eYeGw7bm`>_af01hucTCZr zkjHb3pG04DiAjtP4YfkXtH5&KxN$@L?0D<3uC5DMIXKk^R4O4Mal9zf02c856#+S; z#opv@0&?-j67zNrF0Q!`dWiJTp9vw00R_3BjMe@8Bs3&;sF!+^TkM6kO--ei?_ayD zaEsrYB+K-|8w`G4ll$Y)D*&p|(UkB|5GX(}2y3E<69Wuo+QMl|y?1Ou>onUH)yu;Q zw`PVZYEcs=^?YMI{YxBqL+o3}d*(0s`7B~$*TdT$z)FdEY|PKLg)h5_;_NF}`ta_B z*`w(*$%Tal--B+`NR%qp(im0@=%DX439Wvy8Gfw?hD|~u86asB+s-IxJH!FY-o~El z`@qqWhgtsZ@1*HtHb}aNn&kTWdguaOmXbYi)?J0uv^u0nScc}KN6iod{-mCKliIj} zfky&@f;eok`sw#yIZ`x zyp|S6y}AE*--@7JUtfRx_;Io0r083DW?Wkcmd%vfJ2}7By%qp{y|_CD%_n=)8gg=k z3Q-IZHIQvqUm;tC9mZv${flAOums+&+ne$MAJ1Gk2`*#?h%lu!AI++=52me(j?R_U zp9PuO*>5s3GIF~Lii+5Q<+L*8y^tbLTh9v!2-GoGt0f7)PflL%QQ)|J`zhGGTF8`^ z8TVRF9v(KJGjI~IABubl>k)lTLB!6rlc41pAakOS$L3`8_Zo5Q(3>pw1?X zmzNjhU2m#XpOgRA^>1(K9$H#vS8BMpx&lryOG%k(dtJG5B{n`D6h&~q^*rP*y|^dE z%ep%8N7~x`f83r~Sq+YjDOp;wB_<|*%#M(4prN60Io!7K$A+8!1tLq$&Su;8mhjoz zSXi%|wj3|B+1}k97#UIU@DKsFDeLoR?CBB4b$eM0qDigZmg_q679czd+F=fuNnSW z=m{2lrjHnKE(35DFvJI(ECa6yL;bn)8BA<;OhHhzoAG&e(}Qiq7Naz1b%&WbXle?P zuba2$0gQk^b_4!EvF@)eCz>slwuRwD6X|tfO^^&tW@!-2{QNsG0$_^?{q%`mR7{Lc z%>52noVZ)u-F1fv&r!WS^RDQ|+FI+nyAQ!Kpkhaz*Vg`WW>iOKCpN{5KjDITV6N@W zz__bpaxP@=D$~x2>RoFm`88Cvs9&d~FyICaH7`~Z?%L5<^`+d1Rm*I1zG_)Y2iXUL zQ%mOqXxi)GRQdSS?mhHB-ggEC1>;tns{%J`qMZnMg;Qe<#|yl749hx>h+do?dwF|% z?@#Ta6SKuGHSLA=qknZLHa7M-NGmqgghnVioJ98S|0CX#rJ;);f7|JWo6VLP{IBiF-1x= z98F>rm6e|Y(>QxgWjNxH*0$UQ>_)Eav^hFAYPnkgY%m@J*2z&%&dAH-a$L9oNDh#< z+j|}~4eoBB0s*;(hKFZh9uATdupb7mqb24z{n;JQiyIIP{roAvjs%1*tEhOSso4lZ zr5ne4IPw8L1qCy+YI-KBo)ZI04A~6Q;nO|mb9uwcfHOStDuadwVEz61^9Sc1-?jUh z#&HpOb~Ih0f9IF?YJvSkMFc2Rev2+@SV+F49vCCm)Yi@ooPoa>AK#9H10Zx6n7___ zi@>3%kdV;3PoKWSI@Qxv;+6taw?&3RxlH@*$xUL=GPUL9AE1)sp|)^5LLGsCgaqhhA*{JOz1ngduqZbu#m zi+JMfeVD@x9a)^^nVy6l9{w9Nh{=85{o+Mw_LTVcxhDYGxMcKUOH;%ft9ii%Vt8NY zq~ZC@1BCy91tcLu=`%beh@b2v$BeSFSjLlO1-L#af_h#`^bU5~I=x{5zI_nKZS)O> zHrDp`S*4{hFwk<6YPGwz6{8`U4g~-`n0F4otE;QmW+(f>67tjvD6kjQwKIItV(ilu zE?u1F_#4D{+;}8#b=|5M)V`jn-<6ooS16P*+BXpw`N2P)TRkG2yu5onPkh(yOqqcX zfL*bzR2rHlCOshNvu(!jv9kvQE);`ef)4SedJg3ESKu@t$0ZZ8YWaA9Gwgwr6K_$&sc5bHdb4FIK8`|_m6dY~3vmd~#>`A0r&To zCRINQajx8b2p9nB3Jf8=SgUeF8wUqgDXCPrHYmxCo}RV^m-jI-bAGD~V(#`sBn#jI zK-$2W>a+uxwswF!a9!tco}PKRA=uwcaC^`&433X?!I&q*9c)kahYtyW&!)amV1Fg6 zvx5t3T=hHvz#d0ypytinBirnS2Vr@G$R#_>qa+r4M8g+K{UI+vF#Ih%f%fcOQd0Qf z9~^N)V+B63J=h78T``lw1bs0fn!qqA~|wLncf&p)JLSWd%fQ z$Sz7sO8Qbzu;X78f(Ajg0P<~ndmA38yUKZPd3ALT7!dk=HjNZjXns@=(Qv|K^(7PD z{4Zb9!S%Bl%!Y7*d}&^H7ktwE4<;OqLg5}!=vrp4D6B8&_Wqwgp>t(nXNQ5vRZP8u z@O9|!8g0v?giOS z*DvSx8$px4oi_!u2u#2|fa9}^i}@_I2`JZxhK9^a$2fR+7}ijlMn>I0c0jgqi!2Fd z5UNN92`H+IAVvsk1eL)R0KGB4u%Pw$vC#6E{d=av{LIWC+!svz4&4Bq=5)a#S)7ZK zViOb;VJQ#Y2#^OuP1pD0h@AGIoS-N>Vak2Q{*W-N-@4qXhB&RN0_`odie*nMKLKXl3 literal 0 HcmV?d00001 diff --git a/docs/plot_profile.png b/docs/plot_profile.png new file mode 100644 index 0000000000000000000000000000000000000000..1ec2fe346206fde32b7dd2e66ab362227f16808c GIT binary patch literal 14205 zcmb_@1yoh*x9=7tM37QJP(o6ryCkKg#Z4$BjdV9iDbjHeX#@lXr5gm1mJ$%@knWVO zH`h7u-uu7rjrZ=jV_eTTvW3OkYkg~ebN*_-R#lcIz@xxJp-=?!at|J%P#5Uo^(-zn z{LN{qy$WA0nn@^0pio7TmrtHy!M|yYIlB|ViARM=0KsA4Nxed zwbxsP(k(q>Kyqm{Yy?Pda4nO&xWtvl=|3v)yOxg z%a!_R@yC^`hD=w=XZ)>`3;!4vT5uWoNd!lz#;dYtCiHfWdHVVqgh$I_UozsE>ft7& zipa`yT>rwo^Gk4H!ZRXcr8fk58uZn^b>zE zkf&)>F)|o6A%WL`T$Kc?r4CbutY=`5uiV~kwkDFI;TqoR_P){pt z-hce~8i(2|Dlw6w+-CgY3~c>=MLq3M^;X24%W=N5!GJ?mUA^%Q z>r?;TmZm0E4GkF+69&Gd@KKwZ@Si^)KSs|r5{k_Z6&Wce@r6^`RQi>d^OakV%E-#% zxVyWLxy^*5tS0MHa&zw*8W|;s`HCv%IF_mukTY7)UYhcvIX&$0ou52EowP7F|GMT< z@_gX#^2&;Kz0Z9M3yZ5)uSUhi1?A>`(y59e)Ax#AU$@a=BV(^W;VP&<6VfY)hc|=6 z-8a7MZ@P(~ehv=K>>SF;$xZFopWlb2Xq?&F{C1iB$K+(sQh$#7T%0Bx_E=LhHY!S7 zLE*C4@o>-L#zxMkPhqe!uPQ4A^t^Y`ckkXcx3FlKbBKXeM&vjh7h>_{-(aa%#SssI?T2avs}O2+uQrL%k(k&eO6XrX(`Xyoc>vmU(G|)o>V1y`AZ=g zcB@02xw*M^0yo`6s8ZdgFO7_j9`;811P-p4w8t-k2+Epy2{-II} z(wWvsCN8cRcz!W4F(^Bs6enlr3+c>w;o;$rA3s*r(Yg7lV`IO>enuu+G2!+zCxYnf zF}Qs^;mOH2yHkW)tDRR~?6Bti`QvnUwsD@?)bs!!8`GwOQmOx~kn`Pp8lS|`=v41L z?e85O>`#1*jOgmmb_=vC?P#F{OG`_Ma4-JgAwZ*hoo{9K;wIa^za{GR*7u~3s&ShWA2F&( z9Nj?+TiXRDvBOL7bjIvkZ!K0U%gA7_t*t@XI!-rTx=ly-`t|E)P`0)E6P_ZcfBNZ} zm@uQFqa|R+d3$>&i@3+Eu3FXVLH*CM+Y8(j+5C#xaOwfYKlH_uzen_#1uCnp5qI#) z459BqBXz`{a&T`>?p-)zW?^B}Yf>icR^z(EF56;@e#X1^?hRKtGRa0#nS2YpbW_AF zYI@qxrglpvTO}=`b>{b1Kg=H-lc)i;~LWy5I_cL4iiwd_Q6c2y=sJk>@@cx5-k zg2J>U0d7K(k@Wuk`%X?yQP41;sM+E}%gf6RT-n*!uDoiLP**4Qsi#)z4-O8dVPIew zx_0%q5{5i3F`qF$@;7}{Aki~A7PjMxva+pi;f;$Mp2U(mVqT)uNCkMTe2yREW5M{C znHk`?(7p8)lQ3*AAD1?tMq2GdZtnEv@5Vu8nhW#{3~E|h0wXoFftBAnO}Tk_c^gOe zGW@bugy7PkK|ovQOCr!6Tv}d+ikqI9flU|LDg|p$R8*w(Sp)h^W+rCdz(Ykvd=&i3 zftM&8#7T}X^@}0!U>m~^m75#!g-=3Fkr=1_B zB?~&fL<+^A0V5+bQ@g~J;LDdUD6jM5$ptg9;~O|vZcb%JicTk(r9SIPy}vqCgjqk? z14Wo1>Mg|ngwtN<2L7a^_ghvu3weDq)a)p^|A7F zIrz3b&Z;RLpMrT5lxuZOjT)L)U@oj|&sLh9pO1khaB<;7hU`pB_{`x#TK(-~uU>Izchh9`2UzXx?OvM=I7k;8^V;e9 zr1ynFTCH-ch4RkbyGY@l?E9YcL-`?4Bs-hY=Xi7yc6X%c5p!ED~UDF%_K6Hgt)n9-lh`s zn~%mFuN*XXmynBDzyFZi9qU*kRjq`#P-BiAq=NaNHJzDOvw%l8XHdA(^3@e$O*bp^ z%Ijoih3R&k!3D+5!>>{)Ps>MbV0aOs(XXM&vavh-teByuUr(+m6wXKL64h8+A{;`-?ZJXH#9TqT}- z{la3$*uVXN`9(vz0f(0dR!6!K2Xj2{Xe6$FF@6luZ7VsNOSIo!UPNBDk;jmuPcMY~ zoKL;}{>hY1BlhI;iBPpUjYVS1JE>O!J|_bFlV03cUIm0x^7jFy$hJTxS7}9s&3FDw4G;H^2P^xw&o`!&PQ}J_lKk^?T;T% z2>L7OHuu-d8L!`xy@fGD4_*PdE|0%Ea`F z)2KyNU%w|`w}zbC6u&j%7J{P?fJ4fptg5EwH@Y@I;kjYb@rD&}5|>%`O%!y)amQxVPs5qre<^)y&q9p&>b4-CNDg&44N$QeOR=&cdl9~lHVq!v%QSt4I3}&xCy|RGH@e?hD`S{*g^k;KAFF&@R)s=O+ zrE_}0<4eH#E>0=Cer098u7he#ZS74o`eS~6c6K&)$#CcAiiw9X9;@@!b#=R-f`TI> z7l47>hBLt=`2PKS<2s8L$Kr8SRn^zSx--^_v2XA{*W!Zfr1yS$ z0T7<=KR>F=9&0tDp+zdEth_Jk==f@ z1WB+F2mRP0)2Kgx#xK|VeL^R4Zg(6QB`!2FBqc4iZ|2&YX;ksH(%G=@LxmGVRAA+m zz;eBY5BR5*-PzwweQK93n$8@irEGYNGJg+?1))Gk7c+L+yZmBRgaj>1^#qBWf2Pv? z$9zsW=ZVYJ%Fhx5{Qt5vTb0nErjNRq6v4AInWSuRRCCwy+bvw-rLm{Qc`equIG} zE)gdM+lVCIMT}4S^{LP`wpUAfV&dal5yAyMFMwMjRlwdKC=MZDhV!%I_Rh}Pxp@8T zva-03>NyTQ_xB5dpFpJm0Twwsm;t~Ov1fmo_S+BXr*c5Ol5DD9cXlcQd;$`VVDr?} z?mtWYfM`GI*1SP<3HP1(qGyfRE^8y89f<67a(kU@wcZqT2-K}{DRNqR1VC!I$Vj%( zE3HS?%#0}|B_)9IoJ`W|zODn_=+KZBFtH1T01cp|oLpUZzTYb-G;Svuv*CxL18lpu zHu~}NXA=OPSvfg?3l)HlGseCKc3NCqEa16kRd_No^5j>(F7g9_N}1kl`NFM<9s(mj zNox}`mV~hGaL#rNMOA(~$(0cw5UY8d6VU#_fG=Xzewdixc~Y0DP1PLW#@}#PJUl!G zMOuq%YYarV@FF-o{C!15{MnfgTnzxM54jSlIdb*o%(WC*HEB&eQ#}i+e7bOUcTshhwf6%c zG-_&Uuu>NzTLU`g43GXSEmn6^+SdAITXd!f$NT$ZykXbxX7@epnBtO!W_&WwU;hp+ zsFli-MXPRmY3d`(=hJ~oyFv;9--6fp8leT^>J{;rWho8UqxAS;Z(+m9eogkgTsKhdb@xze}oJ-QM1YAG16y4+C-(9UCj<Tv|*q@KKDe7vs>oHFC8Rq$U-s&HP{}`nIGE@KCFIyNe?-1dFN=2Q# zGqSNiv*qTmpO+glx|Ur~K!`>MOzwDmX+X;Zzg^h{wkmV%7DV)HH6||tk(^kcmT&&) zSBXtY=|V^s)H6^M7VY9d5C%(JH}qN}Zgue2A1VRdb>AJ(w0QnJ1B4vVFSgE(H%crA z*`}wb-BycQp;1$>&D5E?pWZb=ueo{O#f<&=Kr5`X-Jawb)woGi&_OX<)v}z0#U1zq z+s>e5_;1-rUFCjrZ>H?Wf`S~_!NI|}rrpWypa&ies1aAJvz<~5Wbj{ez4B3rJN(8VQK`8;0{Q>yeo5`O) zo6b&;VMeuV#1sBSr0?2jlZdZ)#rNdE{?{k%#ux?#Xf=u{LUG7wN7@Uh9foC3Ku^6K z!Q0%_l%D(C4ofGg>pDxn72k}VKyXCFl{W>;LOy89KPm%%dPP+vgY%}~d904t#Csp_ z^^S~$0HyUkTkwUNn}_WPGK0gx<}(Tk3IXpU9>7eq3ky5bq3qjv`llg4J+fhJ{ce1v zUFRhT-ALqQv$0{Dl!OGKRPstnCIE0ywVD%Ola4Jpy@j;h#QWVpxUmp+{?&XTuyR7z zVSpG29z2MkUUPa*&gg|y&nqC=wHqV6vbI(PzP#nN)UX?f&PnwMfic_?Ngdhvd#b3F zY6dP&AVnD=B+KNL6gaAbOQtvfk~(<+;f&GfYk+7FT>>;*L@<>BY&cHcJE^Eou>>+2gTF(U-XI1{I-*I>@m2RL@RIW$4Y zc{zS!>+FS$Hd>fKmkK|`KYJysd1R5$Jkwj5y(5i~Z_~zIw=gK@xzIjY_}s(|i_9;U zk~+L(*12yL8RvV_dctmS|8PzLP2Wj(Vc0CqWUSJDb-3geoovMBQ~+VI+m<1+5-?cX zdU_T>J!x3jv0wZ_J2Eojw%yKjhmCC>M8Ae@oa=u%)r<#Z7u~uNv@_1#k98h~9b{Q1 zp*B9zT0X;SCD!|WkMy&Z@PwqZ=cM_@8zRkt!q%v((hW0Pp#Aza7<@G?!P3kN4-OXZ z=K*;Mq|2t}W~_xVmnt3da2e3Bz&?Pcj@9uAWG>(Hecz9Hd2|d6CZ#>iAGny?etwy@ z+=vMf1kE(QLt0al3^WYrH+IWE?;sRIPmc=N$oKCLuiO#}1brQzF_;U=sUq@lD3A{r z(Cx8Ig3ilVp|>{_J32TZoL;Ni=}vTXG>qavqRiC{R(jB-%dIH^r#E2HEOaHc!c$b% z(6|m%gOPzD2qr(7wA#U5MZFTR7&xl-a6W0wsb2~9tt0p;I{^EmVO-U^@A%JAhRXnG z=SyO(WJ@BNURjB`H#p&QyvCwch!UWtQO{O;aH0BDjG2~0a@xD*=023cuN^aBZ%r!qd(S@IP&AUP4 zZT^a}5M8^|h^%#5Akn-0B}%$iE9_}J2o2`&tv<-oA2>MkzvQ*>tCn4R`~84RxYW>Ip0#kL<{;_`T=bu-gE(?|N;RH8W+*gb{XQ-gX(>d4<=!b3%&vg3gyAbV^i_p@QO}}U`T3`CbkD?Ba@hxVHQl1r=eFaVLPtj+R_YGE zjx62oxxDkb20Ja2^tKM1T>MyP|M}EP&ki<^OX_n1_lR!Fw;Us{XqtGHFqXJ<4sMPg ziccr{L?{Q_(YoHj`!_cG|7k2O!5dP5NxvT6k-<(i&_rL8R`c(VGEcldhK)^SGa#9L zhgdTBayWAoNX2eDUHlf-*6(4S1zjd9ww)Ap(lvwKglI*N(de!C`V*|s=5MfTLBK)0 zsg@QQL^_{`)-%$|5{Ro#w_H4I#dn=uuTzTf{sS5!*b-{MhoIRefEb0!)A*cGSqVU- zr4WocmyHP}7~7KornD<;Zo!|1Z62k@js5<~NwI0PY#`BK8ZlN(g8>Z)<$x@~Zn?w4 zG7CCd7F;1vVQ}aIM%PTYXWOrmk@Z7A0j=*X$1{vrMx}37eEkCh*H|{-d$r(&)t+p3 z{Ml`f5tH;94|h|2+Rr0;v`7cmQ#i~6aQ^0(a?)DNc|hwA&0sz_^?}s@QFq9S5A;Sq zaT8%7IjgQNwPL~p z(T@NZ$b;cwTX)E4ZEY=Z^9tx~VtHf6x9{E!`S5}HpL23?vb7Ob zT^0}{Fa~ynn``970Zj(MFqDiv8;ndKsLx7ksgFlzuEBl85FeSpR-Fs=}f6p+pDokN*p9b8(O;oxd3@QXzb zrUFnW%bNAm2*_z|#awu*0u18o;ehphDnjgl3!%;aJquE3^kej`QjgETRf>wD=jTmt z7?DVZdCiRwik%QC^?xWRSO9-y_H=)8p=?kuv%LIG&3g6a_>Sh9jXDKeTXyK7wBQ~g zCM_5~85tSi@#4OTfAglrdw;wYSmO{hV)L}iv`x&EYBUL5rXaJ8kB^J&4H?0L#{xlw zDenfd27rPofW2@x_;~~0LHPms7^<9tlCo+hLKrw7?cBo^Iou0$`1GO8B*htWHa2WP z`Wgz13=Jz!_eOE7)gM28pr+PqPMt?|yR$t!S~9qh){j0AmlEW?|B08J8HZ6kN}2E8 z`GJB6&j|5Kot#uPHF3mEO&Njmm6nwuW&Y3ExB*c7() zr+o7A^046n8Ng)!^-JULX&Met4qR~mIW!twWHT=G+|p8BQSs)QBW)nL3EgwrrD`ge z|Mg1-?q6G58)Sb+N5nR3Fo{LMs<|(|dsPVt zxp3I(A2eX~fE8>VHjR88dEgpgDX@iJXPs_(P5M`SJSpx)E!b_)$Y$H3=}xxe>*?9q zBOy!B51w$ywR_1Tj^+%6LbY$dlXh;xzSgM%e8$qnYlg9rFW zsRPfOpO@+VK|0IM|7i32CsIP*>m?W8G3QX^vi1bphIXOB1*m$>LZ>;$^DjeujjWcH z_50{w_Y+&a8J4MRsvhkx4tn!WE|9sRpU6Eew?di*F!F6M%cplea8A7pcBB2y3zjJo z^D|e{mq*gma4Mu^`<-}{qy{DWdwf50H_%gI5j*nG|8DI3`MV9Wxp_FpN!6XGv%A-1;- zYG$~|kx3!V4Flc(V-tOVH6XO)InJM-NPu44ix7kG_pe!mg?=~A6fN}jQur?ziS zwIBa^)L0d~(ntHekM=bWMF*WzU|IlWuly^I>m`r<^aD`2q(Nr_klb)sX3a+l!pN;{ z*CP?Vn3&e6m^VJ{+c@P4@=7&}E{&|}%WtaRuNRcBvl~LP12O^~M_t6^!Y(Y3Z~3#o zk4z@$5$uP;(Aq&3xngPm0=$0s% z4Cq*p2LP}k4NfZJ3cP<$u;L0+KqK#3z%#q<jA|Hs6HGQ>#$>(h=73=Wo9kMccv`V`R^p*UNXoP4ga z7aZU@ySO+&d_zb`2xJd4#BHFEJIm+^$hLq1YRE$YRvf-)8ex0~@2qF$=g;BY-O3;u ze*E-_UQ8?vl0d}Njp!Rv6#3AjK@d}hT@0cNq7^paP;*$1ltwi(JL|SC?7+Oly@&xz zyR%-ifp{2?AOB2`tp?DCv>RVvF=$X$!^K#y8aWpmZ{S{Z18$*RV_%ciuG--W(YB&U1 zKO}fytiTDPc4oKSpztANP-NCa1r-7;9pC-SsgD={WK=B($A87e5#P8W@8xxpz6M|s zhmZ>Ct|h}Sy>ziFb=X{1hx{POG7~9=#8UqJ`9ph-TUJ{7H{xC&lZX5l>{Hq_Ael%g zYH4v1K)O9R>wvIY!9tMgJ`CbfGJ>ZdkkTDBXb-pgNvp_zd)wivv_wBox$R^(a@6u! zm#t6Z#23%uGD|YR99}@BJKu87WH=Kte0l;wipyv;0@x4|`%kuL%AraUyB*r>VY`YC zMza15O*9%Ao?ZNtBIY%3-dw*zCDN859g6r!kky%1l|_BYbWHD+ESln`Pbw#v9nD4_#a0~4Jy-{1jswcznZqPm} zCVhl02lMA}#11YXK|Fx+SvkHgSXmVnz?hN10>o<;B81ls)<(*peuMrR9v$saVE=+} z^*eW7!}HhkSYkMvw>^8gxVSjwBAONnavQeo3p6?oB+H$Xot{^W@BI9 zqvDd1;Tl)=NU^hn^fjmix5K%32(kUy-HlF2Xb0U)YI%KibyimF2#Sxma@+>OO~?!b zk%f+t(Nr~olHUdqc)*aLf}qUeVmc&;K!}OjZ)pjqgt%lsEW5@=EJf-Of`nv%Z)k`2&Pe27mV^8wB$PDAv_)yGE^3=PDX1kD9f4Lq|+-R3+JD^W<;eBq3%A_X%rHtG@&2%??6xTs|jmdY)^%pd0bbUP}bXhop;O zTR?Tp0_~lzs6VwXOtSlJuad}f33w+I1@s~o-D;<{8EEeym%YoqJ9#{@FW|OGbM4wS zT0dj11gPP!a}I!;>4CWlxU7=fRCub=e9Zk0(pwcYDWq`kY3758_0lZOn{T$v3dmGx z>0kFf_c*+doZvcPUQ=!-TPC&u&utB5uWR$ix~v5=077j6dOLo324bg6DuDH&6G9?Y z($H`^eOrWzu&b*J(IemqV)%(~M)oZ*FfxKU0CA!D*G#EXtzu_eQH@L>wPj_|gJR$g zc^!bA(8Gxkb`A*758sc-%o+KHnclgv&YkWLe9}hKomL{Uzef;o&Mp#&Qwo8NqTY%^> zgt&Nkd0_^vN{lX5D}(X{oRJwXHy2USpw&oK8)AYBvU%tQZJLXlTk46tPkkEnDjdHv7tD~?g7q)iXeVek-q}V0Q(34 z1RUn1OL$4@1;|Z_iHWcm7=gnE;1I^d$MY}hJWp(a+!C^TVY!ju8bGfgfO>&{I6%G< z^xR{C{z(T(1-Oi4WivT9ULKy!Ur%kH6>0h34}mwdU-?{dWyB_Z9Z&#_-pbuyPlv0W z??G;hd^>WvIS?JM6j%W_11JV8O^9Fv^8{gL#uK_mx;Vv9_nHglg4hRyy1=*eX4LXu+&RJ_G%(GTlA zIDHF|!bG?LW{#q`F#S*viX`8z6w`NAQc^0k$?R=|)4jAx+1=lF1c@KSJ0QRpG5lzI zOaS*F2};Oh7(&_=nu?mHW-C-TPTYb+mD^S`S%Bs*D)(7pB+`kvv7MbaOV{AT0^%MX zLI7fFkCyY0EeFD2lsLr)Psph>*pGk!2wR)RkG@U_6u{a)eG2e#00Izya&w_6>+ZRr zkb4F{8w{iic3?M4V4jiPD=8~;{<&P&*Q$#~@4jEABEOdcVMXGbrEYPCHZxiw#DI znya34yg77+AA(!E=3-Pb3>X$NH-MZ#7o**;4abIjg)~HAU(i0q65qr$s2F|aTv2u4 z0qN$ZYXg0KP%}GZXF6!~+2$)TuixJC;+qju0G>mC&Fvo})dmAE(|c$dFPf9yULHX2 zEHqS@mz<9Cih%Qm;0SPB5U;{Ku?27`5*dZAO--!=F6qi(0bCU~L=Z2A1ce-W^N=gfGdvd@C!0J}Vs$!uTfI@>j5%>n8 z0Mcr{ZYq;~A~zj_U}unw6d^o{=sNZ1ryKu>OT~uGc<{jrgkT`5aCLQJNC+Mhk@&~o zsfC?{B)u?ku0*AzD8W+8-5Ra1&4G5hHPcE;MkWgxp%T-s+k%2Opr?6xi6Ey(LdnqG z0U_hq;;{m-rKNpVsuEj?0ihfQZI4WasLaHqBoN&U zp#{SUiQw{FT=s&~5=ef_L-42XD}gV93t_?zmzc$ZTmk#WPUD4No+u`vb#_ipiBann zm46!%qSy{t@FhY3) zL5 Date: Tue, 23 Jun 2020 17:28:09 +1000 Subject: [PATCH 3/4] Linting changes --- print_examples.py | 81 ------------------- setup.py | 2 +- tests/__init__.py | 0 tests/test_actual_interval.py | 7 -- tests/test_cli.py | 4 - tests/test_consumption_data.py | 4 - tests/test_file_outputs.py | 6 -- tests/test_header_datestamp.py | 6 -- ...t_powercor.py => test_incomplete_files.py} | 11 +-- tests/test_multiple_quality.py | 7 -- tests/test_open_examples.py | 7 -- tests/test_optional_columns.py | 7 -- 12 files changed, 3 insertions(+), 139 deletions(-) delete mode 100644 print_examples.py create mode 100644 tests/__init__.py rename tests/{test_powercor.py => test_incomplete_files.py} (87%) diff --git a/print_examples.py b/print_examples.py deleted file mode 100644 index 28ff00b..0000000 --- a/print_examples.py +++ /dev/null @@ -1,81 +0,0 @@ -import nemreader as nr - - -def output_file(): - - from nemreader import output_as_csv - - file_name = "examples/unzipped/Example_NEM12_multiple_meters.csv" - output_file = output_as_csv(file_name) - - -def pandas_df(): - - from nemreader import output_data_frames - - file_name = "examples/unzipped/Example_NEM12_multiple_meters.csv" - dfs = output_data_frames(file_name) - for nmi, df in dfs: - print(nmi, df.dtypes) - print(df.describe()) - print(df.head()) - print(df.tail()) - - -def print_meter_record(file_path, rows=5): - """ Output readings for specified number of rows to console """ - m = nr.read_nem_file(file_path) - print("Header:", m.header) - print("Transactions:", m.transactions) - for nmi in m.readings: - for channel in m.readings[nmi]: - print(nmi, "Channel", channel) - for reading in m.readings[nmi][channel][-rows:]: - print("", reading) - - -def print_examples(): - - print("Example NEM12 - Actual Interval:") - print("-" * 10) - print_meter_record("examples/unzipped/Example_NEM12_actual_interval.csv", 5) - - print("\nExample NEM12 - Substituted Interval:") - print("-" * 10) - print_meter_record("examples/unzipped/Example_NEM12_substituted_interval.csv", 5) - - print("\nExample NEM12 - Multiple Quality Methods:") - print("-" * 10) - print_meter_record("examples/unzipped/Example_NEM12_multiple_quality.csv", 5) - - print("\nExample NEM12 - Multiple Meters:") - print("-" * 10) - print_meter_record("examples/unzipped/Example_NEM12_multiple_meters.csv", 5) - - print("\nExample NEM13 - Actual Read:") - print("-" * 10) - print_meter_record("examples/unzipped/Example_NEM13_consumption_data.csv", 5) - - print("\nExample NEM13 - Forward Estimates:") - print("-" * 10) - print_meter_record("examples/unzipped/Example_NEM13_forward_estimate.csv", 5) - - print("\nReal NEM13 Example:") - print("-" * 10) - print_meter_record("examples/NEM13#DATA_14121801#WBAYM#3044076134.V01", 5) - - print("\nReal NEM12 Example:") - print("-" * 10) - print_meter_record("examples/NEM12#DATA_16081001#WBAYM#3044076134.V01", 5) - - print("\nZipped NEM12 Example:") - print("-" * 10) - print_meter_record("examples/NEM12#NEM1201005Scenario1#GLOBALM#NEMMCO.ZIP", 5) - - -if __name__ == "__main__": - # output_file() - pandas_df() - - # print_examples() - diff --git a/setup.py b/setup.py index b58109a..8d00d69 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ setup( name="nemreader", version=about["__version__"], - description="Parse NEM12 (interval metering data) and NEM13 (accumulated metering data) data files ", + description="Parse NEM12 and NEM13 metering data files", packages=find_packages(exclude=["contrib", "docs", "tests*"]), include_package_data=True, long_description=long_description, diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_actual_interval.py b/tests/test_actual_interval.py index ff4ae1b..b12f89a 100644 --- a/tests/test_actual_interval.py +++ b/tests/test_actual_interval.py @@ -1,11 +1,4 @@ -""" Test Suite -""" - import pytest -import os -import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import nemreader as nr diff --git a/tests/test_cli.py b/tests/test_cli.py index df1540b..fa8c812 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,5 @@ -import os -import sys import pytest from click.testing import CliRunner - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from nemreader.cli import cli diff --git a/tests/test_consumption_data.py b/tests/test_consumption_data.py index 969693f..836eb8c 100644 --- a/tests/test_consumption_data.py +++ b/tests/test_consumption_data.py @@ -2,10 +2,6 @@ """ import pytest -import os -import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import nemreader as nr diff --git a/tests/test_file_outputs.py b/tests/test_file_outputs.py index b4073a8..82476b5 100644 --- a/tests/test_file_outputs.py +++ b/tests/test_file_outputs.py @@ -1,9 +1,3 @@ -""" Test Suite """ - -import os -import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from nemreader import output_as_csv from nemreader import output_as_daily_csv from nemreader import output_as_data_frames diff --git a/tests/test_header_datestamp.py b/tests/test_header_datestamp.py index a74496f..0b41119 100644 --- a/tests/test_header_datestamp.py +++ b/tests/test_header_datestamp.py @@ -1,11 +1,5 @@ -""" Test Suite -""" - import os -import sys import datetime - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import nemreader as nr diff --git a/tests/test_powercor.py b/tests/test_incomplete_files.py similarity index 87% rename from tests/test_powercor.py rename to tests/test_incomplete_files.py index a341571..a35647e 100644 --- a/tests/test_powercor.py +++ b/tests/test_incomplete_files.py @@ -1,11 +1,4 @@ -""" Test Suite -""" - import pytest -import os -import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import nemreader as nr @@ -44,12 +37,12 @@ def test_correct_records(): def test_zipped_load(): - meter_data = nr.read_nem_file( + nr.read_nem_file( "examples/nem12/Example_NEM12_powercor.csv.zip", ignore_missing_header=True ) def test_missing_fields(): - meter_data = nr.read_nem_file( + nr.read_nem_file( "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True ) diff --git a/tests/test_multiple_quality.py b/tests/test_multiple_quality.py index 889b2c3..ee4b4b9 100644 --- a/tests/test_multiple_quality.py +++ b/tests/test_multiple_quality.py @@ -1,11 +1,4 @@ -""" Test Suite -""" - import pytest -import os -import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import nemreader as nr diff --git a/tests/test_open_examples.py b/tests/test_open_examples.py index 3c32a19..622fa21 100644 --- a/tests/test_open_examples.py +++ b/tests/test_open_examples.py @@ -1,11 +1,4 @@ -""" Test Suite -""" - -import pytest import os -import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import nemreader as nr diff --git a/tests/test_optional_columns.py b/tests/test_optional_columns.py index bde6c23..da810b0 100644 --- a/tests/test_optional_columns.py +++ b/tests/test_optional_columns.py @@ -1,10 +1,3 @@ -""" Test Suite -""" - -import os -import sys - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import nemreader as nr From b76747b83eb0242ccabbd46e6a0ea6189891f9c8 Mon Sep 17 00:00:00 2001 From: Alex Guinman Date: Tue, 23 Jun 2020 17:50:31 +1000 Subject: [PATCH 4/4] Warn instead of fail if interval row empty --- ...NEM12_incomplete_interval_daily_totals.csv | 4 ++++ .../Example_NEM12_incomplete_interval.csv | 8 ++++++++ .../Example_NEM12_powercor.csv | 0 .../Example_NEM12_powercor.csv.zip | Bin .../Example_NEM12_powercor_missing_fields.csv | 0 nemreader/cli.py | 17 ++++++++++++++--- nemreader/nem_reader.py | 12 ++++++++---- nemreader/version.py | 2 +- tests/test_incomplete_files.py | 18 +++++++++++++----- 9 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 Example_NEM12_incomplete_interval_daily_totals.csv create mode 100644 examples/invalid/Example_NEM12_incomplete_interval.csv rename examples/{unzipped => invalid}/Example_NEM12_powercor.csv (100%) rename examples/{nem12 => invalid}/Example_NEM12_powercor.csv.zip (100%) rename examples/{unzipped => invalid}/Example_NEM12_powercor_missing_fields.csv (100%) diff --git a/Example_NEM12_incomplete_interval_daily_totals.csv b/Example_NEM12_incomplete_interval_daily_totals.csv new file mode 100644 index 0000000..b67e0cc --- /dev/null +++ b/Example_NEM12_incomplete_interval_daily_totals.csv @@ -0,0 +1,4 @@ +nmi,meter_sn,day,channel,day_total,uom,quality_method +VABD000163,METSER123,20040202,E1,53.32799999999995,kWh,A +VABD000163,METSER123,20040201,Q1,106.6559999999999,kVArh,A +VABD000163,METSER123,20040202,Q1,106.6559999999999,kVArh,A diff --git a/examples/invalid/Example_NEM12_incomplete_interval.csv b/examples/invalid/Example_NEM12_incomplete_interval.csv new file mode 100644 index 0000000..3f0f6a2 --- /dev/null +++ b/examples/invalid/Example_NEM12_incomplete_interval.csv @@ -0,0 +1,8 @@ +100,NEM12,200405011135,MDA1,Ret1 +200,VABD000163,E1Q1,1,E1,N1,METSER123,kWh,30, +300,20040201,V,,,20040202120025, +300,20040202,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,A,,,20040202120025,20040202142516 +200,VABD000163,E1Q1,2,Q1,,METSER123,kVArh,30, +300,20040201,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,A,,,20040202120025, +300,20040202,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,A,,,20040202120025, +900 diff --git a/examples/unzipped/Example_NEM12_powercor.csv b/examples/invalid/Example_NEM12_powercor.csv similarity index 100% rename from examples/unzipped/Example_NEM12_powercor.csv rename to examples/invalid/Example_NEM12_powercor.csv diff --git a/examples/nem12/Example_NEM12_powercor.csv.zip b/examples/invalid/Example_NEM12_powercor.csv.zip similarity index 100% rename from examples/nem12/Example_NEM12_powercor.csv.zip rename to examples/invalid/Example_NEM12_powercor.csv.zip diff --git a/examples/unzipped/Example_NEM12_powercor_missing_fields.csv b/examples/invalid/Example_NEM12_powercor_missing_fields.csv similarity index 100% rename from examples/unzipped/Example_NEM12_powercor_missing_fields.csv rename to examples/invalid/Example_NEM12_powercor_missing_fields.csv diff --git a/nemreader/cli.py b/nemreader/cli.py index 8a2b9f0..1ec905e 100644 --- a/nemreader/cli.py +++ b/nemreader/cli.py @@ -23,8 +23,13 @@ def cli(): @click.option("-v", "--verbose", is_flag=True, help="Will print verbose messages.") def list_nmis(nemfile, verbose): """ Output list of NMIs in NEM file """ + if verbose: - logging.basicConfig(level="DEBUG", format=LOG_FORMAT) + log_level = "DEBUG" + else: + log_level = "WARNING" + logging.basicConfig(level=log_level, format=LOG_FORMAT) + nmis = list(nmis_in_file(nemfile)) click.echo("The following NMI[suffix] exist in this file:") for nmi, suffixes in nmis: @@ -48,7 +53,10 @@ def output(nemfile, verbose, outdir): NEMFILE is the name of the file to parse. """ if verbose: - logging.basicConfig(level="DEBUG", format=LOG_FORMAT) + log_level = "DEBUG" + else: + log_level = "WARNING" + logging.basicConfig(level=log_level, format=LOG_FORMAT) for fname in output_as_csv(nemfile, output_dir=outdir): click.echo(f"Created {fname}") @@ -69,6 +77,9 @@ def output_daily(nemfile, verbose, outdir): NEMFILE is the name of the file to parse. """ if verbose: - logging.basicConfig(level="DEBUG", format=LOG_FORMAT) + log_level = "DEBUG" + else: + log_level = "WARNING" + logging.basicConfig(level=log_level, format=LOG_FORMAT) fname = output_as_daily_csv(nemfile, output_dir=outdir) click.echo(f"Created {fname}") diff --git a/nemreader/nem_reader.py b/nemreader/nem_reader.py index 796d6ec..6261399 100644 --- a/nemreader/nem_reader.py +++ b/nemreader/nem_reader.py @@ -29,7 +29,7 @@ def read_nem_file(file_path: str, ignore_missing_header=False) -> NEMFile: """ Read in NEM file and return meter readings named tuple :param file_path: The NEM file to process - :param ignore_missing_header: Whether to continue parsing if missing header. + :param ignore_missing_header: Whether to continue parsing if missing header. Will assume NEM12 format. :returns: The file that was created """ @@ -156,9 +156,13 @@ def parse_nem12_rows( elif record_indicator == 300: num_intervals = int(24 * 60 / nmi_d.interval_length) - assert len(row) > num_intervals, "Incomplete 300 Row in {}".format( - file_name - ) + assert len(row) > 1, f"Invalid 300 Row in {file_name}" + if len(row) < num_intervals + 2: + record_date = row[1] + msg = "Skipping 300 record for %s %s %s. " + msg += "It does not have the expected %s intervals" + log.error(msg, nmi_d.nmi, nmi_d.nmi_suffix, record_date, num_intervals) + continue interval_record = parse_300_row( row, nmi_d.interval_length, nmi_d.uom, nmi_d.meter_serial_number ) diff --git a/nemreader/version.py b/nemreader/version.py index dd9b22c..7225152 100644 --- a/nemreader/version.py +++ b/nemreader/version.py @@ -1 +1 @@ -__version__ = "0.5.1" +__version__ = "0.5.2" diff --git a/tests/test_incomplete_files.py b/tests/test_incomplete_files.py index a35647e..ce92764 100644 --- a/tests/test_incomplete_files.py +++ b/tests/test_incomplete_files.py @@ -4,7 +4,15 @@ def test_correct_NMIs(): meter_data = nr.read_nem_file( - "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True + "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True + ) + assert len(meter_data.readings) == 1 + assert "VABD000163" in meter_data.readings + + +def test_incomplete_interval_row(): + meter_data = nr.read_nem_file( + "examples/invalid/Example_NEM12_incomplete_interval.csv" ) assert len(meter_data.readings) == 1 assert "VABD000163" in meter_data.readings @@ -12,7 +20,7 @@ def test_correct_NMIs(): def test_correct_channels(): meter_data = nr.read_nem_file( - "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True + "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True ) readings = meter_data.readings["VABD000163"] assert len(readings) == 2 @@ -22,7 +30,7 @@ def test_correct_channels(): def test_correct_records(): meter_data = nr.read_nem_file( - "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True + "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True ) readings = meter_data.readings["VABD000163"] @@ -38,11 +46,11 @@ def test_correct_records(): def test_zipped_load(): nr.read_nem_file( - "examples/nem12/Example_NEM12_powercor.csv.zip", ignore_missing_header=True + "examples/invalid/Example_NEM12_powercor.csv.zip", ignore_missing_header=True ) def test_missing_fields(): nr.read_nem_file( - "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True + "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True )