From c335323c9c3aa74afe71061439f0b4535ca40639 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 19:50:09 -0700 Subject: [PATCH 01/23] =?UTF-8?q?=E2=9C=A8=20add=20regression=20metric=20c?= =?UTF-8?q?omputation=20and=20merging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/metrics/regression_metrics.py | 133 ++++++++++++++++++ src/whylogs/core/model_profile.py | 40 ++++-- testdata/metrics/2021-02-12.parquet | Bin 0 -> 30747 bytes testdata/metrics/regression_java.bin | Bin 0 -> 91976 bytes .../core/metrics/test_regression_metrics.py | 37 +++++ 5 files changed, 195 insertions(+), 15 deletions(-) create mode 100644 src/whylogs/core/metrics/regression_metrics.py create mode 100644 testdata/metrics/2021-02-12.parquet create mode 100644 testdata/metrics/regression_java.bin create mode 100644 tests/unit/core/metrics/test_regression_metrics.py diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py new file mode 100644 index 0000000000..626cfacacb --- /dev/null +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -0,0 +1,133 @@ +from typing import List, Union + +import numpy as np +from sklearn.utils.multiclass import type_of_target + +from whylogs.proto import RegressionMetricsMessage +from whylogs.core.statistics import NumberTracker + +SUPPORTED_TYPES = ("regression") + + +class RegressionMetrics: + + def __init__(self, + prediction_field: str = None, + target_field: str = None): + self.prediction_field = prediction_field + self.target_field = target_field + + self.count = 0 + self.sum_abs_diff = 0.0 + self.sum_diff = 0.0 + self.sum2_diff = 0.0 + # to add later + # self.nt_diff = whylogs.core.statistics.NumberTracker() + + def add(self, predictions: List[float], + targets: List[float]): + """ + Function adds predictions and targets computation of regression metrics. + + Args: + predictions (List[Union[str, int, bool]]): + targets (List[Union[str, int, bool]]): + + Raises: + NotImplementedError: in case targets do not fall into binary or + multiclass suport + ValueError: incase missing validation or predictions + """ + tgt_type = type_of_target(targets) + if tgt_type not in ("regression"): + raise NotImplementedError("target type not supported yet") + + if not isinstance(targets, list): + targets = [targets] + if not isinstance(predictions, list): + predictions = [predictions] + + if len(targets) != len(predictions): + raise ValueError( + "both targets and predictions need to have the same length") + # need to vectorize this + for idx, target in enumerate(targets): + + self.sum_abs_diff += abs(predictions[idx] - target) + self.sum_diff += predictions[idx] - target + self.sum2_diff += (predictions[idx] - target)**2 + # To add later + # self.nt_diff.track(predictions[idx] - target) + self.count += 1 + + def mean_absolute_error(self): + if self.count == 0: + return None + return self.sum_abs_diff / self.count + + def mean_squared_error(self): + if self.count == 0: + return None + return self.sum2_diff / self.count + + def root_mean_squared_error(self): + if self.count == 0: + return None + return math.sqrt(self.sum2_diff / self.count) + + def merge(self, other_reg_met): + """ + Merge two seperate confusion matrix which may or may not overlap in labels. + + Args: + other_cm (Optional[ConfusionMatrix]): confusion_matrix to merge with self + Returns: + ConfusionMatrix: merged confusion_matrix + """ + # TODO: always return new objects + if other_reg_met is None: + return self + if self.count == 0: + return other_reg_met + if other_reg_met.count == 0: + return self + + new_reg = RegressionMetrics() + new_reg.count = self.count + other_reg_met.count + new_reg.sum_abs_diff = self.sum_abs_diff + other_reg_met.sum_abs_diff + new_reg.sum_diff = self.sum_diff + other_reg_met.sum_diff + new_reg.sum2_diff = self.sum2_diff + other_reg_met.sum2_diff + + return new_reg + + def to_protobuf(self, ): + """ + Convert to protobuf + + Returns: + TYPE: Protobuf Message + """ + + return RegressionMetricsMessage( + prediction_field=self.prediction_field, + target_field=self.target_field, + count=self.count, + sum_abs_diff=self.sum_abs_diff, + sum_diff=self.sum_diff, + sum2_diff = self.sum2_diff) + + + + @classmethod + def from_protobuf(cls, message: ScoreMatrixMessage, ): + if message.ByteSize() == 0: + return None + + reg_met = RegressionMetrics() + reg_met.count = message.count + reg_met.sum_abs_diff=message.sum_abs_diff + reg_met.sum_diff= message.sum_diff + reg_met.sum2_diff =message.sum2_diff + + + return reg_met diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index 45bae7d64d..0c49281592 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -6,6 +6,8 @@ SUPPORTED_TYPES = ("binary", "multiclass") +MODEL_TYPES = ModelType + class ModelProfile: """ @@ -21,7 +23,8 @@ class ModelProfile: def __init__(self, output_fields=None, - metrics: ModelMetrics = None): + metrics: ModelMetrics = None, + model_type: Mode): super().__init__() if output_fields is None: @@ -30,7 +33,8 @@ def __init__(self, if metrics is None: metrics = ModelMetrics() self.metrics = metrics - + self.model_type= MODEL_TYPES.UNKNOWN + def add_output_field(self, field: str): if field not in self.output_fields: self.output_fields.append(field) @@ -66,19 +70,25 @@ def compute_metrics(self, targets, tgt_type = type_of_target(targets) if tgt_type not in ("binary", "multiclass"): raise NotImplementedError("target type not supported yet") - # if score are not present set them to 1. - if scores is None: - scores = np.ones(len(targets)) - - scores = np.array(scores) - - # compute confusion_matrix - self.metrics.compute_confusion_matrix(predictions=predictions, - targets=targets, - scores=scores, - target_field=target_field, - prediction_field=prediction_field, - score_field=score_field) + self.metrics.compute_regression_metrics(predictions=predictions, + targets=targets, + target_field=target_field, + prediction_field=prediction_field) + else: + + # if score are not present set them to 1. + if scores is None: + scores = np.ones(len(targets)) + + scores = np.array(scores) + + # compute confusion_matrix + self.metrics.compute_confusion_matrix(predictions=predictions, + targets=targets, + scores=scores, + target_field=target_field, + prediction_field=prediction_field, + score_field=score_field) def to_protobuf(self): return ModelProfileMessage(output_fields=self.output_fields, diff --git a/testdata/metrics/2021-02-12.parquet b/testdata/metrics/2021-02-12.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7810aff3e309c193f6bc323562472cbd107e544d GIT binary patch literal 30747 zcmc(|2V9O_{6AhvLn_fiNTn!gr-aXaS6bRqDxRjx9C(^V#rx2 zmy?lwSUmEc;*NbNS8#X-VtR1MBnA+T%Paz5@7l=@ovoJvd=_y*p) zSFTuvwy3yM#B8oY%?w4e5YdlJC-pPJshZ%tjyTdKGd)w;77u4zu;;eE_G` z5)J7-2%d+=%q-^|K@(jSF3M$%aHp*zv(g5GXHAzLU5V<3?)+LO^)Is!SUaX+f2so| zPi@}ItUQ2dRUNPAJ#9g%Cg=S5b~XXCSGRutVLueI%YI#UP%AX;N_<_d`T$)wvQoO= z&<}eP`7XM8q$1vG+QLfXYIx3{WT%r~3X+wHPm%33Dtz|A`(juJvZM`KwR=M>oEwxf z-Ky7soPv%iamJHlstF!6R=#6McpGgi${9eHLhtMz5$Qw*9{Dxfmt>%=m()rru6BbF zU(~zxp3{hdYble(U=eCe+Sqn?Vi*bM81;*XG{DvAZexGCA>_CBl|a$@IG|h<0)&rS+yA*Z&>f| zcexiaRP?~QccZX$Kp|7V>09ltxgK3|2gNTXGph(96y88CWTVbbRq?i;`VPrdiq=J;|CReqg(VOBm z)KSeS^=xV^-*6KI(M+{04St4vJ*(VwjDvwTC9C3sdMrp>4A?96bP7H5f4VQ=V;@|$ zxAi%9GYyUc^Gil0?pek#C-B-7HirR2=Ly-y1VK&G+hMCENIm zP%<*I0|WaP;Xg6t5 zan&Z%Ch=hOO*@6>l@1Aa9Nu?N`e30>N}E?*{P4&*+iu(a!U$KDD1(>#@Lcf^?aK#G5Xv|?19C!A!|nC&TqocRecZAEflolD;km!dupMQn zguLT;Tnl;~S*OnH*22`mz=>@A3CJ_uM{Y7Rjn1wSb&)ZopNH|&RZaTlt^J|o{egg<+ME-yTLl;eD>=diHKgL z!;-1zJ(3-Xy61JT2VUN_2u|>)#5&V1G=CT$0|`2!qXfka8qM|359j#;X=JTwbt9MI zenD>5y@&lMNh_&=@#ZVIeR!}lezX;}aYWY!%rqdTa~D@AY`lfY-PvC6mTE+w-)0@! z8`O-JFfw;4Q}w~2REdzd54gSQ@v=4SpCFK{UfJ*{IU%$w%<+`$BvRhOcse>iM5K9d|#LmEZcL6Pbi$U}R z>qzyuHh~^3Ln4p0x*+(N1+l$w7%6Y-pp;athLx|*q?P3NAcru;yc3BdXk%d7>sZQh z7+_sG`ZhHJw!YaRj<%9vg)#Lvp0_Q=jvn4)wbZ8wy*!QO+jHPO7ToG%)9XOUzlokZ zfE*L{U2ePY;uEk+x7czpVF;*Vt9DUt9)ZZ>iF3)ThG5p`v-k2dA3_L@QOVC3@=A7jCH1p@8)ZQfcHsMy2`UCGe+g}vfDXu zLpO}I>-H$@xP5L#wQd9WjyKlqx-f{8x&tQVhC6}AOsnf&Y%Ls})KfPunMQJHk=!TA z%8?yQ8@lmo02rxKGSAsB#(v50?;zM|zPn9g`20|u9lxF7AJ)ugxX6zTKf9VYml3D< ze0U3h{bCwjk|UX0AN9ju?knPynr1KP5+qbrUz!?SZ1MkgN> zLx=Nh-jq=((jhVoJgDr4^7K~Y2eZBC&e&q+s;qisbLAtAKTRt-8`_o3WZ#PfuO!>P z&T4@Q``ymVHaDRs))IT(Xx6~FtC3->gB|G5xih1!e3XQj(!)hF8J)03EnJSDYZ$#g zwpk=muOC{~K699~l%k0>>Tf0eYS8sZ3I6NqCeTHzY)aqq0{F~TYwAur0s2weRI`d? z*g@uC={sISNU^(ou;9@I65M^oXXWvBc)q)mLbPENsg*qNyYRFbLUP8p=Noo|q2@II zcy}G5EJ!O34yM)ptJupB|*0kN%U+wxczcC%(T7n+3+G2q_%V{(TV+pY?F;t+#-A7 z8uOj;I=(SPALo$jJ5z(Me2&*;xG{vtx1N3%MaTlmppHJ1!(&i!HK;{RISzuJGHtKu z=s|D8vpQ~c&L9Va6q%O{HLacen zr#`<4y3f^&XRIB91Jxzk_nCU2(1XfrHmVVci`~BJ(me}X`C~6cf1EJF57yIO}jyArt^d%vv^9|C?!@$Z$#PV>F^No<)P`X?xUL2y3BMSrAtK0j~p zI-KJ3;Vl6A-%|X}jk8*ruO`6lDgFI3pN5gQ`}Uh>2fM*7^rTSi^*9LS=XGe7Z9+n9 zkMnfgO5tr_<@*}}?I3SYKO23h6ZIUAaPVI<3>6o;cRG<3pf%dJa*tpY=)(Hw2WzXR zVf6!s3NM=;1l;s`B7&`Gx$QV_uHhcJnrZBVgazv&b5DZHy5-f#w)qf)-* zrd`kw(05uXx*P=sS`3*amZN^#GrasmsN373_bda42z0BgPB8HzC^|hC#!h`~FRAp=m>~ z>2#RII=)hH8!ozbOKKX~H#^IG(CC5C!yJ}QQE{MCeQm_SyB?+c1vyCNKY%@VPMvYJ z8-tGVu&lPePT0`*f+Li55)I`}PkI_hBh|QUrj2wBC@u2FrmOUmVDo5XvA^+W*qTI4 zq&ZOn8=rHAzg4*gF>ptVrC|syQ@S1eB!~rs4 ztD4&Z0fQEzM_+$LCWIrs!esR@J<7s$I=B~<=|>hR>bwVL290Zv&8lEYURSznUjytO z3m35s>;!CE!CJlZ)rc&hq%2r}F_s#7G5W^MKIj~m@}}NVj~vT7S<0s>0d_OK&WB#K z*vrO{ajX@LE&GKa$lrVYlEJP|@bI2;Zx}NLq5IP75&rezAXmg% zBHJ>E+^=pb+S5CN$~L?=d&stm@T}MJIfKR`Y({73PE^AbM7jB`ZI7IQ)eN~=sr_BZ zM(B)Y*!yYlI$D`qwSW-7ek6EjkKLOMLb# zXLi7;txYbD@yiI|AIDeAUt%UqxSC1wKPMyHmx$2``?v&?IH8~S@fJD3rZ}V~ZSN;| zcGCZb%&unW*k5qW|H?;n-g0kx$ki_78}1?XSY`s&x`m7K?HPfX2WiyR8@|9tn>!Ra zTbT$nXS>xgO2&&`kTUA~+i5{LNWJTWYM_XPjD>U%z28JuQn5rM< zfYt_!6cwZ8SX0+M37%xUYwM8;$1M7hKGAfDQiz_QG+>#lA=!$2a$8myZX(CHWIMl@ z?#hQyC%=L+SdN)qI}{b<(t*ZT^}DHvQDe(%MX93J3?uPbZf=9qy@+d9b}QEsM$D=} z=cZ)ZG`!x)x%y&f5|kWt=LnUHfc2*xgVr#vCR8&VCwPcW!+SUS3dXS{v`^<`CRI=+ zY#>;vCHgTDH0lI7IZ`OGWG+o(NhvbIS*?nO>wQ#&(ss2%$FNyY{1SWgFdY@)W#x{g zEPKZg*~(Vw3sN1BKNb1bY{?WFs+#C2h#vxz+2i3wxcHA0uTti{w-cxMe0U3h{`ySAMW46O zZG1@CkII#|-`qo)2W9>zYBJ=8(aBp4Nt{nVqnKpk$E!1wD4s;XyPOmA??^@oPG~Vk%>sD=(xfR(pzeK z&{2B;1)y#L{k$%Op4i<>ze0%xJUq7LX}%FmBy<}~#txyoDwld@cMl<#2D6)~I)iAu z`F&TwvlevRHlekVX8?sLE4n?~(+09hMnRq`6KKaiuE2}(L&)R{LGmVa!edk!AT`^D zcE6*tqdrm!p~38mAy~{Ddj50J2-w~1I^wT032q1U-e0e%2aVySH@%0-;NGRR z;aV3;5#Ny9WuJ+9pjjGD2tdOqwV2^`fABESJior?RCzNh-;)^XE1H0^1urSY>lDE@ zANw0}2Rfjo?CQJO7p*9BV_x{?u5mD6=gl`|Dn$}o6wVscr$EX5Pjb;hSt#wO%=^gt zSV*zE;_6r5j<}dVEpyH81B%PsJ#SZ#6BM|mV&AewfO3qe+|k)jD5mit^QovX6qjWZ z6{wN}u92~dd2Q|R;xOZ?FqT#zR|=Kh$ykhd&1*MP@l_yiV}|mI)mNdpTCj9qSrZy! zOL%f<;|TJ0xG_b1l8PvTc--FO^}@d49FNcb8g#TR>|y53ieFRwJ1BOV@6CwBmieK7 zg5nK^^C>RzBgI4Yd0W(QiqD6)0O)^9@e`5C@5cQbpkp_e`rZ}AaDaz?D$lVTX%H_5 zRP0Datyed+mkW*}*SB?b{@GP%r3t!a)EW$I%=dSdo5vt_ZSrFO^$p;EUiUG^8|;Z-WbS!FpbcQcm-Oa2kT?4p1^Ld z?Uy|+^q>NBU0wy%N?2TYG-YE&HQeT7e!GL03@a;Fw}@H!4s|icT1Jocg7CTXjb~lk z(fakFvic&a=))t)8cv#zaQH*`$wRUu@Di=3vMwHjp_rPm4SCH-D|vX=HtBqH-;dGK zS$G`rYaC~)-P;Ni1=%Z2{Q4m*L4RsGv>VQAFJ8-wZx96aW>6Y#GC(tph85eLOVE`Q zWVQ!Q%b|PwYa7?oxe&`+!J^6Cgm&@PTD*1afhKLr=*PMvpjV-If8U)6GHP=v{{-rv_E@o8dkTdT zQ?yU`&Y;^&n*ws%>d=vsdk!o4HG#O5b`Q67CZg8dC3Q=n9Cg*OKiXYC2(v5^@3%eZ z0bBhYTdAHjqLCLLxpLJD5eNlH@`z0U*W@7U%8nuQa}XIG6rf-Ist@+H8LcGu8bc87+}}Ah_G^lNFBEo~@6}6U z%lyzkLGj17^C>R*BgNaSc%L1>DLxkbek~jH={QM=@x4=IfZI~?PB#*OV$9|bm5V*rd9^}gmzLy zo*D;3KB&`)rsK~p5FV;Uu0zaM_;S_KjsADnmX_C*QgTj2pl{Sjq+f6;(%+R;9zT0|IZaX!W2q{X9W+ z6?E7^_aOfv&NMV?J`i$`t{;xuUQgY}SBR2lPmJApl8dtCmbX)vrbGLVVTqa3C9t;8 zyw{O{FB2)B>~P}jN3EU-{k8a>&%sFlK*O{WWG!H4!ckrdgslf$)QxIE$~xf*5uba~ zZ3*bUw7C-&0fYUDqyb=wdf>5@WfB}}WHiK+$I#f7Lv61l%fPJCwYsji3H%&JA2?6e zL$2H)R=9Ei^#~;v+Dg@+bu!nrPjNg)gid94Bduz%nJS`wNAU{Od!qsGmT@OpaBHHV#4;`SOtsJ>#hNq8v|>;{=4#MKqmbCd0@~KYzNcS_n4I zduUGgPa>hPHH)wI4x)R$k`&N_?cAD?i zPh!jb&_6-(o+I-qzV%0nM|$!O*x?kP4{rg`zohsw7G_lzin%>;8V=mAxevZj!OF7g zr|rb=DtwvZ>&KU5Wqv3gLbvhTZX$UMIT`Pa_k2I3zWR|u&YDHe+jfli%`x1M`FSm1 z;lJ@C$d7!!Bho)QGIpFd>-hiKkzed$x*5b2nrJt}tCR8BPa;Q=+siif-fM*@a2e(C zN9W^UbqTkM_4#VB*Ug*VadR3*SB*zi&EUITsma^u4au-@#R#q!N5;^dDj%txX+7{U z?E(Gymbd7gOto?15qvA)g3I|2G8yptbZMM*P$L>XKl%LD=^n`YL_L`hFofFeJ8u(I zT2PrNs%V}ZfafT}N}T)y?B~>Et&r?M-W10&2wwHDok%?Lfu#=RJ%id4$1Bk?=X1v= zc1=Q*(#(Ks<2cd|XL`0$Y8D;&xH^HXZ3c<1FNxZCu@iWC1|R6n_(8A++3Vz`?Z{{G znT_mA2T;w^nI#9`XTl{>rw6a=sj$}gT9ei3qlnKTR<)!$5m5v@zEpU+6tQpDbl*-k zg7$Cj9-rPa1BOF!wR|BND9M1X?7GV&S{~K4sU&m|9;a?F=3yI0i}o(5lv*?ZwLY-7 zOtTAxtxbkZQqpm&M7N$m*w5cb(rgQ6cjm2?S7(G7z9#O2x3 zBNa&9CeuoFG6oLwX&ifB*$Ht;!tE7OC1~sL-gT+0h3FcW4)ycuSXk6jW_$fKC1&Km zXHAJEz7Bci#`MJ6cOccW!pnV2652~yIe8$j9|m|&R7RImW0#|e9vQ3ZfS^l%wEXoK zbbrlVjer?UMOm~Gbla^J2O&Qyr(A>TiPJbgTue-6gai}0Uh zD!4z5ytKs(Y}|&Rm&HBb#B~a7c`|Y5D$WJE~u3jMLtvErB(|kU}1%UsuzxoBuo!cz!WT}Ti z$~mt_^>#Bnv^*-jecLo}5Y=4V1X_^O-c7Df%P9$m9gGTU?oNR-*8fa2Z4e9!E`G{i zpMV$w9$Dny?14RI3JT02wa6`fTDD|_hS2dPeqy!`zuRUO6$!uJ4i7gA_CBl}14b`< z87HqUWFEYq|7jitR5)Y$AVdJvj}EG96`%OBo4gYS|6e&QfE_;}mcuFCCrC4^-#e8HUCK{U#CYL+G82Ti0ajB7(cau*R3EI#}|hmC-h%4<)5)4-DKd1c^Id z+*D$pkP2;}V?MqRW4*L;_QS3$bp6DYhN_S#G{N*-k?rCX(v*FdQpivZvQ%}{8~F!t zx@5_;EE+_KQfQHyW*$kLb1jnCp+~cOeTJ3G0xgS8tcHQN7+MJ`v)Q>9e%&}bbk=B1r z^Wq5*r#=9;Kh~fegU(@=gCi)*FNt4HVE{SF`|j-ETST}kcT!GuG80|Vw#rMTs51vd;a^u}TO~c}y2IK8qqpc!p}(Kce~_&khtj4Czst4Uvf0D4Y;-rF9?a>}0>n7`P^D)V7Oh_ZV|SAkH@#{FnVHiu^i%`T)Ga={ zzXzWYzGSp}X;*-PZL^o&+CPO{b1hdc86hJ)e6BXrsr&&t#oXdV9*rXEP5$(EXkP($ z>}bG{I=+ia4)-5mqtK&$&TjbG2-56a{YsUU9IKtIWs#5{K&xG38yP3Zq4~(=gpaER z(Q&ls9*b8YQlFAdulgOv|wajX-9=P&sGA*rxjKFBDO8Y{s z6mDW-w-pZ7LBREv0-`WA#wNd3%&4mYyd!z0M;nI0muhXhdwCz)Vl%A$P$M0ge>tzY zn}dvC`r<^yopp-{$xpS#kKx^C^PcAC>4OtMld}DFWW_YZ$DNxI-Jg!sXM^b@^m0%e z_h!E(=cDjZ#JOy%<6AA}0Mr0wCwC_k0!$oc({HOL;g$~Doc6G zbHSdRAe^9ouhVx5_@CdVTTV?*=;g^bKEZ|`3ACkIex0op*crDIWMw}9WxVBhv+^{& z61}`Cc6}NutY62OV?O~>#_<$e@g2=3WsJ63rU9-!UAp_TPCs(e4tgWfGK@|HHCwT% z^`OlLjw{eB5w?x(iqr_Z}ziu0e7;o!<-1+=A20bs)ty0d^rv_kASoWII{!k-ArpG|%L` zA1Y}>O|3%5hti@^(}>@vwT5X(H-Y*|Mpz?cNoKU#YPO?J8=Zs8E?1zP68Nn1&!{ec3BbNf2>aMv+%z6q&_lPHK&p zBA(c!@@^L@OwBySJ$(HL4DVE5YI>;|opC<1x#Q_uNNs3pVc6IJk|7+MIv84!{hJzr z5l(!au!oZZp!3k6Mh>tSvG4lK&$y+mLm_ zgTUE!uJP2ZI_P%3ZAQFU zUSYafKS-9`I4>bE3@5nsh&1)B$onV_GL0R9)CoTJv&BU6EB#zTc9@)u!=Ex|#p=>f=Slf16Ge;5oDn?JTP8v+>j$zf@4 zM-6llEA%)@fbZjhMVyjZ=)#q=Jk4sI=+@+C=``07MDc}uJ5@ysiZef66Yz!t%Mjaq zzeqa`IV>H+7uiS-|$EMimtH8~elVm&6n6+H^ML5KZXa_%BI zuIzG6{(g8@a4trVpA1{fr_SKIvITggn;QG2vtj>!4a&_?WQ1!g8irDAhf(;L+DiXA zeBG6214nRo0~}yVy?-*d4juN?+JzonMaD+8X2kmgXxlIr5+EG|-JHi|-kg|3OaZ($ z>t5iKlFS{NI?;Wo&pvemD<4LiF44>g6g-0H;uOnp#gt!9623Q9cAD?ak;KmVp?{)V z=_&jbdw<$Tjm#(K9Y`Z}EAt^P0Q}$5TrjBm+~F-`guycVqL=v2oPqU5q3toVXzvZm z9sC-DC{0$Y#%Spmq}+J*V6j~_@>W_`P>|k&9*(K07>GB3;^xc2IZiX+|Hx>cSa=t@ z^1wkqaX1gM)OsZD7sr8=$YO02U5ENEaRe7u#lby}n&{*2-yrLupmXtUDOZd84v`mzCZ%QaT9jC%Zm988(5E+jZGI;}#PbcT$Y6@>)!&Xt8GUXfC{r@ABWeMunWvQB^t8Qqc?VD+O_?lM`AbPF`l6 zq`(B8>W9AhvKXU2r}X^9G`_=kcP3+7ZVlqYV&D6!bwc0O3nuza6xfPW9;?KI0@vdsIcJ3r_?4!`u(vYDM6+>0;5fw9lyLt-xtnwIUR}+X1Xs6}`-z6|g2lUp#q#5jy0i@+iWC7PDdW z?0I>61~!03QHJ{vY9cT8e(_`i4Xs?`zSOe_DM!;>Fc#_ux!Bzsp0GDVb4$2(4gE9p znB24@$tNB3PK3M{-Zcu2hJp@Do{mA6&~uM%ONu}z+TcS7^B1&7Qz`L^crLIpx$Ifd zw-}2&x1piGtpth&YpVJV4Z_KS772!CGJ+S&w)Bm7c^X=t2R5 z^$(8XPn@~(o)wPW`yL`j85SMkYlp3#mccUdV?gDYyUby{27lgV3wc=Y7dZIJ;xm2Y z3@nX|*$$NDh%tf&3+=);#;0e6oq{`1W8nirnMEzyTsNi_U)lnPuI(pEUrU4XIF+JL z+39FCVa2{{OvON%xJrl)DCvlP z$mXz*NCT<444#Pr?#n}ElRNq#UuS%&qJAHIc~jFL`g91^#y-Dng3qyeJ_MUouFip% zQN!=4d5eMWM(m5}MV&})i-3`ec{5_OQ4?k74@R~UOa?)Ro50Md3=;_*T`S3`SzlQC%v`CwFv6j%yHtC-QGm zK5qj*`M@Ll!(ITtQU>?%hC-BmkBpOTxC%2{7stp}2mpUL2Gz!|QCHVp~$I&_!ON$KVQV`yMqe13j3q-!!)VAYU zHaIU^KY6a96WKj)i6nFFMrW&a@7xJ4LF;WEj!O%aqLr+xcn_-&gA3i(J~qd3RLc=- z9PxAQC&>QKA{k{x})rA%Yslw;=J!Qqo6$o8UtCm9IV{b8l;l}Bjnp>P9+u$h`ZdkJH#T;f z@5N1G%lyzkLG!Z%zoPk1`>7v#$$5jCNHm`haRK1}mgap#&G8NU@dpHUH6_ksedzUu zTeNlZC1{yx?zYPt-Xq#fE)QG!cJO6b-N{~21#T3S6=W2ZXj=53k3Po?+=13S|EDRS z8cz}Fysi#v8s01sis}V6nl~3p6NXXwctwI0O#>ufj(t4M(PAXMt5J-NrQh;W~e*CzLVJ6dEIcva+X5h{Oq z@!13X@w2cpUrs}gE-+wBSeJSw9mz3Yc)I7wAZ*oN*)np!2aT`qIj5digQ!1#;O{3C zf~W6A1Md?Z_{}HXOV7{|kU72CykDaj6dvo{kE9p`at}ej*wG@mDSkDFY&Sj{eo|Y$ zwdXZ@btdC3@#zQ}nb2H+HnRn`robCFu@YolqlCHQM?yzrtOF}8df;B;ve&fu`7ti@ z`@AQ+KS6=pcuJdc8yt}paR@B?1Uqi%(?w-=!BKuP$L+1%==0+SJx!MmaIwp|vW|ZW zO+DEq%(JEh`PC&-8C7)Q4<58np2csz4-G%KaBTzrK!S!xDuJvWzloL(O9~u=gU1{X zP;!hwjg*`^TU##bdpS0|{L2*TzRC6e_B0tm{{Edec{@wc$J=^Y%wp|GrdKR3dUHFH ze%X2BxMMYb=q1(a(A8`>A9N8`L`kaju2L>_drjfQd+1G37dvE z617EJp`<6^<$jYHC<|{Hq-$-4*=6y($D+!>N=KOB>@bSR6D_iaf-4Z6x@#fhsR=|Q zo`Z2es)X7zZ|`lmGL5j9FNaIG22gl%CfgwIFf#U*Xrm>Lqs@Yv3G8pW(bHI78GZla zUoZK6Z$#`g-?JmJWq#q-mQ?v zAXmA7FtU0vwk3DJ*E_muaN}9dnTtOH>D-xJcko&VEOL%wZV`_Hp8}f~vCiWtUWwUA z_iYB6$a()+{=hW2a}@dRRwzWr?D+ofRFWe-4>__WfIB&rms^CLphgE!5CCaefeZ8$c9Yhn3dT;T_ z0{p8R^YM(QF$i4*51BZ+*hq zc7Ya%%6+XQc4Hi_YKbym=*7@og?E`XrtN6g%+OJfk_LEN^_cQAZ!>B%)XZk>9R>RH zwGXKHOTgf)wbA5c6{-k3AsqOE9AgmRJgrV&2S;3*x5Pf|MvfnTx}dl^6d)Sek2;bwO+N_&AJq%)dLP* zizUb6$b(sVHuWOAp{V8ixCi05e#^#5C|?^8Ok5f3ej@jCq}Bu!Q_L|NX^Dcv^29!gx@k8)?j0I$Ji(cDKGs-iQ&a) zWfCoIL1882H4hmJ-6%xjH*0*kWjkQgR33S;Z9go}cJW?ut_taFOo~W~tw$>^UTfLZ zun42t_gtVrrs&tB`FA7iG~b&Ki7oR({{+n+U8MM*vp9e4(|j<;k*TQlHrl% z)0IYqzUMwIj$6&KOZ|_{tSI(iCu#$xUr%ZflqxDJLmSNrwRBmg$C_LSJ9aqL9y@4` z)yR20x@)F}xs`_-*)LHi6edSx4@Wo=*1cGNAmy|yL4SyGXJEWOfyw2nK(vE7Az-7( z_PrZyuyY=(+g|!PVGOs+2YO%a#;|Oi8|s4Fv4&OG^G2IQFr)YbM?P`85d__=1q~0X z5K`TC^H;T+W5?nQl>dJSoqoe?OFQ`2=8Q`F=NiI*o~|x zi7_cZLfzx9g?? z_C}3-U7Da1c0EPf-b=;`)1+c+rJ&k_1t#frQ!Be-&*fHXt<3Pm&beQC70|1XMLehM z6>8)oq!&oqw|hDfJfGhxZ}l=He4=T0U)o}Tg=C+OeR;_k3pLpL)WF7!&>nr~5N)(B z79ZmFj$rA5)h2%7mcD3ApyU0-jdj}*-b4zDuXobKmNBqpQ>pMA31xw#4EyUW*k zIGlx0efId7-KXTSRGD|Cjt}-?xm(v>rP;9!+otDRwH)j*ck@Tgk}}%AeyPCs!pG0! zeXD*FTjq!U37QYx`+?@vuEx%m=B_S39)XOxL%}jb_Jw;{$|DevvYfFNWBP1d$pR z2V*Rn;X%mRM|-gNZLSTw%Zv$B+P64eyetV@9|k{Ka?=>Q``&gdJvxL5?%k?>H^rRL zQ6xg`Q0PYB{usF;vW*qXR@&0On8k)*+O1Nuo7tK$0Vd($E-o0ihJ$n>!(3r0tu4qf;PCrZen=)b#0&I%jv5R||CL=zi7@iAwpO^a|;jLQfIeB`uzoKhnppuiI#R z{Hha{RWCT1e9Z{UVSBPtoXUm3Ye=s$EM|vEUn!_@ye~y)&dJc~+Ixt=y>;8Hai=U6 z-E)N<@+=6gFC4WJGUW+NukqJErZ|E*zZ*YRQecl=Zn_;^o@P&|soYn&r(YWjOcn|x zv+*Lh-@RAwDM%z-8L47;uf3NLAHbAJM(&6)-l*e=JEDhe9-qFv`-nNVXJd78;^1Ly zIgQ~53jtG1s(AIPkX>F_iz#K$btNAxW>;2xE{z?=hhWFC#RswFIZJPwNV*c5T?g;H z5-}k#u$b-Vwl~Eh_sZ&-`zsLAxI&y{gIx$8TLteO@3F)J2N_EyL_ILk@l9k?Ys?6S z+jthe+wX`U$y3g9J8Mdy4S&YQ*I-6?O3Y&yP}q;P9&0aWuhAoT>y|vWk~GJtPugCsGcd&>WR|F|ePN9;e=_qG?zZ|h$-fsqJI(jzO=8RZ&_6-)eip@l7A1ZbC4UyT{wzxUEJ}YB|GKLY{$KJpTz~1m0sBk-hU_o-8??XV zZ`l5lzk&Np{)X-^`5U~y{)xwbdH=)%h`fK|5h&h2@esxzc|wBsk37D>`=<_2^ZuzLoVz#U zJ(j}zr#8lU|J2qT@1NQn;Qdp}le~XwfrR(ZOwoA%+)W~D=y}q6De-$3(iJlqelUxj z%E{Ql%-Ds!i}fg%wS$?t=OI%^J2!g=7p`rbhDW(<@i6>~%h}k$(wu8ECzpe z;AUrsOD_1t)fE?o=dK-{@JlJ_xodOMwTSltT=<&dp}E}6zvcULHWt?Ac4miu4*QDh zkgJyy$sU)RgSDxn*;fc|_D)`ZDI85~%uQWM(Cp1!jm?Z*jq&J5xy&8#=++LFq_lf= z6a=NXyng`mGYYQnuk9q|@)MSyH2({cg`K0ZtEBi`q5OpC2gMJNNG0aI8AtZb&F@)@ zENFC2^LvbnE@*U4^LvbnEogL3^LvcW)z2^MbWZboj7lu%>73^G7?oVm=$z*F7~Q&{ z(K*fUF)Fp7(K*fUF)F>F(K*fUF)AXwpw+Lc-(>c0I;G#(D~T-JEqzt}CT|z+m_&YK z&m^*N*Ys8Oo4j4Pa}xQDy_3ko-P2dqZ}N8G4oc)V_D~`VcTrzezscK$J1LRh(Mx^X zwwT{X{h;`N?CB4SsPoPKFXjIO({uN)&iGs19i8WHEA6aJoQ<8mNOzS^UdGPOjvgcw z?&i)e__8i3RCtTX)-57j_?;ftH}gLzMYf1+5##bEV_!o5IoqUu)}|0d;xN5((Oq+i*& zUs5Kr;&ON}E)$80j(B|8t;+UxZU&0F;`>C^-A$BqT}>R+tTn73O=S$`P$i#k8$V9klQ;qQ^Jc-BeClQd!=j%M_TXt1B}7aVJzSLKRP2rQ zbX^P-q}}xOMC?fMOdNDwOyux<9At4^xNQ+>Ve@^mb2?=QT{mMrb$0{Bz1Ae#cwARg zB|8fh*Mm_4Tx^%oKN#e8S5^*G*qk+7|cSVa|6u-M!k@ z!s5yfB0ppQpK|&Om$JQqo&G*;5mWoU9$(AKz{x<*({`We_c9||8`$ewnJMXd8R!t_ z?3*eo;N`DyWS^L}gei_0FGDEm3tjWs?7RIg5Ch>;B-Qy@88~sDiM%%`bhJ zs}~ErobfvR6*kX*#Hp zod2S3|HW9b>ua3-JlmKUTMRzvyKksw-@|P@m#}&wKDW znVHfbzUOq1^)k@YcG4Hqb}|u_Sn!_MR7u6oK+(>|OwrDrG^d=K2mY?PjxIiC$f?+v zlIB*5(q5#o1fTo380)Ec>F-mKn48Br$V%YXUM6B+@0rZ(b-heQactK3TvJZjL)B)N z82)EXnm7G0uM{=dr*gztPuguR51UiL%y`l#x zD{+!6OOn-bi701E`Zgwg$!;7oSxKX<^REd`M>dhuIX9|x&K}H6E(i+e^Gw8d7`Y= zKcpAnDtY3UHpXaWAoFX+Geod2@KzfVte`=-as|9{yh!N>crzfX_zAOGMH=cQMh^B?E` zSN#I`asGeR&rd(+KhFQJ`i0WZt*!pj{+$1K?<%WB0;fnMntts!@y=_mWs=;}Qk%1a zFT#?9x$kdZczjFJ`YirC_nD9Hn}YO1#*VKBJ0Es4cNH{sa^n29B+MzYMPiGH;2w3N d<{l*t9sF1+{+8nZ<3F+`o5{#D@&6Y1{{wvSQZfJl literal 0 HcmV?d00001 diff --git a/testdata/metrics/regression_java.bin b/testdata/metrics/regression_java.bin new file mode 100644 index 0000000000000000000000000000000000000000..1a99694adf811bfc0ce1dcfe891b241101503c86 GIT binary patch literal 91976 zcmeFacU%Y^2~Yt83MLf9oU<4(XHle^ zU{077GpLwz#`KL<)$QF!J^Q`)$9M1f-M2a0qZHLuYt1#|m}4!DoMrP`8|hmbIGPPf z_Zl)RZa|XP(0)S(xZYhkadEa!vmd%IeW4wX)e%3j{?{9=S2X1r4II!fb!74YsrBr? zeq&%1>B!-+j0`Lz8|^UV*&0P!H5Ik=nAIo3-+>o;rSmJzX3Dw&t}gYkE%tNQD|OKK ze>=dU#?asG*mrULk?n2#8%;7~>s#pQ=^N=q>S@39_4PL3pDzUZrDH5z{DpEeO}|lD z>PCyps1yDRH3lby>Kx-wB{e%XX*MkRq%`xHskt_7BxQVjx2Dt8{TdtAeNCM7qq+^x zmU21KTz#R>0d-W}OzMj90BZkAic*!>P(81;(Tv^qPT6GK;ns!S%QUlRUC@*`<*CX0 z&b{95*Sn83HNiQm#B0}iYHrRv&BNJC)o&e(m1{+D%88Cvnj7Q#Q_{U_s3SFpH5X2d zP|wXipn1Hl1!ZM2mb%N0q6Dw4sB7)cAI#o6j0!1er8zm>Sj`RSto&-dhH5)^ALW(M zP*W0^=eIJbm0w9poJJ6PQW;!!TS?|T_spY`&}DC_R+oaPZ6}vgt~=6{tD3}VLdNQ8 z(u;Rd!>+ESwoja@Y|ReVG>D(1DKvM~oITis8k3f+Jm@{Q#K%;hn$^MTQ2g8()SBrE z&EBmgRI}^LD7NRq);=Ass5L_iHHTjqtA~#-R&QD|OR4r@QNn9%14BbSJ;&2?nPr$j z@xIJzvii{>{ejZ8XUd!g8wl9uwJXN%VojTKFvg@>#2z(Z!nRKa8g@W#Rqeav`|SBo z0}tf3d&jm7dcr?u{+hF5`H1YcE>=c+HhlTkI%Kkm^YK&5EWQVr>SvgYyH>m0tn(zl zCXU^XmmbD?*3%kEXA9Nmr9S=j#jtazqS?6zWL6*)w46Y+h4eyziI9D zYxSos8vOa-G=Ha=HGK*ae>8CpExKX7VfN?-%YR&4wc5O@fnWcDex0UD8eZ=0*Qupr z%O))f+^$@*ztZGdw>DkAOuy~k_27i4OJz0g#@Am2T24vvytuo|+)jbty{x7?hfcrp znqR9JasA4!AN=q6Rn>(?(SluFMW6Wo(UY)y~_20JbwAHUXi`5 ztydW8^}cdvZjXTxC!^xa_Elf#d?~7EeVD(n(6B6z^xkUR#Cp~i*oVxu-=}>`X(8TSJ7bGw3J7nz5vkQ|WIn}X- z8LLxn40(`nQ~yND`cY>-T#G(3Dkv=R`#JL-qo=6q$IflmI!z!sa5YCljWhYyJZtKf z6XT9=jqPt>Hh8>fXHk-9wpC`cTlTe5kNTOu3k-R)4O#j;E(=b@RO@4Oe`~+h)7Spv z|JL8Oc)l0kTPTxA`9h&sp%6$E3Yk!>yDpLngaWZxE|p3Ye7;a>L+%%N%j5!)Oe~b+ zqDUeVNUU*L;EivDe5pb%=F8<0fkI}5t9);nSS}IqWpch)BH)Wer?=2BXF;uZ!s1Qgc5}A;m2`-cCav5JL;0rNIktNsnj2-W| z9a$JIhN|=ay1#LW{`$YQApbn~ANdjq{CDYr)}RG)0qjd65sE|-u~ea8EJ-2|!gysc z3xQB16>BX?C>M%Fd_G?a!&mUdV&-14941PvQK1mZWg-bNeu1|@0Lzq!6;h!@E)Z*t zUr2inkyIfPDI`LvSVW!xCj{>x6Y(W7xl|^AvmjUDYv47+uvb2uh(a!xYdwNkAje}B zatsUafXDh9&&2n}$i(mve4!Y=i7yuF9wHaw0q|9DN0^{Y$_zvz6Uc-DnV24)L_tsR zcW1*)5N=8!k??UBA0yN{!asW;g+w6`z1^-67B2+aFmdcTmlcR!0PBk#K#bX zGNDum_b9__nDrBigfg*AF5*jx4hUr0XNw_xF}yn5G6pE1mrCF*luCsd3DFFR1pZok zvqU7vdNUuufaqsnT_IUHmPo?KN96ch_h=E(3ZVqg!NVoYWyA|SQ-D9v8H`E7gdD!N zR3Mir@F)lc+NYo+6yk^w6Of1{A}pi~flH#*C9w>0z`*#3YX}e8cmc8V&Xy+vyK=IF#pZEUyq4*iR$f7?+0LtNS=%?58@=RQwiNgj+3XTE;~HIl z!`&ZhW?9sroPEHj1Lxq;WyZx;PDV<5wP|TK&m^KUT||`thr}vvW+X3m9sjp6LBt1xogyIuO7$Nt(<$!zS1GGJl*zH|*|k<7Jx^tmJcRI8|X!?LJM8XC)fl<9^+dVCI(J#$-wIPIF6B>oZMw+|+)_ zg9fCgSeO0f-+zUt)-&+blFmQ6I<@K$;IM?P*mmMl>rxt?(i0!N@yKyZsl&pfTO}Xl z6aC#Dt!v!sok>CI-@sELSFcs1`k1^R=9`xvZ*}_i=6+5eDmuDJN0j-W@t9&YN1je~uEgwJGwBmAjV^#YJd&|2V z6PH^Jomg?|MnFY9{-ScK?V5^Hu1706CRLU{Oq@|c9h#!rd;WA_+YQTATfVsk%6iW( z?|)5Nk#T)qMYZ|(0Ow7+{V(4NE#EmLQgv_AG*wD@afPt0*r+qMSfRW;m-Eh}aO4qq1*ShDqbc}U)?K*!$QD~7Sk%g7wfl}js*b4|)i z+HEQ?nlG-X&8aINnOI%6tzLdvLBm2-)tm6Z^WHuc2i;RDUWC`FR)kDey_MJoh95c; zIB}$>YEz3Y6`5v}Du(Arm8)zPmG3x{8OSyBQQfmyse1UfMaAZ95pQc3)tKb z(KV^Q$M{BO8)j#8tIIU)vg-J`xS`n=Z}xroG28m2#jNtdJ;%%&Xtikci&pP$p0qN_ ztr+B7^U11Ksrqr^lF&xoC8;)ezSOn_WnEByufpzfx?#NMq6GUV=aV~^{K&Bno_u*` zcBN~*UOhK^dcUZtUwj~As7bVm!|OQt(uG5JJMC@SbjPDwZ|AYWy@Uf3{2IKee{t8k zr#21C>U+lWrzE)u58WGUdajY%qgAI*ixzcvySCOjaATkMZsAW8BXa~*4uF@=X_G{HnC3b z%}rV3gIZ+yD^|JJZ`;rQ$PLei7eZ$bJ={Me$EL&N%9Zt7H>vLE;JW=-yYOM1&A*Ng zm~zP@-1MdU;V*AqM=ZM>S)Y}@vYSKg&fCY5f_lukA2>{y6V+q!%;@U%2h)0N$*t_v zVcd-#74nIyIKSAQV>j;TbW<=eaz(q$V?93i>ieao;=th3VKM1z=h_`;IwN+7)1ws) z8(xabj4jUc%3?VcZ^A8hNDNy4}VS?kd{#u zLF4uMjGi~@O{ZP`j)^@oBF6=I^)K3x!5ZIcpx*bFV@~(Kwe3L6sQ&-l1dr!59BLM8H$X{8#NnZ zzC?z6iBBg?USf$i5(Z=-QWCQ93OZ5YOT3Y$BY8xgiK>%M4!n@;Si-yvHH{kfm?_V@&drBizE;fYQjZLJ$HOFJ6ZwCoLJQ1u50( zi&$HPP5=l9Xposs4S2E1G#Yb);)tL)^orB=jA#iHK?M(kWONQ|BNm7O_7SlI4k`~-mmfn~$ev8s&3B)A(RL&l*XdLvSu zwX~3-uSmQA#*!8TtdhCcDG8zKetS^)@E0s)Enw0S}TzvrC93;-)iAQk8p@=GnIFNbxA zB~S_&697Q1CHgZ6;q2O&?<-j}Un%LD`L5Rk_tj>$An)8_3<~HB>$iT<; z!8Y;SkSZ6GB`+)TzW3E+ipBSEA@jFh|N6~e5v5%WqEw|E zka|70yT1cR@@+WB^pu81lp4Hjl6(7_NvVUXP0!~uGQUx7)#kG@`kWR19isH^Xmn}c zuTQC#y(%>&9~?C&^<1bYhFs-{*lfR)kp6xT;&U~z^~2QUo|)@sXiS{9Q5zGVD$V1} zDM9pcN){NUp0s3JN%1&uP5KKfYOU31HGgcSW>wNkO-N8@W!EM}nvUW=>h7Z_Xc|@r zQ7^sTs9StKs&Vkoq4@nYn)Bl-sJ;byrTKAL%K4{$D5spApy^vtt!{k3XNmrd>&i!V z)BFZ@aW6GnzAc$`{Oy;qaJcc=2>(0WvjY?(6C&_FZQFkSQV3SV>6 zdWHI4(KcnO!Bgdm+*l2n<2ZtET^`q7U)YQg?%A}#Y)gKqTYVPTMQ9ru8 zrFHGZ#-)Wjrchh2G*mivuToBMY^nKpKTLU1T&vc!DAtVos?f}unx|zj#Kkho@q$*XL=5%?cKjI^R0V+CMkK zbHh~!_M)46i=KP#<}B%Sf6j$&jf{Huk3M~~X`E5%qxnyFvQ14w=D%@YHe!^?%CW&) z#~AX>{dOKd8DvytS#Y+`hXccFEmI$V`_R(;kX2^kBFh^)BW*sh795#7E5~N7s=c{P zzTGxEAY}f#HI3~|95^e3W8Ll7E-31@&3vK#kT1vjADL5VfAzzphejhBJKOduk-i$?CFCO^37mjAfL zQI}Sa+|E?_sEx$+SC5`9IL=C)_;G@Pu)z|a{Kk@vVreR;=J7eUq-&#sBU>JnNc=-M z)rEaZe2d=MExL8^gKy=WGe`1FuK2l+`eO8{_YG=+=h60;U!3!=+ctDwXD#^(?VwB?E zz2l7}4%gkGH$SUOj_R6uAHQ)L*CE?%qu9MI_f!|nkD6xte0|G2Y4jl0TKn!(M@P4u zY~;9a|AOc$>F|Ml!y@`Fi>qx^7HHa!w>9E+{|#3C8sv?gXSh2wF3MxI>GmUQ20p*F z{eDcKXMEe4?U(pp?wjb5yCQD>XWyip&8KfQHmyq9=pqeR9UD7jy;)e$vL_|U>kh^x zSskqzX}5?wt9PDc^tITCn3TItX^-?9w+?>PDlKrQY0?(nv$WPJX)(3+IT^uU107u^ zTV>2>XEWw;opr|9F?U6)b~nlNU1Z3c`yYWQA*%urfG!Bo9g`u7l2}K`N61Z(A|S;? zGAh?~qmUOMh>{jPIzUJT=|GNZ34LYs=71mpW3WS~@FPg)AAM}(EXWGAV?k+$ zBo-}F1Zz^oG4nx!h-3#z7kB`mR{ZRkAs3MyIqR7veD(G@gqG z;b~eYfFc0J0|7(OTLwBqm*_yVQL2)<7PT9?%H&>#YK7Vsen?0v9;Ot6+o8MKU~dUg zi@^10y^#?0Abbuw{WOm9lL>?Rp7iYj4WZnEqhY)b9;yI6i132o03|ezgjPE048kx1 zEfd&^TqnO@z2e_FQPS-Hqo0Gr###b}fNvFpqQZ;=+!DBE(6YdGf~VF=iQsmG8!9JQ z6tEtx@CYjg;|6|0xB(1kk5Ube7PS*RF8CmTWXO|&TmkY&o1WD0zz&$s18_+aHQ=%l zQAkXo!4TXF9YA*g0RbpKMyQn$kRM#chp~pJRLQjQ1VUXQWEVJRRFWvtw1XqW1|S(I z;ehsl=EN)_!GzG$0P6|D2dRaINf4KyaD<)(Km(;D122kzBLEbF`T`h3^+-RLP@E9u z5T;P%LUTj{oyHLQ1$32AKA2JzPlc)w$Pq$9}i~5WrMY_Cg%G&`5~A=wHb3 z30Rm&D+)kQ1QJ4}EhZS0jDTwbl%gOj0CE7NfeC5{2FWUL9gGZUA&?;5W#A>i?Z5&- zoecPs2oqX?@)q?iiOmFO)P=cUBP+9ZKpOzNAofF_$$BxO{4L(nPbM@O8VG@P1K|>D ziD8f;789#UpamHiG0Zby*%&YxIXE$ds7DVPk^?;jY(-SuAiANmr1L1CQv-*<+947{ zd_U(%$dXWI7#!gwYsof%3jp~-Ob|GxsSWVICM?=1g4_ot3S0ys@(8r#GGIb*Ct&ap za0d8+ZWsdOD=UgMgoErvJz-8rcWM?<%ahI8i1?^stBQJ9|D!biW74p zz&6IIeF!1H{Wa5I1VspqP(s2fgc$-|pzkIRhxmXHLVIzMNt_4=M+CrzrhXB6qnJi` zu;?T~!$i=+1A@hjnlR}8HC5BCxr9J#WCidk2Hc0%h#+P{E(2eQBvZ`2Fn4Go^p<#O zSRym;f0Ie!&6qbv0e1j9(lG?VFT!_(o(h=*v)0ZJ2^Xvl&KOt<;g@MIV^JVW*bhPx zv`RM>XcjC5^my@v0@2ob zX!sByX#{1*qqG2aI0pyc04J;OD zr39r@kTxwAk{&f28?YndsSwG)sRdxCojP0>X*OU<-~?eh^yPn>S^vA_7u_Jh)tCoi zL_#iqf~ZohUb3==bu%$sKG>9XawmuL`8A8%Pt3_-Ke6OxJ!j46{>I4E>6>}}g1zin z)=jy&S<6{A=M#+WhjO{u$@N&n=AGa;w5YV0?KshF>C`;7RZzaE;S`103hPbgF7{Vg zu6=V@tKHt4G+N@$%0C%rH1x&^u6qYVmdA8oUb7wD&6^uEx7f7vA=^_vn$!LLXO>sn z`z(HqGi%qySM1&As(7;DTR25do!P!JM^4@Qv+QFBP1#vdh3wmD+t{qz`#C{73{8&i zzRElH#mDG%OFN@e&iB|I@z-K%`+jvC`$2rNk(c*s_M^rc_OaP|4N{Az z8YP*}wQsoQJ7;KdIIr-v1AByNe@=4pP~+6MUhMZ~#U>fU!YzcpVXW?j1C4U!N7>)& zwK0#Kxt{}ws-_9gWO(v`xP<Os#txwxIF5DXT{mP{0eDrhl&#|Pgd+{^QK~8a7@Lr@k1-r_Cr-G zGZunqc1m?@uyLS#!KOftQCh{4mHjF*zZ?#T*j^WC*}h)+=LD&$!pl8SW_PYaF(5E7 zCw*K68Ec9Anes1dCsjP!%Bol`SfqMhKd@qY>&fK{Wch*J&X<&{2JbFEv#yyc_r%bO zQ(I@M`kh@}A$D9?(W_Nzz_S^B18#d9C|~npet`biyE|&!adRNr8 zvaH~r->Dj&y-n3mvcJ@4V`o+0HEmU1%a>NwnBA8t|+)MR;i-(BXa#iv@S zx|Nz%B!2o<{^XsPD*yAH@*3XLz(u*UD=ulK1y1W@sUkAYuX_F`>eG?Cj`z-6SLwH{ z%zW{+sGq@A?|m1>_4{J%?*&z#nNi0C+?@r;I7JTj)v z*}t=4a>>NPm*cZslWSUDd0On;xOrYhvwmyBnp9nkSUBy(kEWIhCUn@NrU6 z`BV3P?=F7k3g>yS)a{x|dW>q8JZqOLw~f2!&<9Zl-3E101W9-|;QVw)z~)jTBTk=fAPsQYti3U7bU?%^lHp#fd0W(Z%m5eH6>No(Jclcefu(J(DQ-#)}4gG$wl z|I)T%w8!HecaOAdAV1k{(}&LOt*+mDAdH#WAw75F)U_r9IDI1mbF&iztJ_9*AHnZ7y#A`uJ$uTl)}Hw6(`(fOgHESo z&h@^Q73+TEZreVqChbk>c|IeW{X~&vc+n$fRi9UrZnPQ|)2c{P8SA+uCTURSnV6jw zu{(~A=vnwVE$(boyD6J477S`-HrQg_?E{0#ef_&ebqY;5SeIt`Qm`T+rvK-vX;WV$ z<}PTEZu73?;AtM|w&960lgd71S7#3}8~!G6Yl>Ni@DU>}w3t+RF@D6nZ3A+Sgjl8Q z%f8bjxCH=Vf8+W(j^%6ak9wtAdVqgNbl7{9TjQ@Zu=ai0XIDl`1Lt{z@rWi#en z<(hM6Ry`WyIkx@Ek?#{S|1+phOe_?k`HjMp)E`LwQJjzz9j!la)J-C8pefDi>RgKTv}MFn~Yc>?wdc!Pxl zdnZK**Z_3J1MU=(X2L%NJA_najm6GAxw;y{lK^eONj02rXaBLovhVyG2BazK|JbqJX+Q}@Z?`#`7wwg5&HHYze? z4Un=>Ut(V#A)KJ})Y60`VsF%Y1lA$249J^g5^%KS{RDsm4*|q0dJOQVXf}{-SHxHA zTJYFa39=R8CXfOWchj&CN;tBk8TbTN31u+T+E<{n4@u&EcqaJ-{agiMsiK?&3W0(g zB_h2*pr@c0fkF@?0Nln5_jeagix&d}feQYYV1U?(A}28bBmk%kfeO$~K#9n#CE;`c zt-uv96iAIv4*@|4_zdI)43LzLjOvlKAUq79NrXCs;+%N~D6N2kfL-xH*C7oMdJu>% zfFgjbA-X_y6j}lQ7WL?70FVMN2tfyt2mlEr#fak9V8*=gw@AV~Lk`vyK8d(VND(M( ziQquI0#yk3E!jOuS~q`+hxC9EV+r31s1(^C3cF*T%7?CE^kg3+@Ht>L=M`^ zP$CrIVp>hW_W~k>>L1)HNQuFBynr1LhB7EjcsSVtNIs7k3-u)2V(1hUn4YeX5Z^Ea zLLz~dXb}o8*cEDS1akyWD1?aN=m@|d!MlOf0F(jV$&ek%azcy*eL=ejm=$^1Z_%5+ zh~2knSfQ{c5Gr(=5iJIW)C2hgUWob1v@Zj}2CpYwDZ;PfGt_Gs7NBm73zh)5ih)}J zdLpd>uuZr%M1AQY00RUl59A8yIC2{q<1%1=-rznF&@>~2UA7DLQ)`1Mmq=z z5LEbFI0nGzfOWL)3=A8>A;l`d$AYBFtUDp80x`$Ecs@`(`pJ+E9zeKH$e7^Mw6GUo zJ@|Dz073%;2q+G7w;Xgk;6(&;!E=#oF~h@L#02L@(uc+cW=fj?Sn9xX39brO5PS;l zz2IMfRO90`oCDjXufohgUuDdb7?Xf8M7$2Fk1cUPeh4A+-=viEAON3ZuQtJupc61> zt?)rBf~zHLP&fmETrv~+bD~Gq4^snD3Yh?4)UwsUyhPxFSs}n6vt+o3K#LK02)PRK zfV-tVASMSKiO36LAmmKOg@PN00YFWF5CiOE&_dvs&<4jFn0u2@t;5@y>8#x6K`d5vl677$ugD&=XqnU zQQ0g@J(kFbr?-z~=4r&S2y!*GG&JVUUf<8edfIM|&E6+$yL=zk&7mor`bOoZi|cBBXaH45p@7-3O z{pJ{Lom|7-)^5A;_E0@8pLM#SppDu{pqF7T+55_PpVtUB@5(8ze9vA}1*<*hV1JQ? zpMNJ)|CeRFfF}k<+fE)eK5`+$BKXNh&as&nxX1fevQCP}aNWa=d8fbi=bTxY>lG z%UypmI!(~qOzV~K( z7lazcl`P~Lyl%yk_I%8HIrcs;p=KWE+2#S<=N|&tu~rj!WrMdH+ud)?o1NC2{qDM$ zwL0-2uX&2PzCqW!oVtOdjK{oMWSY9sjOVoID`!-jzFdGz6|A2JKgq54{q>u_!b?{S zdHB{EfBt5(930<2U;j-K5L)V!qnvzC&wVhs-;3iD+PM0Pcs}Hu6v&^HTy7&`jQ(%JsW1G zS>F34HL2Sb%|72Mb?WtCO{=zN)mhu*ntC&KQ^VS=@hcA=q|tPEs1bjO*34>^Mnz0I zr))XGOu1^wI`yXhL#R*6of^*lhw7A~d77bYp|Z>DnVNGlQ@`a!iMh->ymR(11$5-HSTV_oMo@*ECIAlX$98r=`@_S4TB^hsG;+w=bp0m}^eY z)l6_dqB;Gk6}5L-h=$DJp+`q5+-kArWJv~P)p)ylpYeUQ$@5{F^V09?W^I~jX2uWJ zY?ioFlVYqiyC>HuA1(^j7}Re?HI}|@z3Q$LA z@=j+1P3O#HjlWql^&&H8%^ATiDmuqS?KF84b>;9{zck5pO=H7hnvJR*>Q0?DD)Yye zt2-4mQXlJ}qFxjzsfS6f@j@wNw_Z=3LS-3reu6K%KWtP9So zuC!mR*=Z>orKo>l!J4BFwtsdQ*zU9<#Nw6HY@hA*g?-OCd$sePv8O!0;rTnsTc&Lo z;%dF+f!O=YP1nKiAAMW6VPs=#PXFCgt~PIyyLfQ)_j!*z^qg*W0pUJpqItOMbOhsF+&)%lzd$0G#TUSfm0zaBAR%UEo9ay?# zRH*+RZks+gZmU*?E^l*W`^C?f!)^vIJr;4m_~o0BCS!Y>?^G1DRqWv>4SU>YtAzd(;jHEFCtYPwxc*`{qn_(L_5-1eN~X zC-$9ruYBy9yDwv!7j)WpCahtlLe)(l#; zaBKgZqvsbM+jk*;b;IbxJNTm#Dr=oKx>(;y+`K91>A;yaiPs%X=QLE^7`(~DIW70q zgJI9+MbtAp*YF@_EZPGC=3H4W;lm`r3B4j?N)T?}7?^r}(*ffEBfgPbR%BLIM*gz^j|2~w+} zan0oa^zJ^?5hzJWBm8Ic9u0IHq5?>QF6yLOzyuW*DK${!0f9p+U3ZzVO96&sCmPw_ zq~mtrBFGwKs|%(C5DPiZIXm8|e;{vdPVf)p%?RRus~iZ@1RMpng>Dk;1GXn*P>FwQ z{DfXbw%`Kr20Fz+qGV$gOd1v~1)mB((UOzU5d&mS@LIH!2-8OEZh%Gt5(BzI_N4;m z)fzfEQHdZc_zUblEr$BLv(VlBi=i`a2<b z-~AqWGzjA~%t){wfR)%q$iOKyNgW(?j2c2Bh&^*RL2b#7O|q92*oQ)QQV}>$IIavF zB(ix6V7QKbMlQi^0bYXmezGI+bP#EPgp!?QgnUH%V2lu64%|h&fNZPPwg!RJ00{04@Ov=`kXf}0jH>`S$#zD>16}tL7!e;18$b}YD8gk+e~J}P z0IdpWIjP^z&>&_#s;}!dopvCY|tezBj9tiV}2gV2E`+XRw2HDhK0zaT?0%P zv?>615EuY(ExLv_0$@?}SqOm&K(iK?BxEkqp1=kI0LUP{>4c4)d{{EFYe9zbX`?1F z1A6B;S|Iqj76}E`4&@+JF0x%6?Gv3u{t(2-v0nfwVPFVv*x#Ui22ApgVHyfT`T#&D zi48LWnFMdF4B-~SNXR>AFcruhIlm9;0Xu?G=_HCw0NomdQF6Q<)A0E{{F4#Gk_hh< zftau?@dhRP<67HBan3Hpvr#>}Fi$4yh%AWf(m6S<+hxC-Vg28{mzTI+NPpgx{L z=zIk6)MCh(HO5OX7CI{su6DhFNRmz)IZ=`H1+>&c(m#Q#0h=2bBn(ZvQb>ge%NvG8 zc*ipBBY@%pUPsFwtUz+2ATwr|E69pqzF`*R{6XSSq0s28z=*KQXlXDy2ihd$4!{~} z3n)|DOoBE;ieLwlfsmYqMnZ{Yl0CU_GPsB&M=KyGHF__gDr4=yf!5CA*BnMCOgs$Q zi3Nb{v|0#VA#4U>YFH0UQ)lw<09aD=vhW#5PrGdw6QQL~92_(O%@D1Gh<_yJ2ekmv zP@(gW=%zuOaEydo3e1?%0qoZ%`4t)wq(cHMoRJZrX96jsi2yMA=aF*o4w&`-W3ER$ z6;y+UlrdZ^I+(#|&+6h1RGAR(5sAoAF#mNON0A~MMbi(i7luH~6^#}sDU=g{Jem|} zA!=s{=M0}hd>AB-Xh#G~(CmNBXtl2d-JTG-v07LfgdSbsfke?e!3J#L+~{O7El=zX z{?|;Gc{C;f_XVE;&qDTm>t;ikC17vDIT07ZJQ_AcHU(flcpHQZ$IdJ~QG3V&p@Pth z|4hl$8ysYsHS7V~FeH)ZBwos%Ja@Y*XHhO|s@quZ49^lP(||~uyjx=9nYE8iW^HL< zWW8XboptFGj=e`5YnaO4&hhR=j?~qabjLr|uF}|?QfgRzxm)G^n6pLYRzj1n4ZfBLOUBkM*$%OYId8bj- ziRGM_lb+ma-**ja0+zB`Uo_)}=22!3#?CX(T;J4a&<_=B>A*}@)AkpvhIm=ncQBg8 zX}+^F`_d=kb9T(`#TmWhuwDA>kLHTIJ9+(T*%o6zPc_X=Yix8O zsHII^;2jI#oPYT>38KILngr3m;GA0+gzA#M?HI}Sp8gKtQhgnM_&D=x67+H|UYQfg zE_JXT+-z9NP`1BY|J~#B7sv(ve*>*AJl&#nX^N-6r0GTF`V=nJE20B+An~S}Tvs1X zrCLAwtUhq1QaS2-f@aJPAFA-~vJ$V*%arnbFU{u}(~A4Y22j& zzCsi7RY4uzveqy8L@P?zpq-|9@sX0!SK-vYvOUU?ZVRb^NIlK!q~laG)3;QxF7?B{@&x7<0jc9^{0KHwz<_(9o95e z>f0@#zQ_xemk(EI0>{TvU1n>jngeE3gv!K!)x8NN)q9LfyN7ns+!*+pA~Lc1c0ywk zx?0_WO&ywDKZ0tLRidtcszg2QTRV;Cvqe;vVVe4i;ZgPbYqzP&i2j-x3C%U?4y~#6 zn>H(*>RD?R?=A2P%I!<}6|B&B&OGubgsSYpMw{{HPUtnbyxwp8*XH^s97oNOi@O?> z%&d&HY0oyy9l5_ay7HD`#gc$SohLnIe>4!bT>Q+(^jzD^BbwdoYZ_~wb?I!o17?MV z^Bp=)zGGIj%sM#iYMFU@^mMcFalsZY=qlYytF?Z_mLIF}>SR;v@;o`A{y3X*^hr4b@!tB=YF!i{bb`&)>nnyIL+*(SLbEgjpi?l-yIcIZ_E2{S!>1=*4t+zJ8`e; zr+Tdnmn~m({iwsF`O#)JDi_Dm9n0r;cU$UcIUzQvp2JbcQSpNZ9-kWTwCR3rQcMFY z*9M0>cnT~jSHXanzTJyCjb^46Z(*m~HFoulAF`uUZsT>Wyqcf>u%dDCWR*umn4rn_ zF*ow^Dy2Sp{XMgwdhLReI<819YUEOw-f4zMR$cbFdR_LLtS`8{t$CNJ27B5X*eD_d zb#cKT4irVW1lm4rcj!ddWri!trh1lkzmod?(t*JNkq6yO?kwCnv)7iG5AE(pzv$Iu z;$q+T<_Dwi&D~r*zqVCh(aDIj@h{6`tCMOU%*acM543rDan-5|3G?oMX*|K=NaEV6 zbDPJ@8YfM5cWEU%pO$oI>c2}rY}81@D^3kj`l zlIYTz8C{GbMMd#OPRB*a#4*;0smu$Jp@HmzJwJ%&NGWyU90?EUu%qw-Scb%pxf$sb z(geUx$bYaShNe^qQ3N5?L%b%Zq+nm0Hmg7?ihKf*9xp^1r>lX;Kauo<;suxtB@cNp z5@l?DM%IaH0)LRk0iQZ=$2K7*s(~U1Ujz7vaMFN7iFL-0 z@{?gSf|3Pap>;uj@_EE1{NgFJRRVA$_z2J(vGI?bMKiQW;NyU%p(w^Y$s8Ce5h`8O zU8GtezYT^|eDuwzvoJlB+Jq|yXkMg6SKto8T*49H;7{Rax{Zwlz5*SRAT)q~P!($D zNsiqnyZpd1LX`wSi&+Iip2rX1Y#~3eFttmD+84Xi2uMYkM%weC$jPC&jU4udS_^yj zbcGfFV2V^6$77ZkT37|mSk4hHg zGA(F=xCNaeod@g;MxD(J8hT3F4+QJMD*W70h_V>^fgPtP;SmZ%+Gl{P1!xHn4?#W< z;k4?9HzE{~4aazdf(a>AG+G)Ue2NC3oq37r+ytwQ@c zV1|S{3wV|s{icNk$le3k3poLc9J-1pY6T5JLSImy66^#qOnaGd#?W^n~(}`<<4Y13SP}+bc(9ZyV0EL966Sn~Hh#@)BnG8yJa5b@6gMdRi zl>`RK087AUkpBDx_W-3}EP!pHLGX!MQv#Ta6~>eaM-b2}5fqJ}0saQ-7AlX{8sG~C z_WyH!0x%Pf4~#viAFNWVWw-(q0^`LBV{v|ZM9dKiLb#mZ4r&>-zhy0qKz>Wsn8hV1 zIW!8t?uL*%nUfna=0BxySQqSa0WTN67574em`4EW#^*6dY_oyK&@BL3iG)mu&GSgg zaN4BKfe=t0Pl3lEdMwsC>|b&?JID^+7KC9!ss$QI_P-M+Mc(O!{pt8TLHhwO5$u~@ zT>y7P6M&2YVS~5$ne;(4I8GZp2za4YX54!vWp|W)33&4$0u1gi!qdA*ThR31I?~BGUwr#-Mg2 z11As10SXWv9UUMeXcpQcWZw`(iga0<7Q-C?ssxIU*h5Z!Ad*AKfWIf}4bMz|)6O4% zO@i~AB4d{Q8t&B@N)D%+6Nh_zgU#GJKF654?>N10A;)zM~Sx@{Y+ohm&+VZp+`BV#nobtXYfV#mgnt_y^{8Dy*pX5eX^Eu`6V0KyUwgO zk05S-~52+Jv0X z<{e*k-Qte#3)bQ4Q1kY6_f4i5Wg35NcHgv9qA%-Q^IJw;*2Nk}tenJ}8`FYyRjy=R zKlR?c&zO_kzE?i7o_?EdR+H0+6ZhmMxA=*t$*lKpIq^Pe?C(8Z@t)+qv73^d&we)b zj#>O*vH6QNJIr4fe5^O?(?{-l_iS!{GTUU7>l)VQ{MANd?nkq~KCd(R*1acZeCubd z%YApR91)tD_D?I(kzb#wLSEo%ms~mG+ zlG@oSkeU@ehFaG1lKS%UZ|Y9^E~V+WbCk2+?p1~t+E9&>9jGyn_iIi+=-;|qXiKW# zs#@tXXu8H{$U)76^VubZ?KdkoFEcB33r*0JIp$E|sb;0AiLaE4?V>auJ+4tlr`M-E zsi-EmCCRZv+(g6{ZSW{#gemX3FV?pJs?tc2y<%P?8$Yik1$K!g_39t zX6k9|lb)zA?3zYRWZ7uQ+@D2VQ}&zbqW-w44du2W!EgEeV$JUBP1Ta@N7TH=FO+fI zdeo-D71R)`UR3>*Jmrex`pS~+EvcZMRer;Ep4AMz`i2_lZb`jNUaax1PF9*Z)u^|` zo>#t|vsi8as1Ma)PF=~#i<2~sZaz@|;b6PGI(7K|`-2S&b}yg*G^_<@+^JXH^}Ww> z`)xfGR21;RD8FFAkhLx@#=BfDUaPrUZ2Dk$*iG(ymD#hI5$8k~hnYzRzIuIrNdxn> zF2PI3uW4zqq5td}kDDtk8m(MZw=;2rMW@M^tPIUQSZnJ!d&uYN#ePWwKFYHT7 zj^>Scc(i-^a=k(+@i3S7lPEE}KkNuJ;Pa z8<}TtZ?0kBmQVYfckm9iIn`uHvfGm6pp^vyw^z-ShxF~8FL^FL7+SQTB(lFu5I%w( zKjib{m*JAP53OGg%#Y|1reFCj>}_{>SIvU78comMV>)ksa-w%+>6SJ^yD(m_1;q`|51FMt#chuUfZsVAO8+tkV$-2SqKtR{Bx+^len;gGA1t zlUMqN6$b8D5#Oet%Zr&`7eCI7-L>S$ge0SsxFkRCHe1@K3_8P~XV&xcSi=~^OB)e8}`~Qep5MYZNbY1yM3Dv?=q-3#BE%}@T*S-FSfc7IXv58XKC!} zL&GJ*+Pd7iS3A6u>)WoymYE~&saDOm2wpp)ciQ^e%CN$eumM~BpEqfc`fSsL?nOE2 zsXH1r)$g#N@5qonPZxxTRi(?nb$c5$JuhSIJ+u5n-^((tT^1bLXV-pQ<~rwX4W?F( z|4%yD4z&Y08U&}+kpx&9`mtRDg$&X%w5E|x1M|^_XtL!ASCQ(X(nM~i%f(4W1F{qv z`(P@dQK-Eg*%wJjk@TT>Kw?T~KFEhiC`RIho-^9)OurwUMwB$9@lK8n!WYv%L{)~7 zAhAGBhpZ9_jW&@X#V%>_qPRgjR+oxntb`W;u#IdqL&?gdS|~xtrVnz`Iub>RZg|N3 zF>LhqkprV=DZ@b^zDMkMrT+j`NO{QDKY-5vZaHXcKGGZp8rT=09+ZZN1*oup^2hKVau^dqTdOtSwaT zaw$_AlKre$7(fWXleDTqDlBZPC14qx{m%gt5(Z(lpyDT|cj?MspmbP16l8>AKp;Iv zM^R6cQzGE>L1n_i=qkxy9XoS36bV%{&<%nD0pVa4mS8lXS3qycuOS3-L2d@;1c8Cj zU~ncHsU~&s4ekW`1=J3U2w~`uGTcV^G!PPohh-qoK)FrW9RMF7PFR9=cQjN9z#b`o zK>>!oGx!6sAljl(;SBD%o2=>AXhr}42%zEjTlbw9_`tU*l!Fn6$}=rB7!Y=hdR}TPC=|R zRT_(^tGE%Bv9>sy1W^}OF4O7~vI&4*K(dH+B1Y@t^PjT~a_|^Tg&a%(r6TAev$|v_ zG2n2(d}NpXPu@2y3&<4ee}F0E4|6l*3lbNWo>&*KMds?Sd4f(NP*NB%Oc|!31vju> z5Ifn6j!A+`D57^@BZ~pD1-*p`NuHz=9`q3o1(^y>7FEzf7t-ILl3|ep0cZN91>R0I&ma1pt@a{Ckr2cljUO4EmGg zcwwY|SdpI;M7j+C7eO2j@-ppAu-NE108WG|k}jlH!+uQ`wU-fo$iRT=!7U(_)Cmb^ z%MtDu9)m9lZ2$-?&}HC|kThZdHV`v&Av}J-LU88rZEyg@g<-8QTXF~hd_7Qc?PDQB z^b3Fm!xNBm2+6bw_y+Yuv;tU+XocvFb^ehs za}jZnQzHm5HVeBafdYF7(6_<*GHnD{GKNG>Lm-YHXK2X(wAa{hr2%VK+CDbt}*QsQWrSs`hqva)n@Y&M5k8_B-4Wo;9kvf_VT$`O<;v)koy$ShLYfh(T>l)bM`$&smsvWr%{ z`3<3TeD1hOe*L4_vvTa zt3YElBz>3B*`U!DLsSu*k)ngFPYZ*M({0n)-^PToGwogx_SDxVKld7otzZ82o4-O+ zTJ}`JdzWcj7LWIL@Na&hJ!im8hCS7#=|FY680%68L1t=%Ypj#M+cwvt7^PbEckHR| z26Iadt#0^BV@@kin&tXeziL{#yG-AIf2zzs)9E>t+nevdapNvc^5Fo=J>r}uQ$3$D zb$+I<*CnlVuhnfTy~9;M{RV-4&(f~>A6!~RW$w6IQq|75)YUUtx$pbz(gDj)L3nD?U@! zB^xvcJ^=o?HCE%hcnx*U`!m&3o#TH#r(x;o8Z(XW?C6pn&59_eZr#B%G4-#vZH@9) zb4RM>vaag01zk15TV>RMdEx#e>uvUPnO#V=ZxKw5EuM@c{tEoVBia5V(;Q2Ou&+{z zK}RU_@jWz7ez&PB-Ch0n9gI?+P3^Cl)?@T@*rqgax0iS;eZzOuI%%Sv4uIdBBy>t9XV(26tCW9jmRW|^F# z!wufMO)TaVmYrRYsp@THVGuH?Z!aZJZhL35aZnSJyh}44aG%UJef7-hVrbb2GhZ+L z*UhUO%`ePv=au+)jQM82?AR4jcgsv=mm^tkoUKh&pN$HRO}E*9%y#_nO#2eMjrp5)9kX6&Kl6Ue0imVW99U-`SsuUg%z56ly!bC)<}_?))1xpZB|zNT|Bda1+w zzP8x4f8#hol8yKJ{#}=`Cf0a&+&VX4vrSKa-v<{HM#robc-BiQOW9#4tp8$PQ*~v3 zp)l7++3#+gD11O>*r7LjL|tZ1k5unJFYfq#t)|zkMvAI?4y{e)!L93Ne6s41!tsAU z{aDh2et7{$BeNcOx7i&q{Qbhk8&@Q$wxsk>9qNBom2h|6hl{EPZ4&l)_a1icMw>Rx zHoCv+Xdd!oT8nc1=x*(nZk=>#w9~qFZ`p3j?IqX9ZKIWyBYK-Z0GHSoRhPC7>80NE=-O!EupVKi%}x8%OC7Lu{dV@v z$kdGKZs*s(&>!`hnko&+6OK-8-eeT4lYjf>}t?!d|d+Eid_3mv*i*PZqd|b0U zJ^Dj-#*FIl3~T8l*I9$kXJnbJ9bYtL$Jm}u(;^bCcO3VnpJ~GWMD@6vcK6p-b^|Mg?A>qUsM)JuX8z}}r%;ZNP)&Hr*fD}sSzEIK48qwzhxmI?KrDk*FY#LMG1Cj`&@6NLmqF!F)nOg-$OWxao68kR2cpBlSnJtGk;} zHpn3$q^1FUL?@Q01PDzYsXZxJ(J-RNfx&?_P1@BM9xhAh1FuBbl7P{XdJicMU?`Gy zNy#ry!gnZrQ0U-~HaS9qfiaMFDB8qeL;oZvAaMk>3mGe^Zsan6Qrqo#oBjc%AWQP~ z4=DA&TMkSqhCL@RCHz0x)&jJIE=tJnCcp+az(#NZ$bPj4*puJ6MzVc!1Q|A^Xp?hP zssD$)?~IBn+qNcDfGRQ~L82l-K_v&F4vGPcAc}$_B1%%silBn1m?NNKz^s@>R1}pu zARy+PBj$iPXTQ1XoT9sJ`;PbhdT-p(-tl}*S?sgJ+H0>l*IfHR0|L$gU^667*~DNq!ktWtsm$N{KB zloN_*zalhfE;}S+~7j)3JyrYxOpnE6%jgcnGj9D+X0y?9!Ri_Zju6y zg@fa{Jm4Z!n*e>%7*Qhz`knK~L=^$+qx^}I=qQHbogT_&0DVYFMNA(&prU`cD+?kX zr2Hx60Pe?|yI~B_6r-bhKj3zrVsfUJSOPB*?gGw6U#NEgU8Ihszkn9e6D1J=)hmby zkR%vB@c{JYzk75}SptL!Y6GVTXg{nKKMs1OSQ{pBr!ZpkmP~nKyVVZ{<2crT|lg#R4sZCgNwFguBpz0K(FO@p>>&L)b>x zOPC)}gg~u)*T5K}*a!R%s`+s6DJB^mrs6BC4>ba<82>48dZcb96%`IurR76jfq9{w z2TK5JsT7+4{VT+KXq9ZI?l>e{rW#8sy~9RR>@)~gu%x6Bgz-V(rZ{D(&2|z5Eo8E{~>F)*+<*lJ`- z+~`1VVST_3N@u5zM-yC%n-sJ_zMJN#Ph;%I5HRB6O06AAJ2u| z^Yl9E@QBF;8;w#2C_F184|i(X(m5XVdH;ys(NY-~{kU{r3$DbD^LS#tXy0UP>*2oAmm zmPrl3=^?@+S`q48;#?KefN+zHm&h^Ft%0afd>LL02NtO-aX>-*$9Y*0X=FX<2JLhV z=8yA<>LSrJ)KFOee~Ll*dB+g2SFjb3q@n(B(*_P0p(3^z#s;2F`M6SahQdv45 z#iGABgXjRENp(@Wg}+6i++5%=1zUwLgR_K>;yVFbKtK(Oid34&RPY~zTT>)-VVqF7 zy8ZTj3M!qfrK(F^bk!#J)fKD>6=WUBkW9s!#KbQ&gc@pHbtSFrMYD{YC9^|%h;J?K zE}GjjTRd;TR8e}&Q;F*IDuH2fAFVyj7m1CwEEFuNe@th{pejjzse}GPfsxktBU+MG zZ3YWg+rN|8?obgo?si7vU}Y=y_71AwBx;?OtU-`;o6jfVc1*xP!#&c3nZB?sLHh)dtj73@_n7M8m|6IDc13)|k)`CHLhtz1dBMqPEz*XoO_#M?E)?FWl{jfoLlwO=e0`Sugs z{OP4}Q=%j3+cHORBfwL$U*1mfcGGzALc{sOfoeH=ahLjN4?blfc>d$9XoyjqZoa#N z$obAlo#COc#Ulz9)f@e|zGRzizV^Q8XPTe3Xo$?mcbAA$J;f^D3pKtS6KYIoW4hb4ht}SHS*%bbslRu0_x_p11lQm zd}U3q_B>RuLPtHo#_ys-MP9MwKM_#QOK!05N0|ilbFgCW4B8{h9=U~Gw7629VO_~u z+MJZpzb~(!$dxI>&Ih?uY z7|l*z8pwPPIm?#B>?qalu%2z@WXEh@hBnW?lb!I4Wx6D6VpGMFWVMEE*edn)%(#qf z*+8S|%<^sXWTIYX?3^dF<=S(yn5%a~7|j*t@_gSpOu^3q?5L!Na;LT*N~dLeu>lXe z$OB%?m345M#n3#({_tZ@+^u1nb$ai=@arqt*u-RJ>*0L1U7!jRsBw@H>m89-mtB(e z8RX5(bn=nEvRf*DAs)lFxU>Iv+NfT6PFoT?MX2|yeJ72yZYfy4ApBe7i+6?JS|15m zn;0zam^|>to4qOGcKcgCkGge9?6+X*raNx-8U?e)Xt-_~C^e0=j_r7Rn{?ac&Ud?? zJEL>mG(^BuPQM|>ay)WwB>oMuz8P&x4$@3>#3QvqQF%AyWmoAxS;^E(CUB*UT z^FDKybeKN+t@+bdn*BaGcCw7JxEAODBb-#J=oOSf41M%pwS^~ge)Ag;A_ zY~|8=!}$h{`_9qmYQD$Gu95MQmod8^IG8+Z^&u)O&XM)B`84BD?6KSA088^8)cOn(oUm|+W<+B-pDC!O2)R| zSlJ}E&9{?Jp4Pq_($-tOU18A0E$t6v&UfEpCl2aW;#lgm$-6_W%{H<4Q*P%`{fpe# zj3-^3;?3fgj#?C!75eggaix3r_CD!)%R0^IKK)LD<IMk_sW01o;hU4 zfJ3s`C1%TS5BS->q+7{3CQhopYTos?@8eg0HN6t}dU3*`58dw-zAZ{{Ij6VfQJ18o zwi)YuW0Qvs8vDgGU&Y09@QF!GwzgSBa!CAs^F{4vB+u$?+4thie#7T2-`4y>{Nv#x zM~90}o?bh0`_$DdzXu%})#-_SMNOLa=(08Mn}u)RI(quNt*f+SgHpo_O2>Dew;?UV z?){0zuZNENq&{b2^@^$EAI-7pYVELb!jO{<3)3ARP0;ho5wGpvJN;~2)!q$$!!mrA zsY@39ub1yBV`gN@a0DX8rVt-MK>4{s@s5)b(kWtelzC@Js1#Qe03G0<#_w@41mF%4 zMff6RL|Z6T5ET7VEQZ6F%4d|J1>!KAxJWz!-O>Ar=#WL0jw&S(S>RL3-qk%4IU;(U zAYD|0?;>BzJ(6Nl%KcG9g&eW+v0Qy074fjZjZYktn&iTdxJ2MUFe+s4*K%uH23*n>(XB(~vST?4o8P-miG ztOzk;OYygcfC>Qpi=;9lcQDEHRd6rh2nco~NeqZu2d06z0TO6CBcS)@w<3WT;A3cq zKxn}g@!$mUIH)Fi*8pfCg!a6)?;u?P zO2dfwHn8b|+@QJyz(KIjV+itL1m_9-z$J1>5f}+MdkldpYUou4ocADxnX+AFR@hA3UDr#h_eN(7tU4^>=MU>)6#`+ z0&D}>I&ChxLP@z`7&;Ct!pro&3;Yw;G}u`zJIoLCB&i1Z-oa9#=)(>|0*KTbijNg` zjTHe~2Hp#7AHOvyB}ElO(CUF==2T?p9!(}my5J>13wUza7+ef2m-HQIfn>?Ns0ASd z=me0jSP3e|u2IdNii~AI80;Df=X(ZGjy#5Z)qa+xJD>xLuZ}8rHpCN#P zU=3U~CI!Rcw-=?7XhVYrA@`&tbFi?)Ur`|<9G9|l*ifL`V7^!d)7f3&!K{5ZAg2z+b}>&X>Yr(VHY_uIN<){Pa`#D<#cf#@J>&tpLYM(?+xh z)y=`b@RkRv1cnZm4E_!tR=K%AODTk$NW5W6y|_I9W(pw}jFc80->#D}VUu9y$dWJ? zQ0vr_-vn9Q3Sh${0wT^4!W~!wzb?N-bxK$9yATTw4#U9KC;2J)EhWziWgtR8kV`OE zxQDteigYEs3Z?|cjsMXT;lbez6?YP18R0B>o2CE~gdQpIpm5`eI_wr4RiNtOn8|&> z;c!jB)nnMGRHaA{7NYF*|6f2$Qclt^5fO~}_@{uDn{2SeURG05Wp&fNaY3zH~+VA7Ro%j zjf%cZvxThiZ&gx~33~*OA6kjjFC-dft#2+84XTjd%T3eWb|6tCvD#$eIJ}49oVjyF z^KKXEEa>GZ%ucToT{N3e-$Z?`mTA6*q1ow5VS_PNg5{Ab_1~X6Ag~-WN4PR=x4>p( zL&4gJ7m{_G`)Q{PN|4x{kq8_`J%te)CkeK^&@tLN^oGd!befde+e)B4y{Tr4hg*a@ zkM0)~Z5pkAaqm^Z9*-Wv;-($NUe)6?y$eSPo;~!Ev|N}ajBq(Cln33_VOOselwWRU zSy>q*YMU9OeWa7M(AMCK=J6GY;**Yr+RU_}g5u7-^v-|t7xl37)OzKWW4JSJx5UD5jVFhe6IDp;~~z!;He@jmH?y*~95k_T%g+HTRvuK!Nw zx!D$ht%1Mb^}ZB6lj3yo$o;+=O$W@DX1~5J`mm&#blKM$VU|xz;qbDP8pF$v3BJyc z(CnDEO7bIWp)})av<7G@+=5qM7R2=RX;nrSjv`JuB?~6HRq{mus0x@=vB6Gs$n;+(Kr6I8mN8 zw7tyQ+mfwvm9Y==r!#gw*6i#eQ+7y{Ty}W!Eq1GZDAP~Sl@+e)%|6IoEaK69Qu#@y2x%FLX#ju9TJmF0}y z&Lo`J!L~g6N=|*ZHkiR2ztu}7Yq3Cf(<+D!Ytn!{6tdu8>}*43Y`?*BXXkP9fu=p= zvzc7~(=UfJ-p58T;?4^hqgktD$2QfI6?CzbZ<#$`wsgf&rplv`IW#6mrV)NfPIEF$ z;}P@4W}jTtTpQ=@!c-*;i z!E~Kk)b=B}{qMckG(LQ*r=hz*^5UKA=SChgB*N#meL_Qx_P=iMe-8o8iVkG;Z%*@F3G9y#MVhP19DI4yZZYHM`#lQ`^S6*BgHk)_uedE>1DNYt04g@`jTzSJ{K)n>9+Xk8C z+aGbw&D_wG)x~??H0rs`xn-QYpY84GMpxeH3G8#~A8$2nTBQB( zzM2avZqK%VUAV+6Ir@qH!bw?!FIJyvVv_Ol&^6UVP5nF_j>spUZaO{B^VwI&=FS`6 zcK&&JiBYo(HU_KKZ|tf|0p=IRhqiTwXCMT#vdnmT(DcWv1q->=fwLH zT^k;M<74`gS(X!0cb`E>C~>G%#SM_Y}3Q8TUM#C~_e z?wFkJcuQU1CB;PA`Pr0NnNPY;4ym+B-LPxvl8|-jOP&`z-y2eu<5LoDGQInV;}xZT zE<<|y&)wdjcze0O29 zp7FKrUPW8BJx%x#F}Guj6)K6Zdt4G&v`S3W9JX z{{FQA8!jyyb8Ddboy}*yjL~ax_DPNTm(lj7h_v|t1vC)?0#s;#EuykG<(Uaofs1BH z|BmQ@15cFKqZm{1Pk=t$+oKQ=Q9fP)rVy_oaO6V{5@6uW25&{tiB}!FgrFTD33v!e zX*^L8z><(0Ie7~15Q$Jaj|&9}{Gu6T>;OLNuED{p2NCrkAC2ch%FTskg#19ZA^}DRv@uI&DHlf*C{x05fV~AON6} zA|=#9FNpJi+W}dO+&mYIL-+v?0cb-PChVUt3GPmY6t4WaqLYzn~A0-MQEdnqV&@tR6oSKqh0(OQ@ zfj*)dWrte}h@NlANfQ!M) z`0&CN;3)8eaLSN#k@7^*2^fZ=78Bkdh&XChB{X5-hRh_hQrC7_7_X^K1t=1)2BUr!?b98sK^ zSR>F&_$4LJ0I2%c(~N7H%KjinqS`pBt@5m6oIw77nN?JigOpI7UATXYR4$%4fdNyY z(&BanS15?l(*nUd__om#GB3yt@i!)y`!Wm#)CR$Ox_8`@HyTi6_=$)mVA?;#7{mo) z6>$9G?1ve1O%M@C8Wb1)nMumW`lb3%>?)EXeTl0gRb01_1O0O(2>u#(gP1x+GG z5NcGSrDQ9>Z751kVUpOhM2&INN>W3`BS92k^C=$ryI_Sk1}e1#;X`(cea96?Apil7 z3ULwo2b5g&T6~d;9C4eO3vVQ_A~J1=b6}Nt1lK{VFOqSJ z{XykuiiM;O;u9<*rW)HCtUbR7B*4QHiA=@9_|(}jY@MWC^p8Xa^V1DR8H69!0x~I( zdi)r{>cJqXvXWG0-pXEMnXsEkibisRXw}MSn0hj>;$I zL{Z3w-^MrSfJz(|rX2g1|1!vQABsDPMuh+3E$tryUFg}cB2Z1jZ(&UQg#3R6y2zh^^8+ah zJHUyi*i1AZIC>~K3kL>{_4fg-vd1t2(yn4#ATD;}w?7V3lB>d%!SZ0+^kq_&!Si9j z3UMA}sCbus>-TDsS8D2b6QNi`MU!4rnECCeSglw`s{Z`BNO)tqcxvOd!Wq5BYDgC9 z38XKwgtH$V($aq5E1egcEY(}FS95`Ng=XgTFXlP!XSLt7jx|~oqo%QVVzhWs*(zaY zw_riO?L~dd!W-h9DVro~Mr(@KZvUpgF8G*meaZk$`}W})4p|Y>O*<|Mn;e}g2!DFh zY^y@qx41561OdCBKO1uZSNpA(c9wrIxXk75tJ@27p0iiYPPZR6J^|+t=ax+ zpzuh~XTlEgH;s-L$h1x#8>xAESzpa-EprUdB`uSLRp$!24<9V)rWzs%A2dsH`TQ%L z9_1BUS0}C#UF*GE+UM(FjaOql1-E}r5=B26FX%VCt3gb_M?vFT7bN%k`$_NL%GMt^ zq|_vCRgrmOWs26*pqJW%pZRG{wL7mdB8?Za^0_)Cq;|i&7gONuKwfId>#h5fBxam{1pG5zIfNo z&gJiM#Z=0fAJO&vj&N5@Wja*WluV(dpwdH<&v)R}8?#i7HSIXnQ zukhFUCSzB39K+D>-zp98zgAlyw;60vx_|vlnZ?uA%!?Ffc3>#WSkBrg*BbQ7f6@dO z`SYB;vN9B#hNqa8}6TG4}Lk%Y^cm&Qm(IH-{=?nd+5)U(|p})-GG^SLaVIWqL$Tf zJeJLGm>{#aT*``q-?P`Bk7W&>OqL1GzLQ<=YQy%jU(6gj`GvV=)s_8C9pB8U8y{u4 zE>az{*reNJm3nFy%RU`6J=R!#_EOnNm75#X6VkM&^lYXn*cO+2JX1VRFuKVdgGsN} ziJvt;nc`LZL2@JaO&72Jx1?I8DcR1GCupCD&zfmgQ?1)A_;Q=Ol?(Nbv7(r6^7{tG zGs8n9U4|Ks4HiA=e`Hj>1vB>U8?n5=C@N@5!C~D(lcoKH?gbA^O^+Pc^Qw;bGkdG= zRDSDQYcu<2v9D}itg=X1vtfpl>13JJNK+yZ)D! zoK5TK`Z-A-wl%%B{{Lc;PvQy$Q#lJ^-w}3Mv zH^@S~(s~#At`AuINq#UWtG3m}*-fG^B(`k5;E}lGo@{8_pyprq&kNqs_JD~|tF&g) zc5!}vYt+5m+C?-C`!+N5V0(f8a|5?`AA)9|HP{%^e`?1fwT8#%`e=1JU2?X0WRg>- z&Q}IrU6FpE%XssgO>Kr5h0M-4GVM`ctB{3*atf_)n};&f$Ea-_I4!K%oht!{#(nPb zDy*5%cG>!#dR@A&J2dQaue^~1`hVQ;D#FJ;qe1Q3l&J5cstzW#In+<3I```MZPJ+M zC*Nniyfh&8%2IHFnQvs2S1K(j`teZ|AN|@Gx4?Kk0jR{c|G1_ z^X~91+fOG1t?xE!@%Kjw&%S=_$BOL-@7*~@P#rfbIVkVTnG(O@LxhD!O@ST7~&`eLM6nANR`bCNlDLbgg`5sd_3?gEJ$-uPeA_%H;Mv{~N5>y;7^Dja5Up^1q=Ahyc z(ry3_;z|sTnxGv@$`}wF092?y@lg^liqtQD1MCVAUke`cKx{>CJ)_({YAK-R;XMeD z9v}wTX4;O}V%!rI^-xsHr^t0HStRUEs9>nvmlz`s@&K%mlnCrJwGWWZ@feEg7r<7) zCJ;M>xw!zHgf{%W=Y&XM1Xf;wCDHR1JPZP`B7}=a z67k`}k#(dG0M|jn2#7aGCd4BG_#l86I32FWizjfo?;N^kSNHImO8cr7LL=}ESt$~=wzw;9fPKOH8NO^$? z@lZ1UaHp6v_+Tsqm;z5FvIfyY^*O}V12OZrdxP_!QbnjSP;~9Z`)?2_Q0FM_U;-WH z!8h=$r{;TsCB|B#f1m?NE}=9qa1=nS#2Jy1)tx$|A0SLgtQoVk^_TG9j0QxDYo_@*iXJv8zCKz=uF8 zrf36}01%HD0~P{q1OiX~%OD7Fa}VYS=0P_-*O}8VK@4>S`v|vA;eS-b1x`oF2qQ`X zmPfRKrKQqH`YzTA^NjI<6`_^#R;(a-!oS8U{PXZ-64sGFA&Xu&wEdzX|oY zkN*}=@nfe$fS4HYXEYjbH)Udkf<0IoJT6#C_ zQwEwO#o#d~NOfSJ6Ptw%@1vx@!Q){*(0!U!QWf!Y1+xUz4Sz;S5Rh~H$M^~;LJ}C# z4Sybdq)Xfxwir>kbXhs4(IQD5bZYR_I5lb772X1ZuHS{lupR`iRHumvB$m{bKOv9| z0>YXgG6!rC8d?LNq6Lx3R8j6Y+fO zB58Ifdy(Oz{SBPm$7$q^x-MGQdbVhJvpnHS-{#U)p=%5h`plB{D!NyH-RE-2bIoJg z1-ti(9hyuOr*By=bdGJM=~7xFDH^gux_gM0#B)fI(LQwp&0{XR#J+DrMWvdb1Wb=@ z;()Oe#N}xf!b77@HVmwvr&%d87FIcXX}4d{OY6vuY2u@jI8lfE@jBxKPX(P{UXy&@ zJX|L<-B)n#{Ysq+6Glh}`F0dsx_U;^>)aW^h1TQ5k%pF{-n;Kfss~Osc*MkOEL+o4 z_|5g4z{Mn6J5rD=xF2>!{J<~PFmBo*r%ED`hD^XEGy9!~yND?qILZc*DGF)JQf^ zPbHwe$~SqJxLvH@m(DW#8TK9=vu`WioY*_QM zjGOlv`{EpSlz$aFplp_0Yg{VhQ)R;38xm5QXMR|I;mHx%pjs=od&Dbw(+}mcyANi{ z+m?1Ln|e;8OznYOHZuJPGkVfh=Eb0yvK{q4u=SE=$|JfpVj_|^%7d*sGebVPGHbqV zWS_?z_LmmDXTHqW3D|SQxNOh&=4D1VcCeRX9?C_7J21x$jLWVZX&OM|Yg_%5z45h@ zU9=#OeHtPOaHt-_*e)2$c6>ZZcGoV9@lN!R4LBKd@QhUzTfWvg;77~i(t{ql0gc_) zGIM)PVP-g&urycYX;Ee8)~GU3Q|~d$9DA_G^M}g5irwI^73vzdrYZwXT!uYTVK$8`B$1w0Yq7!6;`+C%c+=DK9qlD6;Fl&izx+l-+if%ZnduI8bH( zZh7|Fl`SnCk}prHY!&70FeSa8M^i_c!Ppj2n_biThrFJXux#8=Ed%ly_0Ih1>hSXp!MFEj;K^iN~pQF}S|huyON{w0r)?5T zuUNeUce;n2Ga0Z~^X|d&ijeqvR_Wzu1IywEK6v4i<2Wf{<%BFI*XdHCpz2bKS=Sp6 znpz&Q$+O3zWJ~o@+aS$wdrQ74V_Itw#<;vSnhUOV_(Mq z4HXp^oZ$qA0hm48lgF+9@I0wBi_4WU6`qBV48bizGb-U%9>r93 zLziq&VhyO5ijN_qgA9aVHCn*|#wl{J+6rk5ktuX-JJ$y62!;hh zjjS267O-=QHUKzDQAV0Dz)=8n4$4x~IO0guM5M$Y9l=Ty7X!MIs>3+-6B!sJDd1s& z7{N61`e7V?q*7DV@qldrxke)=str~TegtcbvzA+1uryG+(a16Pit8Eqfn!r*$!LiI zlfkAaKE_pP_!6iGT3pzjmy+`Xmc@aLQ;&8~ors8tN0<`HfbeZ@%afFj#C;%!fPFzw zC>c(GUAva13kRlTy5L}ot1%Mh{aJ-m6 z3Zk%x+`b?;LXis63FKK6-EarLg_af61+mJWicjHZ;O-EY5z9l~Q1K}i7G0%UKZH%l z3399#e8;a*lyd6e8A-wgwhqiTr-r1KscaD26OY3tQqn$@gM=MmIN)3`r~GU{e*{ts ztS{UWHVprU<==;X#M;9$$hG14;pXTc;!n79x;hF=g|9>8R#%k%`xws+_hn(Z(I%z> zep@LZgen(?0V)twFjkXW!oLTE{PbfMu=Bx6LD7c92xkRwTxb>t5DW>-Ab+&sN-D5u zI8xwcaUd!j8>XN3C03U96(vUbc5!Y%{>Jfi@D?<4inEQ_M!0S)AIWX`3B@{4i7g09 zMV;)QOceV9oHo23=;_~bQYy!aMMumV7*6v4B2L+9qakqKr7LQ4wZZz~Il`Hf9|&d@ zzto;>yHui6uZ?h?TA1O2J1Sy>&G)no9iv1YwZg=6ryLcT#^xCM7WNT#y-;P49=T3) zWpss}wPvnp^?3(D!_-ZBjZZ%i+dmm5ZJ7C4FhH7b&?M=uaLegUf?k`CiknaK61AxM ztl8&jgrF#VnQ+ufnb1A!w#X~LmB70;T~yLcUF5s@g3xb|nOWrURNXSCX@aW7DVpu3 z4wD4+JS^!T5NUTDE)|_#rMx9XA%-^>q|qc`~S8 z#MNfv$mlbAH$E*fXm@<9c3769?o_8rgXT-NYuvYgDjHbb!tD0sQr-17C7MrOgo~5z zEEDx@)TVxlg_{10BoFOjdnQR=KZ?{GnQ>M!*LA+we!~$_JL6Kx$npy5$4_U(PbRsF zQ}6W?Kd1<)_x0{Av+o03L|;NG^nSU3%HuB=P@q#DSAc24D4{I;$Zn34 zvfZ`+a_vY|_AQ=1WAn5#Dgib%3z-*-ADaIsPAYn(Tbch9QGnw+OLjpYF=H{{gZxY5 z^#`S2rm&q)EbuQfI>CM&W>mIz(r8vgzJl$SYUkh7|Bb(G^oxVjx}`GmZmMOgj$1Ps zr-Nnv5`_VuqL#5;cN>+>S!5u;R=Gfa(ISJX)D4yGFkgB=KXV(?wowkV-J4-c?&ir< zcDl$-hjw5MHmqe`%k0<_{;S#T7uDEFqZ%;rx69?5hFg|7S#>$MtEsvFN#RXd$J-XH zmEAaIYs3(Jq%eozs-}?s!#xG5ZYD zs_RYvlmJzxRf+?lb=Zr9jBuJzXs`cD=?dt#{eqHFYyPfBz0S&7qU_YQ|); zIZJa{FEv&EVE1!*#G6I_gTFME-B_r}Jo6m*yUIQf--61g&h=HB+sud=zrR}jf=~`R{Rw z?++Smi@P6of9(pxdqX<3+3a}2aKBCZivAn6)K4?pvsM~&rhd@2!{1h~n{RqXcl-K@ zR<33P4_w{fW<_9wB(=LCH^N(*Z>+RRU2A4-Rjq!^eYUZxbzrDXK*25_o2$3mJsMii zvEioAJ>sS<>DRF4VrXuv;hKhzp5HY0JJrK>?}mh5 z5z9ZgTn^85+i{AdG|BXvbMwTQ1@kZYC7kjYYCgQ5|IUFo&sB}x=|5ca z==0sJvSoum%v$+4_lvAJ`Kg1*);!?G*Q^tt?lfpsV-kGy;)mDmueqh&STQXk$T+j> z&y7{_K^B*;Y%O)_89esKv`^bFP3UCUYQUFC-@A8_?kVh+arAnZ%jQ$NSxk-Vrg7r9 zO46=#-4gDYr&}9+>Ty*2bJFVQ#^J@n>1I{a%EF5*YA24cTNinJrs~9Q&F1y??>}iw zrG9ds^e+1A%R>7^nW@fpQvFgJJ=tg9e!Juk(dAvV#*WSF-Y?a8TB8fGuVTuACN?zm zHHeLxwbreXY(%`)JZWyPY1v6`%iR|+n-2{NJYGE1p}1h!dQ+dWwdeZ{?{GErVb%T7 z!#l=B7HZGxHljA$dga3jeMV@tjL17YboPkcU4r^8XB3Q35#=6kQ*e4zl=s=U8V{vo z(!b0pns@KOnC=%hX4zdom|~NV=O?pzJ9gmyL-L42ty8sYZOlGqtBvpTGJe^i5YP0i zPdW7;n(g>M!b$Px9pZ6oiMtRAAP6DEN%wXtgAknOC~_gJLD~XO9zG&F zhiDZL67f8=2;ihH7)eh5%Sj01&#)O%{ z!T@6-^5%gF(acmWML|BmJRq%N`-9d(&`viQf%j3Qy!nz`06HQQfN_uv=ZO)z?HlzU zU_Ss&;6v(T>fsWQbH<(pC)kItfkp!K_75HlqyoSr<^PGdQwl(Tb&*^f zbW1LIFhE+x3z&EiNuCYv4)X@~53~~hKujEfH|QiF4g3?wFaSK`XF~Vj&Ol8m!988U z0?h>cAl)3pOH5G12Lz5u1MLQo11Q8b4YrPc8yprC2t-Wq)FjprJO`i}tt0%T5+?%* zDek}m?}HRX9q7coLu~+11=0q|fBtj;UWL>M{+jM31La7MBuobWka7rc`?M^|H3#HJ zRUed6SYD-G>FNOO~|(bHVVdz z1Ctg5$r|O>0B=L|4*_Nm^rYHm>Yh5U7NwQVE>q8AY=M}+!nQeWWxcVz)>~hEx(b6`q2^%z?YZ#qO|NsE$A<{$uRGTO{4w1@arZZ-^xM zf;!SMAz;Dq&1il8g8g87Q5GeJC?gMEB<`Dci|1n5@ z5}m>PKzKl&qcjIh8)ky)Fu}|yi?hLffnuZh3igX^2Dj014j_!Ax7?w26HicTGU2D; zpTNvva&hkPE}U-Zf;10q5hFraNs~>CBy5J5NGt%Pc)ZdLY%1Lr0dAJ=l&E6|e+_48 zC^$;6e#o%Wt0Hiv7Dmf`jD<#n$PJ()q<>XzP|BlVrs&oPx@D1@4{}aq9ia5FK`40T z8=+AXZ;7SF3B#pz;rB4poL&r@7Q?K|oOwiAjr zA+tgZj?a+%;XE@u5h8B%4WYSWSX|GEfBb70{X6N@#28g^qnB=mtyeu13YIh%3vYXC zPTgK15wBdSr8!w>I`i@cfsXvHc(&SnvG((ix;pJo3g<6-A(lB-i?TOY360cOixydW z>ljZOuW6F`Lvz`!`o_!Kj51!aXr#84&t>82wAnfhGcu)(+D99%ldaS*5NHZFc1aR_ zxEimqxin65t8t{}cF`lv9sUPJi`yL%x*6`1>KJN@i`mJ#d*hpEl{nan4mgb$$`2nC z_HTYldT7LFL10RZ(YYb_#YaZm7au*lN#j`VAl>70YDArm?bSTlI8S)${72!|(2v4K z;Xc|;&U;qWj}| z3CDUGiXI)mp!;}7k~p!!SYgs89j!rU-E^NtP82_Xn=e{E>YV81gqxDrtqTQ7c~6CJ zC-f90>R;3v)OM^+%J!+kPp=<~J{wFHeQ4EFY~Q1maJkE65ooIAe^m6fFqrq}AO2k4 zx0$%FRFrqZy-nz48dosC_o!+^^JEdCQo zrB2@$oC*+!FWn5>SL zYqV)xG;)bhqYbEd_t*36~n*I9Y*1#E1L8LKsW zF#F~4HYUsHxnERGKc?WQfN9!iK08BawSVs!WvtETjk0;c0{P6n3*wn6`Y<}>*IAcY^W-lpSF&qIY?pnHy3UR_Z_7qBS>k`7M2)2}2dM96{a))cBj-vF z2FE!ux8fwFUj!M~!{@^HP)R&h`QlmEP+Mr2na+k)eO6v)1 zGY>uH`pbchBy@_?Rw{)7&1s6fR%j)b@Af zeSSTTTy1wsD(wDM{l&&ZD&hti$$fl#$;9QCpNAzL4iP&hZ@;^%RZB@;n+?@!XZ$64 z;}R^rFUh2VuXVJ(R!8g1dVDTW*fc{YEa&PVm-U4@w@bWNbsV=#$0g?R;=|ANbj_w7 zHQ1qTs~ca+RxRQ=4Ft5!DU!wo{mc5AsLV7t-!y_Vm4SQOPi)us2NA^Axr(%jVY zS;@^U>NR+Lur_p`MJG?cP1`INTle3vDmUs?s*OquleX_aM>o3s!e(39knOfH zny%(6N7)At4{+-@ag*`o!+YGmtPwu({*mEfbg0`Qzv?8f$f+9+&DB`qoof|+=i`M2 zKD#}(f6f1{(z3yv-a%DQdk0J|onGx9dL?jI*K0j*)i!E(acTec9VgiZmq(Xte9}7A z(ainX?41MNb$V=U+041Zx=X{eH}!4q`-cdoHC#67@HRhPRy(C9NRXk zQtjH@H3J)#czjzQe`cW1$}_Fg-hGbyxu<4VR{rb+i&vsm9iM3?swZwui}kiiEUahr zY0qG3lJ~U{n`h>2NjmmJaQeoatx0Y-?{7Wx;$2el!&EclvU`x@W= z;!pWQ4?R73S4VZ*(Dj+szN%AdhMvE?rM~w_>*1Y5reS^yC8NDJCPfCGpFjHAt=WY| zLwAmmRps|pduW*UxoStx=RU{M7G)nAHKIp$hVOiJ$*lkN>!&DKMS!QMU;y$#T!g5D z4o1rRf`bD7z$uEun-3@v?oy;eQH3JG$T3%RK@>s@oVb9wl#Awz3cw%HodpO1h)CkJ z;S_HmumkG_T!=^kAdl+=GJ0qm0g)n=%cVboK_G5G5DB6TaXuR1Lnll7#%4S)dP!F>$yNf~$`P{jpCEtEJI zKVj!0#SFwpsE=<1>;&@o$Yi5sz%k{wsSKHle(-3zjftn8fOJXiK)27r5h<_30ndO| zfl}zc9e^AjCqbS8FpcjMw1S^djFDWQ;w~a!O%OAXOmOanpfyk+#shALm$6`8;d3xL zd<^OhfLIBA05S+Ffti3R3-ib~{i`$L7XfGk^8tASE{-SqNy%$LZU*iTlpO{OGK>E< z@Ctkx*be|uDwk7y8;T}4WDpFP5b$$&E;oHZQ{c)#=@B;oH_YF2|BDyqh6q5Vs3-zN z0@2rkgYH!20&W5}0PzU2{JaDRi2z`9LR6Sj2%)$Z6xj^}!lqOR z`oT{IoeZQx!L>moD&rX~a2Al1kblRt!gQ2&iVO<8Hx2^QQNa!rjeu?-kV%(j)7gPj zjUP2=8~QT-#-l<1@s9+n0mO+;!+bG)ie5vwL9m|nhRYJf;k;C_0+o{}y5^(f}YqPT(v_Cd)P zEQZp=z?Q&1A;ba)0iuBK2R1e)2`xcr2D(Lg>{87Y`b)STv&xm*;kq`WDFC*yV}aAT z{RsjR+99|KYz`{XtSfW-?Xjz%E1@j|n#b0~rd9S1GfiY8xH5>WFaf-QVCqR?i8Tj- z;>j}tDENX80>UTv1n;7>ykBAve)0hDVUIZL6|W^wGGH)BNPx)4VUGTAW7xAGmf$C_ z>|kt^y+aHFDiHQeKJSkOAWa;JX(*t9rbsz?7$0a95Sbtwhz#Zo0ki~|TH0TrTi{Xo z4fmVa3pYiWZu8pg}x(zQE3ogrQu zSs+;Vg3)d4P@`#o^}Mu+?nkjxZD-LdD}9X*Hy22X^SngIANdO(Z%mQ+3YuxQ9Au(# zPs+Ete#`U+>Xm#tecI(j7lE9+ng0_Q$%-R_j2-??VYXvQG5FL%| zuhCIkPtfGZNX?U}&7_Y@R%@ghz0n8~d=;F_HqbO3VlV2}N~(Kl{ZH}bL4)h>w{K%` zJ+iN~kB_0&?H9*H({Hwv+lYQX${lx2I6-W{iN^5WoxUazSa4-;Dvnd-sSQ-#ECYG&u{Y}1Tr6-$fJhk-W6$5!=OKm24vM)P%dZj!dM98|o z^=9%Oo|moBdLfT_P~n>pzlSks-Qb|HUkIZWc~_o(=d$0Sk}}4A-9(n!ookrqe`>%q z_KD|r+2jU!?DNoUS#g68rD{&)Oj+`Bwx7IOwtmS*dCH&~xp~YQ_R1|)=1oBwJJ)6< zdmy1d+w0Iu_T2r?OserGx#7V)cIUy3G6RFwGV89_m?hf!@`xc7%s$6`O!#qM=A6$s zdA-}MnPCsTSQ<}f@dakSw?@Fy@<9Jv9W}~g?43$W!e%h;FKcB+DQ5D9du}rW3Vhg) z^;Foc4qsTy(XIzJx7i`Tkl&8!dFnb-a(W{x+1!@F`x4piQ5%`9cONs``e?J=&)s5u zW;bL@KtA5ha`kT-utj!RMal+`9n3W8K1H@KS4GAoY-78&UBXsBwqOLiN|qkKHso7D(vFrUmWwYvzw~vx$&qrskU0rEO($%4F=Wq$ z2(w%5%iXirZM0ZpW-WO1$&^qqkPfZePE-)wpWS9fjqk zR*yzmbl#D@)F%4IW0h~8OKh6Hs5(15_M1(Sm&)O+{A}A9Y(-wp0$JmTs>OzG33D8p zhW9+!UA>Lt=-r`$QzmY4D{z&i*U+I}CIg;?eN*m9h`zUcPIm7)BS|S<>=r z;o>PTUw8D+yuNHlP1rfJl!O`V`=#J z+YeTlc6u^m-u_M&kGkB`>~ll4@AZ)Q+D$PtzVzxoe~8JOr&E{pIAYz~W>9o+_-FaG z@k1wNMtt0-zUT9qvWSM0ZW?;D431oOyTkqK+Sa|Z3e}FNxDDAOHBgi!#!C~n1zo0K2{!7?TG5aS@e25VObn-H(jeVqs- zkbLHW5JG%lM`Zk<1)ypOzIFwH1>#U3VIju7_8zk5pe>*g0A}JrCQvx= z9yCaR*o5Qw?XHl2K_);uO;MS{w++AzG6d2SC<6F;EeN4V^8;)P$O1TDcdHKgEx=A- zA8H48@bJitt|NkC1b7}=4?LJ_S@9s!695N-#Zr`t{G*Q|6bDoV(jJ@^s6ws@s?Pyb z1Z;!0*E+^1=ZkAizmd z(f~{YL1`rdhWCUM0&fRajG&bgFoV+|h6L}06v2B zRisP7%mQ;$=YHpbsbQ=)mJSRYmK1~!(mhI>fwL#pP=T4@(|MYO1QcLaaThljRXA*Z zJ_v@x-BWQG7LMcnKmdbd#QdYa2!tLqZQMxcKj}0fp#})nIzbWXS4j2+rw;)Z&b~Sj zjHyJ^n0_oIC_KI$NHgdag=FvGBV6lH@GplI-$Ph6%m%v%MvbKA#AEHrmPs%67!FQVS}4SN3h-`qvY~YGxSXD*Bs3>es5m@PL|)YD%h04f88Q@}5vD@3X&8-_)a%21(>!iicZ^2S0E z0j97eaCLRIjb9}WpSVA|=8h|*<(L`py*R!RGQjGo+pr$2FX@9oqR^dN-bzY@Dk(u? zQ9T|0g|iaG5%3WRLU1b-q8E4=t_|{I;8@`&z~aE_xQ;{1gWtyfr92~8W6t2|b}wvy zkbLCO$i48}04xj0Aj&dO(KRnuf}a83Me|M49}m}mj4hP=6yzfU4!AVbuYy%mj0R#< zMU5f`NG1zDRIz=r$%uiWOSGso5Ihv$h+-N@M~A!#_(E5{FO(c1vI@Hw=Av9*N z3SUW;sXod!Aa4afhMtr35*DB|OY(dQ`7L-n^wpgh0=P6tQNc9P{c)Hr<+}eGHhOOWbflHxplh05v=Ahz< zmG|K(HpuE=?QplW!!T!xj{H5Sslf$PRD@rE~zKtxIcI!Ywq`updAza0gTu8`mKi5E zQbgS3(O!dNrka1W;elIa|9(k)t=2=`NpU*D$qz2+2-HT1v#Re}O?y5}YkHGMIyqkr zCHWtJO6PR!D#@z6E6`gwN^0=9S-qSbdqKTgC&A+ElX_;+GX(+LqNEm0D>PQLvNc%w zaGa*U%V&+A=iG#A+7*h|$vz6#&+TGh=bk6bZ+%kyc2^sN&2H0mwrF|^vp1L+I)9rb zEK1ik9`SOumdnh>T1CrV=69WrEUX zO$G9N7h%AG+d6SOVuh`rCTRtp@szf0`&x2zt-tPxN4b*D(F^oWpJ*r8b23vJx-dbo zdfJk3`rm(|PcKVe*ve4UGnXAQWO!E=5S?i@;nUBk#%OuNgv8IB3a{25mrtjdU zW%n=6XL7x}%MRxpl^ITmkRR67W2hbXI|)qet6B0P_j)nxzSgWsuvS?`nxFscLkpRP z{Z=!DG1p`(#>BDpPpoHGG~UP5Xt(qKQmR@;{je74GG{lgV**z^V4AwmV&z*ZS=Tw; z7~g3t*q3FiWY?}q0~*yk#$I%8#C#r7EU(@a#m=2*&9K{=vt93tS(nyl5B5Ez$!_Uy z&USSj#;k7}$5c7nmbI7fWwz+ou+18OW2-Jz-qab(z8)-t6GrPh}ye4nakC zo2}pf61zb~z(l$Pur&wfu@753mxbN-Wiz8!%bu*SU@Xr`0%rEQEF12Y^E(yU?ngCu zCz|wDnb6>iLAzGzDuG`@wKT^qQpr6bsAyk)K=sR{JK?@xpQ|!PkM0~S`l@=-B`Pwr z&k#}O$P?C+tj38Gb4O+ldRQy@lJS1=v6U*?7box6E?69`b7kESzwZ+ZbZbu=I4!Yl zu0Qoz_qd@iub8+?zs$rwO({7CQO!8{v)Vrsy-OS^SGK~hB*W3A~QS_FYk7?Plwk7>%NqR4Hv6tU3 zAG+_|F~^dE>wWEKN}6Q7=xe^mFs{id9pmW+w+oxL8kXH)+^+IwSw1)3N5+`9m^rK6 zv(6VEw#ZrfzB}tP!*!O;inZ0@wr-Byw`M){Iq%l3@T1-{{U+{>jSrnrH;;AK8fM;0 zs5i>m8fi~qOS#8bQW zEN3<~+1{>b?3q?4r+xh(8ak%6WATI1R~<&TnVOPzz4h}3?F-y8jAhM|+jm=DTx5Oq zV0-z!eyjH|lY=i*yZ3$|?$LREaOgC(nhBl##8)aWG67w_2p{PTmwI+HlVv!MADG(x z?NgJ{r_D};4{Yuf=$F%=_pYYTv$m$4=v{xe;mna?jbqH8)eAZov@OOn+C=?F@R=B$ za|cv<+*OG!aohX0m5*6~v zd@yO)-EHx;rXL0`3zH@+KK*#-vZytQXA*appY;vWaeH>HpfuRw@impX;egMbZ%Qed%M6fKjo90S3j4~T-c z&<_7V0BvM4k$j~f49Rq^X=E!Y=0=DBnhs?=V5wLm$^L&?s{hyCd51-HZEYNJ3Nu4T z5D1_ML{uV0x-e%f*t;SqiXwuD4FPF(L|ZuI~=;Z3xtCd9MG5mR0)d? zDJqsW)p(&chjK$+d4a5i^cz@;NKla*`qbqx@E@epz%f7Ry4=LZ(JezL`hYS9K7`kE zk#GV|imC?bj&b}!tDKk^1RcnVNT-7EUT`bG@`(3|fpa;5=K8j)Uj~0q|Bd2lz!8Ph8SOI1Kp@%gA)Wn;~*y?7J(^&AY^ zQ}T@_V`zbLP~b)1VJ6@(N~LL{=B1#68ePWu3opGv3XAIq&^pS~p|SJi&6cOLREkK}j49yuWU_$2?r%Z|F@M~x+DKmk z0o(AkWPcX}z#;&LQH=>LDHb7@(__7ZC!xVGUKj$bvaWgm*?v#n0SZ%q4G<;77%*S1 z6(0E32D}Vlk!t)fR6=lp(Sg_yqee^z(^p+M4N%YpidGEgJ3mWB2L#p zRk6OsRYF)Ba0p^SGICGO=J2yHZ3JFa^a^`|lu0mgSQL0A{vVtIf*A_)2U8{SJpPL} z1E2^We?3|w6Qt|Lu&(F;Pn>LcZh$WsG*&kZiF*WQ1lSWE8vzl0Ne1sgVRPWTm~i|F zKZO}16Q`3*!jf=3Xc^))kvV~Cf(%o~5o?J>*Koq5{la2{UxV>_iKCB|13*t07k|Ra zMQRL|N6rPJD5B4Z{rCZav5=IMJIx@{%C`j3C2XH0ZB%qC)?z|`i@~Glkhn9noYX>C zE>oWNi1fUHzpV}&=o*BHa4Q;;(mwugLM)L1xkYYml}e_iQttq*P!W`bDlK< z8WU#245Ht_aD1sa?Tw^x1xDg!db~^xWvKN6W?>f5HeyWZ6M|lZ&BDiY(;EE(LRbn_ z`H$h(&@)H~u?eAyAO7%bIBIWnq->`BFymR9G8B?2*)r+oJX4wbOSwEVQl)A>Lq1VH zRI>PUw1LU_?b45%yO^8VA2VsR!$4(z>y7g6?wz70sW$kQ*A9ou6p+4gv%v{l4Fd21v) z<`3>jQfK#&?!Ffx+2h;Lq|J|7xmV;zsZW`&-0$ucxxfE625ntKCB@cvR3$62r48)f zN?1=fWryay5z*B8Zv1TJy$@N6sPfsW=q1KB z4|_?Cd(U^$W42dGI`vDDKK0C%KC8`?Bz`_=bZOM|#&JF#2CMg5OOJ(?872)rtVq7% zD|y$agVKC#vTTa+o<`F~=BR+97XC>uO1Igk zv(w91X5iIKwxDd5c3PDese!bR8Ls<5y{5rL zcFwspb+@eVnDK5im`5Y#vXi4GYtv=d)N7=pO4i&P%IIF7#Vjbl#3XG#%w!L4&mODD zW_xz1V09A=G_?b4G;7*VW!81RqV4j0FDo@VqPCxD#q4m7XU3``z&y{_-cCJr$-~ zJ#)PpWKFSZax=QO?wQ~}j1zms>MmK5P_QFj+3>f1DWQy`nJn@|w|k~RJ_`GuWk%X@ z*OYCFZu!Um;H{e8b-lmM;XKvyCHY-i*cj=Xgx-$2{@q;tH9wwy8Bw~!xYuso_$KOX zv)9d^HO+{To4a?uIkfCfOyi(~u~wCRovqDtI^K9%9$_6fD>uH+x@c?Dnn7c>)LgMO z?0kAn>6Rz9n};v1zF_mlKD=vXSiw&nnq_n-cJE<3%;~ax!n&}}yPYnYHd!6BVd1x3 zOp{ie+~m{TN_MZKVN*@>^Qq2luUZ8+cOCL!@5+K{%`;k0`7}9oZ}Ysog1GQBy%yRp zFY*>x9%{9r$#XTz4R|^9eL4V`7uiqR|gk;Z-)*V*O;{} z3X)9X-5My}dQ2~C;J(s(e)8IVj7NZ-@6qLN+Ix0*?;ckvd+Yfyrb2lx@Q_!70kQjh zEY5g2k1kYQZ0F^D!o#Gx#}%DGVO-6uKdQt|9T zUyojnO6%jB2P8L%jkAfV4c@+Qplok?>yQ&?S8S^8GcU|@qUp^&^2=e;j=jqo$(@Go zlA2o>uk;Gfwe6a{HQH_1_8&c67qn?KJYVynS>XN3h?Kqk5+k&$BTKs{j8}C^85ttU zEtxPbG3tkJVmdDM9UDEyzU8L!o$1l@QPcDe|ZoYBj*%qbIpQ9d6dl~*ZX6*Jb z&mMgbjotk|Y{S~ZnsEic-}wH!4X5JH2mhjO`opz}dOIs3olL{xXY3eMoiTH2y!MX4 z^*vb;No-D+$IOdgl9xx132n9a!eqY+zu%_cpE>1j`seXBQ?E@8n$#<8+E04Z2G4Ki z{OfNU{-4~vMf)rEHJl8QQk~KgMA1T+M0cXlxem5=Y`@s_`KW-hJG$?H#Oicet`N*X zSx);U4x%`Hl@K)2RnSPTzF zgtwvm2rxi&D*zE-WC-#>4*)+nq5(!hmmOpIZE*CU0WcELFzUN*Zj8|n zs?vjC3!+-2)pY=+p6?dkMj0#!CKYFaN{HC6{tiOPDK``_Af(EqX7d!b6&NA1O5zc) zI&_B`K^N)6tYOT6PUr(tNqQZwS%OKS3jmnlBG`@SQ+#h7<`_VRo^0E=I0oV><8RCz?SmG%2#lWy*Odl;ye&DbTLV5;71$sm{PiO~HZ)};^lu@n- zBjUygGeRjX$RFfI7&sr{`fg;_s6)l#!;T8@AoheRXCVPW&W!Rf58)zsybf3z1zZ?7 zpa7Z=EL7M7YyjvG{Sjm?FhcYeIzF%g$WuTe#f6Gjqqm?I#Nz7u0l5>5n|Kk-7UnDT z4zeGFCD^uM8$gZxaFUpd@GKT7EjB*;hgndtfju6uk)&FG6K?QRT|XESQ487%g*EJ+ z4&Kluo`yi;hM7hq5J;dIe6OL~=KL5GNVq8J?_v<4#r2~saWL=>xHHT)rUbGb&Jdw% z!U&VcgI*dA{|^1cq2jtwI7@yKN0^Ds`7UB)o>R)j&?SZw!&5sBMWW-9AtHA9haZ1>7gDr=G}%QT8M%F5fJR6QA_x8PQ>Y%$YM zp>7c^FMJiQzqFU5;j-i>^2W!8ePemsPPKCPe)%}tDrJ*oi843;qsso*Nv3&c_vx?i zFju~<|9VxbT?<)${%GSJ!7ue)r7h=tc8^l@jyjJuI&pKK{G_&< zwDYSc$}V}ylJaD=MXSZj6qmYg(3{h>g`|JfKuPe*mhyq-W~z`8tiJ!LO!*xXUDbTPOLj+KICGumaVcplIpcpbWixWm{C$fzM3u8!nNK{d82)wYQd+PY*(?(X)J)XWXA8mdhe*K^ymsby~H1L^$p&n0yl56P#77FoD7s-SYd6@Nld?xyqp@^60zQagEt)Px09&uYd*X{^fa zJ}w)hDdBX==kq|)ayq~>Ga_#8Y_=vDaEI_G>bJN?3J^-CL*()~|9lq}QT z#;U6$wN|dB%*_E0w7Y6HuuHZzWE}VH(<;{LF}|;gnKOr5X`O7Euss|;tE=w0mAc%w zWKA7Q+4encN_#%AFD;T3F>mK;vG%+5)=uGaLO$u?_} z$qsw4hUsu(F?+$tPV?c~TXiU)S>~4vwfl2{D??*IW3g(iowiHm z3?}_df41v5i&FpWn@nZ8g+}6@$#g%O#T56p2~0LTQj&0TB(tIHsdm+t&YGIHSJ{@I z0+`0<4lp?;3z-McFEbh0O*K2h?Ag1QZZM;6+AwDRE4~J#c3s#ywz_$d&cGYXXZH6l zYk1WuVN%=YiISdGL+4JpJX^MFanA!MW;!U1AC{kw3;Lpb6E%RG>TjvgiUmI=t z+-CQ810phPp6-dCRJFOrR#M~R?6uz3uF2xWH$D4|v72*Z{m28O)b`D{x1YV%rmOv~ z(QDS3JRW90J?ws-?dB@`@T6@sXSw(L)@kUzu_o7in!j9O^txTv3g?9Cv*$M^ZFSy0 z!+*uhJ)2rhdoo{t)@&P>Kl;7W&HwVk)$qlGwel0q+_p~HTI24r(rv+qsI(F07u@!o zQn#LU|Fiq)2Ro(Nr*l2~TzHUp-9N&sYu<}>Qy#weUgr{)<2w1Xk8Q>y?YtbuFX5XO z7pwcv_nTO@K6k|7x&8)gy$<&5dBNW&e95YBHe0ndd=VqJPml-1nI`}4vZN%?BJ5$@ z+adi&FijrJFP0{)5sIy?QTaMk1pu2uERo&_p9Dbhj;8Wq*?a{ zQ|5FG%6hYST+WFu7s471IOCz|p67gU^X{P`-NT}K&nw%wt;egm3r`MMs?+mcVqrk? z+oruo?T^j(s&?pmZpi$z%R=V#OR-K`TFRIN`zM>3$9}L5USO8zbtNV}m??TEuV?<&e)Nzzu~kis>|@I>KU|YxzaVy6NYI2h_P23Hug!v7_LPlJ zNSHq~ND(qocfzfwC!&r`)ERGB-0aY^iShELueYyxJ<*1_U%GHq+XRPsr#$oe^i44A ze*NcXkzEu0Z@!r)lVl}%y^3&1NNSUE>BeKt{=+v?EJmjqFAO%DthZo=*VCsrr|djG zvCs5XXQvjOJh#5>(+<=9a~dia|DP+QxQ!c!d^owH14RVmI6LwXHOx4`LIi=_lMZU3 zu@<*d95ZpJIHaBc(-Go3MEHnkal(W@kXeckfw+Kz5ImQH9R#AnJE4a_odtp!s-@urY49SVQOLRxV&m_8gpMSXL=A|@vGl={`A?CPAcF>!!EYcN6ti$Tf5#C& zj?Gb&1^^*uTet}XdH~=6WRFydC45d_9)^`muY?=}`L1|mP6`3K&Ic(DmNAF{H5{kU zsK@}afgB2K8jeb64ANR;>^P?9?oZ)0AV^y1vU>t3I4->EhoDdR5l4~I)62|{)4i3Tz zcrW-Ss+h!opePesj2b#B(4gvOl6LrTj*k>0AOytY@C%Rw@pKHS4CFW51!i#U;B>;s zQ9qD2!Ut1<6jEQVC12kOFwWpoK#Cx7z;vJ~LY9e#LzsZ{6}b@AeDX|&U?$bWA$=t` zOjkw;-Ng35Ny^~wX#&I)7BUtRj?lRUMiG>UA8X{NoM3?riohC2f$%+i7zuDk64xLJ z7qlx#Ly_$PWMXQNA#>UZ%HBymOXXn@i}{JK1CRsTB(V#46f9UAJ_u(B0s$3bF9g=2 z86)*DHXbO-pu>Wlx(R#|;$VD4AQFv3lf{{-zhR1YU4IYcaRFc`*er+vMiDLLX$H1P zY)7a-q6??^y_Hxi>f68uiAK<=k1$(gb4ZG*iVusxT@*~A6SUPqfP~eIsiB?#$`H+@ znmzm=IE0X$V_zi*1pq-HsSP)tIHIC80s@2sBGi9*iN#L%5T&T3F{T@%Mi#mhQHN7MdJy`8X@iyy<^^8^-I^FC)ZgTVbwu!mTn*3$ z_IqK?q8A7x0Bf-*&_fYR;B*TH2iYZzo@AWj!bj`S6Vh733;@{p@rV0?69WK4`4^Tb z*E$Ry-F!uS2*Uw(!u1f-f(MX78!Qj7QizYBpQ#sth_eNHK^z@{6mbQy9XJrcasVeI zrVLS$!14g2Nkk3?fd%Z%5eS+?s^@|(3h)L0Bn))@$V+%%{RoRUIuygw*kLsAH#ZS7 z0^*|sSjaF3;byEd*g%O2d4g`#Nk=SjR2359U z{0Mv2@hr?AToBe8q3}Ay512?`7G_On8FlUoaxM^f%nD9VP=3jK6e#Mbwcu+wZ6L%h zY*8R`utPfcK0! z+ggE+Mz5QaYT#p8G~Fg|`CD-OFY7!}_%j6PgB zx=U5lm|&q5UkO>;pwg1G!uVlq3&hif&SA0Pm;hT71_>MO*U;3=PEQOQo;_+LwaC<) z+qSnPGp|o`g_TtL{6|}(1wVh08a`NQ*8SZ?xpDhksfo)wRrdIKvZXGJBq#HZG{;pThZK0&_=?%$Fr!8`~TCM8SmZg$5?T1P3k193xnlr|*C^1;oQf?-H zZWd&$^@}k|o%pk)o#cvM``hzX9cHKL{n#l|9@I^v?0mhqs>?4o6gP{u$d1jjF?wEj zQg*g;ipfRCd|5%G40%81f_$yDq2U1KEXmWv0+q?#)$+Tajx~x&S**HgVQ+A2$PJ?k z^>>PXK@kQKzpj_uEqSAiv^b$R@>N@fS)KK8DWJYwcD z$-NT|WpQU3nHPU&B25Usq`aqyRlE<`XlnbS`8VZTn#eu}IP2~Ixq%#LYUiI|l$+`L zzx>;u(bRq(O_kaVc4|{OL}SG&rrI`N@(W+QSMzQ9ja}D(rgT&Ci$Y#2pq0AQAWH4i z_TLrn%`H)vYFG8v+_;~mJ>I@Cb5^}5P$y~(qpG^gzFe+fn)^w(RYrzE(d|=X)`yPOj7{ z4$RPo-d-EH*VD0de(|P~y*)3f->0e7^j^~^TiD5_DQxZj5dr-+HYkpG#jw)~GMO(< zK58q&>1wle)!Kt0-;|!Rn!)Uy(~rGcFrTrMmNDfmbJzxcx7opk14?Fl@6c8^$YraN z?=VccqqbHQL4> z&uEXko@TbHSoU1$V^-$&kPS-h!Cdhi$&~e6%1Zk+W1C!<%WPC=)JG1qV`ZoRP{&@^ zEuDB}FZ;?Qnu&4zwdBl|)ojN~dv&i!efG+O&TP$*3Z~nb3bo_7O@XmHGqmcc0=99s z_1DnUo8Y~YFO`#YUjH!a-L*YybxMMB77ls$N#|3qPdW{5E|>H$yQO$`e2{E#V6X38 z9yHK1pQ)2#cVM>O#qpMYCiAW+oH{GK^`dGO@1K@Dx5-XYbv=-7r1DDCA6nG%*sx!h z8V5U6e2%)MHD})*o-n%Rxy6z_Dz^_YZ!F?{M`WJ;P}Mkfvss&b=L}2FU!@Ka53XBk z4pw@V7Hzh!{e4Z=bBhoglchWQ7|(xev+2Z|?XvBMn`Sw_yf2N2XlDHTr{ujx!yI&r zpY8NEyW`-t^QePsK!)R^eZ8DF9ky~FyD%*_BEY?c^|}tD&M&&qLbJU8fQl!Utu%*a z?CG_>ts1s_+G4I=Q7exYJ$C=B)9|}3p(E#QaSU$VJ-5YC<3-Ipw$5Fx(`=V z^891>kglpX8$WO8GktcnY39+)zRP}`cCpZKTi=k#FK=?$xPD6OsG3Fxw+*OSxFF-e zi<)4I6IZjwd*lu*|MK~s{B@U*lE`t(T~~w-T9>}hG2CxS_}Y^TKCdY`5`Nb!NIqk~ zOT-|b)Z^6~(nhqt<7&5A^~*?=$Bb?H{ym~P86Pcbrk54b(IXNS&3U0sxQscfTj zbbt4AN2-fQk3MG7u5WG8h=HlCiz2>tuc%0^4Rt!}e(gi* z`ejK)s{7e#)`ndNukGbG)#>+jx6+=0X$Z@k202S0{7 zJ4CV+OCdNDej3NUbZ&%0b5cd~_axv53Q+|Pb8-F&MK;cl#E_o^@{|zLZ6^o<>Qvr% z1WqUsMIgI^rro#h7m(v{xdc>G^qodxPD~7nE?huAv410W6LSRQv z&;I}_SONX@51{(*9tUAbQhh9!f{~gBr|aCuI9evSf#r_m9=W@>pjAN1O4$n1WaJl! zlmBj|(nH{3kPXmPCUl=PFta-kxjA78GH9fBbQOpog#sxf=tJoxNhI)lVnZnA1{gs^ zj$F-~%c|%L;Bj;ek3{u@NR9r4obba)e~?Sly~{pzt^s@s-~p*2@>sN;cSy*_DX+rQ zNgImnjmu~7bR_BY2&k>NWROn;kcOkpaGFT`k=uxg(pMcLeVr6I3V;eyF8nDVbh;iN z#0Px`b|Mxb_+0b_X^tm}P>{vA3Frk~guaZVsRY>gj0Wfhg++A#3CDZzTl`q_-yaAE zjFIBBDE!Ar-Qf7BW)i%DZqy^!herSh5Xl08LAoOrXZ_2Dz)b-iC%8Nela9>+r~X7$kfDq41s{>J1Fyra z(m-SU_(P9?JqwrtVgigy-3!06)xh>B$%R@6{0FWdLM6deG3po$# z+eADeio$;5TW8t->6XHE6ZQlfhHZi{Ang?jd@z78E>f`}v4zmbj~hTFthA0WJ!mEx z=vzl*|7Cj71(!mVCO=qY*8~X31_2F3d*}5VXe9sz zEJ~amU=sR9IO2D(6*sXK3WEnwgY}0FfmnHo+cIqfNqVXXX3I( z;J_)pFp+d7B~ps7g)p?L@fN%Ze***q2H_e4#TbcKFs{%XVN5(k$3qW>VZs_ftFU%x zT>mvz;XlQwA#}n2V7I7;7GOEC9XJ@EJ+vJdihdRd7kCyF3~L=k86<^T0Rx1?f(jip z4gY`>J#oS?rBt1X*#zlB0K@+@9t(98i5Vf7fyWeC6WujM%bfTvLK}{`;MrK%8<^Zz11@g(iHpOiX%Udw)D=7e{9p-VKClUq(3ZDqphLus7z$8D^2XeV zf++zsLVAdoVW8TGUIeeE(3ChS2Frt+K$4gO6=F`&aE!7aKh9hgD>xMf39IRUg`ALj zp}TaO8DJc0-GwIC4`YRo0s06jVtnY#!_$vHccnuSs4j8JLefEg*ho!?s~Q33sfruY zNd9BI7`=k>4IjaDytzBg;PLrM;XCCt?rxoWSTOnmr{Rr&xi84E{i1d?wLqcbJAo+-(NK{wXTwcrg=&VL3thxt&+ai zi}YnU ZMAKr`2lsU2fipDjH80ia8(X@z`ybtBt!n@P literal 0 HcmV?d00001 diff --git a/tests/unit/core/metrics/test_regression_metrics.py b/tests/unit/core/metrics/test_regression_metrics.py new file mode 100644 index 0000000000..f73c8fe55f --- /dev/null +++ b/tests/unit/core/metrics/test_regression_metrics.py @@ -0,0 +1,37 @@ +import os + +import pandas as pd +import pytest + +from whylogs.core.metrics.regression_metrics import RegressionMetrics +from whylogs.proto import RegressionMetricsMessage + + + +TEST_DATA_PATH = os.path.abspath(os.path.join(os.path.realpath( + os.path.dirname(__file__)), os.pardir,os.pardir, os.pardir, os.pardir, "testdata")) + +def my_test(): + regmet= RegressionMetrics() + assert regmet.count == 0 + assert regmet.sum_diff==0.0 + assert regmet.sum2_diff ==0.0 + assert regmet.sum_abs_diff==0.0 + + +def test_load_parquet(): + mean_absolute_error=85.94534216005789 + mean_squared_error =11474.89611670205 + root_mean_squared_error =107.12094154133472 + + regmet=RegressionMetrics() + df= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-12.parquet"))) + regmet.add(df["predictions"].to_list(),df["targets"].to_list()) + + assert regmet.count==len(df["predictions"].to_list()) + assert regmet.mean_squared_error()==pytest.approx(mean_squared_error,0.01) + + assert regmet.mean_absolute_error() == pytest.approx(mean_absolute_error,0.01) + assert regmet.root_mean_squared_error() == pytest.approx(root_mean_squared_error,0.01) + + From 9f4cd1edd33e1a3dabac0f524c3b57c1644b5dea Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 19:51:19 -0700 Subject: [PATCH 02/23] =?UTF-8?q?=E2=9C=85=20add=20tests=20for=20loading?= =?UTF-8?q?=20java=20profiles=20and=20metric=20computation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements-dev.txt | 1 + src/whylogs/core/metrics/model_metrics.py | 23 +++++++++++----- .../core/metrics/regression_metrics.py | 26 +++++++++---------- src/whylogs/core/model_profile.py | 10 +++---- .../unit/core/test_datasetprofile_metrics.py | 20 ++++++++++++++ 5 files changed, 55 insertions(+), 25 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d036ffc6e8..dd6a14a31b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,6 +19,7 @@ colorama==0.4.4 coverage==5.3 cryptography==3.3.2 cycler==0.10.0 +scikit-learn==0.24.1 databricks-cli==0.14.1 decorator==4.4.2 distlib==0.3.1 diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 8e8f416adb..3b5b3586bd 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -1,6 +1,7 @@ from typing import List, Union from whylogs.core.metrics.confusion_matrix import ConfusionMatrix +from whylogs.core.metrics.regression_metrics import RegressionMetrics from whylogs.proto import ModelMetricsMessage @@ -12,17 +13,25 @@ class ModelMetrics: confusion_matrix (ConfusionMatrix): ConfusionMatrix which keeps it track of counts with NumberTracker """ - def __init__(self, confusion_matrix: ConfusionMatrix = None): - if confusion_matrix is None: - confusion_matrix = ConfusionMatrix() + def __init__(self, confusion_matrix: ConfusionMatrix = ConfusionMatrix(), + regression_metrics: RegressionMetrics = RegressionMetrics()): + # if confusion_matrix is None: + # confusion_matrix = ConfusionMatrix() self.confusion_matrix = confusion_matrix + # if regression_metrics is None: + # regression_metrics = RegressionMetrics() + self.regression_metrics = regression_metrics def to_protobuf(self, ) -> ModelMetricsMessage: - return ModelMetricsMessage(scoreMatrix=self.confusion_matrix.to_protobuf() if self.confusion_matrix else None) + return ModelMetricsMessage( + scoreMatrix=self.confusion_matrix.to_protobuf() if self.confusion_matrix else None, + regressionMetrics=self.regression_metrics.to_protobuf() if self.regression_metrics else None) @classmethod def from_protobuf(cls, message, ): - return ModelMetrics(confusion_matrix=ConfusionMatrix.from_protobuf(message.scoreMatrix)) + return ModelMetrics( + confusion_matrix=ConfusionMatrix.from_protobuf(message.scoreMatrix), + regression_metrics=RegressionMetrics.from_protobuf(message.regressionMetrics)) def compute_confusion_matrix(self, predictions: List[Union[str, int, bool]], targets: List[Union[str, int, bool]], @@ -66,4 +75,6 @@ def merge(self, other): return self if self.confusion_matrix is None: return other - return ModelMetrics(confusion_matrix=self.confusion_matrix.merge(other.confusion_matrix)) + return ModelMetrics( + confusion_matrix=self.confusion_matrix.merge(other.confusion_matrix), + regression_metrics=self.regression_metrics.merge(other.regression_metrics)) diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py index 626cfacacb..ca160199c4 100644 --- a/src/whylogs/core/metrics/regression_metrics.py +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -1,10 +1,10 @@ -from typing import List, Union +import math +from typing import List -import numpy as np +# import numpy as np from sklearn.utils.multiclass import type_of_target from whylogs.proto import RegressionMetricsMessage -from whylogs.core.statistics import NumberTracker SUPPORTED_TYPES = ("regression") @@ -39,8 +39,9 @@ def add(self, predictions: List[float], ValueError: incase missing validation or predictions """ tgt_type = type_of_target(targets) - if tgt_type not in ("regression"): - raise NotImplementedError("target type not supported yet") + print(tgt_type) + if tgt_type not in ("continuous"): + raise NotImplementedError(f"target type: {tgt_type} not supported for these metrics") if not isinstance(targets, list): targets = [targets] @@ -107,27 +108,24 @@ def to_protobuf(self, ): Returns: TYPE: Protobuf Message """ - + return RegressionMetricsMessage( prediction_field=self.prediction_field, target_field=self.target_field, count=self.count, sum_abs_diff=self.sum_abs_diff, sum_diff=self.sum_diff, - sum2_diff = self.sum2_diff) - - + sum2_diff=self.sum2_diff) @classmethod - def from_protobuf(cls, message: ScoreMatrixMessage, ): + def from_protobuf(cls, message: RegressionMetricsMessage, ): if message.ByteSize() == 0: return None reg_met = RegressionMetrics() reg_met.count = message.count - reg_met.sum_abs_diff=message.sum_abs_diff - reg_met.sum_diff= message.sum_diff - reg_met.sum2_diff =message.sum2_diff - + reg_met.sum_abs_diff = message.sum_abs_diff + reg_met.sum_diff = message.sum_diff + reg_met.sum2_diff = message.sum2_diff return reg_met diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index 0c49281592..ef980ffec4 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -1,12 +1,12 @@ from sklearn.utils.multiclass import type_of_target import numpy as np -from whylogs.proto import ModelProfileMessage +from whylogs.proto import ModelProfileMessage, ModelType from whylogs.core.metrics.model_metrics import ModelMetrics SUPPORTED_TYPES = ("binary", "multiclass") -MODEL_TYPES = ModelType +# MODEL_TYPES = ModelType class ModelProfile: @@ -24,7 +24,7 @@ class ModelProfile: def __init__(self, output_fields=None, metrics: ModelMetrics = None, - model_type: Mode): + model_type: ModelType = ModelType.UNKNOWN): super().__init__() if output_fields is None: @@ -33,8 +33,8 @@ def __init__(self, if metrics is None: metrics = ModelMetrics() self.metrics = metrics - self.model_type= MODEL_TYPES.UNKNOWN - + self.model_type = ModelType.UNKNOWN + def add_output_field(self, field: str): if field not in self.output_fields: self.output_fields.append(field) diff --git a/tests/unit/core/test_datasetprofile_metrics.py b/tests/unit/core/test_datasetprofile_metrics.py index a90d7c9c6a..328af1075e 100644 --- a/tests/unit/core/test_datasetprofile_metrics.py +++ b/tests/unit/core/test_datasetprofile_metrics.py @@ -1,4 +1,5 @@ import os +import pytest from whylogs.core import DatasetProfile from whylogs.core.model_profile import ModelProfile @@ -34,3 +35,22 @@ def test_read_java_protobuf(): assert len(confusion_M.labels) == 2 for idx, lbl in enumerate(confusion_M.labels): assert lbl == labels[idx] + + + +def test_parse_from_protobuf_with_regression(): + dir_path = os.path.dirname(os.path.realpath(__file__)) + prof= DatasetProfile.read_protobuf(os.path.join( + TEST_DATA_PATH, "metrics","regression_java.bin")) + assert prof.name == 'my-model-name' + assert prof.model_profile is not None + assert prof.model_profile.metrics is not None + confusion_M = prof.model_profile.metrics.confusion_matrix + regression_met= prof.model_profile.metrics.regression_metrics + assert regression_met is not None + assert confusion_M is None + # metrics + assert regression_met.count==89 + assert regression_met.sum_abs_diff==pytest.approx(7649.1, 0.1) + assert regression_met.sum_diff==pytest.approx(522.7, 0.1) + assert regression_met.sum2_diff==pytest.approx(1021265.7, 0.1) From fd37caa71d97e29218235ec130dbfdedf8e788b0 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:23:13 -0700 Subject: [PATCH 03/23] =?UTF-8?q?=E2=9C=A8=20track=20metrics=20from=20prof?= =?UTF-8?q?ile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/whylogs/core/datasetprofile.py | 4 ++-- src/whylogs/core/metrics/model_metrics.py | 15 +++++++++++-- .../core/metrics/regression_metrics.py | 3 --- src/whylogs/core/model_profile.py | 15 +++++++------ .../core/metrics/test_regression_metrics.py | 12 +++++++++++ .../unit/core/test_datasetprofile_metrics.py | 21 +++++++++++++++++++ 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/whylogs/core/datasetprofile.py b/src/whylogs/core/datasetprofile.py index d032307e51..c84385997b 100644 --- a/src/whylogs/core/datasetprofile.py +++ b/src/whylogs/core/datasetprofile.py @@ -214,8 +214,8 @@ def track_metrics(self, targets: List[Union[str, bool, float, int]], predictions """ if self.model_profile is None: self.model_profile = ModelProfile() - self.model_profile.compute_metrics(predictions, targets, - scores, target_field=target_field, + self.model_profile.compute_metrics(predictions=predictions, targets=targets, + scores=scores, target_field=target_field, prediction_field=prediction_field, score_field=score_field) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 3b5b3586bd..c229ac0800 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -33,8 +33,8 @@ def from_protobuf(cls, message, ): confusion_matrix=ConfusionMatrix.from_protobuf(message.scoreMatrix), regression_metrics=RegressionMetrics.from_protobuf(message.regressionMetrics)) - def compute_confusion_matrix(self, predictions: List[Union[str, int, bool]], - targets: List[Union[str, int, bool]], + def compute_confusion_matrix(self, predictions: List[Union[str, int, bool,float]], + targets: List[Union[str, int, bool,float]], scores: List[float] = None, target_field: str = None, prediction_field: str = None, @@ -63,6 +63,17 @@ def compute_confusion_matrix(self, predictions: List[Union[str, int, bool]], self.confusion_matrix = self.confusion_matrix.merge( confusion_matrix) + + def compute_regression_metrics(self, predictions: List[float], + targets: List[float], + target_field: str = None, + prediction_field: str = None): + regression_metrics= RegressionMetrics(target_field=target_field,prediction_field=prediction_field) + regression_metrics.add(predictions, targets) + self.regression_metrics=self.regression_metrics.merge(regression_metrics) + + + def merge(self, other): """ diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py index ca160199c4..06fd213dec 100644 --- a/src/whylogs/core/metrics/regression_metrics.py +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -39,7 +39,6 @@ def add(self, predictions: List[float], ValueError: incase missing validation or predictions """ tgt_type = type_of_target(targets) - print(tgt_type) if tgt_type not in ("continuous"): raise NotImplementedError(f"target type: {tgt_type} not supported for these metrics") @@ -86,8 +85,6 @@ def merge(self, other_reg_met): ConfusionMatrix: merged confusion_matrix """ # TODO: always return new objects - if other_reg_met is None: - return self if self.count == 0: return other_reg_met if other_reg_met.count == 0: diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index ef980ffec4..85d128ae8a 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -52,14 +52,14 @@ def compute_metrics(self, targets, Parameters ---------- targets : List - targets (or actuals) for validation + targets (or actuals) for validation, if these are floats it is assumed the model is a regression type model predictions : List predictions (or inferred values) scores : List, optional - associated scores for each prediction + associated scores for each prediction (for binary and multiclass problems) target_field : str, optional prediction_field : str, optional - score_field : str, optional + score_field : str, optional (for binary and multiclass problems) Raises @@ -68,13 +68,14 @@ def compute_metrics(self, targets, """ tgt_type = type_of_target(targets) - if tgt_type not in ("binary", "multiclass"): - raise NotImplementedError("target type not supported yet") + if tgt_type in ("continuous"): + self.model_type = ModelType.REGRESSION self.metrics.compute_regression_metrics(predictions=predictions, targets=targets, target_field=target_field, prediction_field=prediction_field) - else: + elif tgt_type in ("binary", "multiclass"): + self.model_type = ModelType.CLASSIFICATION # if score are not present set them to 1. if scores is None: @@ -89,6 +90,8 @@ def compute_metrics(self, targets, target_field=target_field, prediction_field=prediction_field, score_field=score_field) + else: + raise NotImplementedError(f"target type {tgt_type} not supported yet") def to_protobuf(self): return ModelProfileMessage(output_fields=self.output_fields, diff --git a/tests/unit/core/metrics/test_regression_metrics.py b/tests/unit/core/metrics/test_regression_metrics.py index f73c8fe55f..5ae7574a7c 100644 --- a/tests/unit/core/metrics/test_regression_metrics.py +++ b/tests/unit/core/metrics/test_regression_metrics.py @@ -34,4 +34,16 @@ def test_load_parquet(): assert regmet.mean_absolute_error() == pytest.approx(mean_absolute_error,0.01) assert regmet.root_mean_squared_error() == pytest.approx(root_mean_squared_error,0.01) + msg= regmet.to_protobuf() + new_regmet= RegressionMetrics.from_protobuf(msg) + assert regmet.count ==new_regmet.count + assert regmet.mean_squared_error() ==new_regmet.mean_squared_error() + assert regmet.root_mean_squared_error() ==new_regmet.root_mean_squared_error() + assert regmet.mean_absolute_error() ==new_regmet.mean_absolute_error() + +def test_empty_protobuf_should_return_none(): + empty_message = RegressionMetricsMessage() + assert RegressionMetrics.from_protobuf(empty_message) is None + + diff --git a/tests/unit/core/test_datasetprofile_metrics.py b/tests/unit/core/test_datasetprofile_metrics.py index 328af1075e..406130d5a2 100644 --- a/tests/unit/core/test_datasetprofile_metrics.py +++ b/tests/unit/core/test_datasetprofile_metrics.py @@ -1,6 +1,8 @@ import os import pytest + + from whylogs.core import DatasetProfile from whylogs.core.model_profile import ModelProfile @@ -54,3 +56,22 @@ def test_parse_from_protobuf_with_regression(): assert regression_met.sum_abs_diff==pytest.approx(7649.1, 0.1) assert regression_met.sum_diff==pytest.approx(522.7, 0.1) assert regression_met.sum2_diff==pytest.approx(1021265.7, 0.1) + + +def test_track_metrics(): + import pandas as pd + mean_absolute_error=85.94534216005789 + mean_squared_error =11474.89611670205 + root_mean_squared_error =107.12094154133472 + + x1 = DatasetProfile(name="test") + df= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-12.parquet"))) + x1.track_metrics(df["predictions"].to_list(),df["targets"].to_list()) + regression_metrics=x1.model_profile.metrics.regression_metrics + assert regression_metrics is not None + assert regression_metrics.count==len(df["predictions"].to_list()) + assert regression_metrics.mean_squared_error()==pytest.approx(mean_squared_error,0.01) + + assert regression_metrics.mean_absolute_error() == pytest.approx(mean_absolute_error,0.01) + assert regression_metrics.root_mean_squared_error() == pytest.approx(root_mean_squared_error,0.01) + From afca62a8b7c09285bc8c22d81c7672b9d38cf4ea Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:23:52 -0700 Subject: [PATCH 04/23] flake8 fixes --- src/whylogs/core/metrics/model_metrics.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index c229ac0800..73c55c0234 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -33,8 +33,8 @@ def from_protobuf(cls, message, ): confusion_matrix=ConfusionMatrix.from_protobuf(message.scoreMatrix), regression_metrics=RegressionMetrics.from_protobuf(message.regressionMetrics)) - def compute_confusion_matrix(self, predictions: List[Union[str, int, bool,float]], - targets: List[Union[str, int, bool,float]], + def compute_confusion_matrix(self, predictions: List[Union[str, int, bool, float]], + targets: List[Union[str, int, bool, float]], scores: List[float] = None, target_field: str = None, prediction_field: str = None, @@ -63,16 +63,13 @@ def compute_confusion_matrix(self, predictions: List[Union[str, int, bool,float] self.confusion_matrix = self.confusion_matrix.merge( confusion_matrix) - def compute_regression_metrics(self, predictions: List[float], - targets: List[float], - target_field: str = None, - prediction_field: str = None): - regression_metrics= RegressionMetrics(target_field=target_field,prediction_field=prediction_field) + targets: List[float], + target_field: str = None, + prediction_field: str = None): + regression_metrics = RegressionMetrics(target_field=target_field, prediction_field=prediction_field) regression_metrics.add(predictions, targets) - self.regression_metrics=self.regression_metrics.merge(regression_metrics) - - + self.regression_metrics = self.regression_metrics.merge(regression_metrics) def merge(self, other): """ From fb9b7a8f24c78ab39d5c9525944921b9271e0ce7 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:38:33 -0700 Subject: [PATCH 05/23] update proto --- proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto b/proto index b3de84139e..083464b1e5 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit b3de84139e5fbde1196c71797a62023153bbf8a0 +Subproject commit 083464b1e5fdc200b3118e8621b0e99346d045f9 From d7d49f254ae2f747bc0399f07acdc375d035e511 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:42:19 -0700 Subject: [PATCH 06/23] add model type --- src/whylogs/core/model_profile.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index 85d128ae8a..8c91fd6059 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -94,9 +94,9 @@ def compute_metrics(self, targets, raise NotImplementedError(f"target type {tgt_type} not supported yet") def to_protobuf(self): - return ModelProfileMessage(output_fields=self.output_fields, + return ModelProfileMessage(modeloutput_fields=self.output_fields, metrics=self.metrics.to_protobuf(), - ) + modelType=self.model_type) @classmethod def from_protobuf(cls, message: ModelProfileMessage): @@ -105,13 +105,19 @@ def from_protobuf(cls, message: ModelProfileMessage): for f in message.output_fields: output_fields.append(f) - return ModelProfile(output_fields=output_fields, + return ModelProfile(output_fields=output_fields,model_type=message.modelType, metrics=ModelMetrics.from_protobuf(message.metrics)) def merge(self, model_profile): if model_profile is None: return self + if model_profile.model_type != self.model_type: + model_type= Model.UNKNOWN + else: + model_type=self.model_type + output_fields = list( set(self.output_fields + model_profile.output_fields)) metrics = self.metrics.merge(model_profile.metrics) - return ModelProfile(output_fields=output_fields, metrics=metrics) + + return ModelProfile(output_fields=output_fields, metrics=metrics,model_type=model_type) From 9a30829819d63f6b7c216124933498528e54dfb0 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:43:45 -0700 Subject: [PATCH 07/23] add corner case --- src/whylogs/core/model_profile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index 8c91fd6059..ae5d27dfc9 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -111,7 +111,9 @@ def from_protobuf(cls, message: ModelProfileMessage): def merge(self, model_profile): if model_profile is None: return self - if model_profile.model_type != self.model_type: + if self.model_type is None or model_profile.model_type is None: + model_type= Model.UNKNOWN + elif model_profile.model_type != self.model_type: model_type= Model.UNKNOWN else: model_type=self.model_type From e9b7ee743a46c565ce8962397d85a6e01cbbc200 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:48:16 -0700 Subject: [PATCH 08/23] =?UTF-8?q?=F0=9F=93=9A=20minor=20doc=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/whylogs/core/metrics/model_metrics.py | 3 ++- src/whylogs/core/metrics/regression_metrics.py | 14 ++++++-------- src/whylogs/core/model_profile.py | 9 +++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 73c55c0234..44e67575b4 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -8,9 +8,10 @@ class ModelMetrics: """ Container class for various model-related metrics - + Attributes: confusion_matrix (ConfusionMatrix): ConfusionMatrix which keeps it track of counts with NumberTracker + regression_metrics (RegressionMetrics): Regression Metrics keeps track of a common regression metrics in case the targets are continous. """ def __init__(self, confusion_matrix: ConfusionMatrix = ConfusionMatrix(), diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py index 06fd213dec..fdb0df70e9 100644 --- a/src/whylogs/core/metrics/regression_metrics.py +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -1,7 +1,6 @@ import math from typing import List -# import numpy as np from sklearn.utils.multiclass import type_of_target from whylogs.proto import RegressionMetricsMessage @@ -30,12 +29,11 @@ def add(self, predictions: List[float], Function adds predictions and targets computation of regression metrics. Args: - predictions (List[Union[str, int, bool]]): - targets (List[Union[str, int, bool]]): + predictions (List[float]): + targets (List[float]): Raises: - NotImplementedError: in case targets do not fall into binary or - multiclass suport + NotImplementedError: in case targets do not fall into continuous support ValueError: incase missing validation or predictions """ tgt_type = type_of_target(targets) @@ -80,11 +78,11 @@ def merge(self, other_reg_met): Merge two seperate confusion matrix which may or may not overlap in labels. Args: - other_cm (Optional[ConfusionMatrix]): confusion_matrix to merge with self + other_reg_met : regression metrics to merge with self Returns: - ConfusionMatrix: merged confusion_matrix + RegressionMetrics: merged regression metrics """ - # TODO: always return new objects + if self.count == 0: return other_reg_met if other_reg_met.count == 0: diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index ae5d27dfc9..2ed10bfb34 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -6,19 +6,20 @@ SUPPORTED_TYPES = ("binary", "multiclass") -# MODEL_TYPES = ModelType class ModelProfile: """ Model Class for sketch metrics for model outputs - + Attributes ---------- - output_fields : list - list of fields that map to model outputs metrics : ModelMetrics the model metrics object + model_type : ModelType + Type of mode, CLASSIFICATION, REGRESSION, UNKNOWN, etc. + output_fields : list + list of fields that map to model outputs """ def __init__(self, From 9de055b86eec898d4aedcdbd6330c1259fc22e36 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:51:11 -0700 Subject: [PATCH 09/23] fix message attribute --- src/whylogs/core/model_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index 2ed10bfb34..630ae06440 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -95,7 +95,7 @@ def compute_metrics(self, targets, raise NotImplementedError(f"target type {tgt_type} not supported yet") def to_protobuf(self): - return ModelProfileMessage(modeloutput_fields=self.output_fields, + return ModelProfileMessage(output_fields=self.output_fields, metrics=self.metrics.to_protobuf(), modelType=self.model_type) From 10a9586a7f4ca05632c0b7f852f0540057edd28d Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 20:55:24 -0700 Subject: [PATCH 10/23] type of Model type --- src/whylogs/core/model_profile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index 630ae06440..f840358020 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -113,9 +113,9 @@ def merge(self, model_profile): if model_profile is None: return self if self.model_type is None or model_profile.model_type is None: - model_type= Model.UNKNOWN + model_type= ModelType.UNKNOWN elif model_profile.model_type != self.model_type: - model_type= Model.UNKNOWN + model_type= ModelType.UNKNOWN else: model_type=self.model_type From d48d4a4e86a62dabb0916d4cef9c34332b79fa1a Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:01:00 -0700 Subject: [PATCH 11/23] minor tests --- src/whylogs/core/metrics/model_metrics.py | 2 +- src/whylogs/core/metrics/regression_metrics.py | 2 +- src/whylogs/core/model_profile.py | 13 ++++++------- tests/unit/core/test_model_profile.py | 3 ++- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 44e67575b4..6625046e99 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -8,7 +8,7 @@ class ModelMetrics: """ Container class for various model-related metrics - + Attributes: confusion_matrix (ConfusionMatrix): ConfusionMatrix which keeps it track of counts with NumberTracker regression_metrics (RegressionMetrics): Regression Metrics keeps track of a common regression metrics in case the targets are continous. diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py index fdb0df70e9..edc80bb147 100644 --- a/src/whylogs/core/metrics/regression_metrics.py +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -82,7 +82,7 @@ def merge(self, other_reg_met): Returns: RegressionMetrics: merged regression metrics """ - + if self.count == 0: return other_reg_met if other_reg_met.count == 0: diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index f840358020..f6c6e22193 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -7,11 +7,10 @@ SUPPORTED_TYPES = ("binary", "multiclass") - class ModelProfile: """ Model Class for sketch metrics for model outputs - + Attributes ---------- metrics : ModelMetrics @@ -106,21 +105,21 @@ def from_protobuf(cls, message: ModelProfileMessage): for f in message.output_fields: output_fields.append(f) - return ModelProfile(output_fields=output_fields,model_type=message.modelType, + return ModelProfile(output_fields=output_fields, model_type=message.modelType, metrics=ModelMetrics.from_protobuf(message.metrics)) def merge(self, model_profile): if model_profile is None: return self if self.model_type is None or model_profile.model_type is None: - model_type= ModelType.UNKNOWN + model_type = ModelType.UNKNOWN elif model_profile.model_type != self.model_type: - model_type= ModelType.UNKNOWN + model_type = ModelType.UNKNOWN else: - model_type=self.model_type + model_type = self.model_type output_fields = list( set(self.output_fields + model_profile.output_fields)) metrics = self.metrics.merge(model_profile.metrics) - return ModelProfile(output_fields=output_fields, metrics=metrics,model_type=model_type) + return ModelProfile(output_fields=output_fields, metrics=metrics, model_type=model_type) diff --git a/tests/unit/core/test_model_profile.py b/tests/unit/core/test_model_profile.py index 3f12fa565f..dabd7af749 100644 --- a/tests/unit/core/test_model_profile.py +++ b/tests/unit/core/test_model_profile.py @@ -39,6 +39,7 @@ def test_merge_profile(): mod_prof = ModelProfile() assert mod_prof.output_fields == [] + mod_prof.add_output_fields("predictions") mod_prof.compute_metrics(predictions_1, targets_1, scores_1) assert mod_prof.metrics is not None @@ -48,7 +49,7 @@ def test_merge_profile(): mod_prof_3 = mod_prof.merge(mod_prof_2) mod_prof_3.metrics.confusion_matrix - + assert mod_prof_3.output_fields == ["predictions"] def test_roundtrip_serialization(): original = ModelProfile(output_fields=["test"]) From ba2cd7cfe1b4f1a39caeff24f16491f222240a62 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:11:16 -0700 Subject: [PATCH 12/23] fix model type usage in wrong hiearchy --- src/whylogs/core/metrics/model_metrics.py | 25 +++++++++++++++++------ src/whylogs/core/model_profile.py | 22 +++++++------------- tests/unit/core/test_model_profile.py | 2 +- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 6625046e99..effdaf7afb 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -2,7 +2,7 @@ from whylogs.core.metrics.confusion_matrix import ConfusionMatrix from whylogs.core.metrics.regression_metrics import RegressionMetrics -from whylogs.proto import ModelMetricsMessage +from whylogs.proto import ModelMetricsMessage, ModelType class ModelMetrics: @@ -11,28 +11,32 @@ class ModelMetrics: Attributes: confusion_matrix (ConfusionMatrix): ConfusionMatrix which keeps it track of counts with NumberTracker - regression_metrics (RegressionMetrics): Regression Metrics keeps track of a common regression metrics in case the targets are continous. + regression_metrics (RegressionMetrics): Regression Metrics keeps track of a common regression metrics in case the targets are continous. """ def __init__(self, confusion_matrix: ConfusionMatrix = ConfusionMatrix(), - regression_metrics: RegressionMetrics = RegressionMetrics()): + regression_metrics: RegressionMetrics = RegressionMetrics(), + model_type: ModelType = ModelType.UNKNOWN): # if confusion_matrix is None: # confusion_matrix = ConfusionMatrix() self.confusion_matrix = confusion_matrix # if regression_metrics is None: # regression_metrics = RegressionMetrics() self.regression_metrics = regression_metrics + self.model_type = ModelType.UNKNOWN def to_protobuf(self, ) -> ModelMetricsMessage: return ModelMetricsMessage( scoreMatrix=self.confusion_matrix.to_protobuf() if self.confusion_matrix else None, - regressionMetrics=self.regression_metrics.to_protobuf() if self.regression_metrics else None) + regressionMetrics=self.regression_metrics.to_protobuf() if self.regression_metrics else None, + modelType=self.model_type) @classmethod def from_protobuf(cls, message, ): return ModelMetrics( confusion_matrix=ConfusionMatrix.from_protobuf(message.scoreMatrix), - regression_metrics=RegressionMetrics.from_protobuf(message.regressionMetrics)) + regression_metrics=RegressionMetrics.from_protobuf(message.regressionMetrics), + model_type=message.modelType) def compute_confusion_matrix(self, predictions: List[Union[str, int, bool, float]], targets: List[Union[str, int, bool, float]], @@ -84,6 +88,15 @@ def merge(self, other): return self if self.confusion_matrix is None: return other + + if self.model_type is None or other.model_type is None: + model_type = ModelType.UNKNOWN + elif other.model_type != self.model_type: + model_type = ModelType.UNKNOWN + else: + model_type = self.model_type + return ModelMetrics( confusion_matrix=self.confusion_matrix.merge(other.confusion_matrix), - regression_metrics=self.regression_metrics.merge(other.regression_metrics)) + regression_metrics=self.regression_metrics.merge(other.regression_metrics), + model_type= model_type) diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index f6c6e22193..a9c2c95e9b 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -23,8 +23,7 @@ class ModelProfile: def __init__(self, output_fields=None, - metrics: ModelMetrics = None, - model_type: ModelType = ModelType.UNKNOWN): + metrics: ModelMetrics = None): super().__init__() if output_fields is None: @@ -33,7 +32,6 @@ def __init__(self, if metrics is None: metrics = ModelMetrics() self.metrics = metrics - self.model_type = ModelType.UNKNOWN def add_output_field(self, field: str): if field not in self.output_fields: @@ -69,13 +67,14 @@ def compute_metrics(self, targets, """ tgt_type = type_of_target(targets) if tgt_type in ("continuous"): - self.model_type = ModelType.REGRESSION + self.metrics.compute_regression_metrics(predictions=predictions, targets=targets, target_field=target_field, prediction_field=prediction_field) + self.metrics.model_type = ModelType.REGRESSION elif tgt_type in ("binary", "multiclass"): - self.model_type = ModelType.CLASSIFICATION + self.metrics.model_type = ModelType.CLASSIFICATION # if score are not present set them to 1. if scores is None: @@ -95,8 +94,7 @@ def compute_metrics(self, targets, def to_protobuf(self): return ModelProfileMessage(output_fields=self.output_fields, - metrics=self.metrics.to_protobuf(), - modelType=self.model_type) + metrics=self.metrics.to_protobuf()) @classmethod def from_protobuf(cls, message: ModelProfileMessage): @@ -105,21 +103,15 @@ def from_protobuf(cls, message: ModelProfileMessage): for f in message.output_fields: output_fields.append(f) - return ModelProfile(output_fields=output_fields, model_type=message.modelType, + return ModelProfile(output_fields=output_fields, metrics=ModelMetrics.from_protobuf(message.metrics)) def merge(self, model_profile): if model_profile is None: return self - if self.model_type is None or model_profile.model_type is None: - model_type = ModelType.UNKNOWN - elif model_profile.model_type != self.model_type: - model_type = ModelType.UNKNOWN - else: - model_type = self.model_type output_fields = list( set(self.output_fields + model_profile.output_fields)) metrics = self.metrics.merge(model_profile.metrics) - return ModelProfile(output_fields=output_fields, metrics=metrics, model_type=model_type) + return ModelProfile(output_fields=output_fields, metrics=metrics) diff --git a/tests/unit/core/test_model_profile.py b/tests/unit/core/test_model_profile.py index dabd7af749..e1ad03b159 100644 --- a/tests/unit/core/test_model_profile.py +++ b/tests/unit/core/test_model_profile.py @@ -39,7 +39,7 @@ def test_merge_profile(): mod_prof = ModelProfile() assert mod_prof.output_fields == [] - mod_prof.add_output_fields("predictions") + mod_prof.add_output_field("predictions") mod_prof.compute_metrics(predictions_1, targets_1, scores_1) assert mod_prof.metrics is not None From 400f739c88fbd28430f9509a03b8862dac9c550f Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:19:57 -0700 Subject: [PATCH 13/23] test merging of metrics --- .../core/metrics/regression_metrics.py | 8 ------ .../core/metrics/test_regression_metrics.py | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py index edc80bb147..bdf7edb8c2 100644 --- a/src/whylogs/core/metrics/regression_metrics.py +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -40,14 +40,6 @@ def add(self, predictions: List[float], if tgt_type not in ("continuous"): raise NotImplementedError(f"target type: {tgt_type} not supported for these metrics") - if not isinstance(targets, list): - targets = [targets] - if not isinstance(predictions, list): - predictions = [predictions] - - if len(targets) != len(predictions): - raise ValueError( - "both targets and predictions need to have the same length") # need to vectorize this for idx, target in enumerate(targets): diff --git a/tests/unit/core/metrics/test_regression_metrics.py b/tests/unit/core/metrics/test_regression_metrics.py index 5ae7574a7c..d0474625d4 100644 --- a/tests/unit/core/metrics/test_regression_metrics.py +++ b/tests/unit/core/metrics/test_regression_metrics.py @@ -46,4 +46,29 @@ def test_empty_protobuf_should_return_none(): assert RegressionMetrics.from_protobuf(empty_message) is None +def test_merging(): + regmet_sum=RegressionMetrics() + + regmet=RegressionMetrics() + df= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-12.parquet"))) + regmet.add(df["predictions"].to_list(),df["targets"].to_list()) + regmet_sum.add(df["predictions"].to_list(),df["targets"].to_list()) + + + regmet_2=RegressionMetrics() + df_2= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-13.parquet"))) + regmet_2.add(df_2["predictions"].to_list(),df_2["targets"].to_list()) + regmet_sum.add(df_2["predictions"].to_list(),df_2["targets"].to_list()) + + + merged_reg_metr= regmet.merge(regmet_2) + + assert merged_reg_metr.count == regmet_sum.count + assert merged_reg_metr.mean_squared_error() == regmet_sum.mean_squared_error() + assert merged_reg_metr.root_mean_squared_error() == regmet_sum.root_mean_squared_error() + assert merged_reg_metr.mean_absolute_error() == pytest.approx(regmet_sum.mean_absolute_error(),0.001) + + + + From e8bc612c6cde34f60907ef8c35cdb32271668b50 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:20:54 -0700 Subject: [PATCH 14/23] added another parquet test --- testdata/metrics/2021-02-13.parquet | Bin 0 -> 38359 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 testdata/metrics/2021-02-13.parquet diff --git a/testdata/metrics/2021-02-13.parquet b/testdata/metrics/2021-02-13.parquet new file mode 100644 index 0000000000000000000000000000000000000000..717e42024634a06fcd541cbf8852395c529ae22c GIT binary patch literal 38359 zcmc$`d00*F_y6CdK^iroP?}UC8c144kwR#qLWpE2l$lBy5)mboMDrk}(mbuxq%_a- zc$zzn=X9Ef-|_u^{C@Ax#p}A>zrSAf$L?BP=j^raz3<1q@3o$5AK7iFB*-G@%q#eu z_n@FNlOPK}1A{aB#1gYgTX|W}-FJ9#>9xVV>3U2kJbo|KbrN$Q#@DW9|kt} zcLG>&3_YIihYM>DBvi;%;)Nj%d$-CTxFffB|D~d7)ca}iZOnNNE;77uvRFfcr2U&? zoOmkWg8k>T=I0cYU@HKzvoyHm@ycIbr46VPhUvnAltYzhrH|uUkN##HZiozty?p(fmGI|;{H=NWJoE?YS&bXrI{)2c# zDq5vZHW#$EEI&)CoPnQ=GyY3zy;yyBa2d&T0Um}w+izgqhn#HhT=({p@$u==H<9WU zsBXHyx{%g^0^8R_?F;L~8*ffM+?PBBJ5OGEMUJe7rvcBi_Td-&nsbj)_tGG4Y`(f) zWj6!y?DwBd04 zE8@&~{cf1OybY~v2T`tA?A>kC3Ov`CII0^^2d!7Dex&EW!HJ7&V!3mxpnhMG22HF1 zgN?tOoNb+k6^bwWLnH=aAfezBzMX`b8{d?LVu-lWZRK6&jf*fZXYokmLKhglkmL>V zo5iPXS2F#17>LK}thekArNV&I>Xu;6HoTo5d5X4$k+9*B|NY~4#^G+>5cV(VUJ5hDfCns?ta+HlVhK@AoBmB`x0PEHezZo0f?QS$AOlq1Gdki=)7w zUdqe+fr((wW9f12BMn`S>@elsK7=j8mTQc37tzsjnI=G?2dkOdl#2 z&6@K&cK~tp>NmB78T7JoXxP9tiAOC2x24RFpp3Yb@_zdL&cPp$zMw#Y;jeF-4sR+z z1)s>Iik~!itQx0Hwk*YJD;MmN^oJ1JeZa}@S0|4@HJbq3;`LniN(_M`A=j$rn% zUW~u(wI%i35^!0h9K9-DjLAZ3k7yZG*n;f%wW zs_e}w{O&j*ygz6j%QS>rH5jSL&AhMn^K=1vlCCQ4Fe}Hc+P3V;k4G@_Kv?-W!z3mQ z`$jm0Oki1GZ;rxj_c==0)y?!X<*fY%rH$oU?M@V=CPDb*s@ z$3`cr@k+|aD+S~kCt|=gU4_-J3PbFF+Ngi7h`6yo+WcF?*IUKI_pl@O&Ot@L^8SQ- zrB~A1$5I>)eQO?_&TuLpom-Orx>8hd`6?9_2G&&~?B}h{+_-6PwMCFg$eBsVd!T&TUbsL0X=0Oi7jhn%bnK07MY&g>qhGA)z;qAK+=$FB z$h=P7X4+c#E$GXNHszY_P| z+<`8|D?^X%>W7Czr&pYnBZ2-kfencR<9KPg>}D>3UW_};gzs2Kfq8%ZB=hbjsIE#o zYMk7KF)KN1?s`%oO3ca>I10f`{pBXNQ8H91emr~b`4T=zRIfGm=|$4|TCt(TZfKuz z=#R0 z-5lm4ulrGOXI_M~N)L9wdu(6nng@nI8!|`IT;V4;WoE>7!aa(&o#NFAJX_YWg~cQY z_Ve(tW!9#{Qw^Iz`KRq5wdO+gS94~<2fyWhKWC@lMcP=SyZHpn4;-@8_MgEf&3E3q zCs+t!)@|-8JS}L{(d1^SPlBp_QZcV8$eVa>$}kf2Ma#Lh5bL zDc+G#zBVG7`2kz19v7a zk7R0RB3w=Li_>tZ#p-Y$i>0w%2vR)iE2>AouC2@-s6GWZXg}}mBJeoW@B7d8L1{@c{_k^PgJO8-gC04eDS z0XjDSbp0O!`fstBAFHb59ytitwo3K)dz0Z6yG;H4(H_)&^pe@aq!Q~7JZXKv)(m@N zUe}A&I@D!up zz6ra}qMfkQ^vR=``!qZ?p3}bL-WYQ5wleCl6=P;#{L7cM**M-Xz$v|>7mpQQJEyNb zft)XY@;C!%eeVS;#jXA+PGmg#b4|C| ziVqktB=FMfVJlwk!T?sDhiJk;l@H46h8GzP^$yAHpdZ6|Nt7oQ!z`|?yz+1ixyVCT ztDpD7%{2|jT{ltT1M$rpotq2HR>W^}Z*6gzvgPc5P$t zLqjM2%ZolVRF&-gES5<|<{IJ07gT3K-Q7PhVI2+knVY1U#gF0bL8VJtQ|Yp`?8N#?$xYBY>k>E;twgMq%Hc0WhjP$Rk}%BH>wFRfVHqM1QLr#BkO zwA&;2z|s95i{i|`$@rshP_??*vRaah!tU?;xM3t@T~7YFhtn^Ty~IU3Q} z7v2^1ec!lj8n)=A8lBOJWfgdN`PXwm>Hty82Klig?O06_ zI@b7f4*t5>{C%^Cu>ZdNzbNyk_20&(>#jesx%EG>>7g!7+DOOdpRWHyK>sZ^i{y&K zM&@gw?fpRNjXgE!rWpD5mO?vFZjfthl5275h2G%uu3>0g8J;o`@)Nx%rzWlVhv3Ai zua}keJ79S8qkY){&5*8GCoN?*0_>iQ8%gdAM3Dlc+TCaC!068Mfzew;2n^Jk`TA)Y zVH-o|wRJiTz^HV($8Dq&NmEiUDl@x5-^%)j;`27lUNyaU--dpa+9*8PFgS(rxjEIc zjTB5hNa0F2+=ts6H!tg;E<&XL;lxwhhoPaLJ1JD<9U{+2kqvz>nMmW_w>gA_v1`5b zS<@QeA8TN;}+IYAO0-N!ME`r|#c04_2 znZdgNeI*VQE!|>l48O(d?$`!`2E0w7tS!h@a4b$VtP3NyJpMQ@6N|TBKG~|nKL@!fC8g|=Ls&DGUed7GgcsB2 zM`qhwV28irGB#Z*RD`b;;?*1gJ)IW4>(n9ao%zCN)yYVZZggAqXtD!Y6LKaSFHJ(Z zCRO;V9}P<1Bq-FLuEDy>*@MOTX{eEucw=_B6Kt%>MtvV=!1$8ei`POQ zAm=SHElzrhw&91HYCllbHC@vOsouAI_eG zIVt519M`Yyx$9hq)05)F`LS9|lT;|w>nR5AEUBa^6DsaCxTeEdGJ?LP%A=wj0r2|W zDl3ENaSRLyUtP^xjZc|YHksFZLZ(&j&qSSW*ln|lM^}S_M?G~nEqhUgkI8P9OeZN| zZyi3*!qNaio-_O-*BgOv%$ff*eb z13c}rqE>`b@R0PoCuMU>xRS7fWG_ArW^MJ>xijNfyX!#yahG1aJ5#`7E;EdoRxdvL zHI8Ayj?ImUCuwkx7fxoXde-F9{`|s)e zi!y&&|7~ne82pLNZU2eQ`9snbnsjXb>H0qe^jB>13JM$){J(ikc2RoF|L2GPyb%&y z{l8KhzyBq|{y%>?|2oT`R=R~#_J1-PjLu99(vMC4j6?bV#*xJ+^n+1)#7H{Jh#tqE z-}Mh?_-h=0Nj6AZACY;7L)gdg+C!Y92b4Ps$OfbpI(nF(;V$Bpdn;(2guDu>V4iEBss|}&;<-_Mv*;8>MO|>EQ zR6g|bod5B7tOV=zi@wF!#^TXU?gC%eSAyNwhL}m)G#n`1vSc@0h08nV@OTXwf;e?D zPpD3V3}x|=c-1(jF@`ALVD3bo%T?dDi=~6iyPr`RS6U(6IKXL}2L)DJitioTMBfB1 zeUdr1A{R8In7{XIDMRLk$divPX5q}Kqtslsa>xuhKVuo!5AC1#OUn6-0XuEudCAx9 z7@I;qR=2ko=6Z?_HK*5#n=qWaq5Z-$Mb>c z(bD?uZ38;QNa)7drr@oNd)u=dt>^?~QSw7R=W4nz9fZG+sN1 z2j?Soi~Z~!&oNla$VXu8#uc%xwvwV#_@cqwFST<5U-uea&&z+0Sw8O0vGmmJW6NHF zj@!fNV*BmlhnWtD(A&-JZmmOF0FNqy_Ub&<_%FGFT9_xs~5od)A@j@9T5S-Teln6$;7*)tO6O9FZ++ej#% z&dj;asO|6g|2^0u?7xTa7iIpm(rNUsrf4Y_|HP)sf8u}kymZ-7IyV1w{T~AQZ?UPs zZ}csC)pt0htNeVN&;enUWo+&K6|jB$$U}A39(Z;~sQBetGHe(0J>Xzo3HyVMIgQp) za6r76-PO1gp2+Tsvlsh;=~iwF9|C8gyiVv&@K_h@5J?ueq*4npCv5L*)*6S!yBz8I zK4aK8f_25WMgQE;8*_Cm_uE+D_>{2CTniN`rAbjXD{=y>xx zM^N|)l%8gEFDmaw-)FR>SZX+&xnp7{bF2^QejXuRqfLOIx@F9qH5qu8oR|08&B7fr zbX)G4#QA+K{QNvjL=ww(o>0RM$iF9M5M$PkuC+RReO-~DTz^rQ@?;!E10PkU>D1z0 z;$(Tr^FB~2(`h-+L9ZyZE3D#R?LbFehArn-Q1EB1)3dYsRq*JW=Ocm5LpWqaiZO4i z1L@mg>Xp(}Sf9qnlj+}tN4AA=YMdR#qeSD{5yvL5b1p2+uPKCBgRM2U-;H5%kv=hX zdIU&yNkjNDA0kugJl8y+0e`gEbd*&qHk$(X+BZX(xJK*3H|9~CwB}?hH6Ot)$_xjw zzXn{T8$;H+Q_=fmfXla!4)EojuBBP$JSND7Gt04#0hdAG6}DA^w4N7?|KTTR|Ls1}VkcFTpEZW#dT{?;4)MYSk-Jd7o!o(B3t1?M-7(ja85 zpO1aPJW^7jMEhPA`l!#QGt&yt=`z7l>p(vk#tV-tstrMnv3Sgw$pVBpe4IPUGLEuy zv5%a)hHy7Wy38G&Svbs`sVudU0w+p#FBJ;Zz=EH#lb1y&u&yv_5BXjWq+#X!yD~H= z=&=gS9cuxf<);rj@im~tJqf>`S^3D=Aub)DPQ~?C`r`K0Gy|1?H~Ynh6nqaFep_BN z!~5b>eHRN$an;wEn1iw7_)2){VXOfOjM(SI!aj81^NURclg<)Ir7yE(Iv0R9O|3DE zzB_;>sS`7`3(#s?BzkmbC6JD%mzRZiVdNPe35$qq7??g_AZS~G$GnvS`F2vkRju;a znUBd(J74SIWN5Zd?tL@47h%xb5pET#?N zG*|k`__lfoy_7V0nr#@CZ9Q2=c+`w8G6F+Z%p|P;`PQH&kNWpy!|y>7VgEgSeo^L6 z>%Wc74yQk{srsMT?7uIaahZCV;WNA!oBaW_mtPo6%z-wyj$jjQQ3LI>tirD@4mszJUTk^?0 zVqP7XAAiMa9ZN#P83VJF9184RbXK=~T@6*DMWQmo1Gv6vPx$Z^P(x0m17lmXq!SKQ<&G~AT3W>83Y3@xkcZ?3;gMQsjI&pw9{ka&B}kUBAf z`Q@7vw;iS78>{`Q9}B1;ViP?R(ANt?6I(wy`*z{MCwUTi*O&<_UvnSfywn874_=Z= zf-7Og#s0~)ITd*G>6XS-oukNmTVQm*6a%qO-F$>JHi7qBVxs~bdf_6g_U_}$r||so zLl#Gl^dQISbb;683A}xfqRQAch!mU4%IafjaEziW6;m+@3)gt>AJFKBXUlpdMp`@2 zQh)Le%SbF7&1(H(RgTbbnD0)MUMHNoYid%_-iS=|Bct+;(?H%c`@}$|7w*2=6_~qc z1~;YkoV-Xc<(!ndzdx(B9qL`2Hf~y#gnke5J;Hh`G5En$3FFg26nSFy^8=w3xE|ZB zQSGb5_k3=ft3M4usA*8(X3uui5Zx$LW1WW@&o6A%T?)F9J13PrV~nSLF| zYa41NIkhRcTE!{X1k-_dvf(8U6BF^JyWeh}ZGcHKW7pVlTGBl8dgw5#v5ED?a$eX~iri2TyQJsPtk&TUh9| zi&Rj(FnRvkNEe(`l4qtSOhV^nHoeCe==aISm4<0rB)rkOj?%|H491#^H~knZAUMWC zj?t+er|);J5>}_+62s3uK|A|#oS7-gSbXsB*!+Dni?IKm#=j`@r}f{)X4xCYKe?&) zpWLi_%_#lFnUPM;KfV8lnEqRG?)K%?3gOIxr?iqQW8eB<(N{Hfvu6`(iEdwOXWWO^ zd8M=r4O&sHh3WJ0WrO(OQSRdnb3Mp?G*V(JuN*wZxNp6-=tQ$0N{bXXUufr5t=~{a z#!Z{<<;#lCLCMB|;fd%Dxa1$oA%2(yBKF&5xVF&LNeEiM|W#M0RJu4U|% zs4!gn)$l$U`Sv?8o%I-pvBj3gDfce$a3u=HY1ZJ}+esodj$R$qc5XPwHGs2?UXxyQ zQG(i~rlpe8y&xE8d2Ga$fe;s>l2upJkCx*{XgPbP!1l`V88^2MaI8&L`o36&c5|j( z+y=dHi?c@Lyv#H*B(Nt@&vt;EfO4-LOo41p?EQt0)sW+LQDJ7b5abzBu5Gz91nS4- zwU+ZwBBf3}IdjD@SThX9MSEE^g;b*ro5xHE_QoY)Gri;K|Egx^+g1sOBW z6c87l&O%a^pHI?V8j8d#+Q&yR6FXGpo8Nw_LDBa~T~>R0(c`GORe#?Y>>c8OauYHn zT4hP9aMT0mf_H=LhAzn0DQJ|C=>tE~D}}8}^e50`znSkMtRZn2dt+YmM9R^<1^|X!aO9U7T;WEbWDF{gKhj z<}-lnxQmf9e=5G8+rb-77cI28-Cts3nZ<;Ml_58Fb>QdaXOF!gFcHc*Ji=1#n(_S> z-<^hOb)aF!+1?acfhG>$M>8&r<5}zJGd7p|Fe-UM#agZt$F5nr=XP|Wp$gNXQ&D3m zmmOKMc9w<&-FurWFI3qz7^yIj1*+Vj%mdd~hnPF1J5$ueS^hS{6K=ZskHLd4;!L z^gPOSb%ZgckArW;mi8(Q7NWp(^Q$#l1MuAMo%m-jA9%*gEp>e(1xU$hYP-USpscD* z3!KP-u=Ug19kJ9U(l^)c6JsN^Fu$2KcIv>(tiA`gkh=B=df|(R?XDMXZm_OaUybuV3*l6< zmrzvo7+(9t7Sur@BbVeI0VjF}bzN~ytYF^~I3|R+%_^4TY)#0k%gljlufS zhxYcHn&DVolTd%g91i#W$ldvak!WBbE%)4^2gJ6o5}P&RCOlGQvrVoVgU2&3pQd_o z5S2QD`hM)CXGR+gR*!w1hAep-@nLH^f!*|50!6y<3ghDpvu9I~c1TZcYM}>~r_AX% zI@9aXd>;os7O)fL1b5^0cUAB(-Y$|SlZFR%rnZj;c3^klj^V>`6l5?+GQAc)4kyjh z=Us$&2=Pf>jjk7GkYp{^I;q7&;AZi*_syfg92q5!2&Tb{OLy%1)CREN21j0@R1e%j!om? zw!||$gbB&b(pQ~0h!zYf8f}97#Bi};tG8A(Y|WGGuIynZ3fi<~MGAIG?Rjn@ z!DcLYgup;pN~}tKy4wrQTrSKC&%Z=$chCCSK;;zH1wB&6` zH&_TB8T#78tC)#9zs+8cUBgan6kxT}44A^ivFlYk7hB=;+}AL<&BL(iSW52KmGfvt zTP~=e#Y_k^)e;g~HuLv2=kLi(UvvJ?wD?7tKdt{ZIg>K~Le3rkNzSY^Mrp4vzsUKg z_x}*ne@jlim2%VRmSj-K{j&O`S3Bx`t!1^eaPn~I@ioia zu(Uxmr|0Vg=$gCpo!-$6Av+e(#Ak2&(MLQ;7;IP%~n7Am5k<^@17kp zo50=N$$~%b=VG-t z$~_)v2knlqHvyN*;n+x<{I!Z%u=CNfi(Wa3Z~fL&E%)}LY0wU;@9IEYd+*Ru(`_|C zyR6F)F*yBp8=W$LmekdaYdZ=$EG4jw1_6b5hp z23h-;T^Eq3g?m(+`>d}AaA{MLm)6EA94-z#>yp`nS0lDn_BW2eSkCm7s_{DXkzZJv zUC%;@3cG8@J6aE7T}QW7B!xiNsMb!q1C{7au1%S`RgYd$t6Gl8RRFKc+5KXoLvU`v zb&Cg;g?Ot{3l-yQAjAH2Pw>$kyvbipDJ!eOasPKhr0E`DAyJ!CUiE@7c04QFjg4njzN%}@ z!;W%`1)1<3s6F~lWLT#ey&8hnRlFlY>(;DE*ef`{FrcE{{-IZF|LcWp<)ta@mncH2IAyS7q1;}d!T|M)&UQe(D>}J^-;8H zOt>~UH?LI$E9a{yz#m`EDD8^B(D|qP{}9xFi_U1m zEwS6t6Ih#RQ*i%BC`RA3U+>J=f-xzNBB~kb==-29vxhv5>lRm2n;IsdD)iQ^E6i~? zHNqFe`ke-S>r{>lJsSnxJqnDitWh?lPkWUJloz;B(*aTit%;<>{)tSTX&z};Tq_33ZZu)3!9S!d4x*hFPc zyLdNX$JpENQ56hCu5#MqX_H#)ncUalAJh%o1zdM?yVKQ2GCQNJH!={NDt6_pQz=1< z#kHEHwxc+1rzBfO7uDK#p1a$ypJG-`l`wzKO<_S1*;C45hM21qIvG=oDz5Xrb1l8L*`~Dj%<5% z*z;%yZr2i4co;_aPc1$)zLJUH%Dh=9n6(%Srt4lmc}s&MpYORH2%Z22b+^agwojn2 zSXlh1#sGL8jEUrTA4auwm2bBi#^J{Z_o$#*9G-l5ZD3FLC`OAkU*n?J&n&o#O&Zo# z03R#gL!!eZW;0DjJhY?1R&|O6-?v%3z%8G)-iM5l=VaYY^2Wd?TgR2Td;&(-$#~xC z7yz;P(aJH`S;S|Vs~yvNa9rNuK@S~e2g@#qt7R{u;CQL3X@5SjKFB6ol3S2K^SNH~ ztQq{>T)%F*N>_cUCrTy1uR_fdhi2P7L#UH*(bx9aBuLemN?cdV#*@wISA7$zAx88i z^UKI~>>4Vl9&D(D!+Cnfx11OWiaN-r_+k!UK51b+^P(FKG;3|h`Mq#kIo&8mss;rtmYq zq89mA!y1V*CkjPJ;BM*krPT0BJmT>&%HdlX4tIpJKTRA%gRRfs92~DfInyx3?D|pQ z>W$znwfY7!=PTk|^yeUTr+=;1{1h0R6#K@1kG>D5Fp2-LuEK51B5N}}`amb_0yUSz z3gmN3J-FBQBin!hzW+=Cwjk96Vg6y%y6+_%L|4SDcQv=bf*PyNasgzL_v|a)6H~`T-@(?!IV8|2=dnUjKnsVK<*i8S>`D!zR_#Gu819QXDZ3O zYi1V9leT4*>2{+2M70WoCS9Sk$D)xjv>ru55Gn*{xZ^OTC;C7Q%x>C1fHp!!UIkl9tOUq;5Xq!?Ob9%9NJ0>koM{Uu5(G-Ruoo)2+8v3JH7^J zHK~2(NUve31zEQYhV~&}#@!+bw`O48`s8HErV6m+5=*};8V`oN+T$+Y1Mo)CH_1vp zdd67Q*fRT49mWoC`#8G02zK!qaw>UK;i5&v7MrkPR0?^@7ks-D@iV8lj@|^stn>Hn z(yM@o!}@m`@h6--PEF{_9fF8Qb~^cE>w_;y)@1Bs0(ZYeCsmpY4}m! zVxPu=I(#r;K+!by2VvyYZ`@H1QO!$RZU-%b$EC&xi4+!sd!}Uk-Qr&K{Vb-JF7^{X z#%?vR(CozSgR48Z#WO+NuGD;R?*z)UOeZtn8UH&ve@{*k_TSV07iIpm{@dt$H~%+u z{x2o}rx`|RO8+l({^|Zd1ohvdvoCpgu!i;nY-OEm3%UBSaq_i=3Rga~6!O^p^TydNy^W9K+V#gA|w8Onpp>KV!h z-8}K@u)?bN?~Ukk)0!oRzXw&HSWf1*b%W=wd~+?wY6uoNJ#P>Z2lKFx_f0bix0m*2 z`p`QlVvbHm{g&5c+-=@^<=0^V58A~^#B~)eIssd z9Wreyl-}~R3I@uKea-c*K&}hc4~ccT^wjbEs${QH;Jbd~gS_f6mKW|=Cb>Eetg0BS z9G?`@S3}jwj2o-r{<94yJS~0TQ>JC&*Zy|2v^75SO1%#f+-&z97^sKgGYa;X=tY+4 z6kpX{yQ<->Pg{;_d?MOyDPXMMKL9r}K5cclTS^y;axvIF@`e!)#y-0aZwwDgSf=)& z1Qmm?Zrmc&jvu&&I0{AvKr!Uqflj_E)VtU6`R&92c71P|KYXPVx8B^d(Nn4u;XfT#;GJrJZARrM@($lFlTEI~ zo&%Mm{kTg& zy$WpiUCc7PT!b03Ua?nZ(~wr!x%Ut~(Lh=6&FNlI3{EF1U)Nlr?-qnlx|PL^p-lK$ zTG_oi=(Js}Lh2Is2Ls)+sJCXFs! z?HVprYtRm=gP|{p8-D=Jyg6Z0!E^9bn%0OBj)z*yl%)9E3HaFC)N8=o1=Cv?)^AYw z4$IdaB5X=3fbTmu?&M?Xg_+R&t%p2XVLjhUl5SNL?jmWPShcwjJ=kCQ8oJg3_e<^Y z*JC|+#qP?Q#+(Y!8f|-i;B)uiH|)PBuL%3^Y5I#Ye_H=-biVLp`v1!7|L=zVf2sMs zJ(#5X7k;7hPxt>JsK26;I7}!PsuRfoFK|1Tr z-yHiNdA_;OY3X?{Ch0aWCedG?{OkGtqPt%pO)MMWD*z^wDj_oide^7xdTPkdr+8%3 zQ}rE`Y5K>WZT{4Dr3Y66!fNQFx2{J>irsXY8g%h60gs9RxW0jQnN^G)#z1;3}e%q`CS zfTyjxV|#;WAjB@;_<3@I{^|%h&ZBPhCjZ)k!tQ$sI4>{4Q_9}}#`%k(8>#uo_b~2g zPE9th(v}ujel`|!QlH+uwznIXml--l)4MJ0gymhnbj_jg*b`Z&kr?O=v*{`1e2dTa zZ7110rt8+ZXDKar+L1xW>iVkNHJEatcJ=b{IrPanQOzMlh23k$Uo?lb!SS+$3Gwj; zdb`Ss>(8UpfxR>}cttGq%cBlFGjAfHPVbAT$(_G$d!z>C$y;lR^-CeT(=jQTI0$`j z4*BjqR06%K?ksAnig3N<4^YY<$9z4pjhiK?K$}!#a-i4aBhDHot7=T++Z@hAhm!m8 z+JosNV}&`GDO0V0V=dTr;hkr-&p3KF8tc}0RKnuVg@fqM(v64B%GNBQA74+=^sX5tAuTl^%^R=E-wkC zTpWZ;FFJUY*=S%c`$f)}zM303>z6yIKLv|!`YxeUbm8NTuNt$$)nGSAm+x3q$HW{XRNw)B8kX z7-fWs+CJ1doUYrf_0hKn-_azxh>_pnb;s7Lf<9DaT@fxI5?Tds#J6cu_VnPkBUWLB zL1d^7ue9+IAcIN+&9~ZP07NW~mAc+;hmH=llG7h5@bj*+-e!kf;Mi4`YxR!aF87^8 zOV%64jjuVb)Y=w-?fQjt-_I6c*!SGS8?NWW_QHap_?;9i@jU1?x`~Cjqv1+elV3if z`N3~KJ+I+odspJ-$SRDNVgBlHm-cs5{~kOM_TOLgi)w#b|7}#0!v99Z|E0Vi3}KSa z@@Ar=`cL=&A*lZro$6-gl`VTcuy%4)JtNmL!X3xxZA?O)XjJ#*Kr!Ut@B>DQI&<5Vk98S|Edfqjzn%ljUN0OQZsYZdl**7@pi{W z3_{REu#sNtIM!7=i*0zyKwP>1-K3T60@Ugp7JC)&6O%RY#{{_qt<;v~-)Lne9`>-J zHjED9zFC5QR3D$ z8ulC0P{MH6_>^`#NF336ojouKo_;d(fklkO4XtmZf{hu7j9;wx3h!eiHc}i5XwFl>xA%=kbW;cR zj@P@N^3mKz(u7(W-(1j5Gl^nO|tyIV6I=hCp z*Px^>hu?Yn+p!XQ3UBSFcd*u7ElLmOrnk1OzZh{un4P%(rQ|za6(-`jFY0Ah<1{ec z<+8y-dlAU2x5w2!(BGQ$a8Y@CTM@{=7<7F$h4}Q0KtKQYXv+EX;z7Y)d|9&F`q*+_0^`RupCVEj2qp)&Rc&!wNxagzyZH?)K$f6m zl6K5;Vn0oK;-YLTa36l6Gf98@lT+BeC5kFDVMyE3o^yHWvfSlXg-e?v~*i&JQ!-!z3hq?kp9!8vXj-SJVUPnTulfz7>SQj1pql!4BMD<@%;j zn~bFbJ$oxw3K5>?J+1NIUI~{eRfP|7M2NdjMlWW>E}@$qf4(GX0Q}3iO3WEo5L(r? z55Hnb#>kp!ZQm`+2(jDW5C`ZjUFIKFemYq?h(oNmhYv{%qTY06`5vWSxZs>3#JQiB zcr+~G?t}_2k!ni)npsYNdzo`$?p?Y-fh$WffMhm_Lbt|a9`P9mX%p{AvBS(bPEad(sQGLmkVTS~7sY zRrWFQItkq#8uPRYHX}C=_1SPucdN$MGoj@Y)vY5ayscFU+ z^SvlEX?pj;Tpd=2-uh_Z)r6fPs*#!ZYS1g_!}LhaEcRF*%DVQg4(^$d9=yHT4Bcau zG8y+4KunT!Y0h#M<{WRH%VZ75@xrB;;Fn}famwzNIz@)$$PydVEzE@4bzBqFK?+=A zU(f5;)dLx?&g(h)58*@3{Gr0uB)sz}!t&tVethyXDtzCKKnNHpU9j$_B9}X@if_vp ztdYOGKHZau{`(%Tpk!0gQq%aYWJDJxC`laprZ@r<=XC6Q7YeYaWZ{ClS}xqpHvT~N zD8nYdmC>#~i?GUW@4hGVPxeiV4HUF~ z+v35))eaAPjlX|=mVg~8)X&OVl_2B#(l~3DioD4}`dTg>`2Q659AHf)&(jnXP*G4k z3xb%l&_YwNybYmCOXw&<8a<&TK!Vsk?1fWN(X(JZD+;J6_!b2e6$HyE9x6q!E1-ZP z$bVk|PYxsZeZN1D50cEz&d$u1nVo&Rn~2Rk*Vv>Nc-jfo$65{5*t~~>XSLpXjBTe; z??mW6!!4+1Evt}e^t(}=R$5tyg=i`ceIHqjJ)QhW{!2PKFOWH4)B$t{$*{y#SCvQA zjl1*;oBOF{EndbnO-DW-gK%eqF+70l?_p4dsVE3s9tD;95?@C;jc zM|w#^S{<%zWL0g&%EyLYB<)f)lZJWD@q6rP+pw)e3$-?!Pr%az3WF5FA7KyW*2pWQ zzr}7u#+2QDi>}Hl%2X^y&oo93meEMCe1`e%EeeYEe};<&WZpkEd%X%P9$m{X09axzk=lp98 zWmqO-HU^Gq!UNxO><)?}@Aj2u+CzGIVc z=HHlyV@pZESQ+Rq%wJ-JeZ{A*Hk6mqQ3XcsyWMUMRfgc^LQ0{6A`MRp1xv?^NWy`+!Y-aP*3N)?e7!UmgqjDRNNCJ&S9` z8UQGv+m)S6UScbK*SeU54S?^TZ1C4pZNm2XA21PJlLe)DNB-d2HRJalGqygRJQ(WF zIqb3H@JD=H_UE{&lzI$1TVIcPwc(1P$}KMFJ{sFa0gE-{9^>1iWVIH}sKl?1u77h! zS_-IK6eYfFZo^x0lU6<+r3UTYjFPulreLE=bq_t+GaR1UqVV8L%LmLF9R{;gtH#C` z456OWPy`h2_}lMpa9>v9t5o~9NKF{`hsVaOIpTErvDo8fCy2 z^yG1T{%g!pd+g-AEp>R(o9l&inR?9R!j74jXURY>wK+yMcL#trg^Tk`)0;5cjWZ04 z?HjPfy1mX@KeS?FJ|CYs^~4uk{?_zTxhXjqI&!dgz=~F!mBg`FeO4A)MC`eJa-k|r z9LUICI^iv@Mq5yAY@droN*Py7PA=AnO=oV+k4$Hm9r(AZWJvq&5~j1e_!5-*uckBKTxa{@ymV~YfW%vxx0^AW&lB@0 zs~_Nh4}UwIQId<*U0yTf;;?5J-SUmyi^98D#`>M1?@yFtIXCrhEln)PV~!Qn$ZEG@ zjSXuSJe>6fquR3ABa)i&Bm=$G_eeEZM1=Xrp)Iv|-GMk#L2v_hF7>L_i9g?AS6-Jo zofun(`OKNW%)I;&9v{$VqWGi$pD=`T#H_LoceY<2cW~QV{EOxlR{gv>ES9gjedp6k zJn2zWmdC&+SUR0Pq^j@(=4$E}C-uG_&BCT>DxfndnzG||ZeLM^TNTKzUGTC5`*TcD zhU2qqm_6I@;rp@*OxwET>q3n?_#@fnDqAj8V{v+~^5x9Xjd;)NPWa|l;dC`pTb5xJ z?k9Yo8>0RKQ(Bn8eM)Y{LJuZZOJ(I?`!a3{RxEjeUo3D}o??Km3_3kq*jy|PF3OI0 zHFN0`Z1Yt+_Pm83aod8UpYyBFVkV2=Nh1ga-+Yf_P>#*b2N&|MIGdGhy3$t#2-HRoc zezYpAEWv%=yR309eTc6sHn=RCc@^8FDw^LSCk-8B_Xb~-s==IN%ukLgyM=`t$%S&y z-N)R<>uc7$YQPIiny+u;*Wwx(mf`xBo3OQBC-2KUOT&#R>OA?RTKo#168PdsGrqn+ zJuH#?2v@l3?;L9N279^d+8s>|6jc-mt!ty3a9E<3Ke4d}^P6fA<+Y{;yVvkC_nOy3 zT$UjWHy_`E`#$jAdwl;#Eb3U1>XKcb@bpNz(Tl9AahoEzVoy{l_V`3oWDr^i7{TKz zMQy0VPcU`o94 zx)Jb0c=~~@pRj~SGb-N7f57C7Ht0^ut-u_%@`k!tKEq9`GKy*IoA7-+)GpH-@h4FW zR04&in8NzGcMTISVQv>Ksn+Q3ldaULGak59;(QJ6S?j!NyfMq}*o%@vyhevKh4=6Y zruk!Z*Y40ey-!)I&^XC3HrVVQjjNWY&kjm9)9Cjtw9OsG6WZO0t{tjgVmE_VH4z~0#@)fGCck+3q|Q~TR=Ui|VS)7fPU{*_NMq@)*w>Fh4P z1f~9~=}btZ3$e+E4cYL2;X|c^UeH7DwmT8E0)^ zN`d^V()YelXllNoc9J_(cq^miW#j=Kq>I#qn^%I(zl|?GdO`(`%FBt~>&pQ5l%_># z+;W2MwKgMzDj7ij)ne|i^FslcTW!8ADGrXbP)g>E4}wp#^QPcuOn?RL?XzXc-XQ4Q zqfb*Vy8`%8W4n)wFW8>kARUlu3rkz`i>5Q^;NbRUW4FshfD~2B^9w`RV0URS2R}6r z43>#;T=O;*4AEUV^wlmt)Uo;;QZXt9IQvXpH|$ve+)?k5{rH$8a61>TapKxUFe1m} zw8a?)%nnLU9XO8zb3Mnot{xW&;)9a4FXoD%;=bjiH=IC_vBTS;dQ&Le@OPQO$&(I; z#G&5-a0!KXo~e01p5OwuBqzvhGMo>$$`;=$we*J$w(jNpHcjCF@xdD%|3K)l+D34N zBY->id6?bOk)S5OaQob`e0b^L&1LKVV1kzW@YDTYm@v1> zZlRgc43Oo2UqyN_1A1L~c`CCm7-U|a*l_oc*-*q$-04{74<;Oc{bWvy2s(^>_-T8B z8yNQbYT!E?J{az$^Z>u@23|#lcrNw!0{Ra|T8`l?1n@GgTqQ>Xc!f!?&pF`_AKxoA zwQc3VWpnE)ZFBrUYHa<;MtK9U_{HG(QyNh)#V}GX!Jh|;%@&&|6-2{5VIJ7t3;}qW z_IT%Y8y7HZDftF?Cjc#)hN|jo7ea%XkMa|vZGjZ!?_GQI44_Wzsgo2M2NW<%KO3mA z0C|4&J3+ELxP6`b+p%l$@bG{etQqn`7&NRri8fjn452=mmTnOPtB0%Q?!LSLE;t+o#gxk3h@P4`N9^s75uskGa8X9F9K^IbixQJM|sUGx-Mp_8Rj zN*{EzLxSPXoIkM*GLc}pT!zyAl!ai=eubv<``zIy;gw2Izy~juC_1<*2gC9agBn-* zjswNc_Yalk*n?*Co#hj3n9$f;k)FHJ2dHdrs|p$H3W4Kjlh49|V64+_egE1hFzo(K z=})ej;MniaRXrL*L6CLB9>u14u-vS^LQB^jV&x2<;@_>n{avnSTlAPVgYuZ#)fj%Pe^`Wy|n*qP~xR>eT~sHbO5 z%V_W#<$A`!^WMoDO#qrYU9#79D{KaLM#o5X-0n$FI_tgg`6JcO0$p4#80Gj+4|)`F2M_P+ zZuJ6`^F`BUlzD;;wCOSqvy8zN(Yb}GCVsGX!l>3=5H`xPv956J zYOKpgTW7Bpg+Ly$DBWsVtU3GkDe!6M*FPf3S ze9W5dN-~i!V^)Kc@+>+8)_|r=<-iMVq)ED=F7RxCOu&_$0kCRNp|ee@2|O8b?W}G* z6Uc6!5_o$;Al%bRP4p5)!`#>ThjZ>mLJI3`jqf%CaQ@elsQhdJ7_BTV{z!50|EvoTG#Qvf`1d(PQU;MRF(FlFR3U zb!TrdJ(lyK;pqnHx@0#{6;+zL=uR{U-t1cN*JM+mI#;Kb865~B5;5cPI&t9Boi=R{ zXA4|soIGA%7YNtSYdlgG;|P&uz}tRv0B0=Ji=J(r1>OBVHKird;l`2HnaR_Gkb^iH z;qZbFCpD;BzMU@u%MGq?54gku?Cj^>Sa3Wr{UmTm^Irz@vQswgS>XKVa~(iYUdm#v=b_N&NQvQ|gaAlgaq{i#l_B6jQv8)UUp-*C|A^O8ivpn1 zF7Hbjioe1B%J8x4I zv4K`xLmtmG5}vx55XfwTeVOS^_dvCh#CV6dOwq|p_(UNF5KTh7v zAwmEHoYRUM$3;TU9hn^dDmJ`$Z0R`07EQ40?vaPhT9NQIE5GvY?Fjf$V5GBrDIY3& zR;+H?9R=sASQZT}p@E4j$1AUi4StZ~QrO1-LCdwk?*qx)4PH*x`Jly;I~a-2LFOPA4&lDR9nyV+JH-12cgWZ2;@YMAzTKqIuf~iXJGy0LU;L^JsV{zm zfz%f_mXrG8e&0U1e3#T0S5=bw;$k;aA6Z40TNPTp~p43+dQb~Pve2dgqhaE_L zwKe#E-Reu~tBrA_zS>4Y>Z^yYq`rDYLF%goJyPF~k3J;6s^7vZj6Xq?{3Pow?n^qKq7s%1lnV`jEa5*S} z7abOcAU;|Y5FpS%{5?DZeI*;t5Ca@Bu*eqy?Irq&i|2F{+_jv5U``m@uWMSnV15Eo z1i?@%nitFpXSd4`&5aO!LxeL!I4l7n8ytLzjOiVw(qMb1`@fpqk4($PmEG8S|ze%v(%32C2b`eCUP%YCbDGH)Q;*UZ6zBg zaxYpZvSjnrj_M_CB^xMmFIp(FWE0hn>LqO@8!2*cSgDR@LfviDcZmOIO}|@B?auaJ z$o~h{;`wV7`t|7WDDkv1ESMR^h!PRA$_No7Dk?mdkRpZ?#YgMAM5=)vdAc513(fSj zI@o`QBsVh zVeDn-6vMRj5HNX;!Op=JOpeGtis5MzY8Mh7Xveb;VshDGcEPcMc3hJnrl&iRCz|ak zFGqtm{5;;>54bREI$BRPXg#jK0d^@Ur2-8qVW_p?g z5IlUmg2GrLl;2u_v4~X+sZSa;*hEt|*_437KgVOOzAqjp)6-oC4UkVXEr2S(5??TjBR6$%Af}kx+1;9m|UkA z9~)FJgiKU>4-YGEbfvq!*}r6uBl3G1@q@p~MYZSpcv>5TzsV-*QMS1LlFg~-{?^`C`k}u$ z_1xdc{Z?;1_cz0Sldb3eX4GG@IrrS(jQdTtp8K0gf63M>og+(6;S)`lD_CgO3n)a-~5H6Tj+mi+x9$t*0wwp6QG!jZ7b2H7p5pE zi|Y}}K=Bj9(<~arZ9k54corfbPkMy65j}!wXwu6vlG#L@^C0S5}VtJ6F1km}i;B?@TZlMI`pZM5~fT@8$r>8ngrYHDW(t4nuCYfGb zl@$GepUH0-Ety_i)v-UPr*Jy(KhWQusE@#((_1)8rY9D8sI(sFr%9$Kn#!^V{i)HC z>4`pT*@OO;Kha;@bp!Qdepde!=MMZ2>JPD}SkQW)pY{(u(O>@Qf6hepQK*!k(_2JK zq$kKJ)YzZXQ#q38orv*@2Jz47Ex*&htNc#l;YOhmONXc+3FFUFqW(@|{Vgazr>8`B z;8Femi$5VgvR^-^N8=y8P{wZbj$-}M_;1HckROfzc6@hwvHocMx8o(zi`Q0v$X~2K zvaS|1LO2@=h1G61k?|7uOhP@HqnHa?ge3s+d&e8fA4sgvqEGRwyL=r8;zLRutp-O$ wMsozZtcZvS9ZSL!$a*GvWL;M$ii@kQvm4r9I#x>RfBcggp(7>bLZOrY4|O*c6951J literal 0 HcmV?d00001 From 1d255ae8a3db7164b7b58ef7b8e58af087c45681 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:24:23 -0700 Subject: [PATCH 15/23] =?UTF-8?q?=F0=9F=9A=A8make=20sure=20tox=20test=20in?= =?UTF-8?q?=20python=203.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 995e36d243..3fd08c643b 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ # THIS SCRIPT IS SUPPOSED TO BE AN EXAMPLE. MODIFY IT ACCORDING TO YOUR NEEDS! [tox] -envlist = py36, py37, py38, flake8 +envlist =py36, py37, py38, py39, flake8 [testenv] @@ -31,3 +31,4 @@ python = 3.6: py36 3.7: py37, flake8 3.8: py38 + 3.9: py39 From 074f733f2ab98501fec29fe84ab7451bdc1a3577 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:26:21 -0700 Subject: [PATCH 16/23] style fixes --- src/whylogs/core/metrics/model_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index effdaf7afb..033bbd1d24 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -99,4 +99,4 @@ def merge(self, other): return ModelMetrics( confusion_matrix=self.confusion_matrix.merge(other.confusion_matrix), regression_metrics=self.regression_metrics.merge(other.regression_metrics), - model_type= model_type) + model_type=model_type) From 67a3b5633b8f521ef2a25db499e88db03ce9edd7 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:38:15 -0700 Subject: [PATCH 17/23] small corner case test --- .../core/metrics/test_regression_metrics.py | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/tests/unit/core/metrics/test_regression_metrics.py b/tests/unit/core/metrics/test_regression_metrics.py index d0474625d4..c12c7564e2 100644 --- a/tests/unit/core/metrics/test_regression_metrics.py +++ b/tests/unit/core/metrics/test_regression_metrics.py @@ -7,39 +7,45 @@ from whylogs.proto import RegressionMetricsMessage - TEST_DATA_PATH = os.path.abspath(os.path.join(os.path.realpath( - os.path.dirname(__file__)), os.pardir,os.pardir, os.pardir, os.pardir, "testdata")) + os.path.dirname(__file__)), os.pardir, os.pardir, os.pardir, os.pardir, "testdata")) + def my_test(): - regmet= RegressionMetrics() + regmet = RegressionMetrics() assert regmet.count == 0 - assert regmet.sum_diff==0.0 - assert regmet.sum2_diff ==0.0 - assert regmet.sum_abs_diff==0.0 + assert regmet.sum_diff == 0.0 + assert regmet.sum2_diff == 0.0 + assert regmet.sum_abs_diff == 0.0 + + assert regmet.mean_squared_error() is None + + assert regmet.mean_absolute_error() is None + assert regmet.root_mean_squared_error() is None def test_load_parquet(): - mean_absolute_error=85.94534216005789 - mean_squared_error =11474.89611670205 - root_mean_squared_error =107.12094154133472 + mean_absolute_error = 85.94534216005789 + mean_squared_error = 11474.89611670205 + root_mean_squared_error = 107.12094154133472 + + regmet = RegressionMetrics() + df = pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH, "metrics", "2021-02-12.parquet"))) + regmet.add(df["predictions"].to_list(), df["targets"].to_list()) - regmet=RegressionMetrics() - df= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-12.parquet"))) - regmet.add(df["predictions"].to_list(),df["targets"].to_list()) + assert regmet.count == len(df["predictions"].to_list()) + assert regmet.mean_squared_error() == pytest.approx(mean_squared_error, 0.01) - assert regmet.count==len(df["predictions"].to_list()) - assert regmet.mean_squared_error()==pytest.approx(mean_squared_error,0.01) + assert regmet.mean_absolute_error() == pytest.approx(mean_absolute_error, 0.01) + assert regmet.root_mean_squared_error() == pytest.approx(root_mean_squared_error, 0.01) - assert regmet.mean_absolute_error() == pytest.approx(mean_absolute_error,0.01) - assert regmet.root_mean_squared_error() == pytest.approx(root_mean_squared_error,0.01) + msg = regmet.to_protobuf() + new_regmet = RegressionMetrics.from_protobuf(msg) + assert regmet.count == new_regmet.count + assert regmet.mean_squared_error() == new_regmet.mean_squared_error() + assert regmet.root_mean_squared_error() == new_regmet.root_mean_squared_error() + assert regmet.mean_absolute_error() == new_regmet.mean_absolute_error() - msg= regmet.to_protobuf() - new_regmet= RegressionMetrics.from_protobuf(msg) - assert regmet.count ==new_regmet.count - assert regmet.mean_squared_error() ==new_regmet.mean_squared_error() - assert regmet.root_mean_squared_error() ==new_regmet.root_mean_squared_error() - assert regmet.mean_absolute_error() ==new_regmet.mean_absolute_error() def test_empty_protobuf_should_return_none(): empty_message = RegressionMetricsMessage() @@ -47,28 +53,21 @@ def test_empty_protobuf_should_return_none(): def test_merging(): - regmet_sum=RegressionMetrics() + regmet_sum = RegressionMetrics() - regmet=RegressionMetrics() - df= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-12.parquet"))) - regmet.add(df["predictions"].to_list(),df["targets"].to_list()) - regmet_sum.add(df["predictions"].to_list(),df["targets"].to_list()) - + regmet = RegressionMetrics() + df = pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH, "metrics", "2021-02-12.parquet"))) + regmet.add(df["predictions"].to_list(), df["targets"].to_list()) + regmet_sum.add(df["predictions"].to_list(), df["targets"].to_list()) - regmet_2=RegressionMetrics() - df_2= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-13.parquet"))) - regmet_2.add(df_2["predictions"].to_list(),df_2["targets"].to_list()) - regmet_sum.add(df_2["predictions"].to_list(),df_2["targets"].to_list()) + regmet_2 = RegressionMetrics() + df_2 = pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH, "metrics", "2021-02-13.parquet"))) + regmet_2.add(df_2["predictions"].to_list(), df_2["targets"].to_list()) + regmet_sum.add(df_2["predictions"].to_list(), df_2["targets"].to_list()) - - merged_reg_metr= regmet.merge(regmet_2) + merged_reg_metr = regmet.merge(regmet_2) assert merged_reg_metr.count == regmet_sum.count assert merged_reg_metr.mean_squared_error() == regmet_sum.mean_squared_error() assert merged_reg_metr.root_mean_squared_error() == regmet_sum.root_mean_squared_error() - assert merged_reg_metr.mean_absolute_error() == pytest.approx(regmet_sum.mean_absolute_error(),0.001) - - - - - + assert merged_reg_metr.mean_absolute_error() == pytest.approx(regmet_sum.mean_absolute_error(), 0.001) From a2b33b5ef3ad50342ba3914b02178928862c79f4 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 21:38:24 -0700 Subject: [PATCH 18/23] =?UTF-8?q?Bump=20version:=200.3.3-dev5=20=E2=86=92?= =?UTF-8?q?=200.4.0-dev0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/conf.py | 2 +- setup.cfg | 2 +- src/whylogs/_version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2f58e0a775..3965f5cbf7 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.3-dev5 +current_version = 0.4.0-dev0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/docs/conf.py b/docs/conf.py index 55a335ae22..1298e32ee7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -101,7 +101,7 @@ # built documents. # # The short X.Y version. -version = "0.3.3-dev5" +version = "0.4.0-dev0" # The full version, including alpha/beta/rc tags. release = "" # Is set by calling `setup.py docs` diff --git a/setup.cfg b/setup.cfg index b5e23de22f..468492c380 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ [metadata] name = whylogs -version = 0.3.3-dev5 +version = 0.4.0-dev0 description = Profile and monitor your ML data pipeline end-to-end author = WhyLabs.ai author-email = support@whylabs.ai diff --git a/src/whylogs/_version.py b/src/whylogs/_version.py index 3b75ad6dad..47f3f3b522 100644 --- a/src/whylogs/_version.py +++ b/src/whylogs/_version.py @@ -1,3 +1,3 @@ """WhyLabs version number.""" -__version__ = "0.3.3-dev5" +__version__ = "0.4.0-dev0" From 8dc266ec06a98840c3471cebdf6b5d975b0a2169 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Sun, 14 Mar 2021 23:54:06 -0700 Subject: [PATCH 19/23] =?UTF-8?q?=E2=9C=85=20add=20checks=20on=20fields=20?= =?UTF-8?q?and=20merge=20them=20accordingly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/whylogs/core/metrics/model_metrics.py | 12 +++--- .../core/metrics/regression_metrics.py | 24 +++++++----- .../core/metrics/test_regression_metrics.py | 8 ++-- .../unit/core/test_datasetprofile_metrics.py | 39 +++++++++---------- 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 033bbd1d24..8656f8ae0f 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -14,14 +14,14 @@ class ModelMetrics: regression_metrics (RegressionMetrics): Regression Metrics keeps track of a common regression metrics in case the targets are continous. """ - def __init__(self, confusion_matrix: ConfusionMatrix = ConfusionMatrix(), - regression_metrics: RegressionMetrics = RegressionMetrics(), + def __init__(self, confusion_matrix: ConfusionMatrix = None, + regression_metrics: RegressionMetrics = None, model_type: ModelType = ModelType.UNKNOWN): - # if confusion_matrix is None: - # confusion_matrix = ConfusionMatrix() + if confusion_matrix is None: + confusion_matrix = ConfusionMatrix() self.confusion_matrix = confusion_matrix - # if regression_metrics is None: - # regression_metrics = RegressionMetrics() + if regression_metrics is None: + regression_metrics = RegressionMetrics() self.regression_metrics = regression_metrics self.model_type = ModelType.UNKNOWN diff --git a/src/whylogs/core/metrics/regression_metrics.py b/src/whylogs/core/metrics/regression_metrics.py index bdf7edb8c2..95a98c199c 100644 --- a/src/whylogs/core/metrics/regression_metrics.py +++ b/src/whylogs/core/metrics/regression_metrics.py @@ -65,26 +65,32 @@ def root_mean_squared_error(self): return None return math.sqrt(self.sum2_diff / self.count) - def merge(self, other_reg_met): + def merge(self, other): """ Merge two seperate confusion matrix which may or may not overlap in labels. Args: - other_reg_met : regression metrics to merge with self + other : regression metrics to merge with self Returns: RegressionMetrics: merged regression metrics """ if self.count == 0: - return other_reg_met - if other_reg_met.count == 0: + return other + if other.count == 0: return self - new_reg = RegressionMetrics() - new_reg.count = self.count + other_reg_met.count - new_reg.sum_abs_diff = self.sum_abs_diff + other_reg_met.sum_abs_diff - new_reg.sum_diff = self.sum_diff + other_reg_met.sum_diff - new_reg.sum2_diff = self.sum2_diff + other_reg_met.sum2_diff + if self.prediction_field != other.prediction_field: + raise ValueError("prediction fields differ") + if self.target_field != other.target_field: + raise ValueError("target fields differ") + + new_reg = RegressionMetrics(prediction_field=self.prediction_field, + target_field=self.target_field) + new_reg.count = self.count + other.count + new_reg.sum_abs_diff = self.sum_abs_diff + other.sum_abs_diff + new_reg.sum_diff = self.sum_diff + other.sum_diff + new_reg.sum2_diff = self.sum2_diff + other.sum2_diff return new_reg diff --git a/tests/unit/core/metrics/test_regression_metrics.py b/tests/unit/core/metrics/test_regression_metrics.py index c12c7564e2..46da5c3984 100644 --- a/tests/unit/core/metrics/test_regression_metrics.py +++ b/tests/unit/core/metrics/test_regression_metrics.py @@ -55,12 +55,12 @@ def test_empty_protobuf_should_return_none(): def test_merging(): regmet_sum = RegressionMetrics() - regmet = RegressionMetrics() + regmet = RegressionMetrics(prediction_field="predictions", target_field="targets") df = pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH, "metrics", "2021-02-12.parquet"))) regmet.add(df["predictions"].to_list(), df["targets"].to_list()) regmet_sum.add(df["predictions"].to_list(), df["targets"].to_list()) - regmet_2 = RegressionMetrics() + regmet_2 = RegressionMetrics(prediction_field="predictions", target_field="targets") df_2 = pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH, "metrics", "2021-02-13.parquet"))) regmet_2.add(df_2["predictions"].to_list(), df_2["targets"].to_list()) regmet_sum.add(df_2["predictions"].to_list(), df_2["targets"].to_list()) @@ -68,6 +68,6 @@ def test_merging(): merged_reg_metr = regmet.merge(regmet_2) assert merged_reg_metr.count == regmet_sum.count - assert merged_reg_metr.mean_squared_error() == regmet_sum.mean_squared_error() - assert merged_reg_metr.root_mean_squared_error() == regmet_sum.root_mean_squared_error() + assert merged_reg_metr.mean_squared_error() == pytest.approx(regmet_sum.mean_squared_error(), 0.001) + assert merged_reg_metr.root_mean_squared_error() == pytest.approx(regmet_sum.root_mean_squared_error(), 0.001) assert merged_reg_metr.mean_absolute_error() == pytest.approx(regmet_sum.mean_absolute_error(), 0.001) diff --git a/tests/unit/core/test_datasetprofile_metrics.py b/tests/unit/core/test_datasetprofile_metrics.py index 406130d5a2..f4f8b53955 100644 --- a/tests/unit/core/test_datasetprofile_metrics.py +++ b/tests/unit/core/test_datasetprofile_metrics.py @@ -2,7 +2,6 @@ import pytest - from whylogs.core import DatasetProfile from whylogs.core.model_profile import ModelProfile @@ -39,39 +38,37 @@ def test_read_java_protobuf(): assert lbl == labels[idx] - def test_parse_from_protobuf_with_regression(): dir_path = os.path.dirname(os.path.realpath(__file__)) - prof= DatasetProfile.read_protobuf(os.path.join( - TEST_DATA_PATH, "metrics","regression_java.bin")) + prof = DatasetProfile.read_protobuf(os.path.join( + TEST_DATA_PATH, "metrics", "regression_java.bin")) assert prof.name == 'my-model-name' assert prof.model_profile is not None assert prof.model_profile.metrics is not None confusion_M = prof.model_profile.metrics.confusion_matrix - regression_met= prof.model_profile.metrics.regression_metrics + regression_met = prof.model_profile.metrics.regression_metrics assert regression_met is not None - assert confusion_M is None + assert confusion_M is not None # metrics - assert regression_met.count==89 - assert regression_met.sum_abs_diff==pytest.approx(7649.1, 0.1) - assert regression_met.sum_diff==pytest.approx(522.7, 0.1) - assert regression_met.sum2_diff==pytest.approx(1021265.7, 0.1) + assert regression_met.count == 89 + assert regression_met.sum_abs_diff == pytest.approx(7649.1, 0.1) + assert regression_met.sum_diff == pytest.approx(522.7, 0.1) + assert regression_met.sum2_diff == pytest.approx(1021265.7, 0.1) def test_track_metrics(): import pandas as pd - mean_absolute_error=85.94534216005789 - mean_squared_error =11474.89611670205 - root_mean_squared_error =107.12094154133472 + mean_absolute_error = 85.94534216005789 + mean_squared_error = 11474.89611670205 + root_mean_squared_error = 107.12094154133472 x1 = DatasetProfile(name="test") - df= pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH,"metrics","2021-02-12.parquet"))) - x1.track_metrics(df["predictions"].to_list(),df["targets"].to_list()) - regression_metrics=x1.model_profile.metrics.regression_metrics + df = pd.read_parquet(os.path.join(os.path.join(TEST_DATA_PATH, "metrics", "2021-02-12.parquet"))) + x1.track_metrics(df["predictions"].to_list(), df["targets"].to_list()) + regression_metrics = x1.model_profile.metrics.regression_metrics assert regression_metrics is not None - assert regression_metrics.count==len(df["predictions"].to_list()) - assert regression_metrics.mean_squared_error()==pytest.approx(mean_squared_error,0.01) - - assert regression_metrics.mean_absolute_error() == pytest.approx(mean_absolute_error,0.01) - assert regression_metrics.root_mean_squared_error() == pytest.approx(root_mean_squared_error,0.01) + assert regression_metrics.count == len(df["predictions"].to_list()) + assert regression_metrics.mean_squared_error() == pytest.approx(mean_squared_error, 0.01) + assert regression_metrics.mean_absolute_error() == pytest.approx(mean_absolute_error, 0.01) + assert regression_metrics.root_mean_squared_error() == pytest.approx(root_mean_squared_error, 0.01) From 9bf47cdca10ea15bdf37b21c425d90584c82df39 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Mon, 15 Mar 2021 00:55:38 -0700 Subject: [PATCH 20/23] =?UTF-8?q?Bump=20version:=200.4.0-dev0=20=E2=86=92?= =?UTF-8?q?=200.4.1-dev0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/conf.py | 2 +- setup.cfg | 2 +- src/whylogs/_version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3965f5cbf7..cef94096b1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.0-dev0 +current_version = 0.4.1-dev0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/docs/conf.py b/docs/conf.py index 1298e32ee7..bba7e90bf7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -101,7 +101,7 @@ # built documents. # # The short X.Y version. -version = "0.4.0-dev0" +version = "0.4.1-dev0" # The full version, including alpha/beta/rc tags. release = "" # Is set by calling `setup.py docs` diff --git a/setup.cfg b/setup.cfg index 468492c380..25b1fb2703 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ [metadata] name = whylogs -version = 0.4.0-dev0 +version = 0.4.1-dev0 description = Profile and monitor your ML data pipeline end-to-end author = WhyLabs.ai author-email = support@whylabs.ai diff --git a/src/whylogs/_version.py b/src/whylogs/_version.py index 47f3f3b522..74c21bd0c9 100644 --- a/src/whylogs/_version.py +++ b/src/whylogs/_version.py @@ -1,3 +1,3 @@ """WhyLabs version number.""" -__version__ = "0.4.0-dev0" +__version__ = "0.4.1-dev0" From 1f5d565bd725c566cb548acc367762a1b673a769 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Wed, 24 Mar 2021 14:54:39 -0700 Subject: [PATCH 21/23] set metrics per model type --- src/whylogs/core/metrics/model_metrics.py | 22 ++++++++++++++----- tests/unit/core/metrics/test_model_metrics.py | 21 +++++++++++++++--- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index 8656f8ae0f..d9f0b39f74 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -17,13 +17,23 @@ class ModelMetrics: def __init__(self, confusion_matrix: ConfusionMatrix = None, regression_metrics: RegressionMetrics = None, model_type: ModelType = ModelType.UNKNOWN): - if confusion_matrix is None: - confusion_matrix = ConfusionMatrix() + + self.model_type = model_type + if confusion_matrix is not None and regression_metrics is not None: + raise NotImplementedError("Regression Metrics together with Confusion Matrix not implemented yet") + + if confusion_matrix is not None: + if (self.model_type == ModelType.REGRESSION): + raise NotImplementedError("Incorrent model type") + self.model_type = ModelType.CLASSIFICATION + self.confusion_matrix = confusion_matrix - if regression_metrics is None: - regression_metrics = RegressionMetrics() + + if regression_metrics is not None: + if (self.model_type == ModelType.CLASSIFICATION): + raise NotImplementedError("Incorrent model type") + self.model_type = ModelType.REGRESSION self.regression_metrics = regression_metrics - self.model_type = ModelType.UNKNOWN def to_protobuf(self, ) -> ModelMetricsMessage: return ModelMetricsMessage( @@ -68,7 +78,7 @@ def compute_confusion_matrix(self, predictions: List[Union[str, int, bool, float self.confusion_matrix = self.confusion_matrix.merge( confusion_matrix) - def compute_regression_metrics(self, predictions: List[float], + def compute_regression_metrics(self, predictions: List[Union[float, int]], targets: List[float], target_field: str = None, prediction_field: str = None): diff --git a/tests/unit/core/metrics/test_model_metrics.py b/tests/unit/core/metrics/test_model_metrics.py index abe22e22bf..e3826ae148 100644 --- a/tests/unit/core/metrics/test_model_metrics.py +++ b/tests/unit/core/metrics/test_model_metrics.py @@ -1,8 +1,13 @@ +import pytest + +from whylogs.proto import ModelType from whylogs.core.metrics.model_metrics import ModelMetrics +from whylogs.core.metrics.confusion_matrix import ConfusionMatrix +from whylogs.core.metrics.regression_metrics import RegressionMetrics def tests_model_metrics(): - mod_met = ModelMetrics() + mod_met = ModelMetrics(model_type=ModelType.CLASSIFICATION) targets_1 = ["cat", "dog", "pig"] predictions_1 = ["cat", "dog", "dog"] @@ -12,7 +17,8 @@ def tests_model_metrics(): mod_met.compute_confusion_matrix(predictions_1, targets_1, scores_1) - print(mod_met.confusion_matrix.labels) + assert mod_met.model_type == ModelType.CLASSIFICATION + for idx, value in enumerate(mod_met.confusion_matrix.labels): for jdx, value_2 in enumerate(mod_met.confusion_matrix.labels): print(idx, jdx) @@ -21,7 +27,7 @@ def tests_model_metrics(): def tests_model_metrics_to_protobuf(): - mod_met = ModelMetrics() + mod_met = ModelMetrics(model_type=ModelType.CLASSIFICATION) targets_1 = ["cat", "dog", "pig"] predictions_1 = ["cat", "dog", "dog"] @@ -46,3 +52,12 @@ def test_merge_metrics_with_none_confusion_matrix(): other = ModelMetrics() other.confusion_matrix = None metrics.merge(other) + + + +def test_model_metrics_init(): + reg_met = RegressionMetrics() + conf_ma= ConfusionMatrix() + with pytest.raises(NotImplementedError): + metrics = ModelMetrics(confusion_matrix=conf_ma, regression_metrics=reg_met) + From 1804dae98dc6afb0e5efefa30cf6ae3c73b8e941 Mon Sep 17 00:00:00 2001 From: "Leandro G. Almeida" Date: Thu, 25 Mar 2021 10:56:21 -0700 Subject: [PATCH 22/23] regression fixes due to new model type --- src/whylogs/app/logger.py | 14 ++++++++--- src/whylogs/core/datasetprofile.py | 25 +++++++++++++++---- src/whylogs/core/metrics/model_metrics.py | 9 ++++--- src/whylogs/core/model_profile.py | 6 +++-- .../unit/core/test_datasetprofile_metrics.py | 1 - tests/unit/core/test_model_profile.py | 2 +- 6 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/whylogs/app/logger.py b/src/whylogs/app/logger.py index 524accd7c4..906d31625b 100644 --- a/src/whylogs/app/logger.py +++ b/src/whylogs/app/logger.py @@ -16,7 +16,7 @@ from whylogs.core import DatasetProfile, TrackImage, METADATA_DEFAULT_ATTRIBUTES, TrackBB from whylogs.core.statistics.constraints import DatasetConstraints from whylogs.io import LocalDataset - +from whylogs.proto import ModelType TIME_ROTATION_VALUES = ["s", "m", "h", "d"] @@ -346,13 +346,19 @@ def log_segment_datum(self, feature_name, value): def log_metrics(self, targets, predictions, - scores=None, target_field=None, prediction_field=None, + scores=None, + model_type: ModelType = None, + target_field=None, + prediction_field=None, score_field=None): self._profiles[-1]["full_profile"].track_metrics( - targets, predictions, scores, target_field=target_field, + targets, predictions, scores, + model_type=model_type, + target_field=target_field, prediction_field=prediction_field, - score_field=score_field) + score_field=score_field, + ) def log_image(self, image, diff --git a/src/whylogs/core/datasetprofile.py b/src/whylogs/core/datasetprofile.py index c84385997b..c10ff54a7b 100644 --- a/src/whylogs/core/datasetprofile.py +++ b/src/whylogs/core/datasetprofile.py @@ -24,6 +24,7 @@ DatasetProperties, DatasetSummary, MessageSegment, + ModelType ) from whylogs.core.statistics.constraints import DatasetConstraints, SummaryConstraints from whylogs.util import time @@ -189,9 +190,14 @@ def add_output_field(self, field: Union[str, List[str]]): else: self.model_profile.add_output_field(field) - def track_metrics(self, targets: List[Union[str, bool, float, int]], predictions: List[Union[str, bool, float, int]], scores: List[float] = None, - target_field: str = None, prediction_field: str = None, - score_field: str = None): + def track_metrics(self, + targets: List[Union[str, bool, float, int]], + predictions: List[Union[str, bool, float, int]], + scores: List[float] = None, + model_type: ModelType = None, + target_field: str = None, + prediction_field: str = None, + score_field: str = None, ): """ Function to track metrics based on validation data. @@ -206,6 +212,14 @@ def track_metrics(self, targets: List[Union[str, bool, float, int]], predictions inferred/predicted values scores : List[float], optional assocaited scores for each inferred, all values set to 1 if not passed + target_field : str, optional + Description + prediction_field : str, optional + Description + score_field : str, optional + Description + model_type : ModelType, optional + Defaul is Classification type. target_field : str, optional prediction_field : str, optional score_field : str, optional @@ -215,7 +229,7 @@ def track_metrics(self, targets: List[Union[str, bool, float, int]], predictions if self.model_profile is None: self.model_profile = ModelProfile() self.model_profile.compute_metrics(predictions=predictions, targets=targets, - scores=scores, target_field=target_field, + scores=scores, model_type=model_type, target_field=target_field, prediction_field=prediction_field, score_field=score_field) @@ -350,7 +364,8 @@ def generate_constraints(self) -> DatasetConstraints: Protobuf constraints message. """ self.validate() - constraints = [(name, col.generate_constraints()) for name, col in self.columns.items()] + constraints = [(name, col.generate_constraints()) + for name, col in self.columns.items()] # filter empty constraints constraints = [(n, c) for n, c in constraints if c is not None] return DatasetConstraints(self.to_properties(), None, dict(constraints)) diff --git a/src/whylogs/core/metrics/model_metrics.py b/src/whylogs/core/metrics/model_metrics.py index d9f0b39f74..558865af23 100644 --- a/src/whylogs/core/metrics/model_metrics.py +++ b/src/whylogs/core/metrics/model_metrics.py @@ -72,19 +72,22 @@ def compute_confusion_matrix(self, predictions: List[Union[str, int, bool, float score_field=score_field) confusion_matrix.add(predictions, targets, scores) - if self.confusion_matrix.labels is None or self.confusion_matrix.labels == []: + if self.confusion_matrix is None or self.confusion_matrix.labels is None or self.confusion_matrix.labels == []: self.confusion_matrix = confusion_matrix else: self.confusion_matrix = self.confusion_matrix.merge( confusion_matrix) def compute_regression_metrics(self, predictions: List[Union[float, int]], - targets: List[float], + targets: List[Union[float, int]], target_field: str = None, prediction_field: str = None): regression_metrics = RegressionMetrics(target_field=target_field, prediction_field=prediction_field) regression_metrics.add(predictions, targets) - self.regression_metrics = self.regression_metrics.merge(regression_metrics) + if self.regression_metrics: + self.regression_metrics = self.regression_metrics.merge(regression_metrics) + else: + self.regression_metrics = regression_metrics def merge(self, other): """ diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index a9c2c95e9b..1a2f7daec4 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -40,6 +40,7 @@ def add_output_field(self, field: str): def compute_metrics(self, targets, predictions, scores=None, + model_type: ModelType = None, target_field=None, prediction_field=None, score_field=None @@ -66,14 +67,15 @@ def compute_metrics(self, targets, """ tgt_type = type_of_target(targets) - if tgt_type in ("continuous"): + if tgt_type in ("continuous") or model_type == ModelType.REGRESSION: self.metrics.compute_regression_metrics(predictions=predictions, targets=targets, target_field=target_field, prediction_field=prediction_field) self.metrics.model_type = ModelType.REGRESSION - elif tgt_type in ("binary", "multiclass"): + + elif tgt_type in ("binary", "multiclass") or model_type == ModelType.CLASSIFICATION: self.metrics.model_type = ModelType.CLASSIFICATION # if score are not present set them to 1. diff --git a/tests/unit/core/test_datasetprofile_metrics.py b/tests/unit/core/test_datasetprofile_metrics.py index f4f8b53955..65602ac0e1 100644 --- a/tests/unit/core/test_datasetprofile_metrics.py +++ b/tests/unit/core/test_datasetprofile_metrics.py @@ -48,7 +48,6 @@ def test_parse_from_protobuf_with_regression(): confusion_M = prof.model_profile.metrics.confusion_matrix regression_met = prof.model_profile.metrics.regression_metrics assert regression_met is not None - assert confusion_M is not None # metrics assert regression_met.count == 89 assert regression_met.sum_abs_diff == pytest.approx(7649.1, 0.1) diff --git a/tests/unit/core/test_model_profile.py b/tests/unit/core/test_model_profile.py index e1ad03b159..27e60d7af4 100644 --- a/tests/unit/core/test_model_profile.py +++ b/tests/unit/core/test_model_profile.py @@ -6,7 +6,7 @@ def test_model_profile(): mod_prof = ModelProfile() assert mod_prof.output_fields == [] assert mod_prof.metrics is not None - assert mod_prof.metrics.confusion_matrix is not None + assert mod_prof.metrics.confusion_matrix is None message = mod_prof.to_protobuf() ModelProfile.from_protobuf(message) From a584b2410b89ae42a7cf1363d6c79433b48eb417 Mon Sep 17 00:00:00 2001 From: Sourcery AI <> Date: Thu, 25 Mar 2021 18:32:05 +0000 Subject: [PATCH 23/23] 'Refactored by Sourcery' --- src/whylogs/app/logger.py | 32 +++++++++++++----------------- src/whylogs/core/datasetprofile.py | 10 ++++------ src/whylogs/core/model_profile.py | 5 +---- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/src/whylogs/app/logger.py b/src/whylogs/app/logger.py index 49bfe0fef9..d12a326417 100644 --- a/src/whylogs/app/logger.py +++ b/src/whylogs/app/logger.py @@ -125,9 +125,8 @@ def segmented_profiles(self, ) -> Dict[str, DatasetProfile]: def get_segment(self, segment: Segment) -> Optional[DatasetProfile]: hashed_seg = hash_segment(segment) - segment_profile = self._profiles[-1]["segmented_profiles"].get( + return self._profiles[-1]["segmented_profiles"].get( hashed_seg, None) - return segment_profile def set_segments(self, segments: Union[List[Segment], List[str]]) -> None: if segments: @@ -184,8 +183,7 @@ def _set_rotation(self, with_rotation_time: str = None): self.rotate_at = self.rotate_when(current_time) def rotate_when(self, time): - result = time + self.interval - return result + return time + self.interval def should_rotate(self, ): @@ -193,9 +191,7 @@ def should_rotate(self, ): return False current_time = int(datetime.datetime.utcnow().timestamp()) - if current_time >= self.rotate_at: - return True - return False + return current_time >= self.rotate_at def _rotate_time(self): """ @@ -210,7 +206,7 @@ def _rotate_time(self): time_tuple.strftime(self.suffix), self.suffix) # modify the segment datetime stamps - if (self.segments is None) or ((self.segments is not None) and self.profile_full_dataset): + if self.segments is None or self.profile_full_dataset: self._profiles[-1]["full_profile"].dataset_timestamp = log_datetime if self.segments is not None: for _, each_prof in self._profiles[-1]["segmented_profiles"].items(): @@ -253,11 +249,10 @@ def flush(self, rotation_suffix: str = None): for hashseg, each_seg_prof in self._profiles[-1]["segmented_profiles"].items(): seg_suffix = hashseg full_suffix = "_" + seg_suffix - if rotation_suffix is None: - writer.write(each_seg_prof, full_suffix) - else: + if rotation_suffix is not None: full_suffix += rotation_suffix - writer.write(each_seg_prof, full_suffix) + + writer.write(each_seg_prof, full_suffix) def full_profile_check(self, ) -> bool: """ @@ -417,7 +412,7 @@ def log_local_dataset(self, root_dir, folder_feature_name="folder_feature", imag if isinstance(data, pd.DataFrame): self.log_dataframe(data) - elif isinstance(data, Dict) or isinstance(data, list): + elif isinstance(data, (Dict, list)): self.log_annotation(annotation_data=data) elif isinstance(data, ImageType): if image_feature_transforms: @@ -512,10 +507,11 @@ def log_segments_keys(self, data): for each_segment in segments: try: segment_df = grouped_data.get_group(each_segment) - segment_tags = [] - for i in range(len(self.segments)): - segment_tags.append( - {"key": self.segments[i], "value": each_segment[i]}) + segment_tags = [ + {"key": self.segments[i], "value": each_segment[i]} + for i in range(len(self.segments)) + ] + self.log_df_segment(segment_df, segment_tags) except KeyError: continue @@ -526,7 +522,7 @@ def log_fixed_segments(self, data): for segment_tag in self.segments: # create keys segment_keys = [feature["key"] for feature in segment_tag] - seg = tuple([feature["value"] for feature in segment_tag]) + seg = tuple(feature["value"] for feature in segment_tag) grouped_data = data.groupby(segment_keys) diff --git a/src/whylogs/core/datasetprofile.py b/src/whylogs/core/datasetprofile.py index c10ff54a7b..2e596deb41 100644 --- a/src/whylogs/core/datasetprofile.py +++ b/src/whylogs/core/datasetprofile.py @@ -132,9 +132,9 @@ def __init__( if columns is None: columns = {} if tags is None: - tags = dict() + tags = {} if metadata is None: - metadata = dict() + metadata = {} if session_id is None: session_id = uuid4().hex @@ -841,10 +841,8 @@ def flatten_dataset_frequent_strings(dataset_summary: DatasetSummary): try: item_summary = getter( getter(col, "string_summary"), "frequent").items - items = {} - for item in item_summary: - items[item.value] = int(item.estimate) - if len(items) > 0: + items = {item.value: int(item.estimate) for item in item_summary} + if items: frequent_strings[col_name] = items except KeyError: continue diff --git a/src/whylogs/core/model_profile.py b/src/whylogs/core/model_profile.py index 1a2f7daec4..1d59d97e8b 100644 --- a/src/whylogs/core/model_profile.py +++ b/src/whylogs/core/model_profile.py @@ -101,10 +101,7 @@ def to_protobuf(self): @classmethod def from_protobuf(cls, message: ModelProfileMessage): # convert google.protobuf.pyext._message.RepeatedScalarContainer to a list - output_fields = [] - for f in message.output_fields: - output_fields.append(f) - + output_fields = [f for f in message.output_fields] return ModelProfile(output_fields=output_fields, metrics=ModelMetrics.from_protobuf(message.metrics))