From 805a96bb9a02a1a16bb2c86b13d7169c3edf93bc Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 29 Jan 2018 17:29:17 -0800 Subject: [PATCH 01/35] start type stabilizing JTmv --- examples/EResNN_CIFAR10.jl | 11 ++++++----- src/activations/tanhActivation.jl | 16 ++++++++-------- src/integrators/ResNN.jl | 1 + src/layers/doubleSymLayer.jl | 30 +++++++++++++++--------------- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/examples/EResNN_CIFAR10.jl b/examples/EResNN_CIFAR10.jl index b74a230..c466152 100644 --- a/examples/EResNN_CIFAR10.jl +++ b/examples/EResNN_CIFAR10.jl @@ -1,6 +1,6 @@ using MAT, Meganet -n = 64; +n = 32; Y_train,C_train,Y_test,C_test = getCIFAR10(n,Pkg.dir("Meganet")*"/data/CIFAR10/"); # using PyPlot @@ -9,7 +9,7 @@ Y_train,C_train,Y_test,C_test = getCIFAR10(n,Pkg.dir("Meganet")*"/data/CIFAR10/" # y[:,:,1] = y[:,:,1]';y[:,:,2] = y[:,:,2]';y[:,:,3] = y[:,:,3]'; # figure(); imshow(y) -miniBatchSize = 64; +miniBatchSize = 32; nImg = [32; 32] cin = 3 nc = [16;32;64;64] @@ -35,15 +35,15 @@ for k=1:length(nt) K2 = getConvKernel(nImg,[3,3,nc[k],nc[k]]) nL = getBatchNormLayer(TYPE,[prod(nImg);nc[k]],isTrainable=true) L2 = getDoubleSymLayer(TYPE,K2,nL) - RN = getResNN(TYPE,L2,nt[k],h[k]) - + RN = getResNN(TYPE,L2,nt[k],h[k]) + if k Date: Wed, 31 Jan 2018 12:20:26 -0800 Subject: [PATCH 02/35] affineScalingLayer scales channels in place --- benchmarks/micro/bm_doubleSymLayer.jld | Bin 21018 -> 38746 bytes src/layers/affineScalingLayer.jl | 25 +++++++++++------------- src/utils/testAbstractMeganetElement.jl | 1 + 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/benchmarks/micro/bm_doubleSymLayer.jld b/benchmarks/micro/bm_doubleSymLayer.jld index 0531a145120d1a10218b78c9bf90e7595b7ddefb..894730d2083c05e309392f4d074861c44b546aed 100644 GIT binary patch literal 38746 zcmeHQ4|r9@l|L^b7$N!v2;tv0eSp@0h{?-K^1_0?gd`*gB!)yFx_x;`9ws#ZnnX&_ zV5BUhR<>Z>nz9PPmCsnqHlm!=Da((?}Y?~ zn2>wF@7*(V?wPrBe{*Ke%$d3O&A+Xp+-ooO)_d*C%PV~L*sS!-#A!3^D}A+f<<(X8 zxS8=Yie=06GzbVg@?m1h?I{SBFPu(D7?RWT*d&3RKTWav!ZP(y$wbJ zUK_9Ykt+Xob(ZW(Fm zISX5qb|bC=2O-PjXYN;yd`o7);|^;)J6JU0>nUeXHS0>VEayW9K^F z@V5sD=cyqNf$8R~|D9uGS=^=~*#yO8Vakr;alUYEw3)E^EgcT z^c4N-^^!|j_f#)4#ML16QjnirRaft=s=u9-KA^H1xt^X@1!tVP6i%P`AE?D28i)Zd zcg-YOI=88iLF(7psdN!VL!rG}{gzAEJPD?JdWv%W_P<`Zs7O8O1)7f_dVz|NUtZ~> zc6zyxX;MOrW4prf2q%Ujf2b!6EdEzVfbnmOk$JgISq9PL#LfwZkm#puXcwc!q<6BQ zzN(d;zO2u(9jzBi;c*%|W8~aD#QHuM+DS=uWo3E22iPam{Yi}21RJjApiSrJ4~P2N za&LvOqZ-6U0xg7H3EJHVHIwP7JzgbbxFId{-RiCK)z3`xRh5)gdTYOPyFZj)U0qQ( zGaql>NO!@08rx~4O3ZrFrSv41Gli2RKh#c)N0D7Vx!VLlkam;Q{0&1$j3;Hg5se2& zgf-q;Z)IKY11PR{Y&R&5I&ijw$F6jB;>?raCAtMpY?*EV?k{+XWA=;JzXMMZUq zD4e|~Va35X zxy=Kx58!got*-UWOcO1Qc(Q$kE2lN&)E0^9(l19f^Yzad0_g4C7mVipQhZ-yZdE z^WI*)?s@=youq))quW?~P*B4k)siL)JbkWSaPzvfVaaqUBh2ejc7MIdR$>!lN<8Em zltDvnj%7!rs}%+5KkL#6zhf&^&!9(!KDj-ALV^pFU3fAobHM#0Soi zxp;8~C|GlRY_b=X}6nF1HygN4B`*WnNuNDz~ zYj!HkQ|nt^cTz39V&TOxTv-IMKVVo{d0qYSjgrr2#yb|PNgO@*JNl?P zm>$Pb`cN^--gG4eAy)M}EmFlHH)h{dyz(FNGMxkan^x`Wzzt?*Wv2mmF<8vPAIriP zZdUY1lJd5gk6RYJC^>ZQJMOv{K*ycB8PWT5|1lX}do|?!WhYVqa8dXZ^c;!<>owf4 zY=L;Uk=Rf<&6}IEfK*)#l$6>NWQakjw>Tjs*#$f;Kq&D@bR@ar5}l%-mWP?oc7XZL z#Ns%oH=cZtn8M8Gy-8+1$C+|I;imgCpM4)@dAYGW@V>m$!Yq&LD0rCp+?$yBQX1fC zbY578g@A>Cg@A>Cg@A>Cg@A>Cg@A>Cg@A>Cg}?wqfc-9(pZ)8t7_JAqa=_Tsit>f= zt3%4P2=t#f?OZaN{OMPmoJ0QfmB(KEgWwN*Kkrwqt$xP4s^h`O5ZCr)N#tK|>$;EP z??*gLbin)7&lle3)35%?Y+0(N>m)rX*6Z+yoFx6~trU}|KtHd3RTCUw!OXm@TyjE& z=u6q`L~l`USzcgLOX#@FArD2VN%!H&gx8iGz0#cyc0t`&%cM5=UHcFBrR8U(3pcKi z!!KHt#vWAP7_b4~CG66d@5x*&q+qAICwqyRa#4nfU1Y1oJ)~P+eAPg$j40yYTfXy! z#N%oC3o;?>lQ-WNo76-;QE=ain?vDk8wU3+m=pnk)8D7~ObkYt#&wAQthWV8!wiaj=nOWB;|(1=kSw(=l6T!!*h_uWefz+( zY+arK2Pjy)@Nf=1LuHuUxH2D7Dc}0e=>_l#1y6Rx7r-ulkGB`!t z>$Zt~QX46=Gc!G2dQ~6_9y~Fr1R6>9A3mO43I$Z0iQyA`aFk>lqW`M`d{i>%`fW9c zEz;sCCZG9M9Xvz9-7Vi-1#?LDmsw7=|4zt%O$Ybbz4svJutM~kr?!3Dyh0?N7>ZK; zPZ`*A46YR-#?2bx`K$Cx1sKQv8KLcs8J6>Vp@dEU8!{C~W_z7aXVHl5*dI63NNvE6 z82T44Xpa2t!`Nu2l)OVDs_1pIh~AP_IygpHj92gdTi53es z85hyanf0u9d)jxqu=G!VKiy-VFNjGPY{^vs25W!Td_jUUw02&z{`HSt*!#dGPoZ7d z`{=vrG(R}{WE;}Gp;MndY}o~&fX2>HJF4S#f89oQ!LjkzKSO-(`N>UW503vjcj(%M z5sMKTfV_N3*A__XUjFkdF|(UqIYVc45=;Xj-AW*Y+COftnBY z5bfyLK6LHElx2t%zzXp_Ry?-cEGcaG%QrD}hl(Nz?B`mws{=Q9A?!l?{%1Oi&GQ9Z zlRJ2eABaG_EW04VvFrlkaAE91Q_V}SlU?{M-wn2gyhMJa z{jWR74s5-)oM`6MX0i`K#>350jR(k?@#@_l()V@eJlt0#jxEb2J$S36mR;z@E+w?taWFb(29QQ=R#(=$ZyoS=Sj2!P21N0AJOMiGX9A6V$k$+ z{e)LXaT?7RKEGuYeL1D#pTAG`ptx<#(6tLIz9YM_T@{iZuawlX3um_rul+4$X^Gj7 zgrB+?U9A8N*8Z&df&^!1%@@-0vll_w9i2C?fm5`3`=PlKhA9W@WgzC@x(9|^`{eNx zdKck7dHJ41`xDEXU^mr&%%A_Z9_G-)hIJ)A05{TG1Iaa~9)uf-n?L)ht?)LzDiB-u z)>desWCxBqw?VlWVMM{xyRO&{+bLOf`f3rRWJjLdvK<07^mYbf6zpIG}iq*8F*ffJ8IIR!T_-rWJy=x(WFVxEN^#J&GVtuMk@;!ZAY zeF+Mv3^jKhIRM*4FIqd}QpaAHZ%B7=PF5KSKfCc*L35&mfha;#g7i=Wq?l{_=)#hX&(&68$sI zY1(&HBl!nnk`EBQy5%U*s2AR``~^b8%zote_P^SmD%eHS=h42CkN;@{#s6i|A)HNQq^!m6D$Ro}{_Id2=#`*2pNG1J}!1wW|X+*mplm-}fWNRTg>z#)bH) zv+39;$|DdjYg{Oi8Cv5)*1vwT3y?C0`v0H)buQAq`x+0?y5dKlzGtm3qW+oI7dx(> zu?y|OV^50sC%PwGjnC!w0>8HsaG!FZM z=ubS~AbYTN)xM$YN8WW8A|;l^)=D~kourmsIJ;edCBG^Pm@nX`jtl=$0w~UwU9ju| zDntL;1=!MfE!qVbfBG`C2kyG<(k|@S_X5U)>18vr2h7pWcpaMNApcR-`a8*Qbl%HdM!VtIKVF8 zr|xTi=m#PYufgU=xBLe=9V;ZAFSCN;)iS=iNz!}QOZu~WB*nA29wMNL zE{2MdYySxCtMLCPG`_r{9va-=EA!R*Vgvt@F~4y}{_I%=Rpr%Hv8F#}qQ7UHVPg8D z^J=SatC@zMxKe*1L8UQHnOW)DKZA%>oAIxFxJ>;0iu!wJXEOULHC_?}Vek?Y*cqVl zoa`9Tc!}&J(0C4>_iDU&23g~|cwnjVQh1PQ@RH(qz-jOjlX(oK@shX#4PH_LcTyV9 z$sLx)OXSW=<2krJYP@(|ISkkbOsISPiv_$@;l4-#qw$is;TgPy1m3)^@tkXmByJinkvFw#JO}U7)_C#Us5PF8H$rN>6y6kR z@SJhHLDJwQB=ep>jhDn7hQV_t@TMA#=j07G8ZVJI+h{xo?;p{4@!T0{JQpv}XuK3& ztTA|ralCM&Xu2laI!W=gwH;x%g9& z8ZU)E8)@+3wI$qf$qli9U(;ffS^xhG%NR%QC+Gr1=yX>yE?1y~B`%!v*T zlM|e^CZBQFVhoskRaYNo5lG^PUIZ7=R4`bblP7R-OtSAm zHmt#64$M6uvp+bZa1)#%ia{Ya!$nRL=p!%)aE3CN&}b-^f1;uKWHwhZ7b!oe2*dBr z8P!f99xe>53_wlXK;aaPMNnh-8Tg^fTqJ*8atj8kVt`r2$S^s;Uu<%jt3-93fzl!jH@cPzGMof6mlU56XSfSyPCBll&cF(c3=xK(!t-nx zG{K^O_2Ls5M8V8TvBxuXk3kn~N&k%BQ#Tq0GJ<1qPlxv8@9y9@-0bM*$_NVZ&4r%6 z5XM0-T?pf&w=9?eOs9VtH%#XDgT%UF|4v4U$%dNZoXkKQL4bKO_XJH&1qho#VzTW7 f$;o0K0^oSuIbl6JP-06}`J~PHQ*{_8n(qeyZtdb* diff --git a/src/layers/affineScalingLayer.jl b/src/layers/affineScalingLayer.jl index fc15333..64a2da0 100644 --- a/src/layers/affineScalingLayer.jl +++ b/src/layers/affineScalingLayer.jl @@ -21,24 +21,23 @@ function splitWeights(this::AffineScalingLayer{T},theta_in::Array{T}) where {T < return s2, b2 end -function scaleChannels(Y,s,b) +function scaleChannels!(Y::Array{T},s::Array{T},b::Array{T}) where {T <: Number} for i=1:length(s) - Y[:,i,:] = s[i]*Y[:,i,:] + b[i] + Y[:,i,:] .= Y[:,i,:].*s[i] .+ b[i] end - return Y end -function apply(this::AffineScalingLayer{T},theta::Array{T},Yin::Array{T},doDerivative=false) where {T <: Number} +function apply(this::AffineScalingLayer{T},theta::Array{T},Y::Array{T},doDerivative=false) where {T <: Number} - Y = reshape(Yin,this.nData[1], this.nData[2],:) + Y = reshape(copy(Y),this.nData[1], this.nData[2],:) dA = (T)[] nex = size(Y,3) s2,b2 = splitWeights(this,theta); - Yscaled = scaleChannels(Y,s2,b2); + scaleChannels!(Y,s2,b2); - Yout = reshape(Yscaled,:,nex) + Yout = reshape(Y,:,nex) Ydata = Yout return Ydata, Yout, dA end @@ -65,14 +64,14 @@ function initTheta(this::AffineScalingLayer{T}) where {T <: Number} end function Jthetamv(this::AffineScalingLayer{T},dtheta::Array{T},theta::Array{T},Y::Array{T},tmp=nothing) where {T <: Number} - Y = reshape(copy(Y),this.nData[1], this.nData[2],:) nex = size(Y,3) ds2,db2 = splitWeights(this,dtheta) - dY = scaleChannels(Y,ds2,db2) - dY = reshape(dY,:,nex) + scaleChannels!(Y,ds2,db2) + + dY = reshape(Y,:,nex) dYdata = dY return dYdata, dY end @@ -88,12 +87,11 @@ function JthetaTmv(this::AffineScalingLayer{T},Z::Array{T},dummy::Array{T},theta end function JYmv(this::AffineScalingLayer{T},dY::Array{T},theta::Array{T},Y::Array{T},tmp=nothing) where {T <: Number} - dY = reshape(copy(dY),this.nData[1], this.nData[2],:); nex = size(dY,3) s2,b2 = splitWeights(this,theta); - dY = scaleChannels(dY,s2,b2*0) + scaleChannels!(dY,s2,b2*0) dY = reshape(dY,:,nex) dYdata = dY @@ -101,11 +99,10 @@ function JYmv(this::AffineScalingLayer{T},dY::Array{T},theta::Array{T},Y::Array{ end function JYTmv(this::AffineScalingLayer{T},Z::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},tmp=nothing) where {T <: Number} - Z = reshape(copy(Z),this.nData[1], this.nData[2],:) nex = size(Z,3) s2,b2 = splitWeights(this,theta) - Z = scaleChannels(Z,s2,b2*0) + scaleChannels!(Z,s2,b2*0) return reshape(Z,:,nex) end diff --git a/src/utils/testAbstractMeganetElement.jl b/src/utils/testAbstractMeganetElement.jl index 4dee293..8b38a2b 100644 --- a/src/utils/testAbstractMeganetElement.jl +++ b/src/utils/testAbstractMeganetElement.jl @@ -7,6 +7,7 @@ function testAbstractMeganetElement(L::AbstractMeganetElement{T};out::Bool=false @testset "features immutable" begin theta = initTheta(L) + theta .+= .1 # To test if Y changes for affineScalingLayer Y = randn(T,nFeatIn(L),nex) Yo = copy(Y) Zd,Z,tmp = apply(L,theta,Y,true) From 2b3f1383ddc4bf0b49e9c855bc2200cd5776333b Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Wed, 31 Jan 2018 14:48:51 -0800 Subject: [PATCH 03/35] remove type assertion on Kop --- src/layers/doubleSymLayer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/doubleSymLayer.jl b/src/layers/doubleSymLayer.jl index 887742a..517307a 100644 --- a/src/layers/doubleSymLayer.jl +++ b/src/layers/doubleSymLayer.jl @@ -193,7 +193,7 @@ function JTmv(this::DoubleSymLayer{T}, Zin::Array{T}, dummy::Array{T}, Yt = reshape(tmp[2]::Array{T,2},:,nex) Y = reshape(Yin,:,nex) th1, th2, th3, th4 = splitWeights(this,theta) - Kop = getOp(this.K,th1)::SparseMatrixCSC{T, Int64} + Kop = getOp(this.K,th1) A,dA = this.activation(Yt,true) dth3 = vec(sum(this.Bout'*Z,2)) From 875b591b66a85061a3129bf9fe1d1d418d643028 Mon Sep 17 00:00:00 2001 From: Justin Granek Date: Wed, 31 Jan 2018 15:12:55 -0800 Subject: [PATCH 04/35] Type stabilized connector.jl > apply (except for Q; need to figure out what this is meant to be and update a couple lines) --- benchmarks/micro/bm_connector.jl | 26 ++++++++++++++++++++++++++ benchmarks/micro/bm_connector.jld | Bin 0 -> 824808 bytes src/integrators/connector.jl | 17 ++++++++--------- 3 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 benchmarks/micro/bm_connector.jl create mode 100644 benchmarks/micro/bm_connector.jld diff --git a/benchmarks/micro/bm_connector.jl b/benchmarks/micro/bm_connector.jl new file mode 100644 index 0000000..05e0022 --- /dev/null +++ b/benchmarks/micro/bm_connector.jl @@ -0,0 +1,26 @@ +using Meganet, BenchmarkTools + +history = Pkg.dir("Meganet")*"//benchmarks//micro//bm_connector.jld" + +TYPE = Float64 + +npixel = 500 +nex = 1000 + +K = randn(TYPE,10,5) +L = getConnector(TYPE,K,outTimes=1) + +theta = initTheta(L) + +Y = randn(TYPE,nFeatIn(L),nex) + +# Warmup +Yout,Yout,tmp = apply(L,theta,Y,true) + +@code_warntype apply(L,theta,Y,true) + +trial = @benchmark apply(L,theta,Y,true); + +Meganet.updatehistory!(history, trial) +hist = JLD.load(history, "hist") +judge(hist) \ No newline at end of file diff --git a/benchmarks/micro/bm_connector.jld b/benchmarks/micro/bm_connector.jld new file mode 100644 index 0000000000000000000000000000000000000000..2942fe8c773dbe85f00a33927d59d49be690995f GIT binary patch literal 824808 zcmeF)cbqLnx$pnlJin)ja5?Vw>47;|l1(eXj7C{W9qGIl2#9U^~ zZB|TW&e~?oWkxZynDcLDe?J9G$&%x}_x$el9QPmlJ>Tbfs;XDdnl)?o;LC#^bi(m# z>@jQBt+Dqz{)Dx5#{>4=e~+DRVozIp%Bjbne3ET<*>#s)&Anv#NYUH!85m7Unp5!V>}*0tD}{giKk5z~u-{(Y@&i7SM_ryzALDseZpP}PPg%QW-N~nvT%MP% z6#hw9Ej?c^>-p<=BCfxvr9JihHUED8{*(9jf693M{4K9zXY;yN?n8OLmgiNe|F3$N zLl4_`)zUM#oM$)GPUV~f$j4RJ@xOZBf7f-Z%qHt!2R7rzl{;GQ6D|4iE8{m{`UqKe`hb>v5UI!EqMIp zVU+Lx=WecwD-HawxX+th>WqKY`&{nZ(%Y%^f83=W)AEO#@OfC(tlILRyB>VtUaJo| z@X*5!AIjJLf5m;h^taxB_4|6H-rkzazpve1N9=RJ1w4J1zPA6*zb;(*8vfPq>v)Cb zXQ%l`_jU8TA9nb@tM78)QTsWb+)MNS>HE4gyV3Fi#;*VS?_Y2A6|mvb)T#|O-01J` z=eg_G{j2{$}{5s&UG$&KYL`e@_N?4=~*h3=he#YKcD{D_Zt^~J($hPkA~$f z@cP5c^FLoz*W#*6T4ke1Qr)bTx@IU)?S+482ZM1>A^r1^1FaLbB z{3Gddd-=+@GVRje%k?dnI!ixO-GBW*TrXd-x$4qu&hj_7a_{zDbL!e%_B-jc1J;~+ z+*W2s)@`yhv$SsQiT%1W*8dQuas7Ckm-;+x*6Po04QDQoZn*RgYWaR#ZRskP-_e}5=7a~WUA^4-$A9XdN4Mgg z*rNNme0eMG=Hh=LzPRvT z@Xgg4zc=5L&&BfShUWAY_qShr%JC;3bL#5ds(lv4|NYNPw|?tIpO>2~ zbuZ)da@QjcJn7VRYff5s=K3f7-+i(kRz6Sf;KIv&x`6xi`->mYITwHQ|KbPby8CPD z>Rfe^m+|@CJ@Z^#9^KH4D?Y!+>$a`Fd9}}?SU$i1|35GGTz^T6o)^pCJ}%>VvHV0h z;`kHSF7-BA`U%yTz)-o5Blq$Ave@{GKU#DT{$2myzXRp|pL-qc<*F?&<9WPCzQ!d( zORrO_)_)&Z`l0uFOXamEowoWv^ZVKIeRSQ@lVbJmmpo@K*?Hwh<@x@v@O5(Z$tRw8 z{JPbq{F}_mM{-HJ#;WDt<}80tFOPpSv+k7R*PO8Qee_bd_>u*e`e^y3WcmB5#R@t~qJ#x?T2Od(zRzow(+d2b_8FM@O7|@(HKza>VlC4_kjq`8sV_ zT%IQv@I2|tre5FIi9bv!blr<-w zdiigl%IAIg>t=a;`8n{9{*?J+k1i`Tz5JZ}M}Nw?6W5-2@+oJmzWCo~7OlDT`?@tJ zoN)5dOBZJs4O74Lnyt(~{=TyO_P#vjF6&vlYUw-ba(~;hH}8|9RpEfDZgxG^ z=k1nz+R|PAzWvYr9I*TvP_BEQlTTT@%ic?0Y3pnHKDgNi+PgfT9M{!RH|Bz}{F00K z_ecM2@4#g||8}+fY`grRFQdQb1@@BLvGnWH4c31@UV33z{#ijkBmE~{SlHsq_71$J zIJ5hfX7|4>ZFuUM6Z;d^zuIW%@?LW;{dK)|=?BB*(=YG!=F)3UUiy!{j$6CtnDS~^ zzNRmElq)~FtZQC+y<7jC^p=d;l}r9EmG>jZ9<4{anuXWvTptRm~|(gu=bQS zCmp@?gYfn%$Nnx|);055d#PQtZ0EmQy|QpwdtKho)ywZM$_q>NdfF|2D_Xm5=|8wz zKcn9#?aAdb}S%U4H$0=}W$oUG~E(zmF<6 zZ&mqy>(YD5GD%?=?S#-d%NE(KiXAaS!H%w{!Y5h-EMrya_1lZ@vDF0mIvHp zq<^l=FPHH=X%qbq(f>4lR+>p#3}2xCCF#FQ{|D0jkp7Rz7t#MU8SQV#=>MXeFq#Y3 zUw<*IB5%0TOm-Wo?l+cZQ<*lCVRIRaTPXW2WyY`-8PgS&-Ib)hGJOnJQTA7*zO@Wj zU#Z)cexK4s*$`Qjg+2>l)Tw6NK*CAh5rt7WLZLjPx?x1XMAl(gRM1LdYu;WT2 z+3Z9e{m#l6?M;;vW;8cr9VT=w^bI5&?QN-JD*1M*drX)s&FwX3?!Z2192q@^y;V;b_fgK6_9a)E{Z)^cG2M~<4rCq1 zgOtskWWb2I(jCltG>0fBbcZTOOqkIf#yo~j+1{CTXb)G;=i-JxBr?9q5-iymX`eKnWR9L>I%(Hx_C(UL={ zuT?)_x}UN?Rz}Qdj-!v}cx8*Bns$jJ8)! z=pINP%_+)7OOEJIRXTJqJJv;RVH%L44NxOa>88cpT@oz(L9}cWg?qrsBbZ$d8XUe~LVpf*bkAYkb7gv-w9l6w-3yfS3s)Lnq->%LXkM)BD>FI0RDJt0=Fq-e z*>7%-Rm+n9el`(mmz?@&Kg#&@co(VoYC7%*Z&`!3A|jF>9@ zyESLt!#w7a->bSy(xZF7a>QKw0rkxXS&ykQeu#ZAf0%V2p^gddM^#Ul(R@sGiwA$LeLi;skhaMwlG+)IyAplb{H^XLi-!dh2OFs&F@%W>B+@F&KQ2LbrJm^*uOH8i;0{o%^$fA^q3ZC z|HS!!mi8~wV?z5^_QiK<{t-iyAb{o}0rMa5w#a!~$siUuqrN1rbVJP_;tlv&r zjF`||Q+<#2TI`1bBPO)hW)3~3lCPtFLVI21h}kOp>ruyyZhQJW$b=c~^;LK1EA0)~ z7wrw%uQHO;jnp?gN{bFtWhR@Qm`9HR6PlfwtMuf63Ehn~Urgj&>29LAjP|CSgRU}> zbEVyd^Dv{ind$*etLzsTs29zynvZCA;~WeaF_-@C%%QIgU*gl(C)3A(CnjZ(NzX=(d^4SI`kOO?5DY+Bd7gU&uH(c z?9f*xaxQZRXg*-XTxkx}oJEfTBbtLWm(bivxfsa_-NBrX5i^=Y)c2Ut9ICp-TxkxY z-${qQGLbWyJF^ZWCUl3ZAJN{0{n2BnjO5xxJulE4q5X@F97_GJTrYYwccYG}VnTnE>Iv;VnZtmoGMBk~F^>*Crb>5j%@;j6VXn0IVSn_M zfgCFn*{45Zaz^8|u4u`=c8%r&CN%d|-D9Y9N7KiExzZh@xr}D5vc+&e<%kK* zvCLIEaxsvLiCnbDX&;9kW2HHs^Dv;fKlRE~@&nZO7%`zaLH(j7*E;G&PY#udTx(C{ zI!}_W(vvfqlesR8m`lB@oI=L5KzFL(A7F0VAdb+J|c1Uyhl6 z(L9WEFrj<6>cvF1kD!ho17qJG4L=BcbhhiQRk;5yM( zIRsrf&Q7=uNcWi^DNFoUm3{e z*_v|;^wa}pjOS=wM)MrbLx&y%Mog8NY@f^ZRC;oFp863J=8~VUenRsCxT(rFo_L4t-@JXSC;P-eXu`q+U#9`zqFBsEp)<=G9yuM)a>?9}Ja|oEGR_ z%X$nLF)z@(PU|c>3>Yz?c|GT#c?0V)V5-dIq8(X>4$T`?4;V3FM*AkN1H+q@6J~U8 zVGiBf*dP7d$(80Es#|oJF}zd#hzad^s(Z}n-o-p7%xK=NzC(|(GLeh^Jz5tqy_fUf zM;#+3%;*y5V!(tM?fcoUGLnnt1DY#3vd3KdA7l<)r7!&tF^3L4X0#vHT+x$@fn1Cw ze?;pNnvZfG+66l5zA})Dv8?}?_Vs8!&h?_hfaVkGM@(ow$@Nuwa;P+)Vjdj^%%%Tn z&1H0-A*1`Oazr;NdyJSWUwFDN^-UsNt8a{7|$#Y_%gRy|^>G+$vqOqem8 z&%TxRtEvY~XurmO7%^e~PinrdecS>)^-%iX(7eZl8QnM4PiVfyb)v_JxzxYSz8KJC z)gAiENKU2x9nHrDX6p94To*d@m@s@#a|zA&l|yABn;)p}(EL!@7k|WhjF?KkK>dUn z?T=M2268cx{ZBL(F{8aubype4#YE0%e#*J%e#SmOmkI4Jm_vsi14gv7<|Eo)k}+XM zcai!YBW4W0Qok6Y_2J)_%H*;giVMzfjbECx)KW^?w#jCKpv1KKT>9Y)M0Z>4@lcLnCr zV?c97=FwhBIbp`oP_NA7T61O12h3=$LLK7*?NvDs?bfWrjBXp|Fr&Ge>JBs7tE*o0 zpt}JXGnyN+9vym2n2R^kyxCED3}|;!J?$(r+8Zl7OlWSRxAN&XjQ-LloOg;(#MQ;Pv+3v zO4*`AkD)S=i@D6-TKh!IXl|prUtpwOw6|p*6Xud{r+!9r2jzgVGLwtOF^>r|n!VJw z=rLigw0mnlVn(|Ubu{~OZe=7FGuiB?d5;;*{;FT>9mxl*)RK#y94aHZn91fq_Ctpe zbLk(%9NJ1(`ghWt!+;6R!Rlx9hbRZMhbkw`=nrGxPI?R&i+5%{+QZ3c@1k5AsJkQ7 zk7)0z9MIiOIitI~vabx}SZR;cyt&6p9XVFoqc{)UJ;`YAr5r2my;b*U?xXB5R{GWI z2TYjJdHR?u(;D^deODUDMSC>+V8nzO-7%VTYcJbKy_m`7e#}=!a&4ksjK{M6IBAYw zsU?TXOfK5{Yn{i8@d2va6E0gvy%@-`%%8|Qbd|B>lQKLMOLi1wfVkWzn zu&y$Z%}dz_Gn$vP?iDg)dZn^CS6Xx!Dl@qlUd8%X%Y^nd?1vG}YgP9c(Y;RffbsS0 zhZ+4FSYH{)8RMuq^G0bgR@yhw$6RUO%sy{fX&}eSM9!7wt*n0==b^)Z=IzX3z+CF@ zP~Ty~TPJkNlj=qL ze&#C82UIUwa?z1JW^^CaI`<)&(0y3henjTV@KNS5e2h7?pI|*YG@n%6W5k5!Q|yE3 z)5^t6cAsG%OlUrJ~jl44-3NWg=(vpJ(n1(tnYR3C)*OF9veNjN!|yulko+T=FJRq|rN@8~6WR-z|EUZ=!=KCi3+ZO+7m;GLkp8_x;+SIWA$}7!RRgPOzznb({ zC!@WFve^!=DJ?oQ*HS%UzP56_E_EyO^`zNeT8ul8uP+^DG&f)!dQ2E@sD7>8QFUM3 zNjah0S-I%R5fg@+YA&PMMcHCTvzzLnxVv)1u!nL&a|`97C&yc=ZueZNBl}XnmHH9w zt(6mIw6|fO+sfdW!(8e1Qs3>(JVs2IOMf5sL%*+b#Dso-_QQ<+j?^)tJwSDj@gQY$ zC+Y{wgysF*&UX0-QW{@yZRypOW;xJG7-$0*0OGTcw5 z(ovRxQK{TVWxN&gWTs6PrHO&!fy zLSGvdrkNqU^SoahsJUvb~z>0o}H&!?c~UzqZV1ugf~L*CX#B z&Gn_bk&NhfQnouYcVpZ|`eLgb(Cn&gG2C1^?k+Q$J*Z>GuqXA(bQ{)VysdJ$y)=8X z4in~5-$(s~VPDqo$2zq8D|<{OAE3H9NM=lTV*SC=9U>E&LzO+6!<20&Gp0K$o5N+q zR6IiUa5w4iF71(c59yAQ8STB4Q}N!+uVx{%Ct_J2g!`#H2N4GtQ^jevG@=&Cd_Ca#=6Qxb`PhI8EsJAoFy~b z$0|DvkE4(FZ2FI<{si(9rN{InW&32A&_6{vVt6VU6Xrp6_aF41CKK9cD`#})D4XZW zfcZtrKFaiB>0ZhlnwKeC4Cr5}x;b~H{#E2x%lH~;Un>)uk^Y;gV|=r+d5iQIFu#@h z+wq+;U`BtQ>h@jIy<0}i7~aR+2V_F?A?1wm!_0kzxsOWoapur|QrUl2Mzo((PQ}kF zhcB!&evvseUsAr<^Ob$^tK_e<4#PK#_C7(n=+57vfo_&fO!jLyCv(;Zl&xnU`BHV`q*7jdAgEp8kw)k+}4+EM?K-N zjpoLy%e1XD+sXEta<09W>haq2(OgH_Rfg-TZmb+JV|P8(`{MTOvx7`%uCF{{a|302 zLm6>GcO%uqj?(W$F7B*6qq(tigBIgWSbtO5W4syb(YMNRx0Tx6l^u3y_fUPpjQuUB zqr0W@fEoRss*kslGfuZwZf++#447_D{|<7%gvO~J(e1@P==V`h*zM0gcVzAWGR6ay zi}@gBdnegrb1-#m4^cLU%8ct;Z-bJRnN_RIIaJaj2#{Nj<_8zjor<^g~TRCC4 zk8;Err`4((&-{I*J6aAHaYTQN`u+W+Jywo59k1M-D4U*)9y7KNRKLe?igLyo)2Ym@ zlXLMwWNc1Twr9xUA@tFmsoXwP=HkPYn}*o9Ak7z=$)B&r`p7zHBQ8vV8&l;tQ2~43#5! z!i?rcT4%Avh%=g~xuPeJ*t}T%q9yyvK%TLEiPjbEOO<<^D(%bE?=YZyx#}K!oUna` z`VKox*u0WHW^B$?y{{a|Bc=sT)Qg$izKZj(!;Ew3znb%LtenV2^BT=ZoGLTfyq0~@ zzK)C|Hm_HGTwwnO^#@FqZdAXk4CLZKjyU3k=8f8?L60L&n9;mR^F>eYallyS-pu~k zRt9py;Vqi4jnu~l+P7+5iw-;V7%<_4bESEk_8T$bv_SKA%@-ZHtL(`mCY;f{L;Ez? zE-+I!@8n!;(BXh1n)BEnE&9ry95CX9-Mh4p$54E?>Jevb-lKYp0VnL<%Q_q@6S;Yx z=8BfwRrcg!B%7r5MMw77R|ay#w7`ja#u=OU>%0yFMog76xoAGX^3haP(jl><5Ah#Acnxn69r$FVY#i{?vQ*8(l| zw$hP{9ob{R0pkKk>Iw4#XX?$Db-fnbN=FVjVMg;6t#_54++$eaNIl_nInMNp?fE*t z=*Y#6T=e8(Pc9DR+DJX&gmb0&s;(zgM)FuWk>|?hYkXcRJ906Q6Hb*g*?gV-u&s3D z9wR24Dl@tHhW0I5a*MGtk*CT`p0W8R*IVhx9tRvT{Hp2U1dl1*yB(c$IpN>*km3bI_xTYa=?fgXEfi@ey*}32aJ`O+xEihAWzQ_4!(P4)H2b?hDjOP2=uh@`nWlQcVJ$aEm{o+6_Msjf^C!BCb`vYBX zhaP(j7%}05=7&69wAfa5a#cCx^;`JYim-`;*SA?Wh+cxj2$1 zG=J7U4R+X9268cx&0n;>Xvr;l>?;F#sEp)dBG+c>?O%0%u_s3yaYFMq9xrz2D|_;U zGdAXz>;F5Rq9u0=^wdX8m~lqCO8d6xFyM$&WwQa-gBCmV*jEnZh$AMP(QK&wZDmXL zIN(^B$ZjL;)8n|nM188vP9N>b*WQX15=;`+h9H`G|w&6VN z&|}0AXY8(~bwy7OI9!gAzPY;g8F0je6V7P2)x5)gfsy)z8JlZx-54-dW^%FHj&(R< zMte>5TXfjtgflkR(!9sMGLSRcYjb{OAP*QT6M05+9j+5Sj+Kc#VaDdVT5l^oxfsY1 zr^-yWmg}i>Ij}fQJOg7hNU+gO*IpKsenj2`n zMTZ`H95G==b3@KYTiKF(%%y)Lt#jC6zrdM#(d?-8ePtvUXL7p}=T&y(;z&+7VZIz^ z`etXH(_&valFf~|9vpDOxw5;7<|B@naKagzn`*vCvx{PHkD+oPM;t2?xw$px zVUGhQ^taJm#0fJtw^hH`k~<6w9H>W3m*Y&|+)n3ZbhlUTDhrY5W7b7|0R5_Eoy|n*;v2r5kO0&1txk^u-aK>RD&Bw|_uFceoW?!whIG4O1 z*MS2joY3#DxxR8Fr^-w=chtIKL+&a)xyJz`j+ijxjLiW$r$vVzW93XX2Wp+i0cUIu zQr}^Rp)!(3OqeUpowQ$rwz4DF2I>PQoN%r*2XhWOOgN)IgvVdmlOvi#HP>Q?9(xQp zV8jt;G>370XtBkv(vypU9C5;?<2un}j}zv~ne6VY^~HfaR(6ML&SSu#aw40%urGG# zvBy~19HDgmugL=5FkZ9(&BCe|OEb=&(b-z(Bn?k!LhVYTu@EAV(Z4 zn|pBG=+GafdXFP^_oR-^y_8$@*yDf^GrD_gU57mm7;(gm_C8uyY{@+aG^;h&R9bS2 z9fnHlwLV}(vqtq6JvR4M-Cd4>e!>~MqqV;1$vuvk7uX!5^+iYaIABJzR`bP%>@in1 z_tU&ZhY?3iI8`>sYJH0VBTkrc#`ZX^>(FDu8Qt-kD+Y2gk`w02=KfqC_GlhJ9ew3Q z&S+0yzS5C>Wgr(v@?05D)V>*KG$*Ovphbs01{`ogce2)d445jrUUPAQiF)%u&Dlyv z?$BevSeeM?6zx~+$UP2dPu1Lj5y#4joYAh+I)@P_%$3c9G~Z#wgflj$X|8C=E%pnH z)F<>0);<9T9C5B}PS?CehaCnSDkrizL+gv4JYd9JIg`7GXnkLq$i<1=oXPcKiw;Me z(LPl3uF{i>JvmfH@`UzbTrWl((L9{%!*+p=dWQoxkI?#JOD=X~kD)S>N6hGg_U+JP z^GMZO95CaI=24ofZKzx9aKI5Wnn&~au*DAh%7GlwoW**KI94X|gzaOru3O+heMa+G z&cP8UoJ;?4nrqQvj{yga*qp6(w$hXPQh&VW9QHWiTxp)bb)&89$R2wfDkt(>**uZQ ziyi|Gn9)8-^BoSABRQjaGLN^iBNsin$A}{)%s6BF6zxA$Msl%vs^*FB9sdki>WtW4yLGn%JqpAJ0+9B``4 z>n%F$D+9SWlBdc{p0Rx<_X~Rrm61GRM)NGKZ_zKXr(TTY;zX{^)XlTGkJzHafCEO% zIAe1Tj~4^R%8{HZCvtHnH_u^T3^-Iq@{G-Mwa%h1`FW~$=rLf#u`-b-G|$&Q#fDt8 zWQPIgO7jBtL5E#sPmVZY#^#0E$5-~`+CV*G!i?rcTo*RzF<`_IQ{_Z9k^Qj27CZFV zjYSdJGsV6Sxt>Z(Zn49FW939Hnzw0RTiKF}nQY#!`C>zE z(P57PM@*FyIpd7&J9JJ!^G?pi9z$g!PndB=d!F`j*e`IPK4QX*GdAyHKkOD5sgF2U zHt*Ix7F+ZfG2x8OdpI9^OqIiXH5V&K@`N+C@6&v(qwXtv@_-{wIAfc*UhL2>FjJo^ z?fcmWyGlrWu_O05V8jv4 zCs-m?|f7#u?40d7S7mR8Hig`Ha>T z8?voz$sKy^F`@aa_G_`jhzXmC>#cO;9%JQ5&Xw)ww4cKcJ!YKId|vZoWg=&6zo5Ae z`^u3VzNopOGLk2p(SAwuEjo-i;f&_Xns3lzhXDtSm~g^eIg^X+S9FfUzH%m;^EFp& z$VE#IIAFqDX}-$iLyIkXj5t-AuW5aYzOpALoGLSU#^&qV&tk+0=gQ_ATtBvz9l6I) zIg;lx_bu%=;Ec_;d7PCUIaChhVj^d3v-a~CaK`RC>W9jaoN+Gw?{eMPe2?{Lzpvb) zLyrSSOqg-T_6O`+naC41Kh#{01CE$+t~5W=x?)EzdUB65HWzSRIAQx^)oUH~Vn^;P z6L~K6pYXVF!dy9%i~WVH!vSOENG@iw`Ki_yE!ks_15TK6M*B1E6Due3jOOQ@SJ{v| z3>Yz${0q*(gw3pahZ*fJna7MXnv2wL(V?#lhvIM5 zk2qrUJJzAc9tRvT;f&_@oR2*Qj3xg;a|tufX#c2wj{#F<^C#voV8-}o^(XB9q8xF; z8JoYV@38rs@_-{IoY0ty*8lH09s0^ZZdPf|V~+zy9C6w}^JYWYqQ?<)>2IXD4g-#u zE1QiqpD=EsT+HOKsp>;zCO4a@?{UVsIs4&=VGGrpEoF}*#;sHz(OyA$KyyXqVn>cR z;)EH^m9)-bz!5Xf=o`(qSC$cHG*?l*!+>L{UsZjN112IiuN@bFjq@2b?R-HMGtyFi>x|(_GP!Q>kB5{eUyJ*HXPhk3;1|&S`U!gdGMJB&D%`t{ZCG2(>o2I_lk zZ>a3BLyygkIB!Sx#efOTPU_p8*$;aRIAX$#GrAjVeZT?Ro2cGlz`3%!spdRJw7aO@ zRyuOPh$Bun)4I8`X;m+_W50Ro9Z0~9I(B)`a`ANU3IgEY_Ugk3--kx%`H_Q zF=Ml*>OHz!DSI4n#Dx8=HD_-lJMnFRsGP{f=Jr~j?;wqn76*)& zu-Qv<0cSLOtL|_{yN~J~17@`Qs^6l+p>ie{yZy9o#0k6oIj^!O57^yNb3FzeF_-=U znj01vsZW*RK&}sGGzY0}u|l@e#C_CuBs0>qPd&u z4cf|<++n~0Q)MR4nD4Ir-I3DVL$=tV$NnhwM-2B=&KU1SAML%BJ@(k#M|F$+0t5Ak zBTksHU9J5hCNy6423zbgU_`q{^F5Ae@2k4UfDxOc)o(Fi#2K4oG&i7It2|@8pK?NT zta8zk2h7+V$K3JKVZsUL%5ZfOQO8g@kef%dF9uB5ouz)ngc)bFkI|gReu0^~d93CGCY-Q+ocdj*CkLExMt`=} zjhL}{yy^qCPvCqUFk!~#iJG(6V#4N0>K83}z=#>clQkbPWAhaDL5nj^PvtrW*;hvL z@E_{WXr88=(LY^z#0kSQRG-j1Q@OzoGumgV-=TRn>#;e9^*Ez>j_NJ;IAX%NvUx81 zp+%29CY&&%eV*25Y@V;2Fus6{6S^0w9&o@BC(P(yr1|1V&S;|g-HT<90S8RzUc!FZ zW55C1mujx)$us(wsXtUUFIRoW<`v2o2TYi;f2HP{bD75p=LL4J(!8$>aSNnRnFx84eHxby2_3`V)I7MLsvPHi<#WKiFG)X z{ASeyMjX++g+4}1*uGW$0Y`LiQ{AI^J98Lu#2MRnXwJQJr6W0E#`ZkTjhNBAOZAB6 z-O9GICnwA}WAh&NLH}OmgcFAMF^4nS#JT9O!yY4=_cM-=IZ@zH%gok7`}|m^2?}AM9|b44=^4fXyeBdu%?XoUs42a=-y2j-OF~ z!uVNb`}vi&Dv_#}VycRCj3q z#@wo3t^d88-9Sz_W7tsjjLk;MBlepp2aGr_&~B>vgw5v4Jq|dc-Gcebc1zU`HD_?{UWV%Bly@lIay80Ho%641UW5)R!s=MuEKzB{$4*P2>M@-mWNA+~wmCj^iRUa^7#%_D{dyLrZ zpn8WB`s=G+oXEwQY;K^r`G(TmNVYhZyd!n&c2bTwV!|2C&g_c~T5O9q(Oicf2OQDd zRC5!~=yp*(pu3s!fHRs_^Nlc+C4R2?8yU$Td6;xy|uE(9w(f!y^ZEZoG_!kt@=Hh z+bK8L;*9q8oQoq)mHrNz8!)1A>|fcDBThJ@*^719qC<}nXUuzRo!LiP3>Yz^*_SyS z(CkMYBc}aT@9roA&S(#yj_E+IaOK&C#5L9>+>|jOIG@IO0_LYnj9Le#$*g z=#Ew0A16o5*dDLC!+;~sXz#DN7RSm&o-kMT58!;XCn)>MNVX@^$AnX*J4tgL4w$ex zS^a<+n_l%6dmPa}Q2ibU%xF$kzd=_yk=u2gQ|ZYA&S)Q``G66pN^_d#25cX!?6Aj( zQ|X_sxgH11IHNg3a}Bnb(LO}|79D!*OaDyGb=c#C<3rV-E8W9bhZ)1eRnO=i!5n%t zL3O*pmb%A?8O*_-N$`hqIIu&S)Q_dNDp$Iiq_V>##jr*;RV- zh#9-bYd#d8pd4{R`$W}yj5t>MCuy$OKACkmV!{b$Y@VX|4#&#*sp>ZabLg?hj5FH* z(0u!}l{#|KKSTW)`)4XgG|y7*FyV~m+3FWta*wfij^+}Y=O|n3FyV~tb2Zmv`#k05 z`O=}sVS$;tc>(Kjz>IV0zfg12i=>S!b>t3b9A2!sjLl1w1GXCoWBXy%r{YJHXKX&kI&43#+~a@~&S*cO zx%QK?!;Ilms?TUX%{;o#D!b3igfrSNsNQ^0TJ&F{{$=Sg;Ee58)E{v|^EKAtfDxOo zt3P1=hI0C*G~bpTwpqEy5fe_O{vFMQ@5%wg_mn4`(fxqAAIkPe(&Kv$$`;3KDRYIBr zhYrnsR3BH%!ILqeS);ng{ut$e8E3R>)i1W>jP8EycbshQFDGnIR8A+YG?V?ws`ov8 zbf+kfIAL0+x;;(0;)9hvMzp7^9*buvj~LEWZXYTW&XwcC)ptR5kCXw;qm(^1k7oWX z>W`J?ak4pE2ArRuJUvNf^iNhU#zEOUO*YSz_E|DKhyHWv7oVq`o-f@C_2hu_E7i~E%K6oDdaVqvlilm7w zPC4Rm9`$$04#)Q@+xN+c?FX3upbQ_D(?_KJsPs6a`?%^O#!o8upOWpT<%Hd5l)KN$ zFfsQzX}=&x3|~^7(R@XD!i>%Nst;e4=4-O~hHSC>mU6-_E6?AR-S?%(>4(bg1#-fS zGp3)YpDvW$Po@8v9DXjFU&s+BY<|i7Mfhu(G5kh(_#O4%%kB>{VE0Gn{w&>J=%f3a zax}kL|Gv(wk^!d;m4}Vwys`9~%62n3;kY^VEo9hInysY2f}F2Jzme`LD;>%6)~a{g z$hhrF{WX+_?U=h38AtTjR(-y%v{p{ntK44M?;rzq*C*p#X>UOPhH}DjN9D8=^_``; ziL|@OVOJS;lkVoSN4E!aIAe1Q)y>G#;QlA1KFzWH?y5 zLu7xL>^d^ecUBIEOLK%A(B75#yUFJ6(jF<(QRI8d9s>^dV(#A3uV&86VU2W0OM9$r zkCR>Tc;)8)vd88D$^%X(D32#of1q@yNPDVm*U5Mq{Rh)Ooj!&$s6SMC43DCImh3U0 zJ6rYki84P`hNt5*WcO@5M|%AK*tz%cTc`Tp|HFCZkV(p!JH#S~S;!JHNG!)@QWP-| zEr+-r6RJ@;tmH6Lgrq^zA~8F$l*1^hRZi0(Au&Q)AR_8E>zo#q`O!K9DPgO zxkL`I`)%%BDo2;e`f}Mvd!@SjeHnfvJ6B6Tb8f32tE->LbghgxG5^KfdU}ApU+X;F z%yZdW)uY>a{tjvHW{!h<)ZO388qEXhfTQ252M@|BN%t`KaPVjK=yBP7N)E95tU5j? z&GWLxG)LY2hqN!s5jyjn{{MrRN5*-jn@`s0=cgBtej#ZmN%uNgEh^Jua)j>nYV$_f zS(V9^z+N>!PcGqUU4)@oc+q9nAy-kL<%VD9} zkaO(6L*3g*)>EY2M8?hNcj4yDx1_g`eH=_xcekC}noc;{PUmrZ+1p9__e#^_d}rqG z$6e&$1N5%a?2dcL9=bi%)n3wK+*_S+w2ykQpR7M7{Q+`#5FRW?m<~~!L#4<5Vd{8< z9AfuKb#;{N93y);`lLF3O1jT*&&$E*n16n516`k>dH)Ntdy4Fxj%Ud5rMc~VSv@>U zChUGyogL8qbC{ne>#yP0r3o@#C`TACQipHJ!6njMHn#)1ce&;+`(1T)h3uidQXL0! z@B{As5U-L$OxV9#^WHVG`(ru4&QH`Mbl0jQ4l&)Jx%;UcqW_sXU_x`F<~h2xVV!(*? zojUK_#l5?wNAnxC!-xs1dpJk)TlMf>Il{qx>hwEVKS1LMyT8}mKPWp%I!qYq7gK<{kqMFyRR6zv_PV zH`zmn-KRD0JR=8KKdW|smmWu0J*Rp7JkMd6qptojx1E2f9s0sRXYC8RpRHe1o0nwo zW$7_s-#pL%J-e*um1#ca3rLR%`wMCACUK7Ob?OoJ7gi5&gxy6n4;U9!SFe`|yNj!{ zH9bVr(YZsrgt~_V94^VdH%POTtlr4|!kX^9iE|vHUt044j<8xr^X{^8h;cb}Hqj$= z%j?{)Ft>s3tjPPYx01SAS=MMObwKlGbqBlXR?&Qb^{VRrYBFK9y1IvM4Ygg1d)Rxc zI;<@tcGjWSmDPIEc4ffs+qn03IoyDI8_EH?cW@888>v0oDV$^dPIYe+*~frG^qcBD z;9xU#^)6{Qr>(5Fkln2~$NtvLw~?{1+FtV>4t7w7on)uSIXaB*(|okEtap*!4@iqO z4t8U{hm6?YQ(f;R0}j#bqj^HJuX^}Vp4(5xX|i*G>|w%qu;%U%?j0(-hshz1u=)h| zkCpK_**{(mFnxx(=icX}{k(LTuy+FICv!d>zaaZxl<72Ce@PC%%=|1FzbeDmrN2OS zFO;2&gqB%x}1AgaPLYv{GoI|lICi>PTK3`5N%}sOWFAqeT(d8Z&eSma~t=E zvc3cF#Ji-qM|LpWryk+p0d+$Ad-d=kStn`!AiLRz)xGQ^^dDvCG47-Lle(VG`Qx(6 zKBbP&$n>lnVR)YNIkNhf?7l$1D9wCx^xuCtM8AOMX+dcx$%wtzsr`~NVErcbU};$` zBYWB9)e-wEabC#*jzb?k%{6j95+P{+hD8mh5Bat?B`qb<`Es>#624xta1@krS_3Xhf!&2egby!4-v^`~X>c=Fu#PEmJGm*HGFz|nbX z_cdwGm-Z4Fu9Tzi$?6BP^Fvu*BfHnj5!N>_kFvT+_J1yizvO(DG{2JWR@u2tj&7In z4mrgBU26LqS>HqdmV3XG;X!F0mi{rB9>=HU@EKV>C*upUeo>~EWibC3|8v`&R}SYd zTvR=Jy-Z6>znttYkE=?vnjBzUU7gmJBkZqBuP0qs#*L)iRMzOXQ2VVppDKIkw^yef zW!!~(d&+8WS$|XptoP@9noL+9sO}#m2WSpacMp}FTGr?eQ;*OcqxP7vdz|L|<7GOD z=WuwMdT@s9^f^CM+OuW)2Im(__e~kTE$w$@?`j#Zk)3PlDAP^S{Tye>!OeK5^mo(u z$^P$TdO+F-Wi^||5e}c={At<6;WO&qv(o%i#+PLOWjUDVpX0yR&irz;h_s!#b@X5f z&BK!1e}k;wDErIFYI)gRO@`HFTuavLbN&wI*n6kCw~4fyNwbv&IeJdI7iE9me~tgHdh^TCg0h|@{US0fA#El5Ye@H2>DQB8E6tWNY=u3UcB4Ne z>%FD>DD(YfnkG9Rm))B4Bjw0Tdy;e~%g$+XcqX39{CwHHNcxMVy+l@*N^^zmUN1*j z|4d!qDEqTybu;&dGTkZbdt^U*zk2X{8U8H&Q_Qh`THSv}R?p&d(mgNJ9BE&Y-T7Y_ z{~qIl(k>*cg=G)@lJv@QgoC%LyBo;F?2N;gZ!n>oKtj)pSaExY$|{ySMe zC}WbtKS=k8?EOjFr)5O*ytOO|8)!psn@Ix|w826HCAI|rc_G9#+($vx&AxEE<{S)v+X{O807iITU zJWbYpS)D2US7iTe>CTb83z&a{`9O}ohd+@iN;6B^U(vV9aF48>Wd1kV|Cj8&!1>E^ zxWLQf-&3`qtlxmE$h4;HZz#L(kai<_W7*k+-b(hdx1BoM-HChemBU?R^w5a`;EtdxG<4<>>En+B{|p z^UD7W=9P2ilYUWpaXEaWG|R{-*uz;kd3l~&Nlso_4&E$huPUdlE@xm^OC53cx@!A2 zdCWVc*;p=vGdEFB-BQkYx12mxPTNkNv!fioPtN?HoUuEN!;h#td&yaQ%c&og(+-ly z94c)sryM5FL4UM*nNM;Ly;sk`(P!1Wog}B6EX^0?Ij72ax||c_^owNuP1(CxPW_gg ze2JX-ZD}r*-OJ?EtL3C?W&e6P>jrtuO>*`va>lLF4Q0Y9_p0lMa>`^m4ZCZpXHAjQH^XgZy@T{P<9%wkv+V6F z`_trZ2g`($4plp>KB?}0h5Kj8{yB2aH)Q&z>|88soPMc#(hp=8cf)h8()^g2a<^+` zKg#Jh%NaPlRqckd_Zw+{D|fq>PICGqa>{HO{w7Dy%X*IN{DboshpnjM(2rJ={xn4v>R`+j3iSK~~)Mo#{TobwBL@2#?Phn#l593|#Y$f?WCKi;cbNrqKrcMUmpeHk~G=WNA% zH#u#0IeQN|aI!u?PB~cmGx>nSq6Q%-%a92_jC zeO`9XkoV4z?sB=?6>@kL=hw(IOHRFC?v~^X?2govo{=51kp9l*l^&-quD*9A8P=pN z_fC~Fek4aY?K-vlxg7pd&YB~8FGxS%r19s@T0kDNl&qGOVX~aGj_kin&U}xw+sWY$ zva_qacYisk>7(Q_)9JIM`;HvkBWL}IenNKsA>#`&yv%&n*Xi$hO_?^7?%i^9vh>sC zZeO9lEeF4p)h+Uzdu4iB&Use$UzCHz7uN4*WjSRHx!am@<~q2A?C&e9TJ{c?bB>ZT zE|$~3D<|D9!=3V+KgxQxoV~&#~XsZ6^oY%VTzDzMov?<1!v9=bXkts!S`BuBeTf1;dpmh3+uhmXnGf0piLIav1frCUW)` zvOYvkI$TaZMh>r(Gp@q<79YR2KS{cU<(!VJt(>-Bq=pE|a%hDet~f&UisSu;d%Z?>TaHIpbY& z*pqWUEFV3H^Pvofy=nYh=R|qaN%EFcr8!%^@HNgCS$cf`@@?f!XUbA_1qQ3m+ z+0vaOPd!&oK3^{Q4S5+(3hK-8QM4Coe)vW5fs3WPTyAiM{NVRxeS;j~9yh9Q!3Te- zzW-P9;a|(uZygiJn%#6xVt=LPdWVnISZey)lVHS zcm9+-$;(SVFQ=a(S2;~?^-X!p#d67S%N1tG6E2ree^qVxa;@#J?R(_>+sTXe zljk2S7d}KjSIb8a$0Ot(N6TBVdz^ahPs^jdeCkBG+)1)?vV87TdCwX0*XK%mzFgq~ z`S>^GA(zUT*T`jlEYF`M!>{CaH_KyhlWX58558AE`k*}eF**E`T>UTd!KdU4&&W9M zTgLl*bOCw7BzgHF@}5OyX9aoe%JLRGepU5OljT<1$wzmU$L}xiK1@zOQZ9Uy+yPHG zT7CM-^7t>yE6$dyohRr2w*19?a`3R+;R*TUXXVs?$$RHtef;xvca(z<$Y*z#)193E zqjIJF<*XCsD%0iXZk5;mPCogpyyTzqybaeFf6nBsW$#P!@Ehck&&V^Ml@EM#^7#Hu z-yGFCh4scB`>F2OGf$J9Bi}ae-=86uIAr~CS7*xOPu*bLSN>W4?428qd)>`89(&a- zn~Ys*)lJ79xv%UF`5i`BonQoY(bTa2H3=3@DUAIoKDkM)%_kN!>bKQ6xI_`QoQJJuYuocc4X%IDuA zUs_YHxt@G?SKhd-d}ODwX64=0$L*nhXdm?|pOMqPuK77P%jfRbeDR0WkNiQd|FGQQ zF}c~F<++pJJ^uL?+DNXq@mT+k&D5uDE)Sn7Z}_l$#K~Lt8S4+&S9TB5{3D-~*PS4@ z_^LEvtpCXjdEI52FMGNC>@}L7eXaUWv(%q?<5uJEd0cm_pZD$ZfX(G?R_7~iqdx0H z@__^8c?aqI@xJ=vU&;HQlwba<-1UI>jDLT7EwJ_Y^FFq{Jg6S)SKe)#@qE$4<<2LJ z^~-!!eaZ#uFhdT%BhT1z>iE4oud(geB{vvret-0K<9_oBJB;7E-TFJds@-ju-mAu? zkKA?ajnmu~^OyzAJg*G(zg}gTSIf*Z&v1obQ$MS5KCxHaC;TN7ihxh}q!f z`E+lASEjF?&pYqD;}=Zg{(SSynWi`TDN^^B_+NXQC-1F0KAz7^HvPBg=45^sbDUn@y;4hG_1|9f-|sX0+v)$k|8eW7+iq#P z{E58!HD`ZGi=BOje7X3aM?ds~-~Zc>^V;{Gmi@QK%zxYa_|Nv0-`3>u@qFgZX860W zF7(kyH#TOWS4PKNIPdu2)zwekhu^L-|NXvajIaIw<$vY9=YOc1|K4X;|Gty|w)gz^ zK4pA7pLsjima_TZJ^!_APUbP^o%`zlT)O#jOI@+eAC5g?^Y-_Ue-HXA{y+Ke@ngQH ze-Qrf{`dI5|E>L>_|IGKJheCe_rkpV7sB{M^1q{IE}+k}Px4=j`mai3=I6WMT>mQi zpM1?%u50P9+|fS`)hqga^`GZ4`+xo7*KN7W)U96m*V!xg7J22?M<<`jXD@5=r!431 zT>5|VA?;DSfp!D!2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ&~BjJK)Zo<1MLRd4YV6* zH_&dN-9Wp6b_4AO+6}ZDXgAPqpxr>bfp!D!2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ z&~BjJK)Zo<1MLRd4YV6*H_&dN-9Wp6b_4AO+6}ZDXgAPqpxr>bfp!D!2HFj@8)!Gs zZlK*jyMcBC?FQNnv>RwQ&~BjJK)Zo<1MLRd4YV6*H_&dN-9Wp6b_4AO+6}ZDXgAPq zpxr>bfp!D!2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ&~BjJK)Zo<1MLRd4YV6*H_&dN z-9Wp6b_4AO+6}ZDXgAPqpxr>bfp!D!2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ&~BjJ zK)Zo<1MLRd4YV6*H_&dN-9Wp6b_4AO+6}ZDXgAPqpxr>bfp!D!2HFj@8)!GsZlK*j zyMcBC?FQNnv>RwQ&~BjJK)Zo<1MLRd4YV6*H_&dN-9Wp6b_4AO+6}ZDXgAPqpxr>b zfp!D!2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ&~BjJK)Zo<1MLRd4YV6*H_&dN-9Wp6 zb_4AO+6}ZDXgAPqpxr>bfp!D!2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ&~BjJK)Zo< z1MLRd4YV6*H_&dN-9Wp6b_4AO+6}ZDXgAPqpxr>bfp!D!2HFj@8)!GsZlK*jyMcBC z?FQNnv>RwQ&~BjJK)Zo<1MLRd4YV6*H_&dN-9Wp6b_4AO+6}ZDXgAPqpxr>bfp!D! z2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ&~BjJ!2icL@W5%SZ1JD`h>!2T>VI_qKY926 z&bznFUh9AOJwJNw+W*7(3(v1Ro`=7@ZR{S`$|KL)aNIBMwDH&zCdm~KedoA8JCx5o zIAz?kt4+q+H^es%T+S}tb<3@d(HQ*@%g6)n~lA8-Oa~7`KacPb;kO^ zSL~~Ezt|CD%>fU}Q|{IL`p4AMo{`%uyv6uEdoMKB-#?!`Wh32xaXIyGmesw>D*5@1 zG+$(6^(S^zzh!gvBWI4kr~lH1nlEsoT=j?Yi$i(f-DAzc_sJKIlI#3h-gLWMbg3=J zKiAch<#p9q^ZvEee|fvydR@)Gx|3XJXU!jan|ypnxyI|{r>DscJ~7rTcB1-u+CCz4P1E*LUS5Tgn$UmZ!XHtT}mY zdCI<;UwWB(&KYve`{kFW>-_q=wi9`NRs=aHcy|&oq z-S6FDo>%{XJFosPW5)ma6&D(pF>l*=%9MTg+uO`bm-G1==JPkq7igF>HpeWU|5Izb zfp!D!2HFj@8)!GsZlK*jyMcBC?FQNnv>RwQ@V|2dY1hZ?tMBvsOLMfWz9!w(GN7NS zw%0QMDfe!oe=hwm=Qhq#SHF@L&9Bw&cIHF6Fx{cKzl-zV$m$+xek(0{Oc?Idc|?1^ zI%I#xdwxH+jbDLzs>JQw1SXPh7fbLPX$As~ZJcrd|G^WzoI(L8OeHgHM zoH@oq^91Kl&TaJW#=rLeI^Q`U%w9ly> znmL@K`-eJV^-s0M>R&WgFVL6@{fjy`FUg4JWp!0*o*4ggXA2!2u$o8r0!B3RYM#)} zr}pUP=N!!f>Ixm&1$i#JkUC&QH%W8*y1Dha7uMXN$AAf|MRm_(#8hY&;~qv#=w8p~ zVZexHan3QI?J&n!SS_LRfDsegC3zkbR&UTe>*%b{=Sy+#jdN>hhvrS3qeX`v14cAU z>$$9@vyS!{3+*y`-eW8@%ksU@V!)JpIo-3%ONR-o6?hN2LQe;bg?2^WiymWxW+grk zJqApL)yljd1I7kT#rx5{S?w`kEOe`I4^yF8Rp(htdkmNgtJU;e#8hbBqVozpMl`E) zj=nI^5zQL9@6ZJyLbImk3C&u3FEnqZ3oY$2pjlh@vlVU8q0jf$(S3`d zFw)h!y5}%pwH|W}Xu5n4wCK6W9|)fo(j!| znwM70ZJ~Jw_t6$Q+7|{oPN3OH@2}8eKr@ANbm%eU`Np~z3(Y(E{#Z3=nY%(yr^0Fz zz1O0{gw>`x4;Ty0W_&NS=o$>nV}sSZ^gdr0>8#mY_bT+53a!;WSLo@4W((d|=;&;q z6WT3#KYEO4-mUY15ffHh>D;1gFfgCU$i0N-J$!zl+nPB>OlY>zd4(2Tp{M;+-dC9D zQnM}ZM~5C$q1jINvlZ>oV{FiD&*z~nbhOz)_iUk~J(?YL&toWzbV9Qe??sQXFwyS4 zx?dWYmzwwKeucg;&~|6uTNvr;{d~^`9rJ+cHR^WJdwpS~s}JZr6h^xGAkSmOYFEB5 zxX5-JQPMcy++Nx zd`_XG%|~_aUZatFW~=TU*{EC^cXQ=^)bF@p`!z)LOV^*c}!Rxz~`ewj{y^! z19=aI!s;NMM+^t66Pk~!vlVR%J)I5txkL1P)u3hWF=8rohw42Z6PlViMofj_Fx`uV zi8hDpytHDTwRG0eK0kki-czAP*Pv%^j?{A&9r{9hl?FZIk*K0k?{r$NU&V8rTVp2LU4fHsI?r0#6$UzDLUS7518t$FBbw8>kG9a$rGa@wcLqNf`UWHOs;~PNBPKLo z;(MV*hY=H2XY#&6NBctiW!?7}Fk-^$D}0YaM+b~(&f-2=bm$AKv-#eI?yG!144BZI zqw}n%Lt)DI&gFZdEv(MdxkHb!u=*OGd%m<7a(`WOj{(gEnnz4%zM*+l80d(pK^u5) zp}kP^Y^2Rad|sia112=z~eRV|h1HLC(bPY!43C$1rURa?+pU;29z0#|gqsNH$YMr~nNN3GV z-h-~t)1}oldd?PlI$+G_Kj!#pbX3q73;bVPH5 zp37FWE%bE6gjLk@Sx-kaKjr=CF-@TP8P8#bw$RbB(B7!$OFi>|Dc`$^`-OpySp8i0 ze4+UT-?y-$9VV=P$>*U%j|t5z-isDJ#==Bbzv6oq209idy1H4*$c@ckzC#?&iH{3#;GoK6DLw z=H?#V&pJ9AX!~2;^B6E) z#vDC{!bm5yPwP34sn9&5bBoopG`hk-$HGLLzw3v-ShBUX#(JnLz{sOH%~r$V!s?qx0Q3O!xDUiU3J z3>XXT;@n497-`etdo00y3>dLmQsC~WIPVWgAF`->v=dLi& z5$y`P*JR+_tf+ex+CoRiLbDRjp)FmR?}-s@#T-2*G;h{FN>4@OLA z-=T9?80my&Bi-{DFk&@D=N2QHjWw?dE$z@3209j+cj|qmmbpV;80my&6TK&3wJD7e zQ=!>R_bjGD`!3#780l=Hvu1NWR~1@1>*%bf14c}R#_IjCFwtfUoqG(JCeUoj_eNJ3 z>1?92)w}h4z=Uoq&9k0vGH{-abd!m5^B#RpwxY9vj+pZK*8JRPF{0Uq^9CLBfNm;3 zZ(*cM6Z2|YJ!cC&9UDx{&31Y|qTQbFfgU5KLbHSJS7jATrDiAI zhgE})xyOL1(7c!Tp~Zj^Q=#kWxqu1n`!sjx3nSfR;=I~f@5x%)p~rv`Q=xso-jj87 zHqfPsd9@4Q11-A3K*z#FS0CVepvTZ)WS$Dm2lXC@5$&$LA6=oR1I9wL8$Tyn^cWh9 z%(K;p_>8#mZpO>xZtfvDew0rRL6?(cfGA~Wc%}4ki zSWTeiJYXs`d-A?QNBhD+XYF46d<{nC39aMv_LjCm$2?%fgk~Q-U!g1XbT-i0NZWn+ z9_TP)!s?^C=P(u~+U%!$*^0L4&|@r2wAr8Ug|5(ijL$(^=;&;uvx&~yX?kBmbAY;P zFfca<@;%WNdO8-GgLr?TrDLHzSoh7xrN@8~Q=vJ8?}rY3Va$D~?kB8jo=1lsLxYKV z)*PnyROm5cLVLLGIgFSJ%@N$gfT_?Nse4UY&a;jV7%`zaO7F9Uj`kB6Ij@e^d;A0j z&a;s=pV0Fa28@^*G{^9{7z-0!eUkTMKyxf}v<-UZ*_fX@j_1*$YcMd6n9v-r_hdaC z8cfWy=2Lp!p>HrUFHOu#&8PWZSfRy`&p*Q+3LTvdbVTF%-WVE;%*|){K4>Q}aGs5H zX<}|Zr}tY77%>%^&+~oIqALt^EKGEDg5KjXU@S~@wmOmTi>@%xsnDFn_r?k>It+!? z$$Bp9>Cj+gp7L|k`M&5eV)X@`M@(3qqPfFRXuimM(V@p!m}qmVp37FWE%bEE_fO+{ zVnTm9^TI@D!x`MeRA~BqufmFM(sJIU#5ocqE=yD#hc zQqMe~`3m0$t3pe=|IonwhzZSEd>&S>QOmuoqqBjY$izK!Hs1psMobM>U)6IK6NYnm z-??*J(OFA7426+SSe>W$WgVT+d`;&L6ISPIZZTlO>g&80Ejo-1Cg!Ep1-uV!p`%MZ z^Du#t^Q`#>KR24d=c8-TGtUM(YcAA#EIN#s&|IW@p)k?ro4f}jCNvl8JZtH!qdg{c z-_mm)%_ZszEjsiV3afAP{m?ZSn8yYab2EdV2OWBhg^4zo^8L}G$5?1C<8#nsME4z? zX9Jy0w7FdOLSdpy{dc*KvCv$>_iC_WZVMe9Fcv1-T*>>fLW?dt&^?c#!N@#YeNWG2 zEuD>YHqq|;d>#f&Sp9(aqQ`*dhdR$zv@P^M(!GcY-Bp?gj96W*xh?c`Hqg~f?iV^b z>*-KvuF><^infJ;&PF<+`7!T7j{#Gm{fX{-jF<}TwcIOAbk4fHIyuYxbZDGv!Zq#!XT67pOp}C3AL0=f?nD70Z z_hT$HzuF3Yp5`7#OlY6sy_gz|&+1;*|DESB=6+6d`@Hm+u$sej7z-0!>i)rV z7z-0!{ZscMnt!P)wCK=d#PFi-+n1z6WBxM!zt^tNVnj2q&i#CI8|i9(&0Tf@b;vHL zt`?eGOFN8cCh49nbhLRL=h=nT4kMaHm}4kRbhW7NMNC*N#^)58*K1xCdfF_`=VH~N z(W60pnEW z+j74!ZKt{2VQxJgvD#7RrC}${-Fs#1$*?nx?)~&G_yL(dD9x_Y@5cFur28=LE(50Q z9-5nv%&n)ZJvEP*&^pZ%`n`F6AL-CzwJ-P4e3X0W_fz}*Wx(_?bv2DSCNu|Wo^^CW zdoa(T|G3&8BJH7bTOCFh20CJOxbC?lWI*!?b%pjAbwv9~wMTO-&tt@d)p0zJ;nQmK z8EMg@@tS*dpHl}kC(u}($a^rLIZ1Pe9>d9+C-l?R<_oeqMcV8aY4jM;oT~HGqWA8`-ERWzEbc@F(d z&M{u2Hb0itPv+Lr=34HdMTY_Xb-HJ7;5|_$tbVGFH%fn#bU)`Fx?j;~ZdM13n6SD< z=N8kg>UbOPxm}u}tnQEj{hgfO#r*EV-_Tgyqqf=Ksr>`8dQiHDm}C3{^GES98U8HI z6EdRzi#lO=O6^9{{|%p(0qrwt`#keG(!DI>{7;Sl-wR^b#_?LHajI zyR`&F^f2>3%J8&| z&qz0i`2v3(|NhK^GOj2S#y6|o`qFNLAC>{l?&=C7rlYv`ed%wI>1Xs!vbqN!Vg95H z|B~h<&R>>;g-80kTSRu=Acr_wRy|mrd3FVLy^`!zGQL?>Z^1R0zg60`rC$fTa~s~q zy|+uZz6@wL;QofPpWR4ZPmyk8*?A}PO=PvH9Ad)$mYRomOS={K-y=I))7!|wwlZud z)AsZZvb&?SIM_+u>B%8>-=`j7+*$43FB66jsLcoG*3yIBG_O7+yC0VRBQjxUPv&U$ zQrFnWl;?Zv-0dU#*xgT^KF0IYWOabF2Qo*05a$Qat^K&#VL)>zbL<|b_Snb4;oLhy z_Ku{n#&i_tN6P_@u=@$k(=j~%NokLj<~Ys^!||H;J|+8~mJz$3QF{z%JkMc`eH>uK z?&owrYv~>iu=9EDVGlhH3VSE$zALOw)ZF3-%}JV9SYs&cp3L(YFyatLg=RYMM~fru zd_m{gF5N?i5r?L}xqS(t9lS z&=q<*o9NCZJdY0R!azqHV!{!cZ|i-f6?0eU>C(V_fDwm1?0}g^|t< z>1?7qm+Eu6SQiF*h}C7fU)p6}7kavng9eAp6L!9%&uOya+!pp|hc$ZaW5N+um$L`# z6?!_L`7YlNEA-gM(BOc1HqxnZM4K!0dD#wKVfQs^xtHzHHG1qf7?~$DSMt5EQ&`bm zwAe#e*rx+VObwa=zqdk5*Vr!%baqH5G~eUrDy-#$@ zp3X)(JET+Lh|YGd(f6v*Vz1EA*_zJw>3|W3m*&&& zxi9R~p>RNFBV9UVo=tRiM4RjQ`3ft#TWINQk9HGSbDs5d9|J}l;;2D$J>R>qOJ^0uuq5AXyo1yQ=$1O-wUe=>~d}kdvw;(HTnq*oDW{3k$c%8J&}of zr6cB9^E39+V8y)29_OWwd5s?X7;u0QN7%Vh->X8~V2`=OxrF-bGMqk*c zvwr?95F97zvk}_E9@55w8uUMjF<{XbZO^iKDWV&d1;rq zoxmRFO*+m?Yv!e%d1!FJd{~(1(h+lW3x9vuDXi$%YM1*Kdks3~r8V=cr?Y*!$-wzU z4!DtvuhAj*N=MAI=63#hzz%jBw9I=AI_9M{^L~SodFhaOHqj&O4E6o1 z3G8y7we)MX$Nf^rJX_Nf>ABZrpYzb*fO$636FKBwHqjF~;-0yKe|{S5GB52hpGe2O z8UqdrBi-bX^K7C=h2~EFehRyEsb%ia7xw9Fpi3k3VS|bJsLZQp-Hs zqnmV`Ph`zKU)ZNh1M@*)q)Ug)OB3@EcJAiywb0TI>k0InPh{X;c0gw%JuFQ0sL=d| z-#1o;T{_#NU4u3A*Q)1!wohjVbT-n%|3wqeWk+<@+`~Q!J9PCL?Q$<`>1>ZKbps!6Ea~5p(lf{;mrvI@_hQmhKfgx<=n%pLxIm#t9s9UYeL^M|6{&d-Zcy zg@~r5)zoLQ7|R^h7%D)rFoe?K95? zIy<0CBl97SCeS><&yN*$C(v@9?a{8Vrn8Q0zOYY+!boR_bSg9t^ZUV0gO>S3_PFQ%Lp}HVg@Miv=xn4* zhs>KakMMJ2r?8^Cg_h3tXjfR%O?uAzIA}02AE9}azpn51LoOCmkybiCg!6;^Ef|W zgB9~8E$6*LN7v}FUl{24ADXy7kt6PPp3u)}3wv~vj&onwr?Y_`V8YP^nkV(WvK=~G z(b+CtYMEzyw8Q#8)N{YzU|>GLh{M7}j~X<8;pfK=R%i=*bk@=R*J$A0puxzz$sy+> zG*9vUu~XQkvzG2P=$My!=Gi_S3J3ISHFCenA?Kxu`9zMmS8D#s-muePmwDFGSx0AU z+86feY@i1iG2sZ!NZ+&5V8uLZ>1>ZKb<9gW^K73EIKUyM2F>5}y}N}y+BH}+_k{yG z77pobqDR<$n(u+V!kX?E26|8!>FkhBXrAHk3Ofy2=1n@zvo-At13kcDVWP7mI%}Tg z@2kOzc^57A&^1^y&kpD&Bj?#6U7DC@M|6|s@BBWn(_qCs+oN4!P5TD>%(DYJ8|mzj z&L(Ar_L-Lk=A{GX*+?fGp?RLaufmGXc4>A)4(P0z z!}r0euuEqxo%M7d0}cu!ogLCmCeBA_{=wgEgB|ACE^P}PT{q~NXZv(E&;yJ(#8fz< zJOAYGwb0T{_Bi+0FAVemBMvd)s6q2De*Xo zf_|O~ZDEgg4fdI513f4l(h1Fr{CyU7=xPEj=UGSl!am*Pfb(I4iFtNJmsT(7=doDh z0AqtA=AD=MyTS^)*ei5&zcA1PjD?BLj_6Wj{-*zb`WoyoFRhq&C$PtP*3sFTF7?c_ z13H^%GY{V%tHLgAv4;-p!a!#SbT-mMOgKU_uf9(QtHLgAu~+EmY)$(H`^-xt^NCE{ zD;+U6^YQacV8wYCE%pi>T^D*f+ouB#3lm-1nP1 zdvpse-D}V>uN&+$&jvb9;D~dxpuShOLwB*p{(orT{s3d)i0&-J-&0{lXS=jThc$Za z7Y2Gz80k@=nZ!N{J9M_9vzE^GXxCuPywo!fIGDi5`LHn2BQ&q$?+<&0o-XY(FAdDI z13IEvm_1?#s|LHwE%wkA)^yg>eH={Skn?P!M`#w|@3Fxy^K6fH*vC*9X|t%_Uln%g ztffmG^K4CLBb`n32+d;rUJ5&O*3!KOYvvyNI4F#C>5%!TLGyZj&u*cmvpu>-k9`an zF`-$UJrs869=ZvvIroKqIveQ0|DuuSh7Bg>-41(0S6I{8J{@p~2}fv_U|-lN?9vup zgPwV|PX~-REF95InkD&uST$&w*VxB^gTh3Q(7b{DqMg7#=cR#pc1R~QOX+huSfRxp z)`gxP;;7KPk@uo4?9m=W;eak3F*k4G{e@k+w8y+gU)ZO!fzA%-Atp3S>w9E-v@5LX ze&K-54(U;0wTwQeiw8ztYMjRH7XtR>u+bOK*F80tB)^wAe^F9VlXjW#A*eU#f*tzd$ zNsDsr|1}IrKxlGMpg=MRO^~33B7-0_5(E?}a*)uHv!&#u4I()diU^3U*9^1sk3^``RDz;=e+BgwVcns?)$EKp1x<#2G??F$(<2;>H%YA zpPX@6X)K=~ZDpHWbYzd+${sml!i>X8vjCqTJJ`j9gUM(X)IAp4BaGDhIK9z9FS`>b+eG(uU+ZLMNjTxsO*s=_A%iAGY+w}F#D;rhBF&{qcXD0}oHW;BcQeSsDodhB8xVV`wStHajZOwYeYLl^t@? zle;4f)O#2);Q(``S(ZOvwAe;h*&!FZP530?}ZjSm7d(ifIUn&z>GsQ z%j-Gq%0TX69N~a^#vz&&c>l@Rrtc~vxsM44m~n{hFX(w514is)8sUI?XGQM8eq|!p z4yYH0WV4d)t8Gy)I&ufQ7_ncO$;BbrtjylgR<_BV$}Txz#6Au%;}Bb`=y?`hWryr5 zBRN$L$hmSzZhevGVH-Q>v5P&7*so0FLFJI#`V!BpY?F(QTF;@+7|VqCATX(b}ijsY?A}_FyRneYwKJaU1f({^yJzu^@x2Opjn5{Uunt3Hn|wd35R8V zUEUkrWbDxIVh>a0fShqyY1ZSpl^t@iN3QKtA7I8I`t@~x7kd~n;}FdT?6uO8+m(*o zsr2MxAonI?pMEis2bDu|(Qe2dD?Pc35mV)mY`?5~I_N9A0=;h-{;%~y24t!$GWcF>QoOT9Kw?_tD* z1015+gg=MM7TID4d)Ti`#=e^Kj2R(Kx1G$HBgnjC%GLwr#ve`oK<0?Dk zF7_)2v3G2vtL&2#4seL(Yr3aZ*&%xj zm6>d|=6q#`+{J*s%19pNfPStVlFc^kVT7J~sEp*+w!A-D>|m(uk!$6I(R`ErVY||iN78Gkvp*p3}m1r6<>B>SkBYqs2Bl4A{e5+1^d}6dk#P z0f%UI*L-b@da*<9V!+-ABXzR}_hY-VL-yFk9`-8}d4R1wb-zVd>B$k(2s3r_E#22b ziyph!!vSV&?Zx|yuuHwzBlky`sJp#&Z>WsqJ`O8e`|v)MZF13(JC%VPae%|h*1kLk zd)UW>1I%cK|BW&nuDpMMTZ`{7%C$<;SkLs z+*@hM#SYnH7kiZhaxs(bp?XfSLr$1+h~_ZOJM5sxE(VM^M0>c-d+cJsUS%ZrD>J$1 zj?ldw4A`5DiM}~f=UbIsaxs$ol>@Rpiv3h(viT0r!8Z0Wmww>$pvPWipIl7jVkX<~ z>OPM>9HRRk?~T5)NA6?7_R%_5?2-GG<`~Vju#FxQ4$&OTz34{Rp*u@@3>{kxR z8HZ?op!-_bu1w^NL$p1eYhw>1_DlaX&2`XMcF7rs*g9S39Ck5OW^(I?d=BiO$AH~4 zG*|4AV`U-_u=OLIYokMt0duLJsri6CjM&GFt+TivU1g6vz#*Emb*|VVcd%O-$UPij zMstqtv)Dn8-O3(0VjmM`H0SEx4h}G*IZypoWt&`dq6Drm5$u2jO0FM9HP01_rxAX?Bf7k)O@i+?v5}}FZRfBGWO|b99DKN z)_ZvDVh@LCf6TtnV-Hj5|Afzl9rW16US%W~2V{4N?hBZ4i0z;9yvhzaV8j6q(OjzY z7TegZ3}o{&&9|_N{mKD(h^@|?Gpzv8~iKu&0`W)AHL+tfXFF=8Jx4$)kr`)ga&yV$QB zlyleWd>cK6${sml9}^DIT&Me7WtUt`jvFdwB$DSFk&AQ z4zPWr?(x{g9!Bir0EgJVN$0zjJ#rrt4lv^oTQ}<-TiGS|Fk;3b+FLk3LPx!e0eh7L za_d(1f)V?b19BeWkh=M`?(3k(L1iWnE8T56-^Cur%078eX>Ql~7CQ9Uugv5jnxyl^ zHrZ8n$i*%>-~jD!*hi%&*GB3Y%^f;#u`|LR^|w+Kn%{E{ePt~Ddo|ZVk6jFyO8*a2HU=Jhq(cG_jhaUT-{s8^TE;-^5TLaBoY*#vRtW0G4pw17m^C#sV zCd@d*@Q~&r+J|`$3^>4y<`K=c&|!CkgHnH#_p0pwnfo4_)HXSydtCj1Lu@^vx~=Sz z118Kk#LkmCpD<&1O7)Dbr z=QJ0w^*rx^35OV7P`}tCM@(p6)La`o*u#XamzW!&qaLx33C+vAF9z(Pdqw>Lwq8}X z=+I-p9`;LqjraVkbQoS&j&H~Tn!hnu*&%xz;1GL%*SXpO_0CZJghRA%@;vNdM*Eif zL$v=;E_TQs17>W!%{+SSVjl;X(Y~W|ZR}t~_fPfv#dnp>`_lhwQoH0H_Obne<{b91 z{ULSiVh;ys|IHkF?BTF7ex!M0-WvP=S?gkFoN`1zg>t}%-SMjTae%{0J0X#Qhk7(mU4S$&d(wf zc4t){Vrw?#Ji9b=$XMB$Q}qt|%1kc0xiz1PpHUusR_1wSZ+_Xf(xX{Gxq|`If~uQ^ zq+6JC*vGJ_>fObpU7R`W;{eSP>K9u}DYvnM{iRh;%S>t}cbDVb^0HT%$ek6`@8SS6 z_E%KjuOtW9TSeJ?NwzWI0P||HmHRu$ytDMX$^n`^l{?t_mU7rz_Auc9!#?WAeP!y(*0*KYPx}33AKM2g_YRcZ zgXkYD&7rc5y~D_d%g&L~V?uir_2PGw1Ga;5|GRj!G{;DX8HdYX#?5Y5@j?i^{)m2K={A2Yi1G-uA20ecrHC-fI7`#7n6 zvbk9G){kYx^b=)!iAbcglmx;oY3O zN4D>k0TT{z_y_gK)74tZ$)q5nVOak4XobmL{jgl0@vMfF_S`yzF0eMvcDdsStJ8M~{g-p7Qk z)j5v|J8P)k#Xfe{RK2qneKcz;x6oo6yBM*xj^^#U(xX{Vx#-C~?4w^_bL|ae-jH)& zmYvE-?rx<10J~oyZzAKSGIXTfjEn;uqWvm;Y;CUG#efkrwzkq-8+%_!G|lD|&9(tktsF52ysGxolzJiz|;%0moZxxWK*mHi!6PiS^h_Bh1$ z&Z-COW5PkD+ePyo?Cr{Pu(O-;5Y6t&9{Xtb;2s>H-IMcZzNKt2VS6vmV;2+Jz145+ z!+qGOE)>*QP?X$@ku!r^> z^?MjGW9wY*MT_A))iZX^=bj5>ABWhwQ1$jjvWo%RQFV{zVr7ROTR)+WUF=~WbLn5A z`Gf<^7=Eh0y;No#V)_}+yG(i<;1D~Ps~>(YJ6Fnx?O$*nd)WPz>h@~RVHXqnYt`>z z>w4y~j|t-q>KEOO%7aRK6LmB|sWGJ9A04e3W%Opnr?>cgVr-WPDt9{~~QBzfS*e)ZdZ4_oew*w#WZ- z?EAEX;Zw@J>8Q^nT}zr-$#clT+|tfReF2#klxA`AQgZNl>6VwBmB_2hxP~k z-*G4h*m_&pzb{)KOlto__`iN}a)xQs8e+p$c9;cQG+tVp`F=E30^vuu1`B`vQ z*_l)3xn=mQ^z)LjJ->2VP<9rQW??y4l)R+uFEgpOO~&@JWQ@xzw^ov!HKp5B4zazJ za&K#Cc93aD+=+fy4)>GYgJk~*>5i41lj)x!(-qSEQpW4#@P3(}k^MJi_?K*bD5H5# z|J<5Nh8bk*(=yE@2eZoILb$Q)ZZF$=%HEN3@I#p{l|Ip*{r$1$h1I3mS@!potrO)i zQGZINS7dlsw%?;a?q6f~4OhTBrGHs=Ka^qm560%wb+Ubr9L)CNSlurr^D44`rySlR zJO1BebKMI+9S@L_i+^g&mDiKgt}ln1%dn-4INV*?ohMiR zjqLtjo_)W(=Uut=^wW*q-#<~VJmvIbdG}pgV{Uz*JmX~f`h{|#i{w(b$yeTxXG}Nq z*tu)xl!ts(4tJ8>o#hNy$=2&~)uG&LjakOdf3`i_n77Tdz?kzbC68G~F4C5_;rS~l z&+#RByR)$fXXJha4^EI7!}tCtR&O<+bv?8)Udue)Va&+~4JeQ!F_4ek-pn zXFOO=ee>tW>i6PXw<&k;l&9Y%U;CY$@(*(A2jn3*$DfpYm>yET^UrdM$K)n>#^cII zJR#f9$T^;uE59nQ#PeTQz7(H(SNYM8V zLB^HmR6ZMLoKJbd1?5ulzWHBQx28iKTOVZxLofDxxrELHQeAk%J&D^ z`L5g*@4-92r}}Zn$eoXsSK@2ODch6ek57?H{7_DJhMe|HxzPD?#tY;y;iESz-+8+{{(tkja1TpHIl8vpgT4 zdRBS9=cUIlzo2}|i*nmnI(hyKGU4^JE4Sy6o8SQdHkay$d{&-0uUx^(T^EpM zLHRH~w~F!kDQ3m$Gxc zT;UGs?v&vVviGjM<9&JWJd2F&?dE0Vf?h6m*`j0hFV4T%n45IuJiE!RU3{$m%5P-* z$)zUdu-EKsjk)*y@<&#lypg=(t7DpfZ?1g9*Ol-4rt%WoE6={WeDDC(S3Xc~dzk9` z9j?6J(aL)rD{nkO-gT1tbDk=1!DYX%-1&iA?vgRh4}YqB7T$N6vb#$COZ&>7xkdS; z-%|gb>Py_M{2d(pQTcoK%RBJ+hm{}w%a~!-ERTOlp7^r-*(+n3_x`H9;~VnLH&x&G zAM!LA?jbM!w&qrNNBwu-li&YH^;tfa?HSe{`}5m(MtQFcH_bGb!)ddqe*CP;ow<~E z|D5v8izpA4RQ|zo$}g^@yv8cZ2Yg9hv!?2gt*`v>X3BGKANjq$oOeQ^~T+gg;+9A!=~t#ndSqD6X^3rPHjn#m4KIsm|E* z>*LL{6Q*Y~k01K@4Cc)V2WB&mo%GY0&F{y)yk!4-<}^=_eR=xb3oUA<8k>E8)(clP zi;q$b5dRy21m?%s-*_|N5{0{@=tr_a}GppMMkeO*ZX>zkw5fjW-LJ z_6|o(xAP0D7&Fa;!XX#y+jHXY)BaU|6(@Z8U;hSPHlg~z=U3TV`7t~CC)g|Ylb`B8 z_f~#_tvuEqZBFyZ@fl3(M@x#vv$b@>{H&vw=N&mMNn8vpzEILQZWLbHKp z1I-4S4Ky2QHqdOK*+8>_W&_Oznhi7?Xg1JnpxHpPfo22E2AT~t8)!DrY@pddvw>y< z%?6qcG#h9(&}^XDK(m2n1I-4S4Ky2QHqdOK*+8>_W&_Oznhi7?Xg1JnpxHpPfo22E z2AT~t8)!DrY@pddvw>y<%?6qcG#h9(&}^XDK(m2n1I-4S4Ky2QHqdOK*+8>_W&_Oz znhi7?Xg1JnpxHpPfo22E2AT~t8)!DrY@pddvw>y<%?6qcG#h9(&}^XDK(m2n1I-4S z4Ky2QHqdOK*+8>_W&_Oznhi7?Xg1JnpxHpPfo22E2AT~t8)!DrY@pddvw>y<%?6qc zG#h9(&}^XDK(m2n1I-4S4Ky2QHqdOK*+8>_W&_Oznhi7?Xg1JnpxHpPfo22E2AT~t z8)!DrY@pddvw>y<%?6qcG#h9(&}^XDK(m2n1I-4S4Ky2QHqdOK*+8>_W&_Oznhi7? zXg1JnpxHpPfo22E2AT~t8)!DrY@pddvw>y<%?6qcG#h9(&}^XDK(m2n1I-4S4Ky2Q zHqdOK*+8>_W&_Oznhi7?Xg1JnpxHpPfo22E2AT~t8)!DrY@pddvw>y<%?6qcG#h9( z&}^XDK(m2n1I-4S4Ky2QHqdOK*+8>_W&_Oznhi7?Xg1JnpxHpPfo22E2AT~t8)!Dr zY@pddvw>y<%?6qcG#h9(&}^XDK(m2n1I-4S4Ky2QHqdOK*+8>_W&_Oznhi7?Xg1Jn zpxHpPfo22E2AT~t8)!DrY@pddvw>y<%?6qcG#h9(&}^XDK(m2n1I-4S4Ky2QHqdOK z*+8>_W&_Oznhi7?Xg1JnpxHpPfo22$yEYJ(n{v%HC%kevdnVbRbxgC^igJN<<&sOw zSJxafd}SlK=2ukT^y_k&9aP_ESLG@9k)QdN>Q5+#aR)0;cbGhGU(G##oLuE_x$i;h zC)LAQ-&K9-bL2y(%h}eKr=2C&y=csE`jyJpUL_y7fjZ|myj*$lJJet37UgAcl}A0G z`h34pKI19*>EFt;AC}KQBL8*onC9D$D(5GZKYm7em%qsUUX#;tpOfa2mo&fA^XmWg zb>;8=Q~ASx$!~n5`g!lleLuC<*q`H}@2mgF)XJyx{M%<8%jOIETxQ?TsDHwzRbT!| z-E-J1%BL@&{yxhqud}rBOKZq~;qI%czrot_zHh3()=tWYeM9-j=gIzMs$V$g+GFqg zo-z2Kt;6ZSRn*LbslY2UE!^gF+>iZK%xbjZb1&@krg zrL@2ah5x#*%f`n3xBOQ2R(?o~{(gJKe)2Q@=ibWq+sb3@@n$*3W^?6d*e8GfOl*!b z=bkh1*X3&*KHF96KYQ3QYy7{nM|~IlAN{%h6a9Vt-|=(o|NK7Qds6>g{GYuK{!)zL z-h0+f{348dZcICmbPLM7u=I;av#hjVl4(_3UHY}?uZJ7Z-%z@aj9W>+oy^;lJsH!1 z%HdGzhcS1QOsC=LGMp{VIWnIoZKVD)nJ{0jY_E_W!_Sp7nqMfJtEpcj-L*1YC)4%P z-YCt@GTtgfBBTEe^~!vQ>gI0dD-$`~tG@lC%oy)iF1iQEm= z=%aa*xj*CMGCeV=@k!ISeeO1-_l$_JF{}GbhD^#XXW1FY|5!PI~g;& zIaCkm=HwpCXy#JgVVIja%;@G(Jy*KVs2(t){jBN<&3wup118L9=hvLWfYwsSjAjAu z#f)x2>X^`cjygt6m6>c7(tJjOTpOrIOqKqNx+hkeFR7j|SK3wSqg#!8R;P}34dtRIM@*H*F~6pC zn9!_6y)u#G+Uh6FXx34^7|5~I*Hu4aMzfyk9wR1n>(j3cy zMog7%Yv#~wqiiu?LbI*<5$)HNeWm>d_hQ0~W;>o!>Bs?7>2I(3fC-&fJz_?)1NWoH zQ1Xu4hZ*h8)GHG?quGVI5!zkV_mz>HM(B3axl|c;=XsbaGdb>|xm@Y@&Zn9v=rdc=h02-S<0>?%Du zR3@@LlILPXa}@ViIAu7Jq6^$#8OX6Re2?>(D&5iaF;$vl)OYAHVnTDQ<}3z` zn9v`mxqzuMlZ*Cvo{w=dCi)ru2|N!Y=1O}a&%ubPGLyqenonp?CSyi-it6^%Np<9a z5zY5CSM=mC!bm-pb3f2|hrTk9W2NcoTtssk?@{T=q0~=juF{endJL6`Y<{SFEIRZU zF_r!qn)etmSNb1mE}=VVnz!iCW59&wLd`q$m64nJ&C~f2%+KYJ)OqkL9Sbd8D zBj!^7iRL0^w3l!m9s0^t@=v)3&83`2hoLf+xu0o1pt(%hVZ@B#a-LV2$QkVwya$G# zD<`y9(w~f;eyFs+(D{IIGA8=|DxHg!iEMtUe$kPOk(|)?H9yK!`oH42n9yIXdbnm% zBe|H#MR%>{GrH@Pi-BB>vW=FwxsRGG>42A!*Q)I((?=W^~w&ZEPC8U0O~3m9)! zP9x0J%`M!IuF{i>ft=Ca%K1u1_82gt`L)hD43(K2Z{u^IyPb3BM;NFVBiSa-S4Oh^ z4SfulDl^&M!902l7>mEvT&lEpa(;w~y17epMMtjn)B{FLm6>dRr+bTz957ZUaz=AE z=h0!rgt^k+qjSYTF52I7FFH&kH1~2Jrb_b%?i-<_9xKfsxfesp_o*H*p}Al6VkD0GFE5A*(&ft)ax{v+Ij z5z`1Wb@M3ijS*9&{WI@_<}u}>BNqcXm$}E8M~9&@lZ*BVopb0hpnX#Pgc;3KoI{Tp z&C}|83~2wNdeM<<1NB&$$dfadd!NzsB4#w1_dt&cGn!{LSG45G>6pvq{Byd;V?z5p zbu=&VJaiZ_Rc5kzQRf_{l3!9iVy?6=a}EQ>;wzl5Oypc?Ue$cjkv%5NXkX*`7%C$< zqy4MqLuDf8O7ptrEqaWYE6p3sqaR_Q9!Hp{XS9FgUUZe7oG_#LyUuwGm62S`WIN<} z=+I-pgc;47JP#wL5t_Gn4*JSK&S?I@Jo?EP=;uoFHuqF|a>D$M=FC5(!!Q}myPQLh zX@urI<|`f9W5k3R&HJ23haMwlH2>oBV5+nqs9$vC|E`|-gc;3;+=~u9hRQ_FX#UOT zM2ijs#t|m!_9H$&dW@JbqxqQURyuM(YhKX*t^yr~%1BOV#_3$qkpo7wQ*b}}$r$OU z$(ZRE-FQ8x7|2C4CGT5l$wf!54b)@lPo?`Jrb;)p`o1!d6Pjr>XVFy#a&4rZ&`hg) z9C{3x&`if1Cd`#?dfp!cni+Us^cYM2wE6)fW^^;s$Ao4k)r*$wF<`9BWYf}lj|nr{ znbrR<&cgd(MmMYKMNbZx(aff~q9fM^>cvPdCUP;8?d-fqWg^#R>P0(;&U=iNiEQT7 zoUL?Zj{!59xi~jMN8MNEGCw!x(N%hKsEp)XY3I>>4m}1mpHaUU$hDDrF_m+l<$cj( zz*L#ZabBIvm1aKP2W_P%7bCft$!31utJ0A(8ml=!86*8vnaM@70MA9UAfKnwl8c^P zjN~*z^EsY_euRN~ZK7VxWU~A#dNM{ z$wf!@m4O`5EY3aX&|@h5C3sJC7)BVWr!v2!&U*|P(SDvj28@_6qgk5wL09R?wSjuV zJQ>Y0dXB5~=U{&{qa>#DuvrET?-SCd`#~dCe6aIbf=E zD`-AotTbOxzv##w14c|Ev@7x)jA&NkeJee=7|6AedaBH1x3cc{m4O^FjWAO;tFQ;O z=rLd_{V!_XVZewP&6k*~^yDxZBYm?f_h1-dq+U#9vl^cdU8N`IGPgSOXwhRT{WX}w zSeeKfozuLp4CGkp)?^;dT09T!WOVdBMoehd=03FODg(I~$;Cv@m1Z68M~kj9kW-mk zm(Ps}b7fdhbE8c3i*9|*7ZW+7*?{*&Tj|IVGnx%`E}{Q2_iV)TFkr-7={9B_{Rk8F zjOHskXDdB9V8o1O6P|}2118Lsc2mwx#z4O|Q8yjl8y$K~m`i;#&4`y9k#m{*8uMFEswGEEBh1vpHacI7Rsg7oGE! zk(|-&%6p+hUm3~8M9ye-<9Qe{q1&C$fqoC=SeeOYPu>e%Wgr(Ld6bEMF_Z1LcrWyo zk(@B2-HZ1_hrZJ8tvQbg?LIsgJ%-9iE+%r(?8|#pI&v|RQ)MO>O;`67ExG8)0Tbp* z`)%F}9eRwI(d@@QDkHg=$Yy`ddrTwD)Xf3hU+Kva6Pg1xUu&tmN>2_WOw{c`yaxu% zmEmB`#mY>!hj0&Ow1=whF<``m8O>qbQ|ZYGbLk(h`7pvry_m=u?GZc=J(?q_qsR2$ zG&5Igj?%prBPKN8p^u?5ku#d0`C=^jyWE2gJ;usJ&XwkSy2oRz%;cguTJsJ)28?Kr zVGcb;OqeUpu{u|4sk;$+>Z#Hl$9teZUO8aGjOGO9F;r%9tvONWEIRa+i5yPSd_r?F z=SS$N$4YmK&iTqrHm7ngx)Da|sWOv`=KI`-4n2kuM(PRe54Z;tnx5(rGn&&>50&8&=7%)|4a?zd1IgFLzEcJ_#oG_y~TXPm2`pQ6#m?|^b zoTK|j>FEbdn9-iA^9}=MwCC}@=+K-`9Uc10Ku%~b;2gRUdg@_>=0fg4Ul~ijNOJ*A zRQ4D!p}AQ7T1VYu#EkaGIu|ilnxF98%1ACIaz=9r?^Ef>9>WN8nfs~k37AHhshdkR zZ_!~Y^`FtNOk{VN`l&LL{pITCN`Hmw0nN|Jm?|^bUa7g-L_MSV1#{@oW59?B%~hPk zfT=Q*Yuzt(z8J_6O`m>cAjir~HoxM%(4nt1S8Fa+nrnD&Wh7@b*D{B(GLdUDb$cD> zD?K@4s?6k~y`Im3<_2Yp9%H4skvTLsDHlCCR3>t+3^(h1thBeN?lF~oD|O81e$73_ z+msW!+m!=GOqH2zlI9)y5k~68L^i+Cd0*-7;Jq-T`z_DIi1tqI!B82=d4%pRoeyY! zr)<%ouQYe_T+C?hQN8HN5p$*aJRF9a@Jj6VDG!Jt>CUlQ5Uwo8%F`@f2=g~Z-Y#*nN8SN9QJM@)- zTH$+_CcBrI!&K>BR^M0JS5z-LaxswotD4JbUQ;eQaz^`Co`(?=n%C8Lm6==& zZ)h%|`5Vtgj}bGvzw`V`J5)Vjstj+cA1lpUJh#%49frzC&Y1qed%ewjpnr$^|0!K% zBFA^tPnav?d))KB^yofN4wZ>q%w+qa<{kRVK#rIy-A6nhL&+bjZp@4P?@*-~Cp`v? zn9)wbx$!b#n38*@;vURsrlyY;ePt+f(=h)jX)$6h_36|P)8h;>7C)_=YiCs5%|st< zOW9*UH?!(y78x<2omKT>noYTw$!2!$K{to8n^T6lWW+GHvYm%HOqkJpmN^U<(afv9 zM>n5xMmxW9KxfJ57f_B1O8YtK(JZ7~8>uG@i)zj-#(7MYW^wf`dQ9k+P(K%!Wd8Hi z(Jig)+cKhGmN_)bD;F)t*C5QV!kqy&C2QrjFpL8j9=nDbgL?R z467+;w5ux@Jvm@Tvj+EKcHDz;E&A(7vz~O7k!;py9^D4Y5$%_ii*Y07Yrn#|O=Reh zF`?N^bype4c60U3meQl$ii`mhW;9>ZoJF%W_o3fLIbcS+Ep?1&zplE&h#Ac{)VCOm z+fn}}_hQ71W_$I$ml@p-%4SCyF`?aw`JJWTMFvc}(#N>FazeKU^O!2#p7b%H`IhRn zfqFu}m*(s~GGpp;{@XHCru{f~fb^IT@bRSUvP}+Y>^D%Yv(%65$ zGE+>dBir%Rr<4v8W;9c&pQdJR8ksRnt8AxZZU+3c3^S5vl742KO}g31m67b`WNt2; zN7~PDZeE!%%%>dD%ugT9g34y0NsZ)$c476yBAiFNsB&IR+Qp?uy9E6u>3@DwGdV5I zTwBIvWm=9p`Y$M(l_oWnyt3-C3}`kb zW88wdEoIzFrfqOr`d^oJ2i%$Zt}^c~-JX-0$#!qm!#>jQJE?(ezpZ-SU&aGvI*2;j zLzO*dbcaztTDoIoEFPzvFr2{Lsnk!C_6+IIk^%jB%J~9mE|TtInJ`>R{+WzdNPp#| zrat*s(%m47%I3~Vwd8o0>i%vS?~(2SnI4q(PclCw z{Ub6~n#brrPX9^io*`$Mo|Wb~d{NrJO82G=n9#kYy2pg}9o5r6@jV$oknTh2G5%XQ zO!2aQ-cBjgw9lL_7F%K3-Vox$9hyfx<_P0^Jis`@iArlBy*UHPtku`CbZ8eXN;Ni&r-+uA{os~ z%IS3(-jEU9-eabnnoAm-_qU52XE22Fz$bQr&$_-@H2ZbH$CrDP%-Ho{V`) z=BAPX&D6^O;WW%)`jm3UIGwVcfw|flRgW`C(~`N;&BFOvaW>{>XCC8R%5iR-hq=#4 zJHq%`^%MGel>?gjlpSWPT=WZ&(Jjb%jOZ3p-7G9ECUlFao))9O_@p{=K(mDUZb{B9 zCC$>*F)l+NLtD9+$wj}c`o&!G3fzO{3(Bt2ugG}}XjW1^V8n!eW%?K~p<9LLp|1>I zQI|*QbvO zbEVyYdog0#P<8WVY0+US^^Mf`m@uQ=Sbg^u8PRP*Mz^VQz=(b`)r*mw(0o;Wivi>2 zswd27w_qOqmdXMBR?5XhHeXZSjnGpM7)O|>`>lBn28@-7T(sNp9_WkPsvf>h{~MF) z$*I)0Q{Q92^i9?6_A;aK)X|~GG{RiwcHsPulj_LDKrSY7F_-zBbWgyHerMG)nqA11 zZdcW7BlTh;=Q6(=&&7;xckaW05i^=SG-olQ+mrJcD($y8hY9lt&0gF$LQg$ZMsm^Y z%{`TgoGbmlnh%wUoY8doJm}C@26964ZQchxrb@dX_oG8!8OXWP?XUA0{Q=5pgziA~ z%|SdD9R`eO4(9$!dx+{D1ExxIsOB7oN^=-}44Ba#u71E=X^!AMFk%{^J(Bw>Gr4Gv z(p*CK9iD@sGM0Ye9Oe<4?`qDX$A}rt_n516vE<|;ioU_^I}<~>GCXpU9ip~p}e z$@Vy&j}ASC%2ei#=Q-#r1G$*U8SM!?x6+Y|o?J91>YT@f?j-7!fn1E_+C;rJoUD7( zWVEMf&S6G-s_M0ldM^F%a~?g0O7jEmLyrL?CbT`~(P2V=n)<~+PLnayH>c}-MEgT! zhrTk9i{=cTUm3|6&5tx^(N%hK#Dp2mnLGz0+Ov3H%p)0`h+qMp&5uX#H{N8MwJbDb3iJZ|~sB;bjMoeff(p=G!J*Lu+ z%%QDx?BNsin7|0P*WhUDz zxxdnr%`bRg^cXQ+rG7^9OXe#b*;ht#D)W8br_zyqWgtgPm6>dQ#l4k*oJMG_);V7p z$rY<|moV8B#q?&RJ|M-G+dF7Cxx znaM@_JI#BHm5H1y&D}f)Bj!qb5ATlwQ>FVo_f>jwz=-}{%?C7p;5>#AM(U|DlZ*C` z+&4l`J)^l#^A;VN`>Bu6Q!fT`#DwMn-UmI}0d@2v4Af&~B4-Q_>b}}c-Tp~^^U$O^ zvj6|s`Oly^i*sN5Zjumr$RamcCJsR$asvWH^bjqR+$19~h9MY4GEL4fCJHdnWSQgv z0T$6smNCXdG9ouw zoF~j^UetSw0nJOAmvzj2$w0@FiFPmZdzSQcKLh6pGul`5y-+gJ=2iY&=rCaHV`5%5 zGdEe^^(6ycnCOi5HU9kQO9ncld7b+uJsmKXOtgK2KR>#Xo(?}@;yj~$liv%&PZ&8b z%(Qt+-zl`TFB$1v(!9<4l9qNQJspet@9^h9Uoy}U6Pn3-=SxPqFwy2+-M1M3Lo@fv zn)mp~4z=)}2rp;%1S7_;gv1BUlf6mXtfU#tvGny~> zbC>jV=woD_(S51!1dNzVny>itm2`9}>89}K>SJIYG567Y&7Tt;dQ51(;a*8c`;zWk z{yg8wP||*{xkHcU2i{@8R5H_r_D8)d%(OA@>HpmfhLVxa#l5L?KcksiU1;gTNcS@p z@2BBA=rCf!T%1p<`@UqLGur8NFQ6Huw&)6{*F2&btac>>oiLX)Gw5AHKchO##2nqs zYF{$a3GFQ0!&oxYg?3imFLZP$8R^1Qyqit$T}e-el94V>tmyC2_qTRxJ7cgN)y9nP!Uoy~z zvAAz}hpvx_dD&dtUsT`meGJSa=02Lm^qv28aehDa7%`R1bfH;-_ay@zF{4{j_j5_J zl;*xN$wZr#`5t;S zt7>jBU_v*HcNj3DUrpzsWTG>g)wzduINwE&0aM9Ln>F+SN4t@_tYaQ9q1jmXBf3r08RJMAvUUsR7}0K}xkHZ;b4mMi z-S;H}ozXP-ZplCwnytCF4d2Cp8SS>*E1BqwW;@*rm`Y~aY|qcbfU#uWLHFV>WF95$ zFL{pvQ^`yhx*d6k8Qo5rC(P(~);wU^h4;I1AI-1SwvUc^!d%?*yhD!>b8)^KKM(Eh zYF{$ag^A96G<)##(U%N#Dw*lHr`{K4+V7=#C>ejPx!GHKw4>F9X&-e)v#&Z}#Dp2` zZ**o-oICUwig)|-9(_r3fX)-{fzc-zK8h;&e0#KE;L7}`+Bs_ zEe4Fm{P%qC7#Yw6<`_%5V|88_XmcFj!Bo;7&v!8to}hWccp{DFB(+12_GIR0PfXtvGGN4f2J9;TxIpt(OsW_0JOBPMj`X&x}5J)e2WOc#a=_&I1V z zE4hd1Dz&+q`xr__+FYZ14m}2pn6K5ncpcvzCsRrPC+?xUo_lC-;Jjp_?Me+nU>3c!vqit(tob7>oIMzI&TY=x?X*kmfJkE9vQseuC~5+PnA;`n&nwJqF}Nur!ZKhoR`dX`TunQ~SR&fBgS8Gq?ZHc|_Au zTl5$(mNZZ5en$V4I$%Qo4D)BDe@@1dnRb)7Uoz1d&5L{oGn$t*&#w&BysGw?O4_V* zhoNMCjqjm(U2QSFp)Ry<7&KHoqHJ8pwX_WPM9&SrFmFe`gNpRS4PZ5ucx_PpL^&xphpZeZln$y z%eaZOo63Z7a~i`I+}jej!k^24ej9b%R%VRbaesU1cEC~6WBMiY9c9=_=AD`EBBN)H zW;eCPjAnPzmc{nJ*ETcgE>Ex#)Rfb&D~Mb{9cA* zFvxhUOlXf+n-iozk$dP*R@*bVcMhH_Gn(_&9^J(>=1bKf%82#~=4h`}o2#V1n)7Sv z>joOe(SPC|`WxsQIloC}41ZReTco{JhVe4phIbCs-lg{U$b7Ff_i-;t_XzVxrJsn8 zF@I9JrTW+v{>!oAsKoSpd` z(qTqFujYP;4D)es0UGmyYO@gMCDS6zE${T=5M6=j`Q#7AJKfsf4`7sYMIbaqYh}Nr3Y~j-SlcZSY}K!sQpaR%!0E@ zH=7KY=2qun59YcO9^`n6?Rk6vH84VaIhH|87zModMI)V;VV z?>3{iz@N+9ptr$orQc54U;N)j<_X;>-v3f&3_GdK&YbTm?XRTYL*{*?*_Szn{W(W_ zpxPeHJ&cENuk7zM4~KGp4EG8TSG%J*NBev3p*fZ=>5kJpp*cw%PUSwj)3|r0v}eh9 z_CW2qoL?aGh4e+-EBvF{UB*2$m#ckYRNE_L!f>V9TqPss!mBlR*KzMpGF{L88#%vO zhFh7B$J?3zg+?<$oiN^|Hjhd_Q3kYs<35_l)cNn+dtBNMjStw8cV$HXA#+Scf5biXAFI=+(tI}1{Dr#Ee8u?`X-m4I zzt?&Cf%j8=)cem5JGG3Mrd6BiWI{Wm+M}6WZ84ykgZJn$%*FTS#(8)@uk>h#s56H7 zXmktECEY@rn}wxcgkDtI#biRiIQN#2Zpne>qLG&8Ip07g%;-1d9@-IVvkC7CN2(nLjA%F2c|^Y% z-`R?H7#eE3jr16{`7QS_6ny~mrc4+PQpd7~a1X=p)EUzlwK+oi!XxRUWIR@8^vBUBNOO|( z=uTG0Q@MxsG_@-{gLfE64FHxsUrM*m=%Voq6=_{naO4{qB87BiKw0~lb_6Btzt9u#&VkybcK1v3fQ%2)56Oh?Vb0M$q7InRJjy#Xe^+O;Pw*Yg zMR$0I8O>9g+oz>@hWBWmRR^?_=oh&6vdphY^XfnY?KAh$yr#BqNdFe^-eHar-DJ%J zns?Riy@6&rzOQ*M=|9jsqWMT|(R{29UFkl-PX}s0SLZLJ|5B!s<}2=fjo(P~E&aW; zKQf={WBvC|jF^l0)SRQ8hQ=^R9jC_`IG<7a!kN_>!z^kus|>SAJBRdW=A`H1Ug11y zGjGZHX!Hy4Zoz@Nh1715fhIaEs<~NGS~N?kJqC#8RvngSzM`}% zOScN=mqw7gU3Ea~xQ}rIdW4KkL`!U~NX0!*WU6VPQ1Jx1jLF#}Bb1^?y=k^fke#d(>hpRKjBh;zz zNVPeNduWcP(f?i@19LRTs}uSY=#utC%?tfWeCK58PT~AinM&F-xR2&cwLeS7vw7EI z{s$WEIov~YF6V{k(P++B`wMu7;X>|>mF^;$OS+3S_h>Jn|H!*bd3Tu%m&+LGD`Y}< zrP>!>#l5SgxpttAE=+V7r+fDLfu^Ew&^(~IQSC6^#JiiNY0HEe<1L!UTcsN>?QPOy zxLuvm-N88q%xM3@dD#h?Ck%J0?OifsxSRL)Fu#|3_tAw(?e3T60qHRo{h;RQVeUP` zJq(X>K9Tv~xc8W}82`>a43Dcb+9%ZRp975@b$C*yr=)pCS`3&lKCAP>O#6Rno+nBF zJdNQ6`bC*ulJRBcuSoZ*3|XevrGI0fks{moP|Vi)byn%B`` zr(|nd-K$L}8yK;JVS0WJcCa>>pHni^b_SgXG&AyZFky2h?qPjqzKa>V=w{J*2Wzvc z3mt7{(>$WBsqO5%$1a*VH20Y2ROh*57yUfEM?bH+S+Y%MbVGEnh0XcY2|Jk4&aZQa zb^&#vquZE@^96OUfe|~H(J!QX5!1rzF4h-OJInX5iSr-b+7jxB z?IqQ2Dd~sGg!X6ZPRUH0Wq4PzwyfrLbl6!=^Ddg@)h+ZZsM}~(RCnsqtRy`)v9U7e ztH?S!w5u}5R!KKZ=MghDR@c0Vwc%=u9_wpp?y#|@y0#W`Y+++<&6^mpT{6>V9qwai zU3H;fk8_L|9QUxczB+Co!-mq1koA(j=#6yV#E5xg-fbcqBL^Dk)~1?Aw413NHnD~6 z!p*t2g{*HyV@C6H%_EwI+G7jd)|xl5jdmN&8)&vwchGJ}quHLu+74=uO*FsIyf#WY z>|ofD?_fr=ljenOy0)|C-I8us%{?}ES4T{H(0j_-UeaR1_-oC>-kgt?4%whoY8vY|ZOfYpHEXPj|6(j_!2| z&sAqk=W&j$^VRMGX)eUE%+X(@&RD-#-MEA~cG3J%^IAzCHTPExw05Q1V!|%^t90JE zT4rosqi$jCT6JNo=yA+3U9UDb$oh@4g&njvagP3Gb?48rTe9BPJYX)oMRR|v%oxV2 zTenGfhfIZkQTqw9i|sqPcNgb(bB_KVbqhOa@73I41H1R}oxjSACedi`SJ%-#pmx~A z_Jf+YAC;|%GGfNo-!$)_`MbI>(e=kQZ~ue)*!`#4b!6*FnXvwpI--4AUB`$CGj^WQ zJ@=e!Op;Bs&(m0YLG53Z?j_m44*HihZ(-vVb?sH@F%)LayO>|2(Y~(kV0c5_d{eeC zm+ZX7J2Y?e?j3wrcG0}2ZoeuYuQTUq~3=I@!K`%&%9C%yk37qE+FD$Q$X(M+v*#`-jBU(!vhxtUHjFk@$s z=I!ZaeXtBONIxUz*qur3W|8$-WfLQIG0mp)4w{;}h8gR#YwoZ%2lwWYP3&NOh~^#i z^QqgIu#1iPb>7AL0_qlav9_S*4l}kEV!p8KV0RI9ure(s&Em3-5nD@WURzqWv9^rb zF3Ws*nOBsdF6%2xSGbBgVF&BOc(C^IdB<_S{y-z$*+BCy znhm)(LZ*#m*jSoPWJW)dbF6Qwj+oGGu6eg`3+`X~((Z|SNrznwdu!er zEyF%CVryTu|BW>J$p(hss?+|m)|CEW*+zSax{K}Ksm)<{xNIFE{gJYElx$;0dyM8L z475f&Ovmaxqdi{TJwet^l(A%+HYe%4gYjhg)PdGcSGUoesgBq^OYP5=_7B|0`nl@b zc`~3mpT0mgE|h7kY+WQf*t|p?|0vz1vUVAbEew}y-o;i_w=rI!ZeAs8*T~Mb^q=s0 zyg{ZLrMn4ll^z4e@tU`9lO4=xZ`V9x{Vy~&?o!+Pr2i|Oq`6U)Cd!QU zzp1-u{;qb|DSTY>+CMl)j~z5m@V$S^R)=>_$~NYw)%9m&cvfa?|4VJ2lQoPbbJ3Id z&hyf~D4Q?IjPVt9{Z(1Zvi+uPyd|61#rE5p*WZ`^Lm57zKc>5~jph^1vG%FDh4wRb z9Rs#7q4`|*8ehtQU2IR$y!I{kzLy>YM)PU!f39jyE$h?C_8^(j%%HAem{I+|&ZLgB z$WBc*XP0&^S)W@v%=4(tyaToK(eul^fJ_U@=EAa#X%V%zxTv&?$tDJ@FRpniTtXd} zlxC=`qbvNG=3O+)sr~Y@g^d-|ofW02%YZGkt7={wCYyZ>%O#Ax+F^4qb!%_#ji#}S zb|1~nZ)9zM=?a_b4mJOd(q1J!wy#xp&|RmF=*Fot#_QGH8>GKcw$a?g z9Glqqv*rQKE$RmPTh;aP(iPsO_SnRDJNNG3-2|EMl&!m^y+=0glM$Oq?e3QeGj<-( zy!oK)VB=x#qkBZ{Cd%5AvW<gabFR0^-vhkAazAVElvi+)TW!ZU6 z`qyQ|`kU(JThhKwW5oJ9nl~_F>pjgQ`uEiVTOX=hA8`-cAFEAQHnIK*_ptf7x{a+b z)P?mg)%h#goFY@n4sE~IdHow{zLjn4e6OzkAUoLnQ60=@`rqqJEn9=6pI&w_%%HY2 z$_55(&7^r3-OTEUepa=gO}4RCqcP5|c5`rlPT9m3#1W1Z3*6$Y%HmHDqKq4T$=lsOPZlNum4QemXUE; zTn?9)ZbjL^Cf4ej*U@1ETPx{2qg_>PR~u-94%k7rI``I)5!;y8)I6=ly>+BRk9J+n zJ0WE~qAu#IUX@6m3m_M6Gp=CX@+3w0A4TdM0@aSz?k z)v08r-8MQm+sd$=tZgq_*hce9&Fzk|jg6hv9#he~YTocNq1!{y%7|TT9jJL5yJ!#6+@n31_vnA8t{*D9Xb)36G-K4Y zvWIKl#xA-eG;bV<$H<7C6V&!(S;us$+Mh04B^zgG9w{vfT^mo#j?^fG;WD6sc&51>q+UKq0#EVl&t+y zMhrWtI~adOd)!^t_K0P~){mEMOxQd@^V*5BRd}-6o+2Bka*nmr)rGAyxPO)mE!jOs)-RAA?O1iZ zn0wf`gnO3_w04Es6<(h|@raieVBMBiNU&uWiNY!!Wr&dsf|GhW6C zGTkZLcguW_>=xdqc1ha%Y3x3zHV;ex$Uxnr>ISwkP1M{z#{I`-_a8F;Q>Kn=KS@6= z-E*=wNj9ICofl*m^NaLLvi7p{nK`=G)aG>=-;@b!lbNG=Uv07Vp}O|5Y+)B0UCo))s2~$W5U`jns+hJs_x7tb4~g= zWUFMG?x35K_jB=XZke!y-J<8!xf>#z*qKk=UO+Y$lHG;5w*fmRIMxtgR&d zs?rUU37c!G&04bIq{r3<>e?pKVH&B(F z&ZA^!N7>z(d%H-#tE~S@cCg{qaSz!>vzNMw=GW@R-ptXBR=Wc@FX`!Klk-BK@U!nQUDl8&}FUcCS&_u9MBeKQX@^Z;&1AV)GWx z$II63GGgrxbr)-QsT~H4Mc=LS=Do6!2HJg49UhVKZ?gM1_nzRq!#SEKnPYlNU1*+G zcQHMq_W$A?+85RBSEPGYcCengUoz45b)DBSVeL)LYj5GZdWrP!IX_#vbEQ2`Hn5HHe9b#p zyMTLRW$QA$Qr51L=4$D$qi^CIb6ah1k?q@L7tI9j-y>US?^8FimDJ_|8L*4Zhcu6m z$TqefRojWuW9Kn-{R!DAS?_2b(Lc%kr|}uipOYE=Bz400yt@68ti2+e8DEzjG;gR2 zyKk%gJ2Jg1Gj_4@9`g@m{Uh1J`p4?pC$fX~b9EP+U((qAMje0T9s23M(!Y-m!s(@* zQHGgha~95LqtVX6d@dR1lX(GY7L*-qEkrLYn~Ts^=EY=pan6^JX{a>I$ojIfv;08A zit2>*I_E3N+RD-TTa>yWW>Cox;8>KHkDnpo2%dtn|e<>R~4z%MrFWgPt8ZB%4NL#qCI-vQj+U+kBHV#yqgJk_+ z84s0yjBH^CYln0H2+q+QsZK{pf3)mkC#W;lk5${_WE*S8tNjVGi?tKgO$_Ky*4&&b zYiLg6KGx1q+jE#>{akhH0@+1-A@?ql;Zm8fe!1FQ!9Da>s%uxt26nGe$7^NxI{cGt z-5}kK^i9&-EW@p`HeR~hIKN%SJ7woC*}WI+I?hqKl-<_(Ui3VW%D5B2jlOUV++lZ zntN;?rM5@Q7A9<-qZ06X-PD}IpIWnR>S8dLd7V8(PGq%U_?qcp; zE^AlG=GC%sjSSd9a~58`KH&jp}@}w71CCtukU~ygJ=3%>?Q0wWDPr~Ope^F-ayr(wr%Q|*G zRJ)I4_*}NXlC5uK?R(kzQTl1W?*02+!dy5=b2Fn1vvLo^Z0bghId-r$r{-}k>E@Ab zYz^Vv0?|f5i_3tWCDd_A*;YcXF();5qWOxWCz z`6kkhl-8=1C~-R*HF>35cSH(A?5HnF~sy89cM_LuGeS#L^n5OcJ@ z<9rMrg};~SSZR*K<7M}BSv!mQ1p|#2s_j_mv3rp^NA6$A`PDL9E9=+G_KmXHmbF{) zcD!3Q?v+h!->h{aBm1X@knctA+T^Zk#3ElfNcCq<^ z<{29w@gBR_>T2HjlJ88Bov)?;RyM!qe5!AH|K~DHE$uY2HiK-fBpZy47SDE;}2`+9ooNlx8!Tw`9JR^gGD< zFJ)sV=DW$XyA1nqzMpIzC~HUHQ8J-DR&7s^0o_UJ=E<^qn)GMVEj&lI&zJR!rMXh( zt7N#EzCpIyGTk9N_u*fqe?ZnBrm^;jy7j26JuaJ1$VNwoCuIw}PpLc4$l6OXy(0aq z__{Q2N%yw2@5qQ5&1B9C-%~fS`=Pq^iEMu*^Vc$bD-$-R`nLD4>pY!wGe|#+Y|SQ{ zv&;INvVmqUwVhX*`8mhVg7m_&wg~qYlW}q8OUMRxmt?-QY%eFn^0GBtHrJ8XalV0c zBXBbrx0GpX+5Ls=jFRS;GVUVt9-QyZ`F^tYTiM3if$D(f5c)7_j*|9h**Z?9y6W0EquAe8n7s_~`UM*YK%6L7tncpfCb})|Dymp&x z+%Cfeyi4Z0>3e0u@Bs6F$Zkh=o|g7s^efW5D%}S%VB=%Xzrrc9`MqqJ?|T2bZVr;Q z>7|=R##yDAQ?};fe7=F2`PJRUWmrnuWn{wI@@lt=G{atuVJ?A|ETpQXK3 zy74k3&I=z@$A_hVgnN(5`b6d!G5lTg&Oc@ADf($?UXb~9>EDpyP1!~JmfB&$&O6K} zOZ%RTUHnuwr^q(?Z`9#??qPk}?|c6`tIaOmTrw;n<5DsWm1daiU~O%6cCxvltZyv4 zKj(aFX||W)7t-#++{?I!Z0s-7fwFtBbce}&I36oI$8mmwte-@mA`{lnpwE)FCF6y% zHCA>nmbHji$i|iQ9n$|rw(lHh{T_7_)4ghUpR7G1{Um9gm(3SscvCjslJyT{x6ApL z(oB&FyWgstKgh;3KlJ{7Thq$MAQ=bC`V5@UDBVml%p$urnP!)*`DJGT*;qvSrDR+g zSCL_u%&SSehODh6TkFb*ZasQ^nKzR4jp>oHwwY{iC(~}SwU_MdCmX+&wF9KXe4x5{ z4EIiz=``6zce>h~A;X!{Um){^vVIwJG}owG*u0tZTV%raooaKBwAj8+T}zxlAU%dh z)aglCe_FOCaqm6Z`GoVY>2GBGRyL>kvG=c!c1^}PWMfX*oEztnX(3s&vWu<7)MiQ9 zT1GaOlWqmsTvvvzWNSN_cE(+0V^3M%pFT|HF|vI)^CM;Fc$rR+`5bAlqHmJnPUiRG zW75AWW0sA#Wb*?VK9#kvWcM3sevnOL);ItDXL?y1EXU0$?c8!G4xf)+P>x+xj#*6R z<)o|QO46<`Cu7)<^DSf(CvB^4Y%eE{lC>RWYbQDBS2FA&r{Ivi)TiL!1Gsmn96Ux2 zJzS1FLfWI`IGk{_dU%i%Pn7v=IT^=X#QcwP(iL*dwQ|aJGF~rhx5_kLPPjw1?w000 zS;q+vsz?1z4t)%tlBc{VeU|wRIpsY$_A{A2mmTcls4tmM;of)B{~!lXHC69lAA_fr zjX^RGmdzPtcOG1f`LZ&gTU9-Db=h7=PH=M4`f~7wGHfiHo5;qda`I-fwS}DUb7^*y zX)l@g!Tsd81LPzeeV{rWEXN!o-7zwpEE{LY4yLo!L(Z4o3-C%g{A$^{K~A|zj=NJ% zyh{$gTMoZpnup{_>^`inKO#?gR8GVpPpHQ{B}YFkr#vtH3v$%Ua`G#3?5om!D8m=B z{iPf;-PFCm@7f?a6q~cEr_|)=Ipu`8Ec9MF&E2I zu9N0QIr3IH`EEJxugo8iqhFDyyeez&$QHU!)b2AmdhoQpz1cbC#Q9`neq2h7s>(hYL>cscGi=6A~0{c_?1 za_AFsA}%jq!5yZPIk)kT>L*cjUN_<;YK& ze@_n@-245Sv&k`Q$f4`W&PW+{mcw_EqmGp3R2ff`tv@ioPd5H8(jGxUD$ zDYM8?bIG){96e079FCMj_K{_c`54Ymki##L%`50@WO`AKU3A9Y&mX>uoV1Pf zN69JY%5hP)ACci{=FiLK+B5aOGj22K_m!iXa`>~d{+6u0OD{Zg@16Oj?4CrQE^BAY z&be~jw6pZ?*H@Lb_2tmbWw#+m?IYWN;QUdUUY7A2IcmjOd*AC`EL~fsXJqq;*?Q+g zPM4!@mXEJm>&?T4a_5n9?ACJHLuLI8*=@^sH}mCZ*Uw!|#u0MCk#dQx<&gd5*rVmh z%jBN_ke@%n{7t#t`|^ZuWq08@dOv6IF7g>1HD3L}S30)*sS4KjiHBdUiLNzh|f~ z&~uA*WOom_=K(UkBZq!32XD7v?|#}|?)M8By&SclJYp>_EXZZS+9*OKRrkZF7Q z=6w1g!zJJap(rh7@*;0PD4LwE<9V^G6 zxm~^EB>CM7^m}q}S3We?V!fYp**r2@*<4IE){=*AD9so-{7mNO%5gW#ssAZ&ds>c} zBD=FJ-upSHtt$r|C+9z3-g>t@;!(Nk>vH%~OZ48YyS&_W3%SA8aH4wfgJ zBpLs&*|v%br!g^PHUi+0}dJZrms@EZSohB<^DUb*}K2PujN9Q$j8Ua znctCje~(kG)w{pnOmf4`<%4_4?#FVgS=R2|J7yR8?P$5hZ{%kyuhTo@7RNnTOoa$Zqw}sd3eeb(5^1@@}k(bC>|02JBP|om(9Qvj_W%c!X z?}v_%Z#3jar^y@sAe-al9{0=9ljZv>xZeBC4wKuCldIp+GtBX@`j}6coArD5k90l5 zfd{Ft9JE1i{vK(&Et1Jwqq4OAPbHc)M# z+Ca5|Y6H~lu=ZJ^pfwSj5_)ds2!R2!%^P;H>v zK(&Et1Jwqq4OAPbHc)M#+Ca5|Y6H~lu=ZJ^pf zwSj5_)ds2!R2!%^P;H>vK(&Et1Jwqq4OAPbHc)M#+Ca5|Y6H~lu=ZJ^pfwSj5_)ds2!R2!%^P;H>vK(&Et1Jwqq4OAPbHc)M#+Ca5| zY6H~lu=ZJ^pfwSj5_)ds2!R2!%^P;H>vK(&Et z1Jwqq4OAPbHc)M#+Ca5|Y6H~lu=ZJ^pfwSj5_ z)ds2!R2!%^P;H>vK(&Et1Jwqq4OAPbHc)M#+Ca5|Y6H~lu=ZJ^pfwSj5_)ds2!R2!%^P;H>vK(&Et1Jwqq4OAPbHc)M#+Ca5|Y6H~< zstr^ds5Ve-pxQvSfocQQ2C5BI8>lu=ZJ^pfwSj5_)ds2!R2!%^P;H>vK(&Et1Jwqq z4OAPbHc)M#+Ca5|Y6H~lu=ZJ^pfwSj5_)ds2! zR2!%^P;H>vK(&Et1Jwqq4OAPbHc)M#+Ca5|Y6H~lu=ZJ^pfwSj5_)ds2!R2!%^P;H>vK(&Et1Jwqq4OAPbHc)M#+Ca5|Y6Jft+kjtl z;r0KUKk{-XEb?F7|8Kti-`ssW9^@wHv&!7L%VaxUA4_&iD&%3r=v1e-=`LDBN@%!GlUA=dH)lPEAz0zEQ zTpYjBO1*n?Y%fn8C5!Kr{r=E_@AE|~_ud!xZl7nBUO%<8EatZ!v1)IA{s_%aoq1TV zU%o(oIhTC(*wuRTk1vtM`$1Rfyy(5B(fKs{uim@=^~G}Q*X2R8$Wvz>-a9}0d)>dc zrFrq4>U|H=y(_oZ{PD5sKi#RGXXu)}`-fhte&ROuY}d$TFP0bl zVy)i2&mPjfH*QjYevka|1-ZbpviQB0opbHpyH6LBv#uf^_=PO)&Gh)dzHS};+za2+ z{IRbGp6`CfI=%0{d6GPAjGXHz`NKq6e1E>#^!}o^nI9+5eP!L=cRn~|;Qfzw9@s~H zwBEn(g^yQqz5Dm>+-q~zjAx+{-P1R`5eRK>JvBWogcWpT>Pb;W|a%n*SxcF?>^rR(=Rx(*N5Ms-*?MfHym*8 zwrTWwg~NNMg|6N1-yL4wc>CUb&~&@>%$?WVzrSf;lI~sUKa%Nt=F`9K{^y#pGs--Z z^fOCSV?H-MgkBIAl73lSL8g^B$GD2xt%}2BSWU*^%-4_!^O|bAmJI8mlWs%KN6;JL z#>}^n8SPeTzqJh8aE^96&Ucg%GlrctH@ix^y9~ePKDxb`qaCew`$~_Y=-+6b(d?%# z^mIi3Tb)PDO|?5{pt0yfG*5lBzvCTx%tar{d$fnC&6t5^y3ii3^LzyNj+7SDQOu7X zXrMEu-}CMm=`mnL7jzym97h)(&-)Vw8tKABXLKj>os*Fk0d%3;0gSNZSi_?$DRabfFo`{gR#zB_o~sn3p{E1p;@#Eyj>muz zGn#Aod3|)ueaS$_lIB{zhaN*8Gjn&H-W3MAFwy{BJd10gr6K%%pdxefJ z>zS7g%wr!Db9)=VcS%S4lJ<7?TGG>@WTXq-9lXa-GSY>KF3fbH`3rmMqh(&!F)tgK z_cL*x(M;gyqeb7xz&v6`d#Aor80d((IKPW~eRRw{28@_6qq$q(bA1fV3nN`NF)z$? zS$hw^Kf02E?q}q@Y+`Qi)%Oc6?MenZVJ_+K<2!%-&ve}L7%*bOj4tWBzGR?dA2V}v zzuw!Dj`n?w%nLJZ9^iW=JsmKkd62!7v~;1P3q4&nFwZ}sd5GT=T^|GU!bF$N%=>8` z=Ff??kB<3&tLI(7STfNW%_IEz(e=?YkC@Ot%6oL^O9r~1k@Lbt7iQW{eBGJ<`NyH} zV_;qw>0I3Zo4yk;mP~Z+qj^m4EqV+XG5v>T?v*uv*Uu@mbXmu|(9``4oX3)hE;Ntp z=ev@g4kZ&^*8GD%2U_$PN=CXc(}m^<_Kyxj$w(*6X#UCXgBBh7l7WstVdC6$_&!>6 zB|Tj>GS7W9PwM-wWS|QpT{bb#Xr5x9=t~AVmP~X;`!wGv8R>)>%`@x?ZAnL$4b00X z=7pIqYo6uj_R%sg>zMmK2IgfW^TI^uKH7h=M|2oUMmiPup3}R4v1Fn%nn~;dZAnLa z3>Zshy3jn&chRClkKsQwa<8A6bMpc}7wu2zI4|p&7Y4eYiF5O!e!fN5N6)-4&=Ji` z{5jB~FB#~vk$Fb*GT$p{>9UTwFB$2=OqVsUu!lZ+=4B)EvWa*u z=91V%WTaCcGxM_MJ^eh3 zu4JSWW;E~f4jp<77%^e)qxpdEm2|ZK2?OVakxnHuUDkZ4pI_E8FLZRFr^8PeIWL=- z_cL=|)_lb8i?)xBd0EfApMmqjNT-sSHXrl%S2EDCr0sIQkDhrb8R@c_d7=4){q)f@ zFO0>#PkD#7kB)hvr^^QB5mU)bmo=a9d!R+vN6$R`heqy|P0ah5IX9p4_t!_qywKBy zfi4@Fr;?dAU$7Un=zc=ad10Vq$wZemU-ENHS~_Ab&c9+mB^_PXGcOEu#QYPQDg1sV zEnVnnkD-r|dHN4EU+d?)J_hFfjGUKE%*$rxh2|UfUDDHqkxs?EZ+TbJ(PcgJ!ax^B zx-ikXkM=u$KDv^gE*qFfOnuDE?f3lsp#KR2=lzVF7iQZ0pr2D{>9US_p{L76=7p*F z-jDowN_x65(1npMYs?$Y^!^#BWnSp$P%_hH?Ns_+p`-s(J@*R(-OtE*KNII>YW|!h zEnVp7LQnTIa9%btFHCeUX{TXd=>9`J_sRz55i^=;^>ZA03_oGyJYhyN9ltj^3?(C7 zm}oPIzYnzN`skU5l8G)f)AL=lB^_PXGyiW5yz6J?+zjUTLWiMbqH{?z1Mf>(x~yYf z)-x|mbVf6yevU9eay_wOneXBPv|)>8<@vFX6F4gGxK}=mpbnIl7WtBX5sIr zkCu5^&pcr4V`gq<<>#QqP%_d9Gn(0WkG7AFd0EfAFwkWq^TI?IX1dVS_`UjQnU{6U z!%rAFFHCe{ru%7T=l3gVX@?$T9~1LJGY8)-Y3Y7C&I>(V*38NG(V|0-p=6@XT)an% zzGNuw&8_% zlKnapIZxBdT(^;JDelLJ2`jb%?K$iNgENx7g?b&ZQTN)?ystBoy_tHqkcX`F8`?5@ zUUFt~!Rl;euRT}yMQ0)pndz4SEA{X^_Tx-sZ>HWYx|?)V4?1<)En9g-0uwJZX{3F#C*o$tmJMZ+lsu;fPs3%jIGbF#Cc~VcM~~d#k8`{ zXJ;XI;|n#HFk>CCQ6JJ);k^e8)Z;Ww%#{Hvb$b!dbq2CGQSTP=khR~ts-6=D%+!ms zlAF_B%zHZnIbj~KQ14c9!?>F6PnZX+)EnCBe5W&!yO}&>q2I0KuFc^4F*qYR4VbAH ztk?#$HTW(}Se%t?Yw~^=Fk;4n740QF2O}nDA$Kdep{>R9F*qZ+o5e_WP1hg zgTWce2{Tq_Bik#v4+AF5&hRR}2ct8Q^MHlAw^H{u>Rnrp=VNpxayOHUvy!`wZ0mEs zGm^WBT(CMr)IG_W$;Da8-A49?4RmkBj0Kz1HspISIukihu+Z-|a@dIbF=EDo6&u>d zoWnT5Out*mwh8xO957MO&PMLqrrd{df{A_}uu!kgM(&2qcrK;^GxcsEH>YjRea=Yk zCUQ5E3${MLh3?B(usR#rw$yyYf)$(7Uak3VAje+civEC+x;IhJSO%=r?KOH%z%*c? z-mT;uyT3;pV}*YbXtoS9tu+)U;%W5e`1^}CtuE!4Y}Y};_ZGm^WBoUuCX z^|~iI6S&P*=YobgS%FJr;#3`uhl3s!8-xEu4%L>@BJ_ZI5i zxI6bd6WN=ocPn|wM&I7dbDWu6uwvMQd1oSLtk}@r!Z}RNLaqZg>b9ru2?Ivz-bB4j zu+r}~vb|N$?FMo;l7~$6yWwr@!e*!@%@;wpuL^<#Oy5OI$)#T4e#JR2h7w9 z+Bn3uykh_)K4f}B(OV9hO zp0Q#)K=pzZ8``_MZ@@@BVRjaBx01V!+_m@UdEG#c&O)x(oOU4ZjS20&+>Z&1vyvOe zgLE!o>G^%sorPQnY}C7UFz-9r5 zm$5kQgX(t!Ibm_OKA(9$MogHoI4jvc#CeR)Lax{_d|2nAGn2c8+->A=n9e867!K!N zELgFjeT4URMsmX9v?Dl&(V58ES;!R|+L5|1V{taJ9i_Q6V5VN2b~Nvc2{RV7V>DO2 zA630M^T$*#*f1WedcuNrg5fxwi5$Hh3vIY zYi`Iyzgte_Iap33qn)mtoRw^6s2`k>+|A_bY<=!Cx+h}7g4JoCWge~Y9?nQkn9)AR zc}&=x;Y`h?0W6Os$lXeg=jmL+jP@m-i^-YEc0T7ZV@3Ni?}Zr)R%iMO_hH70t@ppGIh)1(7_p#z zjXp*!)3o*T7wDddZJOHGb*^ITUdVmlkQp1=MXHAZW3PXcdz_WrFkH-Yu%Ug6=U^Hz zQ}5a(I%nVh?@iP*7OZ{#Ql9I~M#v z&379)U8VYvjehtZ&%=lbTlf3i_XAnbuI4-zXSzoHjNymM5tB2MD>k$raSjs}tQdZ* zxrX6d&N;(%)Ul$?Ry||EhW2yyyNTRw@=R$gy$H?RbYQ&TuDnY?$ui+;3#UaJO>C_*>3n zxQBCCd%l-BY#4v9dNyC+{F(k= z=C=0ySJlgq538Rak>T$$VR7bvsNXO@s%($Rj1{BZHT8d&m9S!&NA-$vUgbC+^Jq_2 zZfNs!4(%z*-Au071`G>uZ}+LnVZpi0WLrq}?(Z%_#*7uy)73AF zGWQIbo#|Ps$Hk;QTQ&@fD<^Euw1oO$Nm;OAUW$76Im#6qhNV@{STHQ3x;;-OjL#=y zc2;t?_51?vTbA?7$+Enxm{w5E-IbVId2Smyy-;;~kqoQKjBz#Px;p(CKn=AY~;2HbDPP~-CQ}MZNc1Da~rni9Ol=O zXEOIXS&wlZ^4L%)djHeWbk`52F47**+*!CSyIE`Vlf7MLt%R<77TTwo{~if%>^JoiFQG zWtb()Wzuex@iv(o`3@QGl<98jzm*NcJ@m2Nt6YC4?a$;#@G%(||IO6D@5_=hE+y?b zxHR=;WP2|4RmrQ#u!hX*%DTR^4d`zw%jV?Q%Dg@Kjr4bt730pz4bv{l@hvj#Da$+N zHtwz5FznC#0nELRd?<4mGWjTJpON9SvUJZ=PG7?FW&R3YAnUitm&kOvY?yvd{dd%V zFXLmk_gpkE_bOI(a==_a^U$XUcMcj2Fpt1%8+MbuvFF>tes1dVX6) zhA8W1vTTLhP=AYT`${`NrgzJ5ApL`6JOn>T|491B$$Y%5*iIy$CCk||oFmgLnK6BX zxo=XxoVo9jua@mv8E4OJxLAj2a)ER?gin8|L3Ahr4CO{9ESl zmFYfdf0*0&C;FKGLjPeIF#k0f^6LtIkyt7BEySiSxvUp zWnF{%nli0LeI02p#h2l_GGcnUa(#siuaadw+18h~0d7R!nKo8EY{J~8^<(#S+=F==UT4_ zV8yf-_c$v#zFqzN4q3bJRBjmFr5yME@6FWPKAgk2uX4e#ALlVSOYiTmxrFfm&S7>o za(K7q!h7bnkZZ3WNZ(m{elLBDm@s3(hT$O2VRR;PaoYQIE;<{zYX>uj2{V=f8+CiX z&UYiZVD0@wn8W1EdoOT@NFkr!o z4efa5F<`_rV5Z)+6Le3QV5Fa%h1|7|b00=$BIf}c^{$=BeHfjI>@C!r(@x?(%mWtc z-A49?Pw2jc1uMpr)i2lvv{N)UWT4-TJT`urz#Z^nWRAa6Kk`v|u3-xX| zgZrGBT+lwFxj0~=p0Qv<`z+7F=uG5pCKqQTcdh8&ZX$OxxnLd8KF9lEbS82guu%_Z zat|iVSkXSOxrhlf7H1{50qrdIfYF)A8H=-$yN&F%vw6-01N}H)qTbEqf(`8pJP*SJ z6Mb){-mPSBqu#Z1_>5<>fJ)_R`PUh%y;cP z-ebT(J^qU(&gCa)VZLi$(tCz!8kzGZ>cv^f-A3-(`Rr+$2Ijnxx;IhJ&O)vOHtJpb zGJC@4OyuIM#iM}^eAF|T7S^T^)V47g2?=955 zm3r5{#&ey4JY=NrP1L)Y+_ekzyx@%FgvHs&UHdxkjR7NOXC-$V*)HUs0R#0RBYkh8 z?#-9q+O>UI(LV;C?}?eGfi+Rp~fqFNRyNNtqGxOd;y*eA&Yv1C%F<^8i@+32J-a_45sdpRM8!pj% zMrR@qndvtS-{zhHGxcsEdn@%eV7ye%NzP2}7IL?e8`@?3d@x`fFj1dmX3krwcN@8D zm-9Xt28`4vnV9ot>IEw{v@3WI3{TL=e43`2xo#nME7_{vD`1>pqTkKrg4Nl`L)v%r z{OC;N>?~w&rQXo4uUk%uhwy_I^mk%zRa^d8)-@0R#1JBzF_po2fUa{aDWn&PYy}orT=3c_)BiWm%do%TJA$Kd;+o%s|*YV!dG%)9l)Vqn?E#x5^ zeVfhu3>c`#2`2h^z(T!F)5cuu4R_2Fn^t<+RzITFwzBf|$X6oKT-CL-i2014ilzv$K%BmAd_c=U~8y z$(hNMEX)mA>9+}n8}xqBnaCOIfQ|ZewO_K22?qMnnaKIyv~d23S~=HkidH; zlBa88K06D!TggK<`d<4LKPP7(Pckx>2CUS(joh`H*wcWKx;IhpX7Z4Qez%g_1nt-C z#~H}pNZp&Ldoy)!q3*5JyNx_u?Pk8m8OYv9eaJ-Lo2h#X^&u;LZ=>$DTlgMlApfgI z&QI6Ge8w_hrS5IiyLKyoF8)Om=Z4Jmy@h(Wktb=l@pEtn@)I?3E@2+9Q1@2q-A3+O z<9(fh?2XjDiMlsaAF|M&WM!`Xo7(Ms?|^~&kdgk7iGH47q2H}!Z=*hBxPw0*&PX0I z(Vt{y&ReJtS?PBhxodawb8rUokdc0JX7Y3`%y%2PYj^Q}&OnX>ChB>>LVd_e-)q0& zeWqz-uA9i+OrEZV`RQ7jAF|OO((dMa2Mp9F8JY7Y>O*GwLl*kewKDH*)Q7a+vIl1% zdn5HB6a8)`cMJK6S~=HkX(f@a~-}CbuFi;;d(w}5vZpci(Tga2F%yk=iy4rnw|A2vdoM58g z&E#$+cN^LM!1JAf9G!{m&D6bxdUZCk{gL0p2`2j9OuaZ8c}TmT?{)@qHu{! z8~y2O5AuB&oRQp3?Ujm&iuIS*KP@iOFZjz0;A?-i-E@vQ5GBP)0qMx0GJju$O zw^5&@{hjX{Fi`hK>O&^_lg!KwS?G5w*=zsMcZ3N>`a>rA-b{VE7UqYn^c&iv{2T@h z)Q61pyNTS*WN)E9WToGn_89Lw!9YJw)5P46nSL3tQtvkMB+c%f`u}G%$-vx@iGDYe zCs~*qveF;2(eK(k`kvs7ck>JY6&M)!E2in~(RvIKf1} zo5{sl$=ybtq&bd-WM3EtjMOKYm>V+F_ZI5j zN`1&i-)jr;UCuy`n6XT-()Tv%wlL4bfN{V?eUh2E;;iK63{TU0bR*fDsCzT@GGL|N z&=%ploq;^b$lQ>L{vNml0CfcA9ufZ+)mneQg@keR->Q14dqkd1!V7Ueqz z4AdtXne!&<|Eihu-bQ_r_6+`fV00#OHj0UbD5@rx%h9IIRCGjIX_(s^FvnpPt?Y_ zN!qj7vonyrk$N|gv$K%v1RH&?EzWyQ)4*Iek-M2(2CUTE1j7>ixf?K2?`CqDrj@x# zHs*%3CD|W_f6>S}Z=&wa)XTqU<=iA2b6#7D@19_w-;Lz-Z(2Ctt>lLG9NrIuGm_H; zGyQHMdmHs`SeoD80VDMx6Mb){K4hUkWToFGXv?ra3m6Ao@tt9=DL+UWTS6^eK-TT z8_5YX7Oc)j_Sy@09tLM5cM~~Ju+Xp0Mz&@3+%RCI?oHIYnLK2nKgr76BpY+K9DAFl zfjMua-c95@V4>cvfKB(6Rh;RjchBhw*donZ=^nCqVLVryM;W- z#+=tyfTDd+sIv8iTA@W!AL(jGkM5DzfQ2xpQNqKdk+|>dn5IP z*;&YwtjrDB=y&afyzc}9{pd{OAv1k%pysC#Wy_U;U1Z={}{pqY7Zq3*5JhivqxtG$@-oM52ejpS}3XJ;XIE7{wq4{59M zd*F=Z>6(}?&Pr|*wAI6dmDAHt;PKq28`5` zGn0ob^u3jOw~@QHHt&T26J{(EZ1lUf4)>3b9PZYFyRb#JBK28`?K{gN}2pQx2{ZNTty_JI+TGn31J zmAbc4AJSgI@6iMU{W!ryf5=RK$U?tc$wM~!)74(d_f9a-??$pWQTG<=-bTG^uj1$L zjO66ZWN)GFt<=4Zy4TiYKNv7#a%OVDI-sr3cVZkcQSWAQw~#9~r$yd>ng-^EjP$*U zdN-51h1{*=N!kYNX~0N5IWxIDK`Zm!M()~%dY|Yl^T{Cu2&9G#gwT?_L=R{GvX z-D|JreKBHo7P7ZeAF|Q6t$2CKs&Jv@vJf^L`kdk?c*>GZt)UJMeRLCUSNba>X`5dm}$LXCxR~7L;4I{d&1pOH9s>sI-AK+@usR#rcHzA+4OplT+30s|SKfPq zfquk&eg zw>{ajGms-D%vha`Y;Wa0Oqj7aE4kaq-tacv8!MXW_JrA4$jxc*;+!*+yOrF~_SU&>B6~CSZXs8v?ZbV} zK<;L8w~(9D_T@f|m?iO;zhVeb@4HIT816JzcK%MVKa&}g7d@s*)CUSNba&UKyO~`2+y{A|0TcCZCRc1&GSB@G_YD}ScM~}e*r<2Ihjo9%nCbHW5J5)B=x)D z6UxO|$<1jeYd&Ij7IMXg;S`z7>G?B!2WBkToc3Afota#*VJMpKW^!@b z=QP(%#rt8x>@4JXw&pWdXCuciXs%#&HgY$d!+T@Gg5g}= z6C+k<_#*c>Gr3^Ha31$!#Dp0OHncDC986fToUeYxhW2HihZzf2w68FS2{YPP={p;_ z8)h+w5i{1WsoyYOpqvM6)YI3Qb5?TKF4SCd7P7Ze@7gzXK4Nlaa=A!z-SAD$Va9?D z?PARZELffPE$+jNWt!S0oX3P23s!7s-`2TqBzFtB_PI-$?_Ng6jKx{W-A1;{b-p<5 z3f04ak$S@1=c>*{Oiud_easlHr0&e*g4Nl`_Fdi=BPPtfe-+Qc z#fJ6+`k0({wfe!C$lXHjR&qnTM)!0hIbp`q=YGgMCd|G5Bi`GY$lXk?7=Fw-%vi7v z*r1XEZ8vp zRsG_uJW^5Q1RKFX^2@6)t3-P{KoOxmO3#LVs?dhCz+M=olY-rC= zJz{n?a(kxcy7^hk1=C_=OwVQ>%i_v$3Ffe(Evb6Ih#5=oFU5JZ=Wq@q7HnusYtELD z5i=IF=Q4)@6J{*V@I2lFv4YlxlLqmrrt0_ooi?ta4&`pmBU8V(Kc3& zSg<-9Ic&l`7<=A~d$IMr1@~_$?bV#eiVbZm?!ojLva^wGYx)>ptK4nmZkVZld7W$+ zw^7c{MvkvnKVjUK{014flND`y<$!Sq`q(hNQT2=!TX#qG)6UGH?W!EHy-B$xX}isB zAh+FBFK?OKM)u~nYOb5!rd;148^(8XAKKo^0V7r{`>1dG%IFOHsh-gGCu2oBK=p?8 z-OAxT%pW+n_Fm=ej0dT1@0S(BA<7X8rVps@4IffHV>pbt!)3+z5$c$aP_`pwJW4jS zqv?NCCTwWOsva=+d^~lmCnz_xk1KogNz7sRgmUhl%(+vT!|-YHsWP5{pOqP#GZfB! zPL?ytpO^70S<%i`P8h$ST+Ws4OVYk9BNnu;FgJ_23uM5G?dz)N3zfu}Tzb)fsbDPNZa@D)(3T3O*v3!T~SIYK%=6)ap+BM8!`=N6Bk!+ZLtel&TdAE0>?j_%j)<#~Wlu`;~IKNfwO1R&H2tRSvhwh^>1E^*d#{OWNJC+#?&7 z-z(?)>0_LuY!5Jx;X(3WW&9g+f0qT*qsss3W6I5bKlR@?<5OhDf^ER`RLzHlWWutr za>KL;_b!UhVtxr3mXx-XY-rC>&KQ?CF(Df72AuH zgLl>31V}1Kd#7jbz$Xw#}q%E%WQCZ!60iWPKz3M16PML$*C- ze5b4j(LaPfrVlFT4>Na|wBzt38BeBviVUa9-2JSwoy{D^bCuIqWx;lVa{fAXY!@ll zZ%exrFPHhdGJapCACPY*--@@%@Q|#3k@*p6|ABv(32oj#O#L3!`DI>|yqZjF%xzvv z+18QyrS#X6Z3FTq(l#S+DdVeU*;=-l_H&0_9r z)X}~{zDU+9$v=?g8tRySOdahf%66S>-JdDvpUZST{q7CQ@g`Y+jW<)jMW)+j#`Zhv z_sjAC`61chY+uROQuh01vqnO-E@i)C3&hBfG~N&O|Vu7xj^VO@N=j96cxoL?mi+Ir+T zx0T#5ZlJzxC@Y4ImE$HdZ7vIjEtK0<^j{<6)?{owzgG2%c^l=hEpCT9%8X%W<%($+ z<@hGqFuqwi?IGfV!7PtMZwC)Br-x#v__uwg!p{^_!zeTI3oq8!hb z84HGURIlgBhUqKH;j7fYh8IZt`rKx6yHNG;4Vf@sq@2IWxr_1Axeerand%Mm<;tnb z_#Ig>U8$V0_WWJuuHxMHWca>J*U0=s>OYe96Z+TE$AZmSuhU$&k=ty}|CIjE$=BmA zq}?DR<{OpMO|sk~E2dkOGuEcu?qL2-=6@sO-SqFp-^=g^ndZpyAbn?gi2h$?epuSy znEwx1|1QI$X!lS3=UbRZMl4TKP7BDgkgOOMS8hwoxQxtLF+NxIg7!SlJzu63WLSM} z>l);>WyQRXa(kJK>&lGv737VW>u#)Ewv+ZI+)XBI`;hmO1??c^_&%9GBHK~akHM4Z zpN?nBbQbw+Y3In+J&*pEWV}#@Z_6~B`t|q=*?uGK_cA>s^Iv70XU^2mH$6?3XW&wD z(SR#ZUrEkcRfZXICeB$ydB#REZzALCW!g#3O1PJtv%ie*p?-)=ACUVVDrbL4o^Yg` zb(9Q8%NZY)GjYc8$|syC_dP|2 z+LvU-8Rt{~ii|kpYsx2FAm`w$Zz(T&i7em7%cx%=XW+h7c@Ellna5dIDckpC{l1Jp zkTb59vwkSUkLaWQSh*X2LcUhcz}eR+x1Y+HKa=I>(tafarkm(v`nB>2H_I8f$b|7$ z+m&bGY&_vk)$3jKessE1rdwIfra>gHI`J-%D?`Lj~Ob^I?(f&ln zIT-(-qA8W#_h%>x!x`x{8cgU!a%gqE0x2m zfoGTbU3zE2)>zkJz)@|}N{%l}1gj-}^^RlnzN@@xz!ALJ>I&e^3^Yq z>ue%Fwwb(mOS$5n|GHp-Yjpz)8C?Od&<)>?4`USZvJ-V ziw>3BW_b@j=_B+%Dlf*xPf_0WlX9^$<#At@vwtE_yiT5Vi@f#@dGWJfF!l5K!qT{s zylQoM(MIxTZ;*$7MDFo1xy*5LWnA*p%6orFE^@xS;mh*4ugIe=l%K)#FH*i9&;6$I z5f@X(Tf3L4KI;m(_?2=)JPQxGO7%;=Co3-U1LYrHEg!-KuTfqG@A{$g5kHa(&X(8W zw%04)jhFsHx!fTC_)EFtjq-f_V^cov4tXjra+mTpcgrL0kr)3#p7KYz;T-wn2jyM( z<42Si|GT^Z51N13sh{&|Pmv#fs(jKy^2$ZzflrsaE+LOvTE1)~&;;w#EE z@nK2V>Y4K9 zbEJJqzWZzPP`v3P<(Zer9WIyiR=MR>GXFr{jJsZ~Tye*1m5=$koc{)S=#6suo8{)+ zrkwGj-zx8UkKE&4`ThIkbN(Q=`=dMUi+}DkH`i8A&A%mOJk$=iNuXdp~&!F7jUGRS%VGWqH%l@|h>e1J9ESUMjzjn_jJa z_ce0apUN9=me09GUUi#1<#+PFCoMnq^IYs{^3YY~+t$G?0oieYmuDsp`VVa;?|ZfSOT0$;!>?0bVH@QYx04I+Aotr@{W*Bg zE~>w1SNV_K<2xbzf9(n$j2S6`mc}G z+&^%)6O_0AxLoE$xdJY7%2c*lpHlrN(w03{`9sn!I8Af+oUZyG&XCXejOsgoPOfma zyh7$H&e7cA=c@jaFUqf-FK2&E{rGj|?Y< zd9Cto*D3EeTh4c@-1|29mNsSn!o70i->H7~ACwQiUmpK}T=GG=!=L4)kIH=>o6?^8 zlnwv+`mXEV0xkQ~AOdZ!zUR|GUum*SLXk1LFq94U8KYH!yBs z+`zbjaRcK9#tn=c7&kC(VBEmCfpG)l2F4AH8yGh*ZeZNNxPfs4;|9hJj2jp?Fm7Pn zz_@{N1LFq94U8KYH!yBs+`zbjaRcK9#tn=c7&kC(VBEmCfpG)l2F4AH8yGh*ZeZNN zxPfs4;|9hJj2jp?Fm7Pnz_@{N1LFq94U8KYH!yBs+`zbjaRcK9#tn=c7&kC(VBEmC zfpG)l2F4AH8yGh*ZeZNNxPfs4;|9hJj2jp?Fm7Pnz_@{N1LFq94U8KYH!yBs+`zbj zaRcK9#tn=c7&kC(VBEmCfpG)l2F4AH8yGh*ZeZNNxPfs4;|9hJj2jp?Fm7Pnz_@{N z1LFq94U8KYH!yBs+`zbjaRcK9#tn=c7&kC(VBEmCfpG)l2F4AH8yGh*ZeZNNxPfs4 z;|9hJj2jp?Fm7Pnz_@{N1LFq94U8KYH!yBs+`zbjaRcK9#tn=c7&kC(VBEmCfpG)l z2F4AH8yGh*ZeZNNxPfs4;|9hJj2jp?Fm7Pnz_@{N1LFq94U8KYH!yBs+`zbjaRcK9 z#tn=c7&kC(VBEmCfpG)l2F4AH8yGh*ZeZNNxPfs4;|9hJj2jp?Fm7Pnz_@{N1LFq9 z4U8KYH!yBs+`zbjaRcK9#tn=c7&kC(VBEmCfpG)l2F4AH8yGh*ZeZNNxPfs4;|9hJ zj2jp?Fm7Pnz_@{N1LFq94U8KYH!yBs+`zbjaRcK9#tn=c7&kC(VBEmCfpG)l2F4AH z8yGh*ZeZNNxPfs4;|9hJj2jp?Fm7Pnz_@{N1LFq94U8KYH!yBs+`zbjaRcK9#tn=c z7&kC(VBEmCf&W`KaKp1+xxwS#mhay#{!=@1=9Ky8J1Cp#`E%Q-zQtSQ>|NBqeK+Ng zzfJix^0IrVzTpA#;sfO`_R`!7lx>~;RR7E&@__fr!*tFr<=oRgpmQ(E>i^)w$_tz< zzk96Q@niDjdp}Qs0SlmtUi~jej72ajoWl z@_o6@&*US#?{Tx0cerax+vXn6&;a25qemANPH^^&PzPkwyvpGW4MXWprk3)qaQ#(cK2EwK8Nr~Y~744KxD?Yw{9WISQ&AMKg{<#{K+ zdjB2GiW>hv|NHd!)|UFOpYp%zX~~@zOz`n|G@5 zq<{YNVhc|F^W%T}-}iNHZ~Y%(-`mS3{=VH4{_6+*uf6r(x0g+|=d(2#TP!ZRlKzpa z|MU1fcIv19`PUcMfB#}%+3fa%4`2WPo;~Wf=%0UXmz?_EdFIjA_g!@7d%XY8&NIUL z{E3UU)W!KmE+K`|a_c?-0Kz+aeG%AES@?SnfYgW-QJ|4#zW($!Q-~KVd;TQT2ck3)Yj=Z$=ym0pVnN&G+?G4PSt!JFr3DHn6aF$dOAa9ELgE&_zd%yvG)3Bxd#hYOhx^Q z%^5$({mw$RGwEabJojNkJ4^M58Edbf%{^#eP!1R|VRNQ)G@r3}&sD!-_@Z*ehT%Nb z3pNa2Vjk0gnR>yBcD~LB%vi9ZeVI9on4OhuUtu0IR%iIC<}wy+SZArlLB2Rhd+vZX??db*`cPh35 z%+5+~1KM>wA0uY0*m`}o<|{U|pYpyKor&DdW2Xn^&vC; zZXs7}Sbm}BbSt?H7;oTvFgpvmIqjExuQQRWv-PI>TKk${YK{krU5hchWT!tv)^(aBPPsPv7y~_N3_TP z3K%hAo?xMG_j1pGnR>U7Tc7_O=be$9Fk{8$OuyGXrF)-pbJ`zNuV{Z%?j~}^*8BHs zE;u8(o5?mu^TpZ7-TVOW<7{Mmkmq8=G)*&e#aYP>?N55nkb!>0g00sd()ox98`___ z9|Oj}a1IMLXZWk;5*DXD%>B+lPR>lOeg1Ep$AGEVA5lMGbS84QkQ;{o(D~x50Rwe!q@D*X)EmY}^_+y&X^-(97%*Y(b-Od#<9|hGCEGmeS8QnW zs$QIx+->AAAJ4&z1uKRpX|5Z|746BqkF$|&e)YSNoG?2ZIXp$@GFEKPxPayo7OZGb zRX@4Kk=N8nthPIG$#Oy5OZdjOkXCikqxnjfcG|oE{IXi6;%?FImM6OPII_EGC zSg6NEHJ>nJ!RoYUaKAH>3s$E+lXDm`VRhQGG+(e`TugOure2(l9G=bdF%DR$H?+ld zt{cb+GghZ9!8wcrCh8dr+LC;yGn2cO{GTqRa}C3DxCaYXY-mgKz5@p8-blTh$aR8^ zem5?o=VxajSG4Cchs9~nQ$JwEHo@?Gor~Q-#^S8x@B-#AVZn;c8JE?$f_1<~JuIjB zZX)La3-z?T&UFhptiXF?!i?3~$hIQqR^nb~A$u$JhPE>IV8Z5oq2`*?R#Ckh$q8G} zt7UxL$A)=5>ew)>uX@6U7O7*yumN?f*w8jqKVZV*Y-DfPNarJFEWN%l z{Q)cW=Cn;XhZSv8>R2#rM%|go)oGh+-kYd*Gr9HoEjWi6+0 z6`M1@hUYjlxnka0bIloFt9r%e3^O^0_B!QmAyU*_73&EfqFNRGgfTQ@J^nG z8SP!vG48G0&E$e%AN9MDoG?2J+4kieCd|%8?%IBw!-Uyc$Z>zoXS4&9le3V$je2-D z=P;waNA-x=S;=-F=bf3{Eo6JI<^v`ySg~O^i1)yXZG!eb-U};+gSpR{$Qdiz`!(Oq zz;xYn=_uk`(kt2$9W%2&fNPaat~%K z7*66IXC`|ab^Cn3vR^-rpwv0%mUDV~D~?bE6UXC!B=*w9Yp zK4&I(3)x$#H>{`WzJ}p+-pg6Y-FSxP3RdrD)VI%aA12J$oK~2}G+?GaqL~OLNs3&sIHS#fISv>L;h2qk3>ga>jxU?Og7| zhW`)y_8p{cQN8=VkswJKf|8*@q96=Ga%gfChaeejNg|?-BwxkQ0+PW2kqowg!a2XWzP+oaYd`ySpFdCC{`0A8t<`<^vr4cChVytwVMEmgT674W1YK1^S&{XQ)4E(Va{Q|h^aA?OJlxN z_f!knU#5C!Oyq)w?sCmnJvpMkg7@|?P>+q7TzY8KtL{qPfxa=3GnPhw74O7=aS~?w zrO{ul`vPVx7_U*kTF6y@E$?ZJ<`YoKpfEf$ATbaX%xre3B-==e=heqAq&O0$Q7P2+E zJ9rnSNto$dJ$I+hM@&7;)T@Q;@6!2z8B49-t$snn@OQq8snN()e~;!{1NCYmS2Nk} z)p>Ux-^GL(!~OKp&^^F;^cXNU7IM`+sPn!tkgJJY8vO|8F<``m1>HY*FZ!AvqW&=V zVZ?;FhgRnw(fNc0{iCW!%snjB4c%j$!`xWNt^RR-M-Mag@Py{7k?f!3d}AO-%#B8F zbx-NOYO48ZzJmc{4-@r_r7=9i@4?hq$njat6?D(>jvfZ;)kJR1)Jvn0-Sd10Lt`Qr zv_@w-Uk&8g!$iHB$yN6P_hCY-^%vJkRzp;?3R{QTWj|p?5k*o0oovUVYwUDbuP9N%AK|}YE>H%Y8B3BE!>ObZ^ zjftFlXw>~DI-f9OL2GoM@?MPSKU2LL$q6$SbpO$O)stgmBIm|JHVmKZzK976S`Xb9 zyaN+@_cH%p=XfLaUS|4+elnepjhS4~dg#8y_c3C^+-PJsxz1H1xoYI9pF;C~O1{^a z$Qj*K+>Zeb{g>5`=%?m>42_Z8ny6PZ*)UI|duQy6GT_4{;-x$f&M9ygFX5=1BXy|9+y=aYo zX7y8}n}s^&9vXH3HO*%%jYh79uk$WU=w_vk33Fp18~WLFE?`DOH@o^CQ_XX5u7{C& zHIuC|&Z&C}y1A6Afn0iMb$)K0iTF1GyT>)kMx%(9f&;s)1ZhHAzHwG+d=oipj#Dp0OTBBc3=K^LdXqXn#oL`uE%r!4Uy)lxjiCoR(Y9U*l zTU7TN`fu_MOlXa6am^)k-{O6ZkzCNwEune0Q~@iGz=@MUQIQxqZ)ffJ+wL(H1F1s2@B>mRWIn)Qude|-P-Es)^)fC z^SaD6y7g4A26D#IXymZI&Q&8hqhZ-VbAChSu{7q5)Hie+Gd~Fv{fwo~e@EvFhE2E+ z-FKBex&h_XXq&1YH=~ak3tFS!ocl3g+(LCj_dVtR<(A||PY#Vnj$7$mYoea9pf!fA zIgbV1w$w4BHJ0so_x9rrBx2Ng}3+A72ANrpv$HrXq&*)Dx?{Kx4bz`g&*+ci-s3rs1=9)KgNFG;)!j+sjbwka z>Ivg1%KkK&(a@hxA4{W=?Jt^(nS0RCpFthnnab|0@g}mN|C{;&Bc|%v>X+&{%%3~n zOwQ-2ZWzy3&W&~fbLcK&4&B9M^p_|H^p`4E3%Tkq<2<^{l@r<(oWGJej8`#-8Qs;Y zR}(p3qk6nnhU>?h$kj~tH)<}Uzln2=M)o(WZNBMtwuX4bAANhW1=pImxXqX;UJwL=eriYbFqkDvV z8VlJ!O208b#{KA?U=9;zH1tnuE}?r$**(qqXU3bz{#n&a^*Lq#yi8acld11s7;hqn z7gf(ManH-+&1Cma)eYk-%GE-)>Z|Izi87;MdQJ6${$I>B+Uu&Pe>0Ea4dw8rjF_u$ zsh&s2>)ul?jp=>W4c!OI0Ta3pRj=lnKT>jD~G9NK|d`S-E_(k3;M69Ud`n2Rn@Cy24(A$ zZYIuQK{qq~#z;`;c8gK3F0Smq zMZI+i>KLm_s-7@o>}P&ySIvQQ${ynioI^vmB6BNA zL$|VW!i-^6)eE}Sm|LCm=+;oqjkc!hZY>$nt*x9eW5Ku%_n})?*<(b*upaj}h7Gs} z4c&&Sr;X@i*;v^yZ$cmaca;m~0WyY7Ige>GWxu%$)h(3W_hj6Xxvg*;X&AOu&KS2N zV?nF+?dfCKQQ3W8dJH{uQFDgw2b|vtcOI{iZ5Ph%N`E&Q(6IcFxgU+!@1a~RWcOq4 z`3e1>%JegtF%6Q@(EVKXfC&rwz0{9b(5ic@pZ1Xj4dcGl(J<`Cy_hg#K|eur?w2xS z+@E`YL;trjVy^jj%wbGqOb3wB&>g6H)sw4{oX{{Fq;v5P(jOudy2H2^4fElum!tmo z`lD5^26DpkNBYM}cY=&)C(=JjhQ>&?QgZhopO0dW;wzRoxzw<#Cyw zppO1Y<%|Ul)3cmM!}PrB{zcAX_@{DumHtE-{w>{`(%!;R`fp4Bjx6ZjrH>I)t$)Bg z#*dWkW0^jc@w4$}a{63#|Ah<~Yj&^j@4?72nKUe4QcjagKb6eWFo&T}In2o1OgM{- z=)SJ(W~Gi9ZFbf3+|m}rMP>P>jNg)B339*8%Sc1N0vXea%6=so(5-cKbN+bbbF6Clf%BM$NiZ91NjJ9j+Aal zhGV5amHO#2RsW(K|0XlqdCJvtK64kybfF9v$udm7TE-is-7M2BGT$o0ZPNc;mItJJ zP}&Ifhh% z*jU+p2RD~#d-85FW87Ui{)qmMrD55dypIg~O8-mg_LmvsubKM|9yne*gnYOxM@##o zbVGQI^vBA0g3PDVKTW37W%-K?XX824&z1f>na-CP%doP$RF=kg8TG5j8?GT=EB$rS zFkP?gZlr&cjF@g#uG%f+TbaK@mb+#AJAEwo&_{nS`M&YG`<1I{gg&N+l?~$~${DTZ zM^*Qa$@sWTPe}J9^`~WiR{G~F*=2aJt=Nxrufa~LtBTUvdOWf^6wF3UaN zmI)30a@4EKD;IPtDCg>m^c&+!svE|Yl@n$(^sA^JFg7N#UzK|?u1;=@LG_FUtYz^q1!_90b^q#XLR4=evFtfqoLc9`!S)R+loF$ENHd9wdP`DB3F%Eb=&A%HIfsS zM!&7*tASk5ZO3~tVn)Bc`UTw%+=rounR==7JL+7-)Wb~O(0yO$5*qrbdT31K)-Y zkgG;^yK@hwnt!Bv>|vsw8{Hn<*OYztcSlb7LVJx}^D5Pd%X3x!*I7 zu`!WzV1quDU~YUqE*l@5X?!F_Cj)A-ltQCx*sEuDT=mE=Ei}bVqVO`v20vTx#@3 z@%_d?ZcWsCnd!Tuxff$&B3CoHTFBv#x~DZ#uNHFE4Y4N-jgef<|vsw8{Nq|U-jf_B&Wtq?q#8GjqVh_hrTh8<0MS!;I4-x$dWa}TYa`-|@L7<-thw-)NXH2T$$^}W_i-O&A&--*75 zk$SH4XYhM4qSbt+`l*NREYAOz2Ii`XoF`$SZ;kG6{BHCZF=1{jQy6G-7w#2 z4CHDeS4*A0l;7FINWC>tcbD;fjF@}qFXwxWiCi#Tp}A@#wh4b68!(}}i|;iCa%@cG+`~fM8r|J|r!kOQBlT(`XDp3I zZgqdxcUuGXUPk)WOfHS?9)1@FjF@{^s8@~b@6~+;-F@7Tu`!cd3-zjzTiyNKkD)P= zb7LV_joca@(EYJ7ku#Pa`UiDBG$wM!(rDyr8qqyfBfEd7UJc~fn8+E+|E0!!)jh;_ z8zZ@()&9fm0aFhP^{SEGBRU^3p?g$y-)Lm_80UJJsOKIUb^kc;?_s1~P2|>0y)^nK zbbsn$pRrg3%X}Gk6{uP`c?O= z&U*}vk({v9{&VaBOQV0DK1NJE%+w8?@w?DtY)oWp^e^aq#MHw~y`Z6cQRjUR1NCYo zR};CK$)%oqiTz+~%w$9NGW+OZpq?=Iuu!iW+5MCEG$wLsG;-Cw!oDzK#)9rueh2!- zNUmmbL2Gmq`7XxBM6PCXt5L7I*SHrWCM>o8FXphId!65jp@)%rK|}X%&Nq5;HIl1| zT+QTO7W#(a4SlzliGIe?Xk_;$_cTUwHIoY(`nPnx8p+i}&REdUjdC6XMoc|4>h5jM zHwJRV)R@Vwg}R}8NB6e|>Jbh7ySxWuVArxeF_T*h^{SEme|R4TjE#w$v7j}&&$$l+Mof)HuDUNckD-T!x-|y(n*RNA zG<1`x9x!4?_a*gv8R=Iuxz*}Dlk+`Hn0r{LSKSo+4m5OA^3KLUu10dg+*rs}BfF`1 zH%3gDdlQ)T;Z&M7AFKX}AwlVR?G*66;kF_TN9k*n@&ycCYJCCq z118K^8r_22hY>RtwAx=t^AQu49$KAWSm$zMAy!DjzbE&bA-8bnsy2VuY z|D}Psgc%DOy2ZH{1I9+H{cmYLqBXiDxDR6wGxc5;`e8}kSB>O^)XzXYM=WTyzLL&I%xFDyD>IL|v5>2N72byt6K1p?x>a?)>dCEvdc@Lb zbZcVWbgrS@0XTxg8sY9ePeOo8{W!S^s=#DuxAkPY3M+|wAyv4@3v)#~}RbYDcj zHW^b7{W`n@118Kpbn9{s14b<9*3(=ykuw&wI=4ReVr)$0+*ru1ZUerL0VAfyOm-XU zT*8c{heo~XHsZS&dKjsv#!NQ!8*|@xWN1v}+(W;K&Smu9C1XKDH=w?+Zc4_05v|d0 zrum3|bLG@%WVeO-p)rvQy6^Fh#zZcSZcFAH19?(L=Bk<8>bBzj7#kD0n#tDaw&uN! zfn1H`gr$c@y&AUB{Rs;i`fd3gJxtVdquY*mW59^1F_Q}#`t5a3L_@a&@50!_Oue8r zx*d6EV<5*#nCX`~_kG@r2{XD#zp;?r57ck<)MKO7`JFh29wX*j-b)<`{})pNUY5BkPHPM8}D+0gH%`&uLQgr$c@z3P6*zA!arvfrKWVr)$0j0Fwd zk2v2L$Pp77x;=C*VL|&ZbwB3!VL@wjdup!LQ;(RiG`gSYe87Ym{ZHxlFj3Eqge4AiTUTutO^CL6k+>pKxsVs;(%qMmzLs2he~^A607{!a~2*s8`(q+}{|;84cZmns1HNtC?IdAEa|t{|DvJn90>bwnld_@4(Q*Oubsj z)))@qJ($rN^P!q6jYf8dsqZmjX>^D4Jf+V`((9JB9aP#?t6cYyt7P6uHE9Wsz!bHEIJ45FK#>PZWXKF5^HTtu-2V-L<_tNOQzwvu8Hx{y? zJ6rRyF_Wu>Ts3m5JBROMz=XN>&*k@G!h%-&=V>lsK|_B&-@}NhF_TMUxIpJ4CM@VL zhj7PQ*G zRQG2rXy`9fzZ%F96J{)E7%tcOY9d!Nxiq>f_}%Cm136;CjE3$??m>^Sv5>3oDtPccPyx#&8$sFk|VVQFnLqd(k&Wa%wE(s*$Vy@47cl!c4zlxQBC% znOrSoLw7H~AKiW2gLx7f{a(8JbziHeUJc}GBA0sp0o~&p135NkvNgH~c~7G!hejh; z{Rr>C&=|?BiF#=?ay9-#_g4$qJ;b}wHwJQRq~4mS=f*;=8oBBo<{juUV8Vj#5zPmT zn6RLGRC68^W-L84>h3Yl_b^g#P1JK^A-5WJ|2W^n(8EYQp*8v^*bl}=BfBU0UJn!X z+-PjvnOrSoL;p12#efkLmL9ribiUP74;Xuxs24PJ&+@JwM(Wi>ZY|WqbGk2~ zp?hBSfUz-=tC{SK-;Ew4W;Ap!@Ov5qxiwNxn0r{LTcdlC_ceO5VR%V%u`!XWR{JkA zkEt<}-9P!gJ&e>_6ZL8)R|~l{zQVgO^)OQ}Jv8d>Roz$hqdkYf)M_1x&*;e8k|VyX3aH5VHb z*}um<7%(;_a&9c-s*&CM{LV&C4vmrAYSgR#1Kk@iHfD0c{GsNng>2|P;+^OlBRTc3 zP;WKrRrj&J<1t{yg4XCh;ap=Nw?^t|5*GTc{!_k(8Qo{pF=IjZAN8w&Tx$Ju`sgt< zMshWga}Nvks*&9n`d(}-WJBlvHTM52^cXQ=sr|_`7aI%NeM$X*5p$!Ft8Q|gOIR9> z+)FnF-^GNvhki=t8zVVk!8n!XGg_nnGVgE9Wb0v?nsb;N3)xMhIgg=-iF$48WGJg4P(m!aQ1|`zr6ofVqc$2IeuMHHJRTMa)>xFwCgA zY9uGjjeaJ+kD)P=GnPg-v(8sNIbdu|is)xo%u4Z!8$nG25iwVO5swd1?stc-L8jb80;$4^; zGubVyxoRLsOpTdr=oZm=zbM~p%w+dX<{IN-s#i1F8r|ZY$I!z}-OzoDbB&4YmQcSM z$QcWUB{f$~y7U_&yf&ORF9+V?jf|jOH>HjLUK!6J{)E7{9Ig z)R@VJemTtrjF{0H{qnp6Gg_^$pngWTBKKo#%r&p1xrC+Bt*pLpG_qSo{c0db%#DSd zR@J%OXk=TBdspZCjfq?}atNA_jfrgN*3ewQh*tBO+=B@-md3Ca_hCXqzc&3IChEEN z*I^zbW;FEc@=mnIu%7xE4gChHS0g!NL96u*xd;75oX3E%(QT}`h^euV-FLVT14b<9 zHen8Pqt*U*H5bs(4R9VamL3}QUdByzU$u~{elzt`b#pRi^joMNFk-^oXk`CA&Ncci zRj&qe#9aGZX)ZPjzz=(g7Rj0N2`d>;cwOqj7W8oBDW)qSmnx?$K(a|P4(%GT(2 zP(5J6+!%J`yWf`{GrGt<7%?{*x$1wQa}f&~`klBJBPO)k-&u2Rm+=O2L96{;HRmy4 z?x9h4yYZezBgY@AUrpp{uKnFLm(bAth&pC8bbD|=Cd}x5tbV|VsfUHSq1#jEJ%+|e zZcWq+x}WHt)<`|o{!jT{VSr`8 z`>JmHjW_&4IW_tT)Y0v)>@ha_U#lN6VQw_C`wjPD#Ds?KcbaR>)JyFr%_YoO&@lX- zcVa=hT&NCtNu^Q)j&?@ zj-!tWb7LV_{h#>`=Hr#)3FFP=QtKzGUmE>Ms#gQK8p#>0&Y!Gv0dr%i{ZllT8Z$YZ zs(!?TrP0XlG``!I$pzi%e77-^tY=J?!koq5!I`K zT+q-xs=0!O?lJB~|G08)eL{7^^rW(Xiu0H-qoI3Ra|v_x8SZ&@yn$TM8r^f656^Qi z22(DWUr=^0N{<0EmYQGET>huD9{N|*PmM%wv30 z*?Q>TQa_K%@HQFKJIV$9yUNu-E@&9u*Ia7!AE+KMeyE(Wp!-;L_lfjq7(P`!qM`qc z`hR5lTxKk2=)O?j-0Nfi96o+Yy2+)-hzScC`YALY8r_uKH`RFkmzhUHKeg%!^EAqa zVOsi_vDEr>>X+$phVdFX%&2-sH?y)~oP~Z5%h#Cyx^%OS*Uv^D^PKeO!g-``%;Y?u z`r#Wgp`l-ZI=Th9Z($iRd{fylEk?$)xN=@n`hJ-(FQe?1rH%!|3aaN7Wms9pRheIn z{_3)z56XsRZRXZvZUY%NlIc4#Y=Q%_Y%1fH(r-m>^joVQw#OZr`@VEL<1V_bL1PWoj&BdqDlFdr&zxCbFUbhvwWv%s(vsBh(+Ij^zpE^rXyB zN&gJ>M)$nxhTfDT8m1RGkB0GO&SUteazXzJ=U%0bej*vuYs&FoGW?r<^-X2N{FbsG zWvxbFeLv@TGda(tx|>Ia>b%Mcy(gogpO14G=T~;$U>*xvqg#Oag}4W85#_ij z_h9%Y8N*_nYjlgN?y;czmg*U83Fi87DH$-=ytL{O6Z&PCuP&>cmy@=<^ef1?B6Z9w zDW_FrLBAS#b?JhPn90 zZ$iJ(YW}YJ3B#t$Z$^D{>9&y8=)XrD)0X6|rQb$ojN8)3u)T7`jA;kXVL`v6>hAk8 zMCQ=_fQ)HZ=69pMyNo}QVGrtPKUJ>!L1n);^ZUs(fxN%WzoDLF`91Xmq+vKn+5JHV zjF`|L!g-8`DJRV64p%)MG2V2PvOh+KW2HN8yq+8|R*%GU(_F#J{7&X(>RJWnRf=PUaQWxRs9D`mlaHT~=GdYLiaNWMvyBHb-A-b(*Y>F**} z?_us<>gY$5%R}@Zlm2mOPf~x1jQ(lmjQ&~jbK?!qD@Pk|eo5KC%=|y`6=|>HL>XU~ z?k!oW?<@NcWx$Bm7(Ufp!Z7(8`s-p$Qz&P&>Qt(`>13FmIV|YEs(PAXyf&k9nOXWd zWJW)yazH;9^YcpQ$r$Em9?OE{MP)+YPhLvqrEwW)%h6w+jA;esf@vjXzp@Oglf!ru z*)Xif{QA;uB=g4P@5s1`G%TAchs~tloVz80k(TpCZ$#c&4=K-;~|iGN7HS z9M6*p%Z1EeChdyx`m2~ne+~23Ggs)}I^J@-a=1g5d!)Zt=IVXQ?m^}r!^dTMmW=Tw z<^Sc&^wGbn9A0M*3+6Ydk4pcpOz7XE{~`5{nEOnI&vD8($Nu~frow5Yn@;+#$oy5D zL0TXES;^Hol;fN@7jquxlUDtPa#~R4g>ez-7Lx_t;`G0TOX5;8EG_-=WX!86m(^un zL&o)G+K@Vijpz@^f^l=@u!VG6lDCp^YiZl#4l;e8ybJk9GFN}B?0zcC-qPPXc-CO$W@EN3GoQXPynU&)l%*`d;+%nB03%YrgJtnUl z7nbguGN64+Irq!7l#I(r|841(qrSYfmC0+$ybgIoSvJCrW!gmAfOMPEN57eJwQNq_ zN(S`XD!c7u++NxaxU0XfOW$9lfzb51B_?|ROA29bJ{i#OB{&*SrvNZHlD+i2N zrd2&oFAI)f`iknlPY%w)+}Gvsta4~}**}*I=;u*(^U7f_<9u>p0Xc%hi_u@4`Vw+< zY3Y}hLpZR!vY}r=IbvE#Ijur{H94||ENjWZ^<>yo#?9!X-(1IsK&XfM?b`}bBJ zz=#D0_ffy#D30u_x|<;TZ~zB!1g+M8$$dDA?pNH05eL!lul@**HiloTU-iFH9>j#h zjfFgdh6BIlJof!gxgT9puJ)4?`rp$(Kn`KS5p)MKhXDr~2g&{*&G+K~4q?WEqm2W9 z(D~{h*&WO~8vDo|W9=WpJvfB^P}L(2VnS;iCA-6Pt~B-?&Udiy2<3i^IEaSnNX=C< zc^LbT;vRHI^Bs&hgar-z{;2suOgMxY3yxy{kj@1hz>K5lj?tXQ0gRZj;0T6ec^3{f z7V;>%Kk*%O$0-kB#6e70u>a4RPiQ!beaEZc>Z$i*!i*z0iv9%Ng@ZVZhVDeoc?^w1 zdkPY2gd>=#O06F3i7VQ6<&JW@c8v3)jA9Ld< zIh>=pj0H!|Ro!s3hv7WURTFs#-TAx&2QlFg4x_(-`5q2Y&sflK6#FmKJqbr@zDV^U zEI5jGvHI>3=`r8{MjXTuw8k*3dj_!oQssoB*ms%g{TR_*&UqY0!@euj_dN{MbK@wv z|4N-7Y#b&RG#o{DmCp5HKMrEXf+Ogz(Ybz1IDD<@BN(q!9zsKZz3SCO9>H=0{TpRK zca!n}MjXZdo7Eq}QFMj-a1aegap)G!Wi*Vps-Dnr6y0s=_u~*2Ot*6$N6_7&`Y8H4 znZqG8?7NG8<1l#yt|#8s;aJ zhjFy}lSwf1l-;K?qW?^J5aZ{{eP2k!e)sm+-^WQff z#J*{i`*8%zw5ktJCkLmOBVUoD=)bBwguYLCct+`Fk_G!`RUSdZGP~+=4$jTV9G1D1 zhv%U_uN?9+&Np7apmJGQj^fZamBZq407sTq_RGkGep%&VG#vQ0>IL0$%wbwyIpfd@ z%5g>6x3U~qMH=?6syu>aH8PH3-|DJIH0%qi58}WY%KdA~yq^!@fb~jQu~Sk6|z6!M$-m`V(aNm2~^dK1_}7 z*YweTr#y7996Usp!(=#I4xr)C5vr#nnLk>(AvuKMIP!@yH;$a4x?wz1Ih-X&{w9ac zmhnPauz#3*8D1{a71FMh`6_ae{kO_+yG(abzngrY9Qudsdq~Pu2zPWG)R`vZ9mIkG18jbz?frcLGO z7IJ7SIk=6?II^vB+DZC7>HkCy?JaFzIg(^NOqRpt;1Mz%Dg9B>jv=2){;M21NA{gd z#*xdF!c(J$o4bnoh)bId4*=aqf) zQ(si}FClG7>6enDE09-{!yAw{k%QlrVKeGG%DyP$&eC?5gL})gk1W5CqrZ~<2g%`M z$j8ZeJoPhWI#2dpAp0+r<#L&?li?m|57K`MpONw3vj2TKGUIz=`*L5G!?Q}ifb3gJ z4y}X(GW|qmj7KPso+!hqav)28j!ftZ`8HV|#pmSkN78>L)6DOWy)(=&M{!^w<^DzF zz=|@jD$5$=4anQbksW2+T@Is5%7ZwJ{y^3J5wh=3Uqx*}ts}yUOxQX}`lH;~{eNZ0XOJ;UZaXknv90e~%o#m;U|I zxev$QY4b?eFNZgf{lAy~Pcr;j4xLW_Z*urjIrtBpD8oD-X%CA^znJV_R)$sOz#s7> z8PAaAJUMim937GVZ8`Ggk9Gfa(tTaplDLcP|A~xe;8QY8@yXcvJ}<-KauA2sR36l}XW&gp{-T`THc0R5s z%j$AuSh`E)5cXX||1SK83{T7cf0AFN|F!>&-CyRB{u?r_B1iX-BL~R-KTtnQx(j4M zf1UCG+TF_eQ#m@<=VRaNUsd`wW#12^ogl+m)Q9EJZF1y8X&Zhqc2D2o(w`{vS#pu7 z-5T!y{m%;X%O{qXC+;gJ>?arbtsMQG-2Fhg^${}tNlv(0hU=ui6CaW871_VQWMki* zZEcx0mLKgZ54}aYd*sw3a_dLs`j5%GpOgd7$&cQVBfEZS?Ebq?lDpq3{T=cn9KJ_+ z;_LL^#3?5qJMU(YqjQrNlv6J*7uirw_@0c{$=z?4Y4Rz??y)Up*-rYi zMLuznT;*yx%XRXgo8_r@%H!^q8{aEuxlax}AQyj7+LJOpCs%l0j+#96CHV+W{fhG9 zugW|T-;?frIoC(h|3?mdAwQaI-m$&hH>JG$>vEL^q+e3rfFG@>oY#`uZ^YbpLFu37CJaeD2XQ{#mYnvOM*#a>5mIpR44{*UNz-r@mF5 zi`U$#JaVsm0w>(BJn+0c$jC3q{U@%A+sHeMaSmGkjz0eG|SWPhLoVvY7l}DY^9ua?KUx zV>rcH$}i%&Yb)Qmq1=5Vc_E&^vGO~c%Qv@_VF$V9Zt~bYfd7%g7Q#kdT z%DdnJ?A0MM0xth^2Nm#9DB#1OUkoXk+19{AJ|tObf}#FNIBc(a;it=x&M*JelCw+ zbRpe)13oQ>W?6WwKD3M6?dNj)UoA3LA31WVF%P>;F1py#V|k|sxbj_}%6Dd8(SXPl$_iwl)6xkUNxYm{HT zUY>TR>c9WHT;yKWXT4APp8J(&A5nhjVQJ6F`(BmXOdKx|9);zs%IYx0BD*B$HMe6E~#;+W}@E!H2a z|Ic5Gb^qx$&~2dGK(~Qz1KkF?4RjmmHqdRL+d#L0ZUfy0x(#$2=r+)8pxZ#Vfo=oc z2D%M&8|XIBZJ^sgw}Ea0-3Gc1bQ|b4&~2dGK(~Qz1KkF?4RjmmHqdRL+d#L0ZUfy0 zx(#$2=r+)8pxZ#Vfo=oc2D%M&8|XIBZJ^sgw}Ea0-3Gc1bQ|b4&~2dGK(~Qz1KkF? z4RjmmHqdRL+d#L0ZUfy0x(#$2=r+)8pxZ#Vfo=oc2D%M&8|XIBZJ^sgw}Ea0-3Gc1 zbQ|b4&~2dGK(~Qz1KkF?4RjmmHqdRL+d#L0ZUfy0x(#$2=r+)8pxZ#Vfo=oc2D%M& z8|XIBZJ^sgw}Ea0-3Gc1bQ|b4&~2dGK(~Qz1KkF?4RjmmHqdRL+d#L0ZUfy0x(#$2 z=r+)8pxZ#Vfo=oc2D%M&8|XIBZJ^sgw}Ea0-3Gc1bQ|b4&~2dGK(~Qz1KkF?4Rjmm zHqdRL+d#L0ZUfy0x(#$2=r+)8pxZ#Vfo=oc2D%M&8|XIBZJ^sgw}Ea0-3Gc1bQ|b4 z&~2dGK(~Qz1KkF?4RjmmHqdRL+d#L0ZUfy0x(#$2=r+)8pxZ#Vfo=oc2D%M&8|XIB zZJ^sgw}Ea0-3Gc1bQ|b4&~2dGK(~Qz1KkF?4RjmmHqdRL+d#L0ZUfy0x(#$2=r+)8 zpxZ#Vfo=oc2D%M&8|XIBZJ^sgw}Ea0-3Gc1bQ|b4&~2dGK(~Qz1KkF?4RjmmHqdRL z+d#L0ZUfy0x(#$2=r+)8pxZ#Vfo=oc2D%M&8|XIBZJ^sgw}Ea0-3Gc1bQ|b4&~2dG zK(~Qz1KkF?4RjmmHqdRL+d#L0ZUfy0x(#$2=r-{G*9|&ej&u=^U5_!WG{bPBN z&*U*5E;W|_)xY$ZJHMfP;aST4dn_|nzx*Wm;%&>0<&jI}#s__SEN}m=T;Ngp!s^S7 z)eqWP-m|Uz&yDi_>*d^U$lV5)ADdrnDf#88R~XB)^{+VQG<(Xc*OO0Pu<}^_+?C|y zMQ;4Q+;+!R$NF1NkRvCrHkLnJ|Nm*HY+ss+;yC`ho7OOcp=R2fS@f{kvU{(a4?WO) zC`*Zq)(fp*({9dOn=Oa45cX2YKqO5Gg0dGC1QAFi6w)inf?kOjJ-o0egbE5871X)s zyX&sI{sTe>c0T)lf4_6?J->6$0c{t-KRw{Y6gct`^zGiq^AkQWFvJ<3&hV&ye2tBh z$OS#@0W+%LxA~!mTA*9Ip-=Td508Ok*T8$Dh>u@|&W?ieWB9X=z>cTzZO_1yFW?uy zf!+kh`{$tBJ|I4kjE}yBZ+#8k{fqd(JoJ$SnEwO*_lzpO-k_C}nl^*IKauya2=SR3 z__vOMLOnR^MtnNy_cg$e`r%I<1#?626a8TKAQ-xjxZ?(N)=OyT4D{`-)qJ1wPH@Ht zW}u~U(009szyCscEziq+#~FW;fOh-__g>!5<0Z|Q$6H@lCD1Ao*k;?+jx~WSNpg^x ztn5c*GQBT^gFchxsY6Tx2UB()Fw;i*cIa2m>G?{c)y#_BH1R+t>l-&@@&ByL(LHCV zA<|5AM+>b83T$yp>sG+uTM$r4b-8NOXinB^J@eEGr|QQl$URX7tF@gbTQSGr7q#+4 zuXbRT2kTJ2*p+PVt6h2>vnkGFW?IXh!}jQGG}!JhQi5GQoeJeziJDj(^AmZx14=}( zA6DA@A*HLNS_!oV&iNx9m%P1B@19-9L+#;^ebM!X!@&zB-blMY=u9&gWnCm3?Q}TU zGuWe+X$KD0R+AV=z5x3)%I11=1gl^Xd6Omm;+7fz-&)xfSi1jgEya%Wv#I?TO9!@K zfjZA+-)8XK2_L>8U7mcf#o$TWVbbMQR49hL%E>e7o+Ktd;{fW$mRlAKUYT}KQ+ZP9 z^RMZ7k}Fn>gRIMQ_iZPvhKduzS@5^hy@)0 literal 0 HcmV?d00001 diff --git a/src/integrators/connector.jl b/src/integrators/connector.jl index b18414b..596828f 100644 --- a/src/integrators/connector.jl +++ b/src/integrators/connector.jl @@ -1,10 +1,10 @@ export Connector,getConnector mutable struct Connector{T} <: AbstractMeganetElement{T} - K - b - outTimes - Q + K::Array{T,2} + b::T + outTimes::Bool + Q # ??? end nTheta(this::Connector) = 0 @@ -13,7 +13,7 @@ nFeatOut(this::Connector) = size(this.K,1) nDataOut(this::Connector) = ((this.Q==I) ? nFeatOut(this) : size(this.Q,1)) initTheta(this::Connector{T}) where {T <: Number} = zeros(T,0) -function getConnector(TYPE::Type, K; b = zero(TYPE),outTimes=0,Q=I) +function getConnector(TYPE::Type, K; b = zero(TYPE),outTimes=false,Q=I) return Connector{TYPE}(K,b,outTimes,Q); end @@ -22,10 +22,9 @@ function apply(this::Connector{T},theta::Array{T},Y0::Array{T},doDerivative=true nex = div(length(Y0),nFeatIn(this)) Y0 = reshape(Y0,:,nex) Y = this.K*Y0 .+ this.b - if this.outTimes==1 + Ydata::Array{T,2} = Array{T, 2}(0, 0) # Temporary fix until we know what type Q is + if this.outTimes==true Ydata = this.Q*Y - else - Ydata = Array{T, 2}(0, 0) end tmp = Y0; return Ydata, Y, tmp @@ -36,7 +35,7 @@ function Jmv(this::Connector{T},dtheta::Array{T},dY::Array{T},theta::Array{T},Y: nex = div(length(dY),nFeatIn(this)) dY = reshape(dY,:,nex) dY = this.K*dY - if this.outTimes==1 + if this.outTimes==true dYdata = this.Q*dY else dYdata = [] From 0e086ba3d0528341e2ba45751b8aac4a83474f85 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Wed, 31 Jan 2018 16:53:52 -0800 Subject: [PATCH 05/35] added tensorflow benchmark --- .gitignore | 2 + benchmarks/CIFAR10/tf_benchmarks/Resnet.py | 284 ++++++++++++++++++ .../CIFAR10/tf_benchmarks/cifar10_512_64.py | 30 ++ .../CIFAR10/tf_benchmarks/utils_input.py | 145 +++++++++ 4 files changed, 461 insertions(+) create mode 100644 benchmarks/CIFAR10/tf_benchmarks/Resnet.py create mode 100644 benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py create mode 100644 benchmarks/CIFAR10/tf_benchmarks/utils_input.py diff --git a/.gitignore b/.gitignore index 41b74da..3a84b55 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ *.jl.*.cov *.jl.mem data/ +*.pyc +.vscode/ \ No newline at end of file diff --git a/benchmarks/CIFAR10/tf_benchmarks/Resnet.py b/benchmarks/CIFAR10/tf_benchmarks/Resnet.py new file mode 100644 index 0000000..17b8dac --- /dev/null +++ b/benchmarks/CIFAR10/tf_benchmarks/Resnet.py @@ -0,0 +1,284 @@ +"""Contains resnet model +""" + +import tensorflow as tf +import numpy as np +import time +import math +from utils_input import * +import os + + +class ResnetModel(object): + + def __init__(self, config): + self.channels = config['channels'] + self.k_size = config['kSize'] + self.batchsize = config['batchSize'] + self.epochs = config['maxEpochs'] + self.h = config['h'] + self.num_units = config['numUnits'] + self.num_blocks = config['numBlocksPerUnit'] + self.xtrain = config['xTrain'] + self.xval = config['xValid'] + self.ytrain = config['yTrain'] + self.yval = config['yValid'] + + self.initial_channels = 3 + self.num_train = self.ytrain.shape[0] + self.num_val = self.yval.shape[0] + self.n_class = 10 + + self.strides = [1,1,1,1] + self.activation_func = tf.tanh + self.epochs_completed = 0 + self.weight_decay_rate = .0002 + self.lr = .01 + self.momentum = .9 + self.shuffle = True + + def train(self): + with tf.device("/cpu:0"): + self.X = tf.placeholder(tf.float32, [None, self.xtrain.shape[1]]) + self.y = tf.placeholder(tf.float32, [None, self.n_class]) + + self.logits = self.build_model() + self.loss = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=self.y, logits=self.logits)) + self.loss += weight_decay(self.weight_decay_rate, keyword=r'K_') + + self.train_op = tf.train.MomentumOptimizer(self.lr, self.momentum).minimize(self.loss) + + # Create init op + init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) + + # Only run on single thread + session_conf = tf.ConfigProto( + intra_op_parallelism_threads=1, + inter_op_parallelism_threads=1) + + with tf.Session(config=session_conf) as self.sess: + + # Initialize variables + self.sess.run(init_op) + + self.train_steps_per_epoch = int(math.ceil(self.num_train / float(self.batchsize))) + self.test_steps_per_epoch = self.num_val // self.batchsize + + # Time it! + start_time = time.time() + + train_index = 0 + for ep in range(self.epochs): + + for s in range(self.train_steps_per_epoch): + train_batch, train_index = self.next_train_batch(train_index) + train_data = train_batch[0] + train_labels = train_batch[1] + _, loss = self.sess.run([self.train_op, self.loss], feed_dict={self.X: train_data, self.y: train_labels}) + print(s) + print(loss) + + self.validate_model() + self.epochs_completed += 1 + + end_time = time.time() + print("Total time: %f" % (end_time - start_time)) + + def validate_model(self): + # Run training + avg_train = 0 + n_ex = min(2**12, self.num_train) + n_steps = int(math.ceil(n_ex / float(self.batchsize))) + train_index = 0 + for s in range(n_steps): + train_batch, train_index = self.next_train_batch(train_index) + train_data = train_batch[0] + train_labels = train_batch[1] + _, loss = self.sess.run([self.train_op, self.loss], feed_dict={self.X: train_data, self.y: train_labels}) + avg_train += loss + avg_train /= n_steps + print("overall train:") + print(avg_train) + + # Run Validation + val_index = 0 + avg_loss = 0 + for i in range(self.test_steps_per_epoch): + val_batch, val_index = self.next_val_batch(val_index) + val_data = val_batch[0] + val_labels = val_batch[1] + + preds, loss = self.sess.run([self.logits, self.loss], feed_dict={self.X: val_data, + self.y: val_labels}) + + avg_loss += loss + + avg_loss /= self.test_steps_per_epoch + print("val: %f" % avg_loss) + + def build_model(self): + with tf.device("/cpu:0"): + # Reshape + X = tf.reshape(self.X, [-1, 32, 32, self.initial_channels]) + # Opening Convolution + K_opening = weight_variable([self.k_size, self.k_size, self.initial_channels, self.channels[0]], 'K_Opening') + X = tf.nn.conv2d(X, K_opening, strides=self.strides, padding='SAME') + + # Opening Batch Norm + X = tf.contrib.layers.batch_norm(X, fused=True, is_training=True, scope='Opening') + + #Opening Activation + X = self.activation_func(X) + + # Build Units + for unit_num in range(self.num_units): + X = self.unit(X, unit_num) + + # Average Pooling Layer + X = tf.reduce_mean(X, [1,2]) + X = tf.reshape(X, [self.batchsize, -1]) + + # Fully Connected Layer + K_FC = weight_variable([self.channels[-1], self.n_class], 'K_FC') + B_FC = bias_variable([self.n_class], 'B_FC') + + # Return Logits + logits = tf.nn.xw_plus_b(X, K_FC, B_FC) + return logits + + def unit(self, X, unit_num): + with tf.variable_scope("unit_%d" % unit_num): + for block_num in range(self.num_blocks[unit_num] - 1): + X = self.block(X, unit_num, block_num) + + # Create Connector Block + X = self.connector_block(X, unit_num) + + return X + + def block(self, X0, unit_num, block_num): + with tf.variable_scope("block_%d" % block_num): + # First Convolution + K1 = weight_variable([self.k_size, self.k_size, self.channels[unit_num], self.channels[unit_num]], 'K_1') + X1 = tf.nn.conv2d(X0, K1, strides=self.strides, padding='SAME') + + # Batch Norm + X1 = tf.contrib.layers.batch_norm(X1, fused=True, is_training=True, scope='bn_1') + + # Activation + X1 = self.activation_func(X1) + + # Second Convolution + X1 = -tf.nn.conv2d_transpose(X1, K1, + output_shape=tf.concat([tf.shape(X1)[0:3], [tf.shape(K1)[2]]], axis=0), + strides=self.strides, + padding='SAME') + X1 = tf.reshape(X1, tf.shape(X0)) + + # Second Batch Norm + X1 = tf.contrib.layers.batch_norm(X1, fused=True, is_training=True, scope='bn_2') + + # Compute return value of resnet block + X = X0 + self.h[unit_num] * X1 + + return X + + def connector_block(self, X, unit_num): + with tf.variable_scope("connector_block"): + # Convolution increasing channels + K_Conn = weight_variable([1, 1, self.channels[unit_num], self.channels[unit_num + 1]], 'K_Conn') + X = tf.nn.conv2d(X, K_Conn, strides=self.strides, padding='SAME') + + # Batch Norm + X = tf.contrib.layers.batch_norm(X, fused=True, is_training=True, scope='connector') + + # Activation + X = self.activation_func(X) + + X = tf.nn.avg_pool(X, ksize=[1, 2, 2, 1], + strides=[1, 2, 2, 1], padding='VALID') + + return X + + def next_train_batch(self, index): + """Gets the next batch of data for training + + Arguments: + index {int} -- Current index in the training data + + Returns: + batch {tuple of numpy arrays} -- First np array is the batch of data and second is batch of labels + new_index {int} -- The new index in the training data after creating batch + """ + + start_index = index + end_index = index + self.batchsize + final_index = self.num_train - 1 + + # First batch + if self.shuffle and self.epochs_completed == 0 and start_index == 0: + self.shuffle_data() + + # If we are reaching the end of an epoch + if final_index < end_index: + + # Get remaining data in epoch + remainder = end_index - final_index + x_remaining = self.xtrain[start_index:final_index] + y_remaining = self.ytrain[start_index:final_index] + + if self.shuffle: + self.shuffle_data() + + # Get new data in next epoch + x_extra = self.xtrain[0:remainder] + y_extra = self.ytrain[0:remainder] + + # Combine data into single batch + x_batch = np.concatenate((x_remaining, x_extra), axis=0) + y_batch = np.concatenate((y_remaining, y_extra), axis=0) + batch = (x_batch, y_batch) + + # # Increment epochs_completed + # self.epochs_completed += 1 + + new_index = remainder + else: + batch = (self.xtrain[start_index:end_index], self.ytrain[start_index:end_index]) + new_index = end_index + + return batch, new_index + + def shuffle_data(self): + """Shuffles the training data + """ + perm = np.arange(self.num_train) + np.random.shuffle(perm) + self.xtrain = self.xtrain[perm] + self.ytrain = self.ytrain[perm] + + def next_val_batch(self, index): + """Gets the next batch of data for validation + + Arguments: + index {int} -- Current index in the validation data + + Returns: + batch {tuple of numpy arrays} -- First np array is the batch of data and second is batch of labels + new_index {int} -- The new index in the validation data after creating batch + """ + if self.epochs_completed == 0 and index == 0: + # Shuffle validation data at start + perm = np.arange(self.num_val) + np.random.shuffle(perm) + self.xval = self.xval[perm] + self.yval = self.yval[perm] + #TODO: Ensure we are validating on all of the validation data + start_index = index + end_index = index + self.batchsize + + batch = (self.xval[start_index:end_index], self.yval[start_index:end_index]) + new_index = end_index + + return batch, new_index \ No newline at end of file diff --git a/benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py b/benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py new file mode 100644 index 0000000..949ffaa --- /dev/null +++ b/benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py @@ -0,0 +1,30 @@ +"""Runs benchmark using tensorflow and 1 cpu thread + Runs 1 epoch with 512/103 Train/Val split in 8.56s +""" +from utils_input import * +from Resnet import ResnetModel + +if __name__ == '__main__': + # Load Data + ntrain = 512 + nval = 103 + train_data, valid_data = load_cifar10("../", ntrain, nval) # Note this does not evenly distribute classes + + # Set config + modelConfig = { + "channels": [16,32,64,64], + "batchSize": 64, + "numUnits": 3, + "numBlocksPerUnit": [3,3,3], + "h": [1.,1.,1.], + "maxEpochs": 1, + "kSize": 3, + "xTrain": train_data[0], + "xValid": valid_data[0], + "yTrain": train_data[1], + "yValid": valid_data[1], + } + + # Train model + model = ResnetModel(modelConfig) + model.train() diff --git a/benchmarks/CIFAR10/tf_benchmarks/utils_input.py b/benchmarks/CIFAR10/tf_benchmarks/utils_input.py new file mode 100644 index 0000000..6bde335 --- /dev/null +++ b/benchmarks/CIFAR10/tf_benchmarks/utils_input.py @@ -0,0 +1,145 @@ +"""Contains helper functions for running tensorflow benchmark +""" + +import numpy as np +import os +import sys +from six.moves import urllib +from six.moves import cPickle +import tarfile +import tensorflow as tf + +def one_hot(labels, n_class): + """ Return one hot encoding of labels + + Args: + labels: a vector of length n, with values in [0, n_class-1] + n_class: number of classes, typically 10 + Returns: + One hot encoding, a (n, n_class) np array. + """ + + return np.eye(n_class)[labels].reshape((labels.shape[0], n_class)) + +def maybe_download_and_extract(DATA_URL, dest_directory, extracted_filepath=None): + """Download and extract the tarball from Alex's website.""" + # https://github.com/tensorflow/models/blob/dac6755b121f1446ec857cd05c2ff53b2fd26b90/tutorials/image/cifar10/cifar10.py + + if not os.path.exists(dest_directory): + os.makedirs(dest_directory) + filename = DATA_URL.split('/')[-1] + filepath = os.path.join(dest_directory, filename) + if not os.path.exists(filepath): + def _progress(count, block_size, total_size): + sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename, + float(count * block_size) / float(total_size) * 100.0)) + sys.stdout.flush() + filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress) + print() + statinfo = os.stat(filepath) + print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') + if extracted_filepath: + extracted_dir_path = os.path.join(dest_directory, extracted_filepath) + if not os.path.exists(extracted_dir_path): + tarfile.open(filepath, 'r:gz').extractall(dest_directory) + +def load_batch(fpath, label_key='labels'): + """Internal utility for parsing CIFAR data. + # Arguments + fpath: path the file to parse. + label_key: key for label data in the retrieve + dictionary. + # Returns + A tuple `(data, labels)`. + """ + # https://github.com/fchollet/keras/blob/3e933ca0ed1c526c0a9b8643ca84129db96ecc17/keras/datasets/cifar.py + + f = open(fpath, 'rb') + if sys.version_info < (3,): + d = cPickle.load(f) + else: + d = cPickle.load(f, encoding='bytes') + # decode utf8 + d_decoded = {} + for k, v in d.items(): + d_decoded[k.decode('utf8')] = v + d = d_decoded + f.close() + data = d['data'] + labels = d[label_key] + + data = data.reshape(data.shape[0], 3, 32, 32).transpose([0, 2, 3, 1]) + return data, labels + +def load_cifar10(dirname, ntrain, nval): + """Loads CIFAR10 dataset. + # Returns + Tuple of np arrays: `(x_train, y_train), (x_test, y_test)`. + """ + CIFAR10_DATA_URL = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" + + maybe_download_and_extract( + CIFAR10_DATA_URL, dirname, extracted_filepath='cifar-10-batches-py') + path = os.path.join(dirname, 'cifar-10-batches-py') + + nb_train_samples = 50000 + + x_train = np.zeros((nb_train_samples, 32, 32, 3), dtype='uint8') + y_train = np.zeros((nb_train_samples,), dtype='uint8') + + for i in range(1, 6): + fpath = os.path.join(path, 'data_batch_' + str(i)) + data, labels = load_batch(fpath) + x_train[(i - 1) * 10000: i * 10000, :, :, :] = data + y_train[(i - 1) * 10000: i * 10000] = labels + + fpath = os.path.join(path, 'test_batch') + x_test, y_test = load_batch(fpath) + + y_train = np.reshape(y_train, (len(y_train), 1)) + y_test = np.reshape(y_test, (len(y_test), 1)) + + x_train = x_train.reshape((x_train.shape[0], 32 * 32 * 3)) + x_test = x_test.reshape((x_test.shape[0], 32 * 32 * 3)) + + y_train_one_hot = one_hot(y_train, 10) + y_test_one_hot = one_hot(y_test, 10) + + return (x_train[:ntrain], y_train_one_hot[:ntrain]), (x_test[:nval], y_test_one_hot[:nval]) + +def weight_decay(weight_decay_rate, keyword): + """Adds weight decay using l2 loss to all variables with names matching keyword + + Arguments: + weight_decay_rate {float} -- The decay rate factor + keyword {string or regex} -- The keyword to match variable names to + + Returns: + [Tensor] -- A Tensor containing the total loss from weight decay + """ + costs = [] + for var in tf.trainable_variables(): + if var.op.name.find(keyword) > 0: + costs.append(tf.nn.l2_loss(var) / 2) + + if len(costs) == 0: + # No variables match the keyword + return 0 + else: + return tf.multiply(weight_decay_rate, tf.add_n(costs)) + +def weight_variable(shape, name): + """Creates a tensorflow variable with given shape and name initialized using Xavier Initialization + Arguments: + shape {array} -- Array with the dimensions needed for the variable + name {string} -- Name for the variable + + Returns: + [tf.Variable] -- A trainable tensorflow variable of type float32 + """ + return tf.get_variable(name, shape=shape, dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer()) + +def bias_variable(shape, name): + """bias_variable generates a bias variable of a given shape.""" + initial = tf.constant(0.1, shape=shape) + return tf.get_variable(name, dtype=tf.float32, initializer=initial) \ No newline at end of file From ea193840bd94cebc333a04c9b6599be4c596b235 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Wed, 31 Jan 2018 17:09:26 -0800 Subject: [PATCH 06/35] stabilize JTmv --- examples/EResNN_CIFAR10.jl | 2 +- src/integrators/ResNN.jl | 4 +++- src/kernelTypes/convGEMMKernel.jl | 37 ++++++++++++++++--------------- src/layers/doubleSymLayer.jl | 16 ++++++------- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/examples/EResNN_CIFAR10.jl b/examples/EResNN_CIFAR10.jl index d494eab..686c9e5 100644 --- a/examples/EResNN_CIFAR10.jl +++ b/examples/EResNN_CIFAR10.jl @@ -1,4 +1,4 @@ -using MAT, Meganet +using MAT, Meganet, Juno n = 32; Y_train,C_train,Y_test,C_test = getCIFAR10(n,Pkg.dir("Meganet")*"/data/CIFAR10/"); diff --git a/src/integrators/ResNN.jl b/src/integrators/ResNN.jl index 86d26bd..1c6ddbb 100644 --- a/src/integrators/ResNN.jl +++ b/src/integrators/ResNN.jl @@ -1,4 +1,5 @@ export ResNN,getResNN +import Juno """ Residual Neural Network block @@ -166,8 +167,9 @@ function JTmv(this::ResNN{T},Wdata::Array{T},W::Array{T},theta::Array{T},Y::Arra W += this.Q'* Wdata[:,cnt,:] cnt = cnt-1 end + dmbi,dW = JTmv(this.layer,W,zeros(T,0),theta[:,i],tmp[i,1],tmp[i,2]) - @code_warntype JTmv(this.layer,W,zeros(T,0),theta[:,i],tmp[i,1],tmp[i,2]) + dtheta[:,i] = this.h*dmbi W += this.h*dW end diff --git a/src/kernelTypes/convGEMMKernel.jl b/src/kernelTypes/convGEMMKernel.jl index c9216bb..41f6c12 100644 --- a/src/kernelTypes/convGEMMKernel.jl +++ b/src/kernelTypes/convGEMMKernel.jl @@ -74,38 +74,39 @@ function Jthetamv(this::convGEMMKernel{T},dtheta::Array{T},dummy::Array{T},Y::Ar return Z end -function JthetaTmv(this::convGEMMKernel{T},Z::Array{T},dummy::Array{T},Y::Array{T}) where {T<:Number} +function JthetaTmv(this::convGEMMKernel{T}, Zin::Array{T}, dummy::Array{T}, Yin::Array{T}) where {T<:Number} # derivative of Z*(A(theta)*Y) w.r.t. theta - sK = this.sK; - nImg = this.nImg; - nex = div(numel(Y),prod(nImgIn(this))) + sK = this.sK + nImg = this.nImg + nex = div(numel(Yin),prod(nImgIn(this))) # compute convolution - Y = reshape(Y,nImg[1],nImg[2],this.sK[3],nex); - Z = reshape(Z,nImg[1]*nImg[2],this.sK[4],nex); - Zk = zeros(T,nImg[1]*nImg[2],this.sK[4]); - aux = zeros(T,nImg[1],nImg[2],this.sK[3]); + Y = reshape(Yin, nImg[1], nImg[2], this.sK[3], nex) + Z = reshape(Zin, nImg[1]*nImg[2], this.sK[4], nex) + Zk = zeros(T, nImg[1]*nImg[2], this.sK[4]) + aux = zeros(T, nImg[1], nImg[2], this.sK[3]) + ### reshape the kernels for gemm!: - dtheta = zeros(T,tuple(sK...)); - KK = Array{Array{T,2}}(sK[1],sK[2]); + dtheta = zeros(T, sK[1], sK[2], sK[3], sK[4]) + KK = Array{Array{T, 2}}(sK[1], sK[2]) for k1 = 1:sK[1] for k2 = 1:sK[2] - @inbounds KK[k1,k2] = zeros(T,sK[3],sK[4]); + @inbounds KK[k1, k2] = zeros(T, sK[3], sK[4]) end end - shiftX = [0;-1;0;0;1;0]; - shiftT = [1;0;0;0;0;-1]; + shiftX = [0;-1;0;0;1;0] + shiftT = [1;0;0;0;0;-1] for k = 1:nex - getColumn!(Z,Zk,k); - multConv2Dblock(Y,KK, Zk,aux,shiftX,shiftT,k,doDerivative = 1); + getColumn!(Z, Zk, k) + multConv2Dblock(Y, KK, Zk, aux, shiftX, shiftT, k, doDerivative = 1) end ### Assemble the kernels from gemm!: for k1 = 1:sK[1] for k2 = 1:sK[2] - @inbounds dtheta[k1,k2,:,:] = KK[k1,k2]; + @inbounds dtheta[k1, k2, :, :] = KK[k1, k2] end end - dtheta = reshape(dtheta,tuple(this.sK...)); - return dtheta + dtheta_out = reshape(dtheta, sK[1], sK[2], sK[3], sK[4]) + return dtheta_out end diff --git a/src/layers/doubleSymLayer.jl b/src/layers/doubleSymLayer.jl index 517307a..c17fc0a 100644 --- a/src/layers/doubleSymLayer.jl +++ b/src/layers/doubleSymLayer.jl @@ -194,19 +194,17 @@ function JTmv(this::DoubleSymLayer{T}, Zin::Array{T}, dummy::Array{T}, Y = reshape(Yin,:,nex) th1, th2, th3, th4 = splitWeights(this,theta) Kop = getOp(this.K,th1) - A,dA = this.activation(Yt,true) + A::Array{T,2}, dA::Array{T,2} = this.activation(Yt,true) dth3 = vec(sum(this.Bout'*Z,2)) - dAZ = dA.*(Kop*Z) - dth2 = vec(sum(this.Bin'*dAZ,2)) - dth4,dAZ = JTmv(this.nLayer,dAZ,zeros(T,0),th4,Kop*Y,tmp[1]) - - dth1 = JthetaTmv(this.K,dAZ,zeros(T,0),Y) - + dAZ1::Array{T,2} = dA.*(Kop*Z) + dth2 = vec(sum(this.Bin'*dAZ1,2)) + dth4, dAZ2::Vector{T} = JTmv(this.nLayer,dAZ1,zeros(T,0),th4,Kop*Y,tmp[1]) + dth1 = JthetaTmv(this.K,dAZ2,zeros(T,0),Y) dth1 = dth1 + JthetaTmv(this.K,A,(T)[],Z) dtheta = [-vec(dth1); -vec(dth2); vec(dth3);-vec(dth4)] - dAZ_out = reshape(dAZ,:,nex) - dY = -(Kop'*dAZ_out) + dAZ_out = reshape(dAZ2,:,nex) + dY::Array{T, 2} = -(Kop'*dAZ_out) return dtheta,dY end From 72410c5a035ebfb765e60fb96b9d44f8b81e9c82 Mon Sep 17 00:00:00 2001 From: moumitaTora Date: Thu, 1 Feb 2018 10:30:56 -0800 Subject: [PATCH 07/35] removed a line. first commit woo! :) --- src/activations/tanhActivation.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/activations/tanhActivation.jl b/src/activations/tanhActivation.jl index 1c7ab8a..62ea44e 100644 --- a/src/activations/tanhActivation.jl +++ b/src/activations/tanhActivation.jl @@ -18,7 +18,6 @@ export tanhActivation dA - derivatives """ function tanhActivation(Y::Array{T,2},doDerivative::Bool=false) where {T <: Number} - A = tanh.(Y) if doDerivative dA = 1-A.^2 From 4fdb0902b2a15bb981a5ca14bb759b079083da07 Mon Sep 17 00:00:00 2001 From: DavidBegert Date: Thu, 1 Feb 2018 11:15:57 -0800 Subject: [PATCH 08/35] change path to data --- benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py b/benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py index 949ffaa..9b73bc2 100644 --- a/benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py +++ b/benchmarks/CIFAR10/tf_benchmarks/cifar10_512_64.py @@ -4,11 +4,13 @@ from utils_input import * from Resnet import ResnetModel +DATA_DIR = "path/to/where/you/want/your/data/stored" + if __name__ == '__main__': # Load Data ntrain = 512 nval = 103 - train_data, valid_data = load_cifar10("../", ntrain, nval) # Note this does not evenly distribute classes + train_data, valid_data = load_cifar10(DATA_DIR, ntrain, nval) # Note this does not evenly distribute classes # Set config modelConfig = { From 5152ff3399add6cd71163488dca78f5635c19cb5 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Thu, 1 Feb 2018 14:48:32 -0800 Subject: [PATCH 09/35] push local to switch pcs --- src/activations/tanhActivation.jl | 2 +- src/kernelTypes/convGEMMKernel.jl | 20 ++++++++++---------- src/layers/doubleSymLayer.jl | 1 - 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/activations/tanhActivation.jl b/src/activations/tanhActivation.jl index f79d1c8..d68b351 100644 --- a/src/activations/tanhActivation.jl +++ b/src/activations/tanhActivation.jl @@ -22,7 +22,7 @@ function tanhActivation(Y::Array{T,2},doDerivative::Bool=false) where {T <: Numb A = tanh.(Y) dA = similar(Y) if doDerivative - dA .= 1-A.^2 + dA .= one(T) .- A.^2 else fill!(dA, zero(T)) end diff --git a/src/kernelTypes/convGEMMKernel.jl b/src/kernelTypes/convGEMMKernel.jl index 41f6c12..f24c56a 100644 --- a/src/kernelTypes/convGEMMKernel.jl +++ b/src/kernelTypes/convGEMMKernel.jl @@ -15,11 +15,11 @@ function Amv(this::convGEMMKernel{T},theta::Array{T},Y::Array{T}) where {T<:Numb nex = div(numel(Y),prod(nImgIn(this))) # compute convolution Y = reshape(Y,nImg[1],nImg[2],this.sK[3],nex); - AY = zeros(T,nImg[1]*nImg[2],this.sK[4],nex); + AY = Array{T, 3}(nImg[1]*nImg[2],this.sK[4],nex); aux = zeros(T,nImg[1],nImg[2],this.sK[3]); AYk = zeros(T,nImg[1]*nImg[2],this.sK[4]); ### reshape the kernels for gemm!: - K = reshape(theta,tuple(sK...)); + K = reshape(theta, sK[1], sK[2], sK[3], sK[4]) KK = Array{Array{T,2}}(sK[1],sK[2]); for k1 = 1:sK[1] for k2 = 1:sK[2] @@ -34,16 +34,16 @@ function Amv(this::convGEMMKernel{T},theta::Array{T},Y::Array{T}) where {T<:Numb @inbounds AY[:,:,k] = AYk; AYk[:] = zero(T) end - AY = reshape(AY,:,nex); - return AY + AY_out = reshape(AY,:,nex); + return AY_out end -function ATmv(this::convGEMMKernel{T},theta::Array{T},Z::Array{T}) where {T<:Number} +function ATmv(this::convGEMMKernel{T},theta::Array{T},Zin::Array{T}) where {T<:Number} nImg = this.nImg; sK = this.sK; - nex = div(numel(Z),prod(nImgOut(this))); - K = reshape(theta,tuple(sK...)); - Z = reshape(Z,nImg[1],nImg[2],sK[4],nex); + nex = div(numel(Zin),prod(nImgOut(this))); + K = reshape(theta, sK[1], sK[2], sK[3], sK[4]); + Z = reshape(Zin,nImg[1],nImg[2],sK[4],nex); aux = zeros(T,nImg[1],nImg[2],sK[4]); ATZ = zeros(T,nImg[1]*nImg[2],sK[3],nex); ATZk = zeros(T,nImg[1]*nImg[2],sK[3]); @@ -64,8 +64,8 @@ function ATmv(this::convGEMMKernel{T},theta::Array{T},Z::Array{T}) where {T<:Num @inbounds ATZ[:,:,k] = ATZk; ATZk[:] = zero(T) end - ATZ = reshape(ATZ,:,nex); - return ATZ + ATZ_out = reshape(ATZ,:,nex); + return ATZ_out end function Jthetamv(this::convGEMMKernel{T},dtheta::Array{T},dummy::Array{T},Y::Array{T},temp=nothing) where {T<:Number} diff --git a/src/layers/doubleSymLayer.jl b/src/layers/doubleSymLayer.jl index c17fc0a..f23e764 100644 --- a/src/layers/doubleSymLayer.jl +++ b/src/layers/doubleSymLayer.jl @@ -195,7 +195,6 @@ function JTmv(this::DoubleSymLayer{T}, Zin::Array{T}, dummy::Array{T}, th1, th2, th3, th4 = splitWeights(this,theta) Kop = getOp(this.K,th1) A::Array{T,2}, dA::Array{T,2} = this.activation(Yt,true) - dth3 = vec(sum(this.Bout'*Z,2)) dAZ1::Array{T,2} = dA.*(Kop*Z) dth2 = vec(sum(this.Bin'*dAZ1,2)) From 9de97af972dff42a272bf1859e36dc5af8c38eba Mon Sep 17 00:00:00 2001 From: Justin Granek Date: Thu, 1 Feb 2018 16:25:32 -0800 Subject: [PATCH 10/35] Type stabilized the connector.jl functions --- examples/EResNN_CIFAR10.jl | 4 ++-- src/integrators/connector.jl | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/EResNN_CIFAR10.jl b/examples/EResNN_CIFAR10.jl index efa3f01..9afcbbc 100644 --- a/examples/EResNN_CIFAR10.jl +++ b/examples/EResNN_CIFAR10.jl @@ -85,8 +85,8 @@ solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y_train,C_train,Y_test,C_test) # Profile.clear() # Profile.clear_malloc_data() # Profile.init(n = 10^7, delay = 0.01) -# @profile solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y,C,Y,C) +# @profile solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y_train,C_train,Y_test,C_test) # open("/tmp/EREsNN_CIFAR10.txt", "w") do s - # Profile.print(IOContext(s, :displaysize => (24, 500))) +# Profile.print(IOContext(s, :displaysize => (24, 500))) # end diff --git a/src/integrators/connector.jl b/src/integrators/connector.jl index 596828f..5824642 100644 --- a/src/integrators/connector.jl +++ b/src/integrators/connector.jl @@ -1,10 +1,10 @@ export Connector,getConnector -mutable struct Connector{T} <: AbstractMeganetElement{T} - K::Array{T,2} +mutable struct Connector{T,TQ <: Union{Array{T,2},UniformScaling{Int}}, TK <: Union{Array{T,2},SparseMatrixCSC{T,Int}}} <: AbstractMeganetElement{T} + K::TK b::T - outTimes::Bool - Q # ??? + outTimes::Int + Q::TQ # ??? end nTheta(this::Connector) = 0 @@ -13,8 +13,8 @@ nFeatOut(this::Connector) = size(this.K,1) nDataOut(this::Connector) = ((this.Q==I) ? nFeatOut(this) : size(this.Q,1)) initTheta(this::Connector{T}) where {T <: Number} = zeros(T,0) -function getConnector(TYPE::Type, K; b = zero(TYPE),outTimes=false,Q=I) - return Connector{TYPE}(K,b,outTimes,Q); +function getConnector(TYPE::Type, K; b = zero(TYPE),outTimes=0,Q=I) + return Connector(K,b,outTimes,Q); end @@ -23,7 +23,7 @@ function apply(this::Connector{T},theta::Array{T},Y0::Array{T},doDerivative=true Y0 = reshape(Y0,:,nex) Y = this.K*Y0 .+ this.b Ydata::Array{T,2} = Array{T, 2}(0, 0) # Temporary fix until we know what type Q is - if this.outTimes==true + if this.outTimes==1 Ydata = this.Q*Y end tmp = Y0; @@ -31,29 +31,29 @@ function apply(this::Connector{T},theta::Array{T},Y0::Array{T},doDerivative=true end function Jmv(this::Connector{T},dtheta::Array{T},dY::Array{T},theta::Array{T},Y::Array{T},tmp=nothing) where {T <: Number} - + # ??? This doesn't seem to get used? nex = div(length(dY),nFeatIn(this)) dY = reshape(dY,:,nex) dY = this.K*dY - if this.outTimes==true + dYdata::Array{T,2} = Array{T, 2}(0, 0) # Temporary fix until we know what type Q is + if this.outTimes==1 dYdata = this.Q*dY - else - dYdata = [] end + return dYdata,dY end -function JTmv(this::Connector{T},Wdata::Array{T},W::Array{T},theta::Array{T},Y::Array{T},tmp=nothing) where {T <: Number} +function JTmv(this::Connector{T},Wdata::Array{T},Win::Array{T},theta::Array{T},Y::Array{T},tmp=nothing) where {T <: Number} nex = div(length(Y),nFeatIn(this)) - if length(W)==0 - W = zero(T); + if length(Win)==0 + W = zeros(T,1,1); else - W = reshape(W,:,nex); + W = reshape(Win,:,nex); end if length(Wdata)>0 Wdata = reshape(Wdata,:,nex); - W = W+ this.Q'*Wdata; + W = W .+ this.Q'*Wdata end dtheta = zeros(T,0); From c8027c9a2ac36ea243a4ad732ddc940e4183a440 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Fri, 2 Feb 2018 14:16:54 -0800 Subject: [PATCH 11/35] batch norm optimized --- .gitignore | 1 + benchmarks/micro/bm_batchnorm.jl | 54 ++++++++++++++++++++------ src/layers/normLayer.jl | 65 ++++++++++++++++---------------- 3 files changed, 76 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 41b74da..521b563 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.jl.*.cov *.jl.mem data/ +*.jld \ No newline at end of file diff --git a/benchmarks/micro/bm_batchnorm.jl b/benchmarks/micro/bm_batchnorm.jl index 1c044e2..e38f39d 100644 --- a/benchmarks/micro/bm_batchnorm.jl +++ b/benchmarks/micro/bm_batchnorm.jl @@ -1,23 +1,55 @@ using Meganet, BenchmarkTools -history = Pkg.dir("Meganet")*"//benchmarks//micro//bm_batchnorm.jld" +const history = Pkg.dir("Meganet")*"//benchmarks//micro//bm_batchnorm.jld" -TYPE = Float64 +const TYPE = Float64 npixel = 500 nex = 1000 nchannel = 3 -L = getNormLayer(TYPE,[npixel,nchannel,nex],3) -theta = initTheta(L) -Y = randn(TYPE,nFeatIn(L),nex) +const L = getNormLayer(TYPE,[npixel,nchannel,nex],3) +const theta = initTheta(L) +const Y = randn(TYPE,nFeatIn(L),nex) -Yout2,Yout2,tmp2 = apply(L,theta,Y,true) -@code_warntype apply(L,theta,Y,true) +function benchmarkJYTmv() + funcName = "JYTmv" #TODO: pass funcName to history instead of calling it "hist" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) -trial = @benchmark apply(L,theta,Y,true); + #Warmup + Z1 = JYTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp) + trial = @benchmark JYTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp) + Meganet.updatehistory!(history, trial, "hist") + hist = JLD.load(history, "hist") + judge(hist) +end -Meganet.updatehistory!(history, trial) -hist = JLD.load(history, "hist") -judge(hist) \ No newline at end of file + +function benchmarkJYmv() + funcName = "JYmv" + _,_,tmp = apply(L,theta,Y,true) + dY = randn(TYPE,nFeatIn(L),nex) + #Warmup + dZ, dZ = JYmv(L,dY,theta,Y,tmp) + # @code_warntype JYmv(L, dY, theta, Y, tmp) + trial = @benchmark JYmv(L, dY, theta, Y, tmp) + Meganet.updatehistory!(history, trial, "hist") + hist = JLD.load(history, "hist") + judge(hist) +end + + +function benchmarkApply(L, theta, Y, history) + funcName = "apply" + Yout2,Yout2,tmp2 = apply(L,theta,Y,true) + + @code_warntype apply(L,theta,Y,true) + + trial = @benchmark apply(L,theta,Y,true) + + Meganet.updatehistory!(history, trial, "hist") + hist = JLD.load(history, "hist") + judge(hist) +end diff --git a/src/layers/normLayer.jl b/src/layers/normLayer.jl index 22ecfe1..8d944f8 100644 --- a/src/layers/normLayer.jl +++ b/src/layers/normLayer.jl @@ -31,7 +31,7 @@ function getTVNormLayer(TYPE::Type,nData;eps = convert(TYPE,1e-3),isTrainable::B end end -function apply(this::normLayer{T},theta::Array{T},Yin::Array{T},doDerivative=true) where {T <: Number} +function apply(this::normLayer{T},theta::Array{T},Yin::Array{T,2},doDerivative=true) where {T <: Number} # first organize Y with channels nf = this.nData[2]::Int @@ -41,17 +41,15 @@ function apply(this::normLayer{T},theta::Array{T},Yin::Array{T},doDerivative=tru dA = (T)[] # subtract mean across pixels - Ya = mean(Y,this.doNorm) - Y = Y.-Ya - # Y .-= Ya #TODO: This line is more efficient, but tests do not want Y to change. Why dont we want Y to change in place? + Yout = Y.-mean(Y,this.doNorm) # normalize - S2 = mean(Y.^2,this.doNorm) - Y ./= sqrt.(S2+this.eps) + S2 = sqrt.(mean(Yout.^2,this.doNorm) + this.eps) + Yout ./= S2 - Yout = reshape(Y,:,nex) + Yout2 = reshape(Yout,:,nex) - return Yout, Yout, dA + return Yout2, Yout2, dA end function nTheta(this::normLayer) @@ -79,26 +77,27 @@ function Jthetamv(this::normLayer,dtheta::Array{T},theta::Array{T},Y::Array{T},d return zeros(T,size(Y)), zeros(T,size(Y)) end -function JYmv(this::normLayer,dY::Array{T},theta::Array{T},Y::Array{T},dA=nothing) where {T <: Number} +function JYmv(this::normLayer,dYin::Array{T},theta::Array{T},Yin::Array{T},dA=nothing) where {T <: Number} - nex = div(length(dY),nFeatIn(this)) + nex = div(length(dYin),nFeatIn(this)) nf = this.nData[2] - dY = reshape(dY,:,nf,nex) - Y = reshape(Y,:,nf,nex) + dY = reshape(dYin,:,nf,nex) + Y = reshape(Yin,:,nf,nex) Ya = mean(Y,this.doNorm) - Y = Y .- Ya + Yout = Y .- Ya dYa = mean(dY,this.doNorm) - dY = dY .- dYa - S2y = mean(Y.^2,this.doNorm); + dYout = dY .- dYa + + S2y = mean(Yout.^2,this.doNorm); den = sqrt.(S2y+this.eps); + tmp = mean(Yout.*dYout,this.doNorm) + dYout ./= den - tmp = mean(Y.*dY,this.doNorm) - dY = dY ./ den + den .^= 3 + Yout .= Yout.*tmp ./den - Y = Y .* tmp - Y = Y ./ den.^3 - dZ = reshape(dY-Y,:,nex) + dZ = reshape(dYout-Yout,:,nex) return dZ,dZ end @@ -116,27 +115,27 @@ function JthetaTmv(this::normLayer{T},Z::Array{T},dummy::Array{T},theta::Array{T return zeros(T,0) end -function JYTmv(this::normLayer{T},Z::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},dA=nothing) where {T <: Number} +function JYTmv(this::normLayer{T},Zin::Array{T},dummy::Array{T},theta::Array{T},Yin::Array{T},dA=nothing) where {T <: Number} - nex = div(length(Y),nFeatIn(this)) + nex = div(length(Yin),nFeatIn(this)) nf = this.nData[2] - Z = reshape(Z,:,nf,nex) - Y = reshape(Y,:,nf,nex) + Z = reshape(Zin,:,nf,nex) + Y = reshape(Yin,:,nf,nex) Ya = mean(Y,this.doNorm) - Y = Y .- Ya + Yout = Y .- Ya Za = mean(Z,this.doNorm) - Z = Z .- Za - S2y = mean(Y.^2,this.doNorm) + Zout = Z .- Za + S2y = mean(Yout.^2,this.doNorm) den = sqrt.(S2y+this.eps) - tmp = mean(Y.*Z,this.doNorm) - Z = Z ./ den - Y = Y .* tmp - Y = Y ./ den.^3 - dY = Z-Y + tmp = mean(Yout.*Zout,this.doNorm) + Zout ./= den + Yout .*= tmp + Yout ./= den.^3 # TODO: look into doing both this division and multiplication above at same time + dY = Zout-Yout dYa = mean(dY,this.doNorm) - dY = dY .- dYa + dY .-= dYa return reshape(dY,:,nex) end From 9f12ad28ac72fd24a01b9eac93e8b9143669f236 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Fri, 2 Feb 2018 14:46:51 -0800 Subject: [PATCH 12/35] stabilize --- benchmarks/CIFAR10/profile_cifar10_512_64.jl | 6 +++--- src/layers/doubleSymLayer.jl | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmarks/CIFAR10/profile_cifar10_512_64.jl b/benchmarks/CIFAR10/profile_cifar10_512_64.jl index 2a1dd1d..76da26c 100644 --- a/benchmarks/CIFAR10/profile_cifar10_512_64.jl +++ b/benchmarks/CIFAR10/profile_cifar10_512_64.jl @@ -1,10 +1,10 @@ -using MAT, Meganet, BenchmarkTools, Compat, JLD, ProfileView +using MAT, Meganet, BenchmarkTools, Compat, JLD#, ProfileView # Macro Benchmark on CIFAR10 n = 32 miniBatchSize = 32 -path2data = "/home/klensink/Documents/cifar-10-batches-mat/" +path2data = "/home/klensink/.juliapro/JuliaPro-0.6.2.1/JuliaPro/pkgs-0.6.2.1/v0.6/Meganet/data/CIFAR10/" history = Pkg.dir("Meganet")*"/benchmarks/CIFAR10/cifar10_512_64.jld" Y_train,C_train,Y_test,C_test = getCIFAR10(n, path2data) @@ -79,7 +79,7 @@ solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y_train,C_train,Y_test,C_test) Profile.clear() Profile.init(100000000, 0.001) @profile solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y_train,C_train,Y_test,C_test) -ProfileView.view() +#ProfileView.view() if true Juno.profiler() diff --git a/src/layers/doubleSymLayer.jl b/src/layers/doubleSymLayer.jl index f23e764..dd367c7 100644 --- a/src/layers/doubleSymLayer.jl +++ b/src/layers/doubleSymLayer.jl @@ -188,14 +188,16 @@ end function JTmv(this::DoubleSymLayer{T}, Zin::Array{T}, dummy::Array{T}, theta::Array{T}, Yin::Array{T}, tmp) where {T<:Number} - nex::Int = div(length(Yin),nFeatIn(this)) + nex = div(length(Yin),nFeatIn(this)) Z = reshape(Zin, :, nex) Yt = reshape(tmp[2]::Array{T,2},:,nex) Y = reshape(Yin,:,nex) th1, th2, th3, th4 = splitWeights(this,theta) Kop = getOp(this.K,th1) A::Array{T,2}, dA::Array{T,2} = this.activation(Yt,true) + dth3 = vec(sum(this.Bout'*Z,2)) + dAZ1::Array{T,2} = dA.*(Kop*Z) dth2 = vec(sum(this.Bin'*dAZ1,2)) dth4, dAZ2::Vector{T} = JTmv(this.nLayer,dAZ1,zeros(T,0),th4,Kop*Y,tmp[1]) From 8feb91f228134eef4d092c4eecfe01a66774bede Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Fri, 2 Feb 2018 16:53:03 -0800 Subject: [PATCH 13/35] replace Kop with function calls --- src/layers/doubleSymLayer.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/layers/doubleSymLayer.jl b/src/layers/doubleSymLayer.jl index dd367c7..c245ce7 100644 --- a/src/layers/doubleSymLayer.jl +++ b/src/layers/doubleSymLayer.jl @@ -193,19 +193,23 @@ function JTmv(this::DoubleSymLayer{T}, Zin::Array{T}, dummy::Array{T}, Yt = reshape(tmp[2]::Array{T,2},:,nex) Y = reshape(Yin,:,nex) th1, th2, th3, th4 = splitWeights(this,theta) - Kop = getOp(this.K,th1) + #Kop = getOp(this.K,th1) A::Array{T,2}, dA::Array{T,2} = this.activation(Yt,true) dth3 = vec(sum(this.Bout'*Z,2)) - dAZ1::Array{T,2} = dA.*(Kop*Z) + KopZ = Amv(this.K, th1, Z) + dAZ1 = dA.*KopZ + dth2 = vec(sum(this.Bin'*dAZ1,2)) - dth4, dAZ2::Vector{T} = JTmv(this.nLayer,dAZ1,zeros(T,0),th4,Kop*Y,tmp[1]) + KopY = Amv(this.K, th1, Y) + dth4, dAZ2 = JTmv(this.nLayer,dAZ1,zeros(T,0),th4,KopY,tmp[1]) dth1 = JthetaTmv(this.K,dAZ2,zeros(T,0),Y) dth1 = dth1 + JthetaTmv(this.K,A,(T)[],Z) dtheta = [-vec(dth1); -vec(dth2); vec(dth3);-vec(dth4)] dAZ_out = reshape(dAZ2,:,nex) - dY::Array{T, 2} = -(Kop'*dAZ_out) - return dtheta,dY + KopTdAZ = ATmv(this.K, th1, dAZ_out) + dY = -KopTdAZ + return dtheta, dY end From 527ed6fa448b04734502f1bb52015f14d1be3a07 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Fri, 2 Feb 2018 16:57:38 -0800 Subject: [PATCH 14/35] undo changes to driver and profiler --- benchmarks/CIFAR10/profile_cifar10_512_64.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/CIFAR10/profile_cifar10_512_64.jl b/benchmarks/CIFAR10/profile_cifar10_512_64.jl index 76da26c..d42b3cf 100644 --- a/benchmarks/CIFAR10/profile_cifar10_512_64.jl +++ b/benchmarks/CIFAR10/profile_cifar10_512_64.jl @@ -1,10 +1,10 @@ -using MAT, Meganet, BenchmarkTools, Compat, JLD#, ProfileView +using MAT, Meganet, BenchmarkTools, Compat, JLD, ProfileView # Macro Benchmark on CIFAR10 n = 32 miniBatchSize = 32 -path2data = "/home/klensink/.juliapro/JuliaPro-0.6.2.1/JuliaPro/pkgs-0.6.2.1/v0.6/Meganet/data/CIFAR10/" +path2data = Pkg.dir("Meganet")*"/data/CIFAR10/" history = Pkg.dir("Meganet")*"/benchmarks/CIFAR10/cifar10_512_64.jld" Y_train,C_train,Y_test,C_test = getCIFAR10(n, path2data) @@ -79,7 +79,7 @@ solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y_train,C_train,Y_test,C_test) Profile.clear() Profile.init(100000000, 0.001) @profile solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y_train,C_train,Y_test,C_test) -#ProfileView.view() +ProfileView.view() if true Juno.profiler() From 35bf3a07b3fa2b898280d799278a26362d319ec6 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Fri, 2 Feb 2018 16:58:26 -0800 Subject: [PATCH 15/35] undo changes to driver --- examples/EResNN_CIFAR10.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/EResNN_CIFAR10.jl b/examples/EResNN_CIFAR10.jl index 686c9e5..cba6f74 100644 --- a/examples/EResNN_CIFAR10.jl +++ b/examples/EResNN_CIFAR10.jl @@ -1,6 +1,6 @@ using MAT, Meganet, Juno -n = 32; +n = 256; Y_train,C_train,Y_test,C_test = getCIFAR10(n,Pkg.dir("Meganet")*"/data/CIFAR10/"); # using PyPlot @@ -9,7 +9,7 @@ Y_train,C_train,Y_test,C_test = getCIFAR10(n,Pkg.dir("Meganet")*"/data/CIFAR10/" # y[:,:,1] = y[:,:,1]';y[:,:,2] = y[:,:,2]';y[:,:,3] = y[:,:,3]'; # figure(); imshow(y) -miniBatchSize = 32; +miniBatchSize = 64; nImg = [32; 32] cin = 3 nc = [16;32;64;64] From 30e1ec417097ea00d67e6ba54e87717db6707989 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Mon, 5 Feb 2018 10:20:54 -0800 Subject: [PATCH 16/35] made benchmark accept function names --- src/utils/Benchmark.jl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/utils/Benchmark.jl b/src/utils/Benchmark.jl index e207182..721af23 100644 --- a/src/utils/Benchmark.jl +++ b/src/utils/Benchmark.jl @@ -16,16 +16,23 @@ end Appends `hist` in the JLD file `history` with the latest trial and metadata contained in a `Benchmark` instance. """ -function updatehistory!(history::String, trial::BenchmarkTools.Trial; pkg::Module = Meganet) +function updatehistory!(history::String, trial::BenchmarkTools.Trial, funcName::String; pkg::Module = Meganet) cd(Pkg.dir("$pkg")) if isfile(history) println("Appending trial history: "*history) - hist = JLD.load(history, "hist") - push!(hist, Meganet.Benchmark(trial)) + + hist = JLD.load(history) + if haskey(hist, funcName) + histFunc = hist[funcName] + else + histFunc = Vector{Meganet.Benchmark}() + end + + push!(histFunc, Meganet.Benchmark(trial)) JLD.jldopen(history, "w") do file - write(file, "hist", hist) + write(file, funcName, histFunc) end else println("Creating trial history: "*history) @@ -33,7 +40,7 @@ function updatehistory!(history::String, trial::BenchmarkTools.Trial; pkg::Modul push!(hist, Meganet.Benchmark(trial)) JLD.jldopen(history, "w") do file - write(file, "hist", hist) + write(file, funcName, hist) end end end From 260dfce22210d3abdf41bc7ea1ab8ad17ccab951 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Mon, 5 Feb 2018 10:21:26 -0800 Subject: [PATCH 17/35] stabalized single layer functions --- benchmarks/micro/bm_singlelayer.jl | 104 +++++++++++++++++++++++++++++ src/integrators/NN.jl | 4 +- src/layers/singleLayer.jl | 93 ++++++++++++++------------ test/layer/singleLayerTest.jl | 2 +- 4 files changed, 156 insertions(+), 47 deletions(-) create mode 100644 benchmarks/micro/bm_singlelayer.jl diff --git a/benchmarks/micro/bm_singlelayer.jl b/benchmarks/micro/bm_singlelayer.jl new file mode 100644 index 0000000..8190b44 --- /dev/null +++ b/benchmarks/micro/bm_singlelayer.jl @@ -0,0 +1,104 @@ +using Meganet, BenchmarkTools + +const history = Pkg.dir("Meganet")*"//benchmarks//micro//bm_singleLayer.jld" + +TYPE = Float32 +K = getConvGEMMKernel(TYPE,[16,16],[3,3,64,64]) +nex = 1000 + +nLayer = getBatchNormLayer(TYPE,[16*16;64],isTrainable=true) +L = getSingleLayer(TYPE,K,nLayer) + +theta = initTheta(L) + +Y = randn(TYPE,nFeatIn(L),nex) + + +function benchmarkApply() + funcName = "apply" + Zd,Z,tmp = apply(L,theta,Y,true) + @code_warntype apply(L,theta,Y,true) + + trial = @benchmark apply(L,theta,Y,true) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJYTmv() + funcName = "JYTmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + + #Warmup + Z1 = JYTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp); + trial = @benchmark JYTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJthetaTmv() + funcName = "JthetaTmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + + #Warmup + Z1 = JthetaTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp); + trial = @benchmark JthetaTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJTmv() + funcName = "JTmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + + #Warmup + Z1 = JTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp); + trial = @benchmark JTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJYmv() + funcName = "JYmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + + #Warmup + Z1 = JYmv(L,copy(Zout),theta,Y,tmp); + trial = @benchmark JYmv(L,copy(Zout),theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJmv() + funcName = "Jmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + dtheta = randn(TYPE, size(theta)) + #Warmup + Z1 = Jmv(L,copy(dtheta),copy(Zout),theta,Y,tmp); + trial = @benchmark Jmv(L,copy(dtheta),copy(Zout),theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJthetamv() + funcName = "Jthetamv" + _,_,tmp = apply(L,theta,Y,true) + dtheta = randn(TYPE, size(theta)) + + #Warmup + Z1 = Jthetamv(L,dtheta,theta,Y,tmp); + trial = @benchmark Jthetamv(L,dtheta,theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end \ No newline at end of file diff --git a/src/integrators/NN.jl b/src/integrators/NN.jl index 7d3ee0f..6f28a2a 100644 --- a/src/integrators/NN.jl +++ b/src/integrators/NN.jl @@ -36,9 +36,9 @@ end # ---------- counting thetas, input and output features ----- function nTheta(this::NN) - n = 0; + n::Int = 0; for k=1:length(this.layers) - n = n + nTheta(this.layers[k]); + n += nTheta(this.layers[k]); end return n end diff --git a/src/layers/singleLayer.jl b/src/layers/singleLayer.jl index 28fa27e..3b6b24f 100644 --- a/src/layers/singleLayer.jl +++ b/src/layers/singleLayer.jl @@ -1,17 +1,16 @@ export singleLayer,getSingleLayer -mutable struct singleLayer{T} <: AbstractMeganetElement{T} +mutable struct singleLayer{T, TK <: AbstractConvKernel{T}, TN <: Union{NN{T}, normLayer{T}}} <: AbstractMeganetElement{T} activation :: Function # activation function - K # transformation type - nLayer :: Union{NN{T}, normLayer{T}, AffineScalingLayer{T}} # normalization layer + K :: TK # transformation type + nLayer :: TN # normalization layer Bin :: Array{T} # bias inside nonlinearity Bout :: Array{T} # bias outside nonlinearity - # singleLayer{T}(K,nLayer;Bin=zeros(T,nFeatOut(K),0),Bout=zeros(T,nFeatOut(K),0),activation=tanhActivation) = end function getSingleLayer(TYPE::Type, K,nLayer;Bin=zeros(TYPE,nFeatOut(K),0),Bout=zeros(TYPE,nFeatOut(K),0),activation=tanhActivation) - singleLayer{TYPE}(activation,K,nLayer,Bin,Bout); + singleLayer(activation,K,nLayer,Bin,Bout); end @@ -28,17 +27,19 @@ function splitWeights(this::singleLayer{T},theta::Array{T}) where {T <: Number} return th1, th2, th3, th4 end -function apply(this::singleLayer{T},theta::Array{T},Y::Array{T},doDerivative=false) where {T <: Number} +function apply(this::singleLayer{T},theta::Array{T},Yin::Array{T},doDerivative=false) where {T <: Number} tmp = Array{Any}(2) - nex = div(length(Y),nFeatIn(this)) - Y = reshape(Y,:,nex) + nex = div(length(Yin),nFeatIn(this)) + Y = reshape(Yin,:,nex) th1,th2,th3,th4 = splitWeights(this,theta) - Y = getOp(this.K,th1)*Y .+ this.Bin * th2 - Y,dummy,tmp[1] = apply(this.nLayer,th4,Y,doDerivative) - Y,tmp[2] = this.activation(Y,doDerivative) - Y = Y .+ this.Bout*th3 - Ydata = Y - return Ydata, Y, tmp + + Yout::Array{T,2} = getOp(this.K,th1)*Y + Yout .+= this.Bin * th2 + Yout,dummy,tmp[1] = apply(this.nLayer,th4,Yout,doDerivative) + Yout,tmp[2] = this.activation(Yout,doDerivative) + Yout .+= this.Bout*th3 + Ydata = Yout + return Ydata, Yout, tmp end function nTheta(this::singleLayer) @@ -62,62 +63,65 @@ function initTheta(this::singleLayer{T}) where {T <: Number} end -function Jthetamv(this::singleLayer{T},dtheta::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} - dA = tmp[2] - nex = div(length(Y),nFeatIn(this)) - Y = reshape(Y,:,nex) +function Jthetamv(this::singleLayer{T},dtheta::Array{T},theta::Array{T},Yin::Array{T},tmp) where {T <: Number} + dA::Array{T,2} = tmp[2] + nex = div(length(Yin),nFeatIn(this)) + Y = reshape(Yin,:,nex) th1,th2,th3,th4 = splitWeights(this,theta) dth1,dth2,dth3,dth4 = splitWeights(this,dtheta) - dZ = Jthetamv(this.K,dth1,th1,Y) .+ this.Bin*dth2 + dZ::Array{T,2} = Jthetamv(this.K,dth1,th1,Y) .+ this.Bin*dth2 Kop = getOp(this.K,th1) dZ = Jmv(this.nLayer,dth4,dZ,th4,Kop*Y.+this.Bin*th2,tmp[1])[2] - dZ = dA.*dZ .+ this.Bout*dth3; + dZ .*= dA + dZ .+= this.Bout*dth3 return dZ, dZ end -function JYmv(this::singleLayer{T},dY::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} +function JYmv(this::singleLayer{T},dYin::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} dA = tmp[2] - nex = div(length(dY),nFeatIn(this)) + nex = div(length(dYin),nFeatIn(this)) th1,th2,th3,th4 = splitWeights(this,theta) Kop = getOp(this.K,th1) - dY = reshape(dY,:,nex) + dY = reshape(dYin,:,nex) dZ = Kop*dY dZ = JYmv(this.nLayer,dZ,th4,Kop*Y.+this.Bin*th2,tmp[1])[2] - dZ = dA.*dZ + # dZ = dA.*dZ + dZ .*= dA return dZ,dZ end -function Jmv(this::singleLayer{T},dtheta::Array{T},dY::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} - dA = tmp[2] - nex = div(length(Y),nFeatIn(this)) +function Jmv(this::singleLayer{T},dtheta::Array{T},dYin::Array{T},theta::Array{T},Yin::Array{T},tmp) where {T <: Number} + dA::Array{T,2} = tmp[2] + nex = div(length(Yin),nFeatIn(this)) th1,th2,th3,th4 = splitWeights(this,theta) dth1,dth2,dth3,dth4 = splitWeights(this,dtheta) - dY = reshape(dY,:,nex); + dY = reshape(dYin,:,nex); Kop = getOp(this.K,th1) - dZ = Kop*dY; + dZ::Array{T, 2} = Kop*dY; - Y = reshape(Y,:,nex); + Y = reshape(Yin,:,nex); dZ += Jthetamv(this.K,dth1,th1,Y) .+ this.Bin*dth2 dZ = Jmv(this.nLayer,dth4,dZ,th4,Kop*Y.+this.Bin*th2,tmp[1])[2] - dZ = dA.*dZ .+ this.Bout*dth3 + dZ .*= dA + dZ .+= this.Bout*dth3 return dZ,dZ end -function JTmv(this::singleLayer{T},Z::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} - dA = tmp[2] +function JTmv(this::singleLayer{T},Zin::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} + dA::Array{T,2} = tmp[2] nex = div(length(Y),nFeatIn(this)) - Z = reshape(Z,:,nex) + Z = reshape(Zin,:,nex) th1,th2,th3,th4 = splitWeights(this,theta) Kop = getOp(this.K,th1) dth3 = vec(sum(this.Bout'*Z,2)) dAZ = dA.*Z - dth4,dAZ = JTmv(this.nLayer,dAZ,zeros(T,0),th4,Kop*Y.+this.Bin*th2,tmp[1]) + dth4,dAZ = JTmv(this.nLayer,dAZ,zeros(T,0),th4,Kop*Y.+this.Bin*th2,tmp[1]) # this not type stable dth2 = vec(sum(this.Bin'*reshape(dAZ,:,nex),2)) - dth1 = JthetaTmv(this.K, dAZ,theta,Y) + dth1 = JthetaTmv(this.K, dAZ,theta,Y) # this not type stable dY = Kop'*reshape(dAZ,:,nex) dtheta = [vec(dth1); vec(dth2); vec(dth3); vec(dth4)] @@ -126,12 +130,12 @@ function JTmv(this::singleLayer{T},Z::Array{T},dummy::Array{T},theta::Array{T},Y end -function JthetaTmv(this::singleLayer{T},Z::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} +function JthetaTmv(this::singleLayer{T},Zin::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} dA = tmp[2] - nex = div(length(Z),nFeatOut(this)) + nex = div(length(Zin),nFeatOut(this)) th1,th2,th3,th4 = splitWeights(this,theta) - Z = reshape(Z,:,nex); + Z = reshape(Zin,:,nex); dAZ = dA.*Z; dth3 = vec(sum(this.Bout'*Z,2)); Kop = getOp(this.K,th1) @@ -141,13 +145,14 @@ function JthetaTmv(this::singleLayer{T},Z::Array{T},dummy::Array{T},theta::Array return [vec(dth1); vec(dth2); vec(dth3); vec(dth4)]; end -function JYTmv(this::singleLayer{T},Z::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} - dA = tmp[2] +function JYTmv(this::singleLayer{T},Zin::Array{T},dummy::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} + dA::Array{T,2} = tmp[2] nex = div(length(Y),nFeatIn(this)) th1,th2,th3,th4 = splitWeights(this,theta) Kop = getOp(this.K,th1) - Z = reshape(Z,:,nex) - dAZ = dA.*Z + Z = reshape(Zin,:,nex) + dAZ::Array{T,2} = dA.*Z dAZ = JYTmv(this.nLayer,dAZ,(T)[],th4,Kop*Y.+this.Bin*th2,tmp[1]) - return Kop'*reshape(dAZ,:,nex) + ret::Array{T,2} = Kop'*reshape(dAZ,:,nex) + return ret #TODO: @lars or eldad rename this variable as I'm not sure what to call it end diff --git a/test/layer/singleLayerTest.jl b/test/layer/singleLayerTest.jl index ce94ee4..cf009ef 100644 --- a/test/layer/singleLayerTest.jl +++ b/test/layer/singleLayerTest.jl @@ -24,7 +24,7 @@ for TYPE=[Float64,Float32] nex = 4 K = getSparseConvKernel2D(TYPE,nImg,[3,3,1,nc]) Bin = randn(TYPE, nFeatOut(K),4) - nLayer = getBatchNormLayer(TYPE,[prod(nImg),nc],isTrainable=true).layers[2] + nLayer = getBatchNormLayer(TYPE,[prod(nImg),nc],isTrainable=true) #Do we need this to be .layers[2]? L = getSingleLayer(TYPE,K,nLayer,Bin=Bin) @testset "singleLayer (conv/Batch/not trainable) $TYPE" begin testAbstractMeganetElement(L,nex=nex) From 8cb8428591d18b056cd24d7cce289aa160670533 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 5 Feb 2018 14:35:10 -0800 Subject: [PATCH 18/35] remove Juno dependancy --- benchmarks/CIFAR10/profile_cifar10_512_64.jl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/benchmarks/CIFAR10/profile_cifar10_512_64.jl b/benchmarks/CIFAR10/profile_cifar10_512_64.jl index d42b3cf..553e42f 100644 --- a/benchmarks/CIFAR10/profile_cifar10_512_64.jl +++ b/benchmarks/CIFAR10/profile_cifar10_512_64.jl @@ -1,7 +1,7 @@ using MAT, Meganet, BenchmarkTools, Compat, JLD, ProfileView # Macro Benchmark on CIFAR10 -n = 32 +n = 64 miniBatchSize = 32 path2data = Pkg.dir("Meganet")*"/data/CIFAR10/" @@ -80,7 +80,3 @@ Profile.clear() Profile.init(100000000, 0.001) @profile solve(opt,objFun::dnnObjFctn,[vec(theta);vec(W)],Y_train,C_train,Y_test,C_test) ProfileView.view() - -if true - Juno.profiler() -end From 2efbe1c28b54620579ac176ef255d3a985a11b40 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 5 Feb 2018 14:36:37 -0800 Subject: [PATCH 19/35] Update EResNN_CIFAR10.jl --- examples/EResNN_CIFAR10.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/EResNN_CIFAR10.jl b/examples/EResNN_CIFAR10.jl index cba6f74..4837774 100644 --- a/examples/EResNN_CIFAR10.jl +++ b/examples/EResNN_CIFAR10.jl @@ -1,4 +1,4 @@ -using MAT, Meganet, Juno +using MAT, Megane n = 256; Y_train,C_train,Y_test,C_test = getCIFAR10(n,Pkg.dir("Meganet")*"/data/CIFAR10/"); From 60c4f576cec488c5f9a085afca7d7188bcde4764 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 5 Feb 2018 14:36:58 -0800 Subject: [PATCH 20/35] Update ResNN.jl --- src/integrators/ResNN.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/integrators/ResNN.jl b/src/integrators/ResNN.jl index 1c6ddbb..f47bc87 100644 --- a/src/integrators/ResNN.jl +++ b/src/integrators/ResNN.jl @@ -1,5 +1,4 @@ export ResNN,getResNN -import Juno """ Residual Neural Network block From 5bb807f0ee0040db855f9d6807a3cda56d5484f6 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 5 Feb 2018 14:37:42 -0800 Subject: [PATCH 21/35] Update EResNN_CIFAR10.jl --- examples/EResNN_CIFAR10.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/EResNN_CIFAR10.jl b/examples/EResNN_CIFAR10.jl index 4837774..efa3f01 100644 --- a/examples/EResNN_CIFAR10.jl +++ b/examples/EResNN_CIFAR10.jl @@ -1,4 +1,4 @@ -using MAT, Megane +using MAT, Meganet n = 256; Y_train,C_train,Y_test,C_test = getCIFAR10(n,Pkg.dir("Meganet")*"/data/CIFAR10/"); From de6680d0e3275895f48419aa4a88be4e2cd2a528 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 5 Feb 2018 16:22:23 -0800 Subject: [PATCH 22/35] pre-alloc dA as zeros --- src/activations/tanhActivation.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/activations/tanhActivation.jl b/src/activations/tanhActivation.jl index d68b351..b8fe0bf 100644 --- a/src/activations/tanhActivation.jl +++ b/src/activations/tanhActivation.jl @@ -20,11 +20,9 @@ export tanhActivation function tanhActivation(Y::Array{T,2},doDerivative::Bool=false) where {T <: Number} A = tanh.(Y) - dA = similar(Y) + dA = zeros(A) if doDerivative dA .= one(T) .- A.^2 - else - fill!(dA, zero(T)) end return A, dA end From 83f6d03e5a8f55ca929bb8562cca633e17d59ae3 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 5 Feb 2018 17:07:52 -0800 Subject: [PATCH 23/35] setup benchmark for convGEMM --- benchmarks/micro/bm_multConv2Dblock.jl | 28 ++++++++++++++++++++++++++ src/kernelTypes/convGEMMKernel.jl | 1 + src/utils/Benchmark.jl | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 benchmarks/micro/bm_multConv2Dblock.jl diff --git a/benchmarks/micro/bm_multConv2Dblock.jl b/benchmarks/micro/bm_multConv2Dblock.jl new file mode 100644 index 0000000..fb4ae1c --- /dev/null +++ b/benchmarks/micro/bm_multConv2Dblock.jl @@ -0,0 +1,28 @@ +using Meganet, JLD, BenchmarkTools, Juno + +history = Pkg.dir("Meganet")*"//benchmarks//micro//bm_multConv2Dblock.jld" +file = Pkg.dir("Meganet")*"/benchmarks/micro/vars_multConv2Dblock.jld" + +x = load(file, "x") +K = load(file, "K") +y = load(file, "y") +t = load(file, "t") +shiftX = load(file, "shiftX") +shiftT = load(file, "shiftT") +imIdx = load(file, "imIdx") +doDerivative = load(file, "doDerivative") + +trial = @benchmark Meganet.multConv2Dblock(x, K, y, t, shiftX, shiftT, imIdx, doDerivative = doDerivative) +@enter Meganet.multConv2Dblock(x, K, y, t, shiftX, shiftT, imIdx, doDerivative = doDerivative) +display(trial) +hist = load(history, "multConv2Dblock") + +if false + Meganet.updatehistory!(history, trial, "multConv2Dblock") +end + +y = load(file, "y") +Meganet.multConv2Dblock(x, K, y, t, shiftX, shiftT, imIdx, doDerivative = doDerivative) +y = load(file, "y") +Profile.clear_malloc_data() +Meganet.multConv2Dblock(x, K, y, t, shiftX, shiftT, imIdx, doDerivative = doDerivative); diff --git a/src/kernelTypes/convGEMMKernel.jl b/src/kernelTypes/convGEMMKernel.jl index f24c56a..a376624 100644 --- a/src/kernelTypes/convGEMMKernel.jl +++ b/src/kernelTypes/convGEMMKernel.jl @@ -1,4 +1,5 @@ export convGEMMKernel,Amv,ATmv,transposeTest,getConvGEMMKernel +import Juno mutable struct convGEMMKernel{T} <: AbstractConvKernel{T} nImg :: Array{Int,1} diff --git a/src/utils/Benchmark.jl b/src/utils/Benchmark.jl index 721af23..7b13246 100644 --- a/src/utils/Benchmark.jl +++ b/src/utils/Benchmark.jl @@ -11,7 +11,7 @@ function Benchmark(trial::BenchmarkTools.Trial) end """ - Use: updatehistory!(history::String, trial::BenchmarkTools.Trial; pkg::Module = Meganet) + Use: updatehistory!(history::String, trial::BenchmarkTools.Trial, funcName::String; pkg::Module = Meganet) Appends `hist` in the JLD file `history` with the latest trial and metadata contained in a `Benchmark` instance. From 5b8e0b0dbdc3e2327df409ee1225896fca0c8dcf Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Mon, 5 Feb 2018 17:24:14 -0800 Subject: [PATCH 24/35] remove import Juno --- src/kernelTypes/convGEMMKernel.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/kernelTypes/convGEMMKernel.jl b/src/kernelTypes/convGEMMKernel.jl index a376624..f24c56a 100644 --- a/src/kernelTypes/convGEMMKernel.jl +++ b/src/kernelTypes/convGEMMKernel.jl @@ -1,5 +1,4 @@ export convGEMMKernel,Amv,ATmv,transposeTest,getConvGEMMKernel -import Juno mutable struct convGEMMKernel{T} <: AbstractConvKernel{T} nImg :: Array{Int,1} From a8d31a1ad2b63190eda04b9c17039b2a4c00f6b8 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Tue, 6 Feb 2018 14:50:09 -0800 Subject: [PATCH 25/35] improved apply and JYTmv --- benchmarks/micro/bm_resNN.jl | 94 ++++++++++++++++++++++++++++++++++++ src/integrators/ResNN.jl | 40 +++++++-------- 2 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 benchmarks/micro/bm_resNN.jl diff --git a/benchmarks/micro/bm_resNN.jl b/benchmarks/micro/bm_resNN.jl new file mode 100644 index 0000000..ab1f2e0 --- /dev/null +++ b/benchmarks/micro/bm_resNN.jl @@ -0,0 +1,94 @@ +using Meganet, BenchmarkTools + +const history = Pkg.dir("Meganet")*"//benchmarks//micro//bm_resNN.jld" + +TYPE = Float32 +K = getConvGEMMKernel(TYPE,[96,96],[3,3,64,64]) +nex = 20 +# Bin = randn(TYPE,18,4) +# Bout = randn(TYPE,18,3) +nLayer = getBatchNormLayer(TYPE,[96*96,64],isTrainable=true) +lay = getSingleLayer(TYPE,K,nLayer) + +L = getResNN(TYPE,lay,4) + +theta = initTheta(L) + +Y = randn(TYPE,nFeatIn(L),nex) + +function benchmarkApply() + funcName = "apply" + Zd,Z,tmp = apply(L,theta,Y,true); + @code_warntype apply(L,theta,Y,true) + + trial = @benchmark apply(L,theta,Y,true) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +### +function benchmarkJYTmv() + funcName = "JYTmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + + #Warmup + Z1 = JYTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp); + trial = @benchmark JYTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJTmv() + funcName = "JTmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + + #Warmup + Z1 = JTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp); + trial = @benchmark JTmv(L,copy(Zout),(TYPE)[],theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJYmv() + funcName = "JYmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + + #Warmup + Z1 = JYmv(L,copy(Zout),theta,Y,tmp); + trial = @benchmark JYmv(L,copy(Zout),theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJmv() + funcName = "Jmv" + _,_,tmp = apply(L,theta,Y,true) + Zout = randn(TYPE,nFeatOut(L),nex) + dtheta = randn(TYPE, size(theta)) + #Warmup + Z1 = Jmv(L,copy(dtheta),copy(Zout),theta,Y,tmp); + trial = @benchmark Jmv(L,copy(dtheta),copy(Zout),theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end + +function benchmarkJthetamv() + funcName = "Jthetamv" + _,_,tmp = apply(L,theta,Y,true) + dtheta = randn(TYPE, size(theta)) + + #Warmup + Z1 = Jthetamv(L,dtheta,theta,Y,tmp); + trial = @benchmark Jthetamv(L,dtheta,theta,Y,tmp) + Meganet.updatehistory!(history, trial, funcName) + hist = JLD.load(history, funcName) + judge(hist) +end \ No newline at end of file diff --git a/src/integrators/ResNN.jl b/src/integrators/ResNN.jl index 7e5b7bd..06983ec 100644 --- a/src/integrators/ResNN.jl +++ b/src/integrators/ResNN.jl @@ -5,20 +5,20 @@ Residual Neural Network block Y_k+1 = Y_k + h*layer{k}(theta{k},Y_k) """ -mutable struct ResNN{T} <: AbstractMeganetElement{T} - layer - nt - h - outTimes +mutable struct ResNN{T, TL <: AbstractMeganetElement{T}} <: AbstractMeganetElement{T} #TODO limit TL more + layer :: TL + nt :: Int + h :: T + outTimes :: Array{Int,1} Q end function getResNN(TYPE::Type,layer,nt,h=one(TYPE),outTimes=eye(Int,nt)[:,nt],Q=I) - h = convert(TYPE,h); + h = convert(TYPE,h); if nFeatIn(layer)!=nFeatOut(layer) error("ResNN layer must be square!") - end - return ResNN{TYPE}(layer,nt,h,outTimes,Q) + end + return ResNN(layer,nt,h,outTimes,Q) end @@ -48,7 +48,7 @@ function initTheta(this::ResNN{T}) where {T<:Number} end # ------- apply forward problems ----------- -function apply(this::ResNN{T},theta::Array{T},Y0::Array{T},doDerivative=true) where {T<:Number} +function apply(this::ResNN{T},theta_in::Array{T},Y0::Array{T},doDerivative=true) where {T<:Number} nex = div(length(Y0),nFeatIn(this)) Y = reshape(Y0,:,nex) @@ -57,9 +57,9 @@ function apply(this::ResNN{T},theta::Array{T},Y0::Array{T},doDerivative=true) w tmp[1,1] = Y0 end - theta = reshape(theta,:,this.nt) + theta = reshape(theta_in,:,this.nt) - Ydata = zeros(T,0,nex) + Ydata::Array{T,2} = zeros(T,0,nex) for i=1:this.nt Z,dummy,tmp[i,2] = apply(this.layer,theta[:,i],Y,doDerivative) Y += this.h * Z @@ -115,26 +115,26 @@ end # -------- Jacobian transpose matvecs ---------------- -function JYTmv(this::ResNN{T},Wdata::Array{T},W::Array{T},theta::Array{T},Y::Array{T},tmp) where {T<:Number} +function JYTmv(this::ResNN{T},Wdata_in::Array{T},Win::Array{T},theta_in::Array{T},Y::Array{T},tmp) where {T<:Number} nex = div(length(Y),nFeatIn(this)) - if length(Wdata)>0 - Wdata = reshape(Wdata,:,sum(this.outTimes),nex) + if length(Wdata_in)>0 + Wdata = reshape(Wdata_in,:,sum(this.outTimes),nex) end - if length(W)==0 - W = zero(T) + if length(Win)==0 + W = zeros(T,0,0) else - W = reshape(W,:,nex) + W = reshape(Win,:,nex) end - theta = reshape(theta,:,this.nt) + theta = reshape(theta_in,:,this.nt) cnt = sum(this.outTimes) for i=this.nt:-1:1 if this.outTimes[i]==1 - W += this.Q'*Wdata[:,cnt,:] + W::Array{T,2} = this.Q'*Wdata[:,cnt,:] cnt = cnt-1 end - dW = JYTmv(this.layer,W,zeros(T,0),theta[:,i],tmp[i,1],tmp[i,2]) + dW::Array{T,2} = JYTmv(this.layer,W,zeros(T,0),theta[:,i],tmp[i,1],tmp[i,2]) W += this.h*dW end return W From 4254b24816ec4f95e7cf01c4b92b214eff640e35 Mon Sep 17 00:00:00 2001 From: Keegan Lensink Date: Tue, 6 Feb 2018 17:07:56 -0800 Subject: [PATCH 26/35] remove Juno for real this time --- benchmarks/micro/bm_multConv2Dblock.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/micro/bm_multConv2Dblock.jl b/benchmarks/micro/bm_multConv2Dblock.jl index fb4ae1c..8044997 100644 --- a/benchmarks/micro/bm_multConv2Dblock.jl +++ b/benchmarks/micro/bm_multConv2Dblock.jl @@ -1,4 +1,4 @@ -using Meganet, JLD, BenchmarkTools, Juno +using Meganet, JLD, BenchmarkTools history = Pkg.dir("Meganet")*"//benchmarks//micro//bm_multConv2Dblock.jld" file = Pkg.dir("Meganet")*"/benchmarks/micro/vars_multConv2Dblock.jld" From 9f4139da0482564de3bf2802bcc27b176c47fcf7 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Wed, 7 Feb 2018 13:59:41 -0800 Subject: [PATCH 27/35] type stabalized NN a bit, still needs work --- src/integrators/NN.jl | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/integrators/NN.jl b/src/integrators/NN.jl index 6f28a2a..9484a7e 100644 --- a/src/integrators/NN.jl +++ b/src/integrators/NN.jl @@ -5,10 +5,10 @@ NN Neural Network block Y_k+1 = layer{k}(theta{k},Y_k) """ -mutable struct NN{T} <: AbstractMeganetElement{T} - layers ::Array{AbstractMeganetElement{T}, 1} # layers of Neural Network, cell array - outTimes - Q +mutable struct NN{T, TQ <: Union{Array{T,2},UniformScaling{Int}}} <: AbstractMeganetElement{T} + layers ::Array{AbstractMeganetElement{T}, 1} # layers of Neural Network, cell array + outTimes ::Array{Int,1} + Q :: TQ end function getNN(layers::Array{AbstractMeganetElement{T}},outTimes=eye(Int,length(layers))[:,end],Q=I) where {T <: Number} @@ -21,7 +21,7 @@ function getNN(layers::Array{AbstractMeganetElement{T}},outTimes=eye(Int,length( end nout = nFeatOut(layers[k]) end - return NN{T}(layers,outTimes,Q); + return NN(layers,outTimes,Q); end @@ -43,7 +43,7 @@ function nTheta(this::NN) return n end nFeatIn(this::NN) = nFeatIn(this.layers[1]) -nFeatOut(this::NN) = nFeatOut(this.layers[end]) +nFeatOut(this::NN)::Int = nFeatOut(this.layers[end]) function nDataOut(this::NN) n=0; @@ -159,39 +159,41 @@ end function JthetaTmv(this::NN{T},Wdata::Array{T},W::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} - return JTmv(this,Wdata,W,theta,Y,tmp)[1]; + return JTmv(this,Wdata,W,theta,Y,tmp)[1]; # TODO: Why calculating both, Can be more efficient? end -function JTmv(this::NN,Wdata::Array{T},W::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} - nex = div(length(Y),nFeatIn(this)) +function JTmv(this::NN{T},Wdata::Array{T},Win::Array{T},theta::Array{T},Y::Array{T},tmp)::Tuple{Array{T,1},Array{T,1}} where {T <: Number} + # WOW THIS IS HACKED BIG TIME. Need to find a way to type stabalize W (not ez) + #TODO: Make this type stable - Some internals are not stable + nex = div(length(Y),nFeatIn(this)::Int) if size(Wdata,1)>0 Wdata = reshape(Wdata,:,nex) end - if length(W)==0 + + if length(Win)==0 W = zeros(T,nFeatOut(this),nex) - elseif length(W)>1 - W = reshape(W,:,nex) + else + W = reshape(Win,:,nex) end - - dtheta = 0*theta + dtheta = zero(T)*theta nt = length(this.layers) cnt = 0; cnt2 = 0 for i=nt:-1:1 if this.outTimes[i]==1 - nn = nFeatOut(this.layers[i]) + nn = nFeatOut(this.layers[i])::Int W += this.Q'*Wdata[end-cnt2-nn+1:end-cnt2,:] cnt2 = cnt2 + nn end - ni = nTheta(this.layers[i]) + ni = nTheta(this.layers[i])::Int dmbi,W = JTmv(this.layers[i],W,zeros(T,0),theta[end-cnt-ni+1:end-cnt],tmp[i,1],tmp[i,2]) dtheta[end-cnt-ni+1:end-cnt] = dmbi - cnt = cnt+ni + cnt += ni end - return vec(dtheta), vec(W) + return vec(dtheta), vec(W) end From 8d70b9ce17d2b393f7076cd341793f4be8b51631 Mon Sep 17 00:00:00 2001 From: moumitaTora Date: Wed, 7 Feb 2018 17:10:58 -0800 Subject: [PATCH 28/35] fixed convgemm boundary --- src/kernelTypes/convGEMMKernel.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernelTypes/convGEMMKernel.jl b/src/kernelTypes/convGEMMKernel.jl index f24c56a..aab8b64 100644 --- a/src/kernelTypes/convGEMMKernel.jl +++ b/src/kernelTypes/convGEMMKernel.jl @@ -144,13 +144,13 @@ for p = 1:2:2*kernelWidth if jt > 1 @inbounds t[:,1:(jt-1),cc] = 0.0; end - while jt < nImg2+shiftT[p+1] + while jt <= nImg2+shiftT[p+1] it = 1+shiftT[q]; ix = 1+shiftX[q]; if it > 1 @inbounds t[1:(it-1),jt,cc] = 0.0; end - while it < nImg1+shiftT[q+1] + while it <= nImg1+shiftT[q+1] @inbounds t[it,jt,cc] = x[ix,jx,cc,imIdx]; it+=1;ix+=1; end From 54eddd5bc7e95e82144d4c30b44c9d9a8315ddaf Mon Sep 17 00:00:00 2001 From: moumitaTora Date: Wed, 7 Feb 2018 17:11:15 -0800 Subject: [PATCH 29/35] fixed sgd print values --- src/optimization/sgd.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimization/sgd.jl b/src/optimization/sgd.jl index a1b31e6..a2feab5 100644 --- a/src/optimization/sgd.jl +++ b/src/optimization/sgd.jl @@ -72,7 +72,7 @@ function solve(this::SGD{T},objFun::dnnObjFctn,xc::Array{T},Y::Array{T},C::Array Jval,pVal = getMisfit(objFun,xc,Yv,Cv,false); if this.out; - @printf "%d\t%1.2e\t%1.2f\t%1.2e\t%1.2e\t%1.2f\n" epoch Jc 100*(1-para[3]/para[2]) norm(xOld-xc) Jval 100*(1-pVal[3]/para[2]) + @printf "%d\t%1.2e\t%1.2f\t%1.2e\t%1.2e\t%1.2f\n" epoch Jc 100*(1-para[3]/para[2]) norm(xOld-xc) Jval 100*(1-pVal[3]/pVal[2]) end xOld = copy(xc); From 0ccd0a875720aebb18f2bd2290b1c7dffc19e724 Mon Sep 17 00:00:00 2001 From: moumitaTora Date: Wed, 7 Feb 2018 17:11:29 -0800 Subject: [PATCH 30/35] added new convgemm derivitive test --- test/kernel/convGEMMKernelTest.jl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/kernel/convGEMMKernelTest.jl b/test/kernel/convGEMMKernelTest.jl index efc8727..c1f2627 100644 --- a/test/kernel/convGEMMKernelTest.jl +++ b/test/kernel/convGEMMKernelTest.jl @@ -32,4 +32,22 @@ for TYPE=[Float64,Float32] # println("derivativeTest t1=$t1\t t2=$t2") @test norm(t1-t2)/norm(t2) < 1e3*eps(TYPE) end + + @testset "new derivitive test" begin + nImage = [16,16]; + sK = [3,3,2,4]; + K = randn(TYPE,tuple(sK...)); + Y = randn(TYPE,nImage[1],nImage[2],sK[3],2); + Z = randn(TYPE,nImage[1],nImage[2],sK[4],2); + Kernel2 = getConvGEMMKernel(TYPE,nImage,sK); + AY = Amv(Kernel2,K,Y); + ATZ = ATmv(Kernel2,K,Z); + + v1 = vecdot(Z,AY); + v2 = vecdot(ATZ,Y); + + v3 = vecdot(Z,Jthetamv(Kernel2,K,(TYPE)[],Y)); + v4 = vecdot(K,JthetaTmv(Kernel2,Z,(TYPE)[],Y)); + @test v1 == v2 == v3 == v4; + end end From b4a6f5c583c527d88aaff9b69d928efdda379385 Mon Sep 17 00:00:00 2001 From: moumitaTora Date: Wed, 7 Feb 2018 17:19:02 -0800 Subject: [PATCH 31/35] updated new derivitive convgemm test --- test/kernel/convGEMMKernelTest.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/kernel/convGEMMKernelTest.jl b/test/kernel/convGEMMKernelTest.jl index c1f2627..8b16941 100644 --- a/test/kernel/convGEMMKernelTest.jl +++ b/test/kernel/convGEMMKernelTest.jl @@ -48,6 +48,8 @@ for TYPE=[Float64,Float32] v3 = vecdot(Z,Jthetamv(Kernel2,K,(TYPE)[],Y)); v4 = vecdot(K,JthetaTmv(Kernel2,Z,(TYPE)[],Y)); - @test v1 == v2 == v3 == v4; + @test norm(v1-v2)/norm(v2) < 1e3*eps(TYPE) && + norm(v2-v3)/norm(v3) < 1e3*eps(TYPE) && + norm(v3-v4)/norm(v4) < 1e3*eps(TYPE) end end From 6ef58ee448443f7a10dff00870e6182409d66ee3 Mon Sep 17 00:00:00 2001 From: elliotholtham Date: Wed, 7 Feb 2018 17:30:39 -0800 Subject: [PATCH 32/35] Speed-up the GEM convolution --- src/kernelTypes/convGEMMKernel.jl | 100 ++++++++++++++++-------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/src/kernelTypes/convGEMMKernel.jl b/src/kernelTypes/convGEMMKernel.jl index f24c56a..ad6cfc3 100644 --- a/src/kernelTypes/convGEMMKernel.jl +++ b/src/kernelTypes/convGEMMKernel.jl @@ -119,60 +119,66 @@ for c=1:size(Z,2) end end -function multConv2Dblock(x::Array{T},K::Array{Array{T,2},2}, y::Array{T}, t::Array{T},shiftX,shiftT,imIdx;doDerivative = 0) where {T<:Number} -## y = K*x -## K - 3X3 array of Arrays -## x - a vector of length |nImgag+2|*cin (zero padded) -## y - a vector of length |nImgag|*cout - -nImg1 = size(x,1); -nImg2 = size(x,2); -cin = size(x,3); -cout = size(y,2); -OneType = one(T); - -kernelWidth = size(K,1); -# y = reshape(y,nImg1*nImg2,cout); # it is supposed to be of this shape... -k=1; -jt=0;it=0;jt=0;jx=0; -for p = 1:2:2*kernelWidth - for q = 1:2:2*kernelWidth - t = reshape(t,nImg1,nImg2,cin); - for cc = 1:cin - jx = 1+shiftX[p]; - jt = 1+shiftT[p]; - if jt > 1 - @inbounds t[:,1:(jt-1),cc] = 0.0; - end - while jt < nImg2+shiftT[p+1] - it = 1+shiftT[q]; - ix = 1+shiftX[q]; - if it > 1 - @inbounds t[1:(it-1),jt,cc] = 0.0; +function multConv2Dblock(x::Array{T},K::Array{Array{T,2},2}, y::Array{T}, tin::Array{T},shiftX,shiftT,imIdx;doDerivative = 0) where {T<:Number} + ## y = K*x + ## K - 3X3 array of Arrays + ## x - a vector of length |nImgag+2|*cin (zero padded) + ## y - a vector of length |nImgag|*cout + + nImg1 = size(x,1); + nImg2 = size(x,2); + cin = size(x,3); + cout = size(y,2); + OneType = one(T); + + kernelWidth = size(K,1); + # y = reshape(y,nImg1*nImg2,cout); # it is supposed to be of this shape... + k=1; + jt=0;it=0;jt=0;jx=0; + for p = 1:2:2*kernelWidth + for q = 1:2:2*kernelWidth + t = reshape(tin,nImg1,nImg2,cin); + lower = nImg2+shiftT[p+1] # Move outside of the forloop for increased speed + upper = nImg1+shiftT[q+1] # Move outside of the forloop for increased speed + for cc = 1:cin + jx = 1+shiftX[p]; # Moving these outside didn't seem to help + jt = 1+shiftT[p]; + if jt > 1 + @inbounds t[:,1:(jt-1),cc] = 0.0; end - while it < nImg1+shiftT[q+1] - @inbounds t[it,jt,cc] = x[ix,jx,cc,imIdx]; - it+=1;ix+=1; + while jt < lower + it = 1+shiftT[q]; + ix = 1+shiftX[q]; + if it > 1 + for ii = 1:(it-1) + @inbounds t[ii,jt,cc] = zero(T) #@inbounds t[1:(it-1),jt,cc] = 0.0 - faster unvectorized + end + end + while it < upper + @inbounds t[it,jt,cc] = x[ix,jx,cc,imIdx]; + it+=1;ix+=1; + end + if it <= nImg1 + for ii = it:nImg1 + @inbounds t[ii,jt,cc] = zero(T) #@inbounds t[it:nImg1,jt,cc] = 0.0 - faster unvectorized + end + end + jt+=1;jx+=1; end - if it <= nImg1 - @inbounds t[it:nImg1,jt,cc] = 0.0; + if jt <= nImg2 + @inbounds t[:,jt:nImg2,cc] = 0.0; end - jt+=1;jx+=1; end - if jt <= nImg2 - @inbounds t[:,jt:nImg2,cc] = 0.0; + tin = reshape(t,nImg1*nImg2,cin); + if doDerivative == 0 + BLAS.gemm!('N','T',OneType,tin,K[k],OneType,y); + else + BLAS.gemm!('T','N',OneType,tin,y,OneType,K[k]); end + k+=1; end - t = reshape(t,nImg1*nImg2,cin); - if doDerivative == 0 - BLAS.gemm!('N','T',OneType,t,K[k],OneType,y); - else - BLAS.gemm!('T','N',OneType,t,y,OneType,K[k]); - end - k+=1; end -end -return y; + return y; end From 8aedbeffbb1b24f1149cdd3b01eaa87256ddefbd Mon Sep 17 00:00:00 2001 From: elliotholtham Date: Thu, 8 Feb 2018 17:23:15 -0800 Subject: [PATCH 33/35] Speedup Kernal by removing reshapes and passing directly to BLAS --- src/kernelTypes/convGEMMKernel.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/kernelTypes/convGEMMKernel.jl b/src/kernelTypes/convGEMMKernel.jl index a818b2a..09c68b4 100644 --- a/src/kernelTypes/convGEMMKernel.jl +++ b/src/kernelTypes/convGEMMKernel.jl @@ -130,14 +130,13 @@ function multConv2Dblock(x::Array{T},K::Array{Array{T,2},2}, y::Array{T}, tin::A cin = size(x,3); cout = size(y,2); OneType = one(T); - + t = reshape(tin,nImg1,nImg2,cin); kernelWidth = size(K,1); # y = reshape(y,nImg1*nImg2,cout); # it is supposed to be of this shape... k=1; jt=0;it=0;jt=0;jx=0; for p = 1:2:2*kernelWidth for q = 1:2:2*kernelWidth - t = reshape(tin,nImg1,nImg2,cin); lower = nImg2+shiftT[p+1] # Move outside of the forloop for increased speed upper = nImg1+shiftT[q+1] # Move outside of the forloop for increased speed for cc = 1:cin @@ -170,11 +169,10 @@ function multConv2Dblock(x::Array{T},K::Array{Array{T,2},2}, y::Array{T}, tin::A @inbounds t[:,jt:nImg2,cc] = 0.0; end end - tin = reshape(t,nImg1*nImg2,cin); if doDerivative == 0 - BLAS.gemm!('N','T',OneType,tin,K[k],OneType,y); + BLAS.gemm!('N','T',OneType,reshape(t,nImg1*nImg2,cin),K[k],OneType,y); else - BLAS.gemm!('T','N',OneType,tin,y,OneType,K[k]); + BLAS.gemm!('T','N',OneType,reshape(t,nImg1*nImg2,cin),y,OneType,K[k]); end k+=1; end From 76f4477058dd578ed56680443d1ead823ab2a3d3 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Fri, 9 Feb 2018 10:57:41 -0800 Subject: [PATCH 34/35] added level of abstraction for batchnormlayer vs NN --- src/Meganet.jl | 1 + src/integrators/batchNormNN.jl | 190 +++++++++++++++++++++++++++++++++ src/layers/normLayer.jl | 4 +- src/layers/singleLayer.jl | 4 +- 4 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 src/integrators/batchNormNN.jl diff --git a/src/Meganet.jl b/src/Meganet.jl index 30d1aaf..3e63a76 100644 --- a/src/Meganet.jl +++ b/src/Meganet.jl @@ -22,6 +22,7 @@ include("kernelTypes/convCircKernel.jl"); include("layers/affineScalingLayer.jl") include("layers/normLayer.jl") +include("integrators/batchNormNN.jl") include("layers/doubleSymLayer.jl") include("layers/singleLayer.jl") diff --git a/src/integrators/batchNormNN.jl b/src/integrators/batchNormNN.jl new file mode 100644 index 0000000..7e1e789 --- /dev/null +++ b/src/integrators/batchNormNN.jl @@ -0,0 +1,190 @@ +export batchNormNN,getbatchNormNN,initTheta + +""" +batchNormNN Neural Network block + + Y_k+1 = layer{k}(theta{k},Y_k) +""" +#TODO: Can probably optimize some functions using the knowledge that we only have a norm and an AFS layer. +mutable struct batchNormNN{T,TQ <: Union{Array{T,2},UniformScaling{Int}}} <: AbstractMeganetElement{T} + layers ::Tuple{normLayer{T}, AffineScalingLayer{T}} + outTimes ::Array{Int,1} + Q ::TQ +end + +function getbatchNormNN(layers::Tuple{normLayer{T}, AffineScalingLayer{T}},outTimes=eye(Int,length(layers))[:,end],Q=I) where {T <: Number} + nt = length(layers) + nout = nFeatOut(layers[1]) + + for k=2:nt + if nFeatIn(layers[k]) != nout + error("Dim. of input features of block $k does not match dim. of output features of block $(k-1)"); + end + nout = nFeatOut(layers[k]) + end + return batchNormNN(layers,outTimes,Q); +end + +# ---------- counting thetas, input and output features ----- +function nTheta(this::batchNormNN) + n::Int = 0; + for k=1:length(this.layers) + n += nTheta(this.layers[k]); + end + return n +end +nFeatIn(this::batchNormNN) = nFeatIn(this.layers[1]) +nFeatOut(this::batchNormNN) = nFeatOut(this.layers[end]) + +function nDataOut(this::batchNormNN) + n=0; + for k=1:length(this.layers) + n = n+this.outTimes[k]* nFeatOut(this.layers[k]); + end +end + +function initTheta(this::batchNormNN{T}) where {T <: Number} + theta = zeros(T,0) + for k=1:length(this.layers) + theta = [theta; vec(initTheta(this.layers[k]))] + end + return convert(Array{T},theta) +end + + +# --------- forward problem ---------- +function apply(this::batchNormNN{T},theta::Array{T},Y0::Array{T,2},doDerivative=true) where {T<:Number} + Y::Array{T,2} = copy(Y0) + nex = div(length(Y),nFeatIn(this))::Int + nt = length(this.layers) + + tmp = Array{Any}(nt+1,2) + if doDerivative + tmp[1,1] = Y0 + end + + Ydata::Array{T,2} = zeros(T,0,nex) + cnt = 0 + for i=1:nt + ni = nTheta(this.layers[i])::Int + + Yd::Array{T,2}, Y, tmp[i,2] = apply(this.layers[i],theta[cnt+(1:ni)],Y,doDerivative) + if this.outTimes[i]==1 + Ydata = [Ydata; this.Q*Yd] + end + if doDerivative + tmp[i+1,1] = copy(Y) + end + cnt = cnt + ni + end + + return Ydata,Y,tmp +end + +# -------- Jacobian matvecs -------- +function JYmv(this::batchNormNN{T},dY::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} + nex = div(length(Y),nFeatIn(this)) + nt = length(this.layers) + cnt = 0 + dYdata = zeros(T,0,nex) + for i=1:nt + ni = nTheta(this.layers[i]) + dY = JYmv(this.layers[i],dY,theta[cnt+(1:ni)],tmp[i,1],tmp[i,2])[2] + if this.outTimes[i]==1 + dYdata = [dYdata; this.Q*dY] + end + cnt = cnt+ni + end + return dYdata, dY +end + +function Jmv(this::batchNormNN{T},dtheta::Array{T},dY::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} + nex = div(length(Y),nFeatIn(this)) + nt = length(this.layers); + if isempty(dY) + dY = 0*Y + end + + dYdata = zeros(T,0,nex) + cnt = 0 + for i=1:nt + ni = nTheta(this.layers[i]) + dY = Jmv(this.layers[i],dtheta[cnt+(1:ni)],dY,theta[cnt+(1:ni)], + tmp[i,1],tmp[i,2])[2] + if this.outTimes[i]==1 + dYdata = [dYdata; this.Q*dY] + end + cnt = cnt+ni + end + return dYdata,dY +end + +# -------- Jacobian' matvecs -------- +function JYTmv(this::batchNormNN{T},Wdata::Array{T},W::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} + + nex = div(length(Y),nFeatIn(this)); + if !isempty(Wdata) + Wdata = reshape(Wdata,:,nex); + end + if isempty(W) + W = zero(T) + elseif length(W)>1 + W = reshape(W,:,nex) + end + nt = length(this.layers) + + cnt = 0; cnt2 = 0; + for i=nt:-1:1 + ni = nTheta(this.layers[i]) + if this.outTimes[i]==1 + nn = nFeatOut(this.layers[i]) + W = W + this.Q'*Wdata[end-cnt2-nn+1:end-cnt2,:] + cnt2 = cnt2 + nn + end + W = JYTmv(this.layers[i], W,(T)[],theta[end-cnt-ni+1:end-cnt], + tmp[i,1],tmp[i,2]) + cnt = cnt+ni + end + return W +end + + +function JthetaTmv(this::batchNormNN{T},Wdata::Array{T},W::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} + return JTmv(this,Wdata,W,theta,Y,tmp)[1]; +end + + + +function JTmv(this::batchNormNN,Wdata::Array{T},W::Array{T},theta::Array{T},Y::Array{T},tmp) where {T <: Number} + + nex = div(length(Y),nFeatIn(this)) + + if size(Wdata,1)>0 + Wdata = reshape(Wdata,:,nex) + end + if length(W)==0 + W = zeros(T,nFeatOut(this),nex) + elseif length(W)>1 + W = reshape(W,:,nex) + end + + dtheta = 0*theta + nt = length(this.layers) + + cnt = 0; cnt2 = 0 + for i=nt:-1:1 + if this.outTimes[i]==1 + nn = nFeatOut(this.layers[i]) + W += this.Q'*Wdata[end-cnt2-nn+1:end-cnt2,:] + cnt2 = cnt2 + nn + end + + ni = nTheta(this.layers[i]) + + dmbi,W = JTmv(this.layers[i],W,zeros(T,0),theta[end-cnt-ni+1:end-cnt],tmp[i,1],tmp[i,2]) + dtheta[end-cnt-ni+1:end-cnt] = dmbi + cnt = cnt+ni + end + return vec(dtheta), vec(W) + +end diff --git a/src/layers/normLayer.jl b/src/layers/normLayer.jl index 8d944f8..be6d4f7 100644 --- a/src/layers/normLayer.jl +++ b/src/layers/normLayer.jl @@ -15,7 +15,7 @@ function getBatchNormLayer(TYPE::Type, nData; eps = convert(TYPE,1e-3),isTrainab L = normLayer{TYPE}(nData,3,eps) if isTrainable SL = AffineScalingLayer{TYPE}(nData) - return getNN([L;SL]); + return getbatchNormNN((L,SL)); else return L; end @@ -25,7 +25,7 @@ function getTVNormLayer(TYPE::Type,nData;eps = convert(TYPE,1e-3),isTrainable::B L = normLayer{TYPE}(nData,2,eps) if isTrainable SL = AffineScalingLayer{TYPE}(nData) - return getNN([L;SL]) + return getbatchNormNN((L,SL)) else return L end diff --git a/src/layers/singleLayer.jl b/src/layers/singleLayer.jl index 3b6b24f..875955e 100644 --- a/src/layers/singleLayer.jl +++ b/src/layers/singleLayer.jl @@ -1,6 +1,6 @@ export singleLayer,getSingleLayer -mutable struct singleLayer{T, TK <: AbstractConvKernel{T}, TN <: Union{NN{T}, normLayer{T}}} <: AbstractMeganetElement{T} +mutable struct singleLayer{T, TK <: AbstractConvKernel{T}, TN <: Union{batchNormNN{T}, normLayer{T}}} <: AbstractMeganetElement{T} activation :: Function # activation function K :: TK # transformation type nLayer :: TN # normalization layer @@ -125,7 +125,7 @@ function JTmv(this::singleLayer{T},Zin::Array{T},dummy::Array{T},theta::Array{T} dY = Kop'*reshape(dAZ,:,nex) dtheta = [vec(dth1); vec(dth2); vec(dth3); vec(dth4)] - + return dtheta, dY end From 5f943d045f6412264e5e4f3adfc375e62886c1a9 Mon Sep 17 00:00:00 2001 From: davidbegert Date: Fri, 9 Feb 2018 11:18:46 -0800 Subject: [PATCH 35/35] gave doublesymlayer batchNormNN instead of NN --- src/layers/doubleSymLayer.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/doubleSymLayer.jl b/src/layers/doubleSymLayer.jl index c245ce7..5f48cf3 100644 --- a/src/layers/doubleSymLayer.jl +++ b/src/layers/doubleSymLayer.jl @@ -5,7 +5,7 @@ export DoubleSymLayer,getDoubleSymLayer Y(theta,Y0) = K(th1)'(activation( K(th1)\*Y0 + trafo.Bin\*th2))) + trafo.Bout\*th3 """ -mutable struct DoubleSymLayer{T, TK <: AbstractConvKernel{T}, TN <: Union{NN{T}, normLayer{T}}} <: AbstractMeganetElement{T} +mutable struct DoubleSymLayer{T, TK <: AbstractConvKernel{T}, TN <: Union{batchNormNN{T}, normLayer{T}}} <: AbstractMeganetElement{T} activation :: Function # activation function K :: TK # Kernel model, e.g., convMod nLayer :: TN # normalization layer @@ -205,7 +205,7 @@ function JTmv(this::DoubleSymLayer{T}, Zin::Array{T}, dummy::Array{T}, KopY = Amv(this.K, th1, Y) dth4, dAZ2 = JTmv(this.nLayer,dAZ1,zeros(T,0),th4,KopY,tmp[1]) dth1 = JthetaTmv(this.K,dAZ2,zeros(T,0),Y) - dth1 = dth1 + JthetaTmv(this.K,A,(T)[],Z) + dth1 += JthetaTmv(this.K,A,(T)[],Z) dtheta = [-vec(dth1); -vec(dth2); vec(dth3);-vec(dth4)] dAZ_out = reshape(dAZ2,:,nex)