From 9993506bf454255b4ae1c28d87ac181bf946d2e8 Mon Sep 17 00:00:00 2001 From: Jake Reynolds Date: Wed, 21 Feb 2024 16:29:41 -0600 Subject: [PATCH] notes --- neural-networks/types/bi-gram/index.md | 1 + neural-networks/types/convolution/index.md | 5 + .../types/multi-layer-perceptron/index.md | 23 +- .../types/multi-layer-perceptron/mlp-2.png | Bin 0 -> 36550 bytes neural-networks/types/wavenet/index.md | 196 ++++++++++++++++++ 5 files changed, 214 insertions(+), 11 deletions(-) create mode 100644 neural-networks/types/convolution/index.md create mode 100644 neural-networks/types/multi-layer-perceptron/mlp-2.png create mode 100644 neural-networks/types/wavenet/index.md diff --git a/neural-networks/types/bi-gram/index.md b/neural-networks/types/bi-gram/index.md index bd34143..e8aa59e 100644 --- a/neural-networks/types/bi-gram/index.md +++ b/neural-networks/types/bi-gram/index.md @@ -1,6 +1,7 @@ --- title: bi-gram language model description: bi-gram language model +order: 1000 --- # bi-gram character-level language model diff --git a/neural-networks/types/convolution/index.md b/neural-networks/types/convolution/index.md new file mode 100644 index 0000000..74f4699 --- /dev/null +++ b/neural-networks/types/convolution/index.md @@ -0,0 +1,5 @@ +--- +title: convolution +description: convolution +order: 997 +--- diff --git a/neural-networks/types/multi-layer-perceptron/index.md b/neural-networks/types/multi-layer-perceptron/index.md index aa7d378..15022b1 100644 --- a/neural-networks/types/multi-layer-perceptron/index.md +++ b/neural-networks/types/multi-layer-perceptron/index.md @@ -1,6 +1,7 @@ --- title: multi-layer perceptron description: multi-layer perceptron neural networks +order: 999 --- Multi-layer perceptrons are the beginnings of a neural network. They consist of multiple layers of [neurons](../../neurons), [normalization layers](../../neurons/normalization), and more to create a neural network. It's important to have a solid understanding of what a [neuron](../../neurons) is before diving in here. @@ -295,8 +296,9 @@ Xtest, YTest = build_dataset(test_set) n_embd = 10 n_hidden = 200 -C = torch.randn((vocab_size, n_embd)) model = torch.nn.Sequential( + torch.nn.Embedding(vocab_size, n_embd), + torch.nn.Flatten(), torch.nn.Linear(n_embd * block_size, n_hidden, bias=False), torch.nn.BatchNorm1d(n_hidden), torch.nn.Tanh(), torch.nn.Linear(n_hidden, n_hidden, bias=False), torch.nn.BatchNorm1d(n_hidden), torch.nn.Tanh(), torch.nn.Linear(n_hidden, n_hidden, bias=False), torch.nn.BatchNorm1d(n_hidden), torch.nn.Tanh(), @@ -322,9 +324,7 @@ for i in range(max_steps): ix = torch.randint(0, Xtr.shape[0], (batch_size, )) Xb, Yb = Xtr[ix], Ytr[ix] # forward pass - emb = C[Xb] - x = emb.view(emb.shape[0], -1) - x = model(x) + x = model(Xb) loss = torch.nn.functional.cross_entropy(x, Yb) for p in model.parameters(): @@ -360,8 +360,7 @@ def split_loss(split): 'val': (Xval, Yval), 'test': (Xtest, Ytest), }[split] - emb = C[x] - logits = model(emb.view(emb.shape[0], -1)) + logits = model(x) loss = torch.nn.functional.cross_entropy(logits, y) print(split, loss.item()) @@ -378,11 +377,8 @@ for _ in range(num_examples): CONTEXT = [0] * block_size out = [] while True: - emb = C[CONTEXT] - h = emb.view(1, -1) - x = model(h) - - probs = torch.softmax(x, 1) + logits = model(torch.tensor([CONTEXT])) + probs = torch.softmax(logits, 1) ix = torch.multinomial(probs, num_samples=1, replacement=True).item() if ix == 0: break @@ -403,3 +399,8 @@ for _ in range(num_examples): ``` === + +### Critiques + +One critique of MLPs is that the first step in the network crushes all of the data right off the bat, since all of the input embeddings are fed into a single layer. It might be better to work on smaller groupings of characters in multiple layers, so that they may more intelligently come up with relationships. More on that in [wavenet](../wavenet) An example of that is at the beginning of this article, and another example of that crushing can be seen below. +![multi-layer perceptron](./mlp-2.png) diff --git a/neural-networks/types/multi-layer-perceptron/mlp-2.png b/neural-networks/types/multi-layer-perceptron/mlp-2.png new file mode 100644 index 0000000000000000000000000000000000000000..03573c16e96a377baa6f2ac03bf5a35fd38733ae GIT binary patch literal 36550 zcmd3O1y>ziwk;5x;7)LNcXxMpcXxMpm*5Tw7ThfmoZ!LT-QDtvulwHa@!lVJj8WrM zoo%&k?YZV$hbTpP33yl>SP&2pcqvIyWe^b1PT(?yh6KLRS&PpF9#pMFL=>e&L4MBA@5V-PAulVzom3E*Eq^P%Y>edp(B(Vc&_{OYPIN{b|joE=cyfOh(` zYGl9@dDh`(LV@4p+eP4rNi1H-ItrYU7ECkH|e+(UzaMp%J>19zan zg#%n5AfJ-ML7;$76yOpq0Q;X*(9VKS|9fxxkx@uRL`n+yR55WjGqZQGbZ|Z92#N=) znzvHbbk&rTbOb4cyw9xf&6A+S%H>aC`C*|C56oxc~T? zftc{0EUq?u#F}!7gdz^kW`ykYjP#7e{IG+eVU{Cl_ zuaU8Xn=2nN@kc}d`}eQ+H1o9jPfPYL|LzvhL57bM1}1t&hX1V_D9ZcsEw`eTr-8X7tpPD+u5B$1H7!9j%tksMvXliR$WN<4!D2??R0!GZ&Y1<~?Y z!GWZY90`KJlZ0vmn)1I3pyf4w6#M@tt@k&Q=lyA>O2*;S&4E0ffv0Dy${0l-86K#T zr__+VGB*>L@A03}G#aPvUoaHu%vVKq%s!8oMyL5(NBSP;t3vtweuh|ik2$s(+tn@W z^>{8cj{JVltl^)rjUnXw`j(K8!VYZNh_Anf2|k>!Ih`)nc{(|)cYg0nr_~Bx@A5HO zsL{LH)+>h(_GdHeAuVZlReI$kAsxODCKsJ+k)BF8lBOq6bHfaDBI;<|MGC5Arew0nNFt@ zUM82leZJNat+CebmP9UxjC5M;)kZISgFZ=HsaI(#g{^Go zS8Dt)-stwTT&gc!I`zV0r=)~Fn8;PxyL@=iuv$vq=Yd#PL+2NCo|G`cP3DdJ=%$6GE4>+U8Q}$>qE&xYBH= zRBI^2w)>3%n#N2$H9P=$lH|7dyA3@DjiNgFhTj7o(9aNHeo(1Y%2fMX#G=uZt~6nw zARz~}>2$+|K0$rAD3{Oc1C}yDLW>ay4Gum#+Z~6`y=^~f-9zlp0-+GYb0qqqSt`;A z4`4-EYbKo-PZx<9#&p{*)mQZQ=Em5swHrNM?_sgqVm;rVRdTbg5?$TjtNXeJ*Ank= zn~8nTUNIdzTXE-d=R2P+v z!(mfa?*kfv7Z4CYAN+e5F8^dQ{Q88gqmNFls;IudU=UE9N`y#u9g(tUND~{ChM$s< zfVc>|W1#odTkvl!`d~o#Eh+1}E+%4m{lK7fvKgn#zHC*MZ5#%@u4bK4rWW@5{ffhA z)=t9fSt3@Za*1qE9-j}6Ip#%P+Cd~Ti7b8K^w1I`6OEbv+3?ry=*4=Q)mEnN=VO{i z>p#-)W7oW79=e)N{`X536`j7nqqD1B2a}n0xW+=D5Gxo5JB30Z>tGXjz-#cQ@Xw++R8DGxtgB;JPLI9d2HL=f;=h=!sQ zVGDs>JUlGPrfPGF9Z@%97V`(|s;Ce!P*h{Hi!N`qT|W2JZLa-|x-GJQ-yRIA!rHy> ztV=z;zI{dKqTy{GeM*QZ%4|J<)R$;5|E-LN(roUf8aT_$L>`+~L(-ql=%Id_hBstF z3*XPFJM4xGSoU_egQVmeU!2JeryqF-KW`8EJmtd$W_WXk<{@aPa>1s0=0S(%p=hYo zL2Q}w1Oo8?RBC7>EJg3M5$6?mxSwfP?foK<&V!n%p4+t{&-=7%B-8xt*)7SDI>hVd zua}dKZY(Josa)HhV3k&(hL)i5*6-p*5V z{pXl+S*%J@T3$!HW&`Zrx0aW$YvJ%;<`T(wCilj(*ebJh%)*AR8=Oty@=k&@POl|< z?@kufaGM9cuu@NYWHRWlO!o>^atCq}U$4i*kO~HANugUmgEfvS6(>}QyI5bDlADz$ zsg+C{f3};*;lk0f!-4;6Wr}CX+EO-+R3J`J$9d%c{$vryAB^++z_M{zwAhI0E(a}C z99ER1^xHI)&Q%Fw$^G-m4;wC*{W(i}#h1+x#Ku@`7HihsvZTH_%SF}FQX`Hyv^a5$ z`1v{$iMa50)-2RHOQASe3;ZautRdySG9tY=T9o1uE2nsIIQwm0l2TM1a%ZC{%Pr|s zm#l`FZz)JV4!_~D{?15hS(zpcnaI?@I`wfPz!O-Tez$78lq`NyyvwO#i^IsUWK}3n zDw_XxQ#2|Zty3Hy9^KD*bFR?n)Tz=p>1+XoFJ`Tdcfn$V%7|G%oIZ|FDSu>W0GrKM(h>tr~r~L{T z!GqGN`%ZI1>KW-66W?UiJ-fW`l%-jzsihw@-)4NX#9icZw5X019uzawJsf@;)O)kl z2XEJ!a=DfS{KlI0RnKug+Sc01sM1Wa{fwSRPpmc2KDKBu$XNJ1=cdRR^xhN?O2ew> zyG58lIORS3zM0em~bq&3lvy2u+Jj5oGar^deOwzDAUpaw;R zx7>u&VDM-u0a%zTln{ZxiUN&)2hh zcqPOzEN~-y-@?eA7kI&kMXEc$^<(^UDT*RjuZ%(-G(Gmgft0))-zBNS{g5}){9dPb6Ig%Xx?*ha-LuNRNINYO<>*m; zetXr=oPtGe{j@dSXflMrhB+qxE_@yqi20@JT#b|}?%%ML5QzY^?sD6NnCRa~);{v1 z_9|7X zeZauBlD57&TjB`!oqGy9mBt1O1_OyQNcdiq(5?Jn4Ty9@uPJbPJ%q|&LcPL3?OMA1fo zfiy0v1OqF~DUWifO0jj_beAwUy8@Vtat3)*}N5+~B@ zHEC#$)$ESqrIG@5cd8mvWJDlkr)Q#MAmJuHbZwZ&@!40R-|AvtObnku&y^Ud!Uz*Z zf%eIeIi<*r_3J^aYq14;a{`L6( zO)6moTs3C*(ddf!sAx4*MNiHnH^(TNfX_2NShCzkOkrt^Y> zVu~07Uc^@>6G&kZkt8k`b6?sBM+-+|JSPJFO}=^sxT6OYw32n zsMO(cktZ%KvC?JJqF!e#Hn6i;XOfklpYQ%~uIH8yp!Ybm;&%Cb{w{G?MLmn*M4_El zUj=E|4zHUs_w!Y*xYT@ae&b)zcfNh0Fxmj9Qe6aG9ya-fDlL{cv_jaF0B}$nB0`ZY z`4x2JpWvWFM3=x}v9sJ{L#bQ>JyRwZgU91Rs8#=UQfwrdOh{Tv%Ns`OaAzk3aF~en z+FhsgT)3N2G9?SCX_C;0h)GD)H0z89rs~aPblZF(fMZs}=sJPWknUrb!bxqBK(``; z@!n#gp@kv}ydsfE#ESy{jL>{W)@1Q|v$A#?wVE>7{tMubxSj3om10yMrs4>&i0O5> z4**CV-9i7d@5YPwS0;l2zv=IEbO4ugq^b5n1d$REx_C+j%b~!4oyYY?p;Anh%3!9_ z>0sEcG^_PgYc~o1sZjevKtrwnjoItk6j&$VJA-kb;6sWWxB3p}zN4<=-j-=sF#`y3 z_jk4imGoK;A%n73w=dUd215eimlgonw$*6|vP?cVy3SIgHZ%gTdYlAcL`EKoZpKobN^LCU94`a5pl+2+OUgT9G~BQ-FhM;%0?MV* zrEt+VW9m3MHoT!$2v-$y_{?S+kOl?@F3FO(LWCpGkzpC8hopsso88bQvY9Ba7Lr=^ zrc9rsT`tbf;NMv?G?$p7>RB+EqAXy*iY{q|CdV7+w!}7%Ep%!*BiGls^h9-HGyMY7n%JqOw?8=q78SrzPZ zl4>!*V~Gl;YbdPzA@DW8JCCtrh>U^*?T`ue7Mi0mG2ZiV`4QObM7Ne|D_Tw?#n)OS zm!`EBlX6oE_1PG-m+ksLhc*3k0?3C7GPVW9=YQWiB!iC|^)_0g9kS+C@?^KgMPHHF z?uCH&vG6QbZWFa`uht2IfD1yS3S93BsyB<388s>SO9#eo3tm6-I{L){q|F68L1snN zXxVl161l(fP?{p(YAlDJEEIl6A>G&O_WSba0_%8(qgY8kRZU~5mgF>1 zuJB2vd^!@u?m@iLWx6ry$}#}8HsunX2;^x-Hl|kUV_fT56`maJS8N*2)=l^J8u z>yW+~2*I;ur=9gSIMqssRX-5k;?{1`6an~JhYfJ;7d(!d)i!>x8Ft+s1i3Uqri*K$ zXLeR=P|=wdQ9qKyE(UFoMOi%4ezm%rgHJztWZGspLU&#iDY{f_HXW4HF=pcVroTP) zPSX z4dd*H4AZEL0kgVykj$M(;97?X_o9=C$qc=YJyzE43uypmN{-2DsCajk6UlIHF0y1k zzq#fNqfTBzp&#^);$Uo`hwLPkJSSA?RRSvb-#MoXHxK_SziQWnoUIIxeFp7DOJ1eN z*CxQ848m}rbI6k_v|R-@C#6=fwyf2D@+%Y$bG+RASz>Ck!n)r*#;cGKT*eifwADh& z=cg0{jTF8F$ zt|Hj|H13NBylzu)%D!pl(v=b9Tp()_`X4-bZbgskwxy6cx4_>-{1w17Xwo~)d) z-~7cruRH~B?%qObWT?z)+%LncYkG|G$|#qGJXVc)V!!mckWt^rxSjZ- z>EX++>c}+rKBRs%J zsE|pHcgSYD>~Q}+nVpuuBoRwT%H@-=he-%@-rG(fBukz6)!>532JTw!C8CY#!a>Q! zY+exG79xq32OreqVIWc%ujJ_%5e_lDZKE?zeMX^#BH(|IWaSGV&Vf4cvwnD5n>A02 zmrR6>OPSQ;{7HCB`grv_0e?q=Yd54z?*v`F#@Uj^*+&wo$SLgRbT zodC=##u!&;T>3EbNck$1j&D5C#fl-<_b=X90DSl>P1*w7&bnN9dK{iW2s!Lzyc}(C z)idRD1DlwrM8Rf>ax3aRw1UKXqno?<6G5jNG@E&2QJR4gC%YvNtXTn(Dp3UC<;Hih z!=^{O)wfHqS6^i=QAmXeR%ULL0KCj-M0LOTUVE155tUD&A6h+1+B<*HRvTc z&~?r~^V#M6{_o&T794}Q2Z)T#BQIC%Dz$D}1mOa|cx)mJ=aI8O6foj8v=hWeaEnfD z<9vi)5m(pb;E;>9_WCU*qWWZ@X;ye;;)T$Cc0$~Wr|viOE%mC{%Nifimy~CXS&2G` zH$A`1)(p5^jN9|KMJ1O;Ex0b`mPp@Or9}$2ay^nbS9&RhsI3h@nf+LXrjzZ ze!!;*?CsWpO1z11ZhhXYp`NfY)1JEgL!)k@^?j2!l-spm;*t0804sjCH;cIq5yQQJ z?6)w2#HQdS8Cn0m4G%7T3wgPs2C+^LE)@};<{D0uh}Jj4qI7sH{u+`P{*dSmpk zlQoVjGiE#k4V>#rWeh`Ap#ck5U&Qkx}UZ~w?D=j2YUO4yuQX;knv_W0&_Y|LHl zSag}wY(0)yrF(Y zQSjxfU;~&OW6#ZxMGqH&yZk#M5?R zy+tZACg~$xi%k#L#Wb6RY(zsui2{E9uXDa=%%!8Di1V?+5r5bF=Jhj`XcXjzVD^Z( zVswJD)o!0&Fi(%cE~m>9uFmFrrn^P?UKfn;>4e;i*oN)ZTcoHa5Pbp$Gkp#&hw`2@ z=k0J_l^+X*y1k{0$gHYjL4JJW_q+IfJZ>9-)_AdZ-%X>$>;gk`Kzw?OoJb_+$GW#~H1XyZoX*|F>8ATq4|?tc*#0`P5w0KT26>@2vS6pfcQl<02;W zO-2i?inOv#SOJj+eEii8-q16nSVK1AzTuw24^L?7o!o>Fi`-RX@yYnZ@xKazynS^;xd(w zy*9Ej;DSUNWzVNX2=1k zz5Ia<4-VV2%b%1e6zDqkXs}_x@?v{0rNR#4qjwM7GZzN5u?pW7op|NsNsKsF(C8qt;9rSkUXzB)E|6I>( zfUx6yB>7hxle`BwHOIHOyzn3+Aa^qST$HGF__^=PonjK#Q?fVv3{5LVyhF$aLRH+eJoNgKk_*2K`?L`2q0^-H(0WR1E^&nv_jx)TX5-)YRT!_|vui1@$A6sk3R+!xb$LKM>zx$%8ygG(SHYUIQ6v_lnpZWKL912t zcvEWuc(wrE6N<&-!Eorj61UY#@uEUkhA?RI{`v&lBe}d7w8MkS=Y=1w$GSXA86aYTGUjGlm=~9bN^&_^(ZUuxQbEU$ zXUReJm!DT}@9944;Viw}w8RP!HKzforZ`EgbUH%4IH66PK<}(&!+#bKU8>L4n{+BQ z>xzP-t2pD(Vm^3*xGEw$K$Tqn@bs1WmnU+0ngDlmK=^q+>p|5|-^W!iEGc=~dZDJe zM(U?^Nucbt{Y**J{oBJvWoxNShPu;J_#UtCW3Bk81dZyqL~)EHcBgGoIzmFh0l7@( zQt@al60+}mqiHf>Ny;;j2)N&BBx11&ov_JezZU%YrX&)^X1}UQVpd!urDao6z*-O1 zM5|SASy)&|%?oq|2VTv(1h=Wv>&Ci7wFyv8g=`jTa=mL_EH2i&j=4;v@W84~hT@Nr zcO;~>0F@#?oy9^vj#iBhpc`;AXmfN9xZ@KD?FeJYN=5Yr$=fwbf3WCA13Sn{mM;mXFf;|{Hj-W zvHt4r4uT5KbTk#_{r&y2Z1!ZfT;T_8@dx>Votr(VT5l?43Xm&W44s@Isl>q6`J_z$ zLo}XFaR0YxG}vuY1l2mN3ZTrX=_-Ggrtg;8$%mZG;|~E8sd}@qvi+5I*Q4MMtAdTA z3KRD#0xPLi&f&0Lf_F{4GqCBK32f6+TDCQ=7&e=Ea2*ROl@ie`Rx30}tp-!I`3jJwY%nT+gb^3mmO^|fxs`!yKoWp@FjTn2IdNb z9{%BIl^cb*CUueQ4=zR|stRiuOhv@Q_dVC1e-g_CpLwmxCPSOBPb9@O^Cy~v{G=C| zXbvVSVvl;Ywi*?J`m(8Ca0HthY+M=!;!r*D(Q*^baI*5#wJ+i1!IeutmneSy?v^tC zx#>2MLj4RrQTyqG6MQ8MX^(<{Xcx93J z9Z?aRyPLlckzHgZ$VSWCjL%bZ;fnHISK@PKzsq)eMN%dAM3)=qJ&1x%3=TeaycHDY zx2L`q>dycW?%MzX&EqjH{dwcj*&0$^D%ZH2d{7GzXVq0yc^W3AeF0UBmX^p37L)2O z^1L4Non9_zl$szwqyi{BfiW=QkZmAj(y59!A#TTWN;jsqS9Q#cyT9RhJoV$#Gyr`< z6(0_hUUgpg^2~7DfROnL^mdI7Nl5_%i0P9nbr>ixv9-q&L(tqN1a#F5aWE#$m$@5N;Tf!lNC2Ie+-cxCkV(UD67E=hkksPcgZ&g@w z5ifN@do;^7ECU@G8U{v5B)Q$}#TNZ4*&p9|F6?Sl8Xw=x+vn`%(e6Q4*XJ*xbLj{+ z4Gc}{cqYrx`b`+o1t((MYL#xAG8L6NBP#_4&+ceGy1L|eel_!2?YC~-PL#B2jyJ|P zo3~R+P6qnNj%8btj$Y9Zh7tu&&u9Q7+OC1~j=C8P;BjCAPOl zaJihnJw8W^7|^S`%RU=DFC1^h1ZZ+)Dl0x;jknFa!O9MpE!~;1qfaJnp?(>C&yxCS zuz#5xoK=Dhz$LTHx|!$GhRM09azWw`R^E7vOAG$0Fk#2tKLQ56N2t5eD_52qSpvy3 z3Pi^Q@Z5c{2{nQPQ%DFg@>0~MB!r4rk~mXbmGymuw+I{pPt?XVXSLG&GVJpwah<8K0&Mz6o?_;c|R-nf2BN zs5E)i9TJY@9y~6L06mY`wUVsb6R1QMl=Z61_O`?*e?4#Up7B`%C zx^=XkhWzlz;g3Ldv~?Pg`h^`B@UuKfCYBut_7(g39M3p2O zz@4H%WZaRoJhh-g*5KRv4fZMhE9#060Z7~KHQLCYeV8y@8mWecfv>X}tfa3>%2hj! zZZ#zb=3^odm^i5UK13hmSM&8B*8nVnMt~QO;qsdb82I8oro#Mzv|orA_byeJyavEm zi!NnBOinfd#Qy4H`kmXhNj^`zd4fLuZw~6s6SR#h2trAd*~1`!^=N#@hmQL!@^|Lm<+&Tx9do4yS7MC#PtqtwT8K3L(gzr) zL*D#N1IE19TR(dOy2QkYFlW3q5Y+F_f$hse*@JCBV9;UKeltkVAC03I?NO_fmz)Hw z-$jBT_knDZ$14%rj`p?pOdh9rU~f3>Y3el2hv{s;C!${7HRmTcxnqas1KB%qg7nFJ z2t&nPnV-zO-AoaqiC`x~sIz0T);YQ&(ln!Kpgr-y%<9tz8`A^4^?w>D8Dr1*LI_@K z{%c}-`974r18>_G2aC=QdUFdG9t*mqj}=F_IsFm4WpcOS?F=WQU^nnbV&U{_r777z zF?NK#|0ME!bzvSn>adB5cOPdXuO#lt==V7hX!Pk$E(BhV69II++wq<1YH0QkFtvCn z?vHO5yULG=vmPCNcBhFeW3&@%0tpQs_Qb^0+hzs2AMLJ?FcMp$wZFX}cv?^p)m@&> zL)yOb!cr_*Op)4VLF~PVHxD$v{fuucm6YIocKZ1ZPXRj;2!IXXP!nmf?z;7 zRT%#D5vKNa!%ns7_iXly_HK!^dUySovn4AEO|mZ^D;6XutIhgb`O1%TNy794UK9ad zo{#G<$)nv#R;n5qW-kLzUXI?uahym+85Qw zf=PuL|Ip*?cS*UO1eYNwiU|%LR6>}@>?NH#k@Mx?Twhw+xkZKqcJi!()jw#x%J8qz z+@rE&Ny!5XKmoOPHRydPVoSv{NGo{j;BH@xioIEb8!CFb_y$)NOTx9`e_Q zcuvAMA9GRJC!`iX7AH^xX(*t7hECP>vG9jpW$67ioRrICZ3fHf6aJXtAW{-SScZ|Q z%#6#8l$bx%-{r;dTp-teZi;NKVTAtE&ZR53DZ*qp-KNo_h;64YmK?0QFyNrGkB_jP ztOq#|vTjq+clyn_l6rj@P5U?yg-Xk;aD%MZ&G|iFwXI)$KIp<2yaqm2d`ReF?4WeC zo5or*Itu5+9X+zzI^jhn7~r~Jf~qy#jgOAlYV@FKBpF~QVF|O0fAUpqYdIG?1%;Ot zEwb`bbHjS{2`ACmxo7W_j6(pswhehWiJPW|-|KKycv*-Wd!~_SSTEWQ>@Qbz4S##Z z{q$HSd3mvDqVGp$7^LA$b|@t5s1Bot#1V#KRLNZ|WFQdj^SS%T9WxDY5nwJ(gIi32 z1} z(ee;rI9~TmBB{-OY=)Di5Cdyf&=|?mpoH#ma#kvw)ou=~Mxfh>WTH>~-!_lxLu-e% z{vmmj`nv9{P!Xyx^U_6J#`|x3S&IqSMBBjIeH)#$zk1)R$ZUJ{G(QZ-$D~4h1b+&h zp<4yhJ18^Z?Au&Me{J^b+;T=5Jj)6MEZ&Ma=_=4|ApR8;aDbNzlAHC{A^xVu8!*;i z?h?o9KH%l?A~p$8g_Q!;wZej(24tK?fcQ2ZnP?y{kvm8CH859AlH{8l#La%o>L3N8 zIjhF`oKB07t5HaAlC9-&ol$t|pm_t{aRK~31fXWrY}M@*P|yq05yzJil4Y->JUnNY zb=qHaKS$kmcFsc9{5~O9i)m%LY%h8FcvXrfO(RDs&{FZ@1OX9~JJ)+-lo~ZTTAZQb z;l@DBqGT>N+w;SPgE~%>sQ`FUkqLKC{CVigk6(i>$u&Jry2-es@DtdqSo zX^gH-ncgy?>1MQP!x8|_b=~J;YdkY!NIEPmEU20jkI(B`RFB{PRrAe!I1#i|I!$>& zsnO^ecy~9Yy#*Mg$T?%~u8}U@l|kgN1Ms+Mb>b4)JmK5{%<6&LbKA6pOE(7a58azi zr`ZtDofq5PXuKcJ;{buO2ngoWY_d+nX$P30xG@qTj(`5eJ)cDqDC zM9txL8|?~o+L{p+9dh+{%QeKLj}|M6>1%-IKahXbdvluN?+)^{y5VjqAkI}GQUn5C z(WsQc*E&2T&X${g&aVMPg;JF!lkIvZw%^OW+BE_JKQ179ODHNTR=ooHet$GNjcPjp zhJS^Rg_$64-f(XGScj5>V|T71M3`7>IMaa&VOW9#s|geYC;}$OO+hki(zDjdR9-J< zAgVNyeKR4JWiFFSh5U~9LvDumr~r8yDuf2;BLo4~=lMcrPLX}%Ev-5X1W1CuDR_dd z+pY{30(Db{Dsa#aqF$Hjt*tFod*1;1gXCo9@PXj#3KU1 zfvAK)G)Qz>?fZZ`_R{P1t7JdII(X7&dBD;NhK|} ze&NW%Yc{i)boLl6HZ)kXAb@~rioA~}v`lYh&R9p|IqJ?Ezw!95Kg6KRP*w3lfT}yn zuCK34rctLYOA4jt+RXbqS~3Du`cYP&j*_)jqp=v{Ec#}}X=!eO?heX7NHcF`bLyf~WxoeHbtP{Vdl9%7yo+C&HOMTj}aOt3ynB1uRYN1r_t|l!WR1#B`(nir~75K zJ>Ps?gvF;~ZXRX9>`4<-EhFrwAH-xXJ(RmXkNF=kf5!BxP>bFwrID#h7h<2{#&$%R z3{Wa7OswUk5b_9=^fU6`JXQLirsmZwm%uKsq&WF6uv?wU9C^Sk=a7bJwZ}^&W0?wGrvDvAF1m9_7y@QT!sL_`?(u8`X!YLN zlN$(KmEr{`qZ_Ds07H7CU6bF#-g(p*Iy9Ld=%G`aN-VWNCm@J{-xV9B?tZfm`e~ya zGr#IHKBS)wbZuy#B?n^h?V70JXRa~wazqFQa(S`JPaIXuZ*^^+$ln&QJSB&S$chuh zenA@^>hvB?#?@yCe#8r$Ps4n2!vR^aSL!Hg{vyO6bTHpL*CR_o@IrQZMtWSJv=Ud= zoIQxR*od)P?Z7ODE&jx6m>DIp_co&ZrtY?y0VZ)zu;4b50bCyn^@k#bo{ zv%nRNpuIY*L-Y>tOaW>Q7By6{n@ehW&+pRcy+f-1@jP0O_`;2t)$v z2>TGjmQFVH`Co>6j*7BPboDv0%ED)Kg%;wOQUz>4RIUXCYr;BLe0Bo<)YCSM?_70a z0!!_bmU~$@D7o3`Dt07mFydqss~w5}_Sqc4Bvlm4WDY*13b+3&Z=2OHrNsDZ?z;-X z_Frv=H>7O68)R<41CHs>f5_XFOZTdz^V=qIC6_PZl13ynF_ZhOjt~~PqFXJ^ifM8A zeV6F>`VVZ(Y74|H5BA8(GMzlzM(WbQD^>3W!@B@48z!PLq{ov>R(O+o zYdHBg8*Gv5gSyfr-{6jH4r&-PJxmNThgVtF+ z`meNYY-dwOvsVhaGmW@UksMchyqsqd{jx(X*Qm?3S|m2)_zM7GJp@;bI>5$lMX2A( z1BpdrV_fUZN#0^_r7h?rC-YFG=k(=#S#P$eST2U$xI}-Y)OG2`dN~TUY^TSw-kC4X zFWtY!@3z#UF-`-_cn=y7&0zH+6i@J*EUQj3AAx>d8#{#NsW6h9-Mr`@t1W`{Fh{`Ar%LJ5H^r?#-zR2` zY2Ck%$ih{(o;hbL1Pb>D8lMq6#kn(Z6MV&mxJgm7BnlG?-8#wLnIhvMyPKXwc8JPa2!^9MSu$vMJ z+N<(h*yfkDb`XG4z z-~*mDROMX&k-cap-!ZCi=1Az6vHf>oCQ}~OHzMb^Mv^i-nM1C+zPgfo&Q9S#m3+kFz!V33m4e?C(;RIc{zkD2+HVH8I!*`x`VL^?};Z zpG9s1naBKYAIOMiT;VulVCHHfv^~HLuZFl1Uk=o<=8LG9AEz;#gx!S#@g_W`nuD91 zF)_^^z?>%i;Lc`tb0|`NqC;-=(5EEPrMXP7WC>gM?-ACzEps@2Rd)Z8W*23o7S7pK zzjNK$V*iy|eL6Nw4tcZ32MBLev&O)QLYNhC+iXfu~N+xs3ny(E^ZAt zQswS7)mRkBQ1Z|ZozOrHon8v#|C(t3Dn;OsZEzz6x*IBBJ%-6XDb0)oF8dKNjK*vp zZs6q!0YR=da4DDnHB-1KGo)8w=wN#&mnCscU{JxqJ^MnDQ-sJRnVWZ_GX>6Z^SKR+ z>yRXWgrm6M%()T7N+0*+b|-HM8iC9}mPn`<^n;T?y4|c@i55{8xMx7EM+0sC`cm!r zDIe-BR(a`~=&;G`P_Kr~JH4cOP&d%h1bUp884BH{%hhx8t2A1kgLBr9NfClzae(%a z?pRN7VD5)4C))g3$OOwEb4-ijT(98VoC8_yKg6F&#*Dd+UUbyS{?(rl2_9CxccNQ} z@@owU5T!_#l!J@OM@$Vi#{Do0GVi?!B6V;h@_tJH^HcGt9!s?})b3*1x9i+3C+y)7MT9(20#6L!*)E>>PtB zYnEdZfZOz})}^UB>(-^SH{2jH$k(u?FxOL}-5$Kw_w|IqJclTDD&+n;x9juxl$eqh zp>m)ccd1@l16^LdvG0}>?Mh`(SDF*CVwNCmtD!r+7GwO3nqVoAU3o5Z&~q8Rm)1~9VuJF>^4iI z2Bp$TA6?>l#ytpn-!Ji<=}OmeDHNH>nKIDtR)5M8<&ZSpjNs41!itpdui3XQMrFme zv^0J6Vmt;{=yRbb$_mMFYZUgHwU+;C_)@o$ux6>7>{j08GxX0iBCRcPgyFTP23xhS^0Te^np zXQR)U(s6H*3|m6pX&{XUf0k0p*+bcQgA+#tAar5X_;DHT5z4OhU|FoM$AchlD35E( zt;2UjjBjKJc5J)VweI+uD3Oi}gS)RS#s%Jj4)0Q&GlnaU?oWqD4tD{+OugnsJ9|Hi zg+F*GRw|U%T(t^Xv14g^e^o~t{;HOm8>Rk>U@Mvnj^uFL-tizJVlk{LMzASIt$J)K zu2ssWp_X|;Pa9F+jXaeRj3a+lH16mwt^P!%8-Cu^VBdZ8d~R*F^?DP5>V_L8WNS+y z8{IxKLLuLHr3nu|J``$?qmiB!Uh$oVODxG(`??50tilT_}Inls)Hor$tmSDh+~)C2^*)+ z^PXE(NbT=4kgEC1U<)DyM&d6~kDVC#uV;CU&#T^^vw2DFsQYUTsp%OXOvEUg`PmcU zSSAcqSZ|00A~+1o2XhK32_qIYQ#35K>}K@p^zh@6^N=DmbWQ5pS1E7yxSW+QlU{Oo zp0$MsxVQ9wXrz2~iI5#)n)gPToFyn@Qb$zUgNG5)>WqHkaEfE}*an&9T7To|z&;+g zw#75IA0Ke*nUhzhQfkDucV<{vL*EUtc8!InEBL0U=t{4~#W*kg+2{8HXO;a@pL zXuwx_?ZIwN!%(IEg@gkU$->3uxXf(JoO(Igg}KTKK_6xSro9te!d_qWidGCYntT!( zpqVh(Ln!sF_WLmOGm3KO$cF;XU!VYHNGB4~@5U;!<)I!FtKmo^bs~wt=LM?J)O=wv zY%sZxL7+pL>;h?O9)%r0rP&SSx66!A!#1rO+J`gNS3@w@megP2Z-`8wAV~}CH|_Oh zAG+oTeFq}UA+m!t21pI;z{IJcug;zoV!^d5LbG1gOea&$d+xQx58~XD^c`ZndSqAO za4U+OD(<@q`5S7Y?%#3%|3d_wTi9xLz^@`@SJ6n9^Ty?2(&W#Qo+ZBGZ?lOQS5koG zqZJi^%MCd=M#u}H>5MYJ?Mt21XCJWyTDilthw`IobdzcMq_TNQJ!gY>F?|GmCo7iZ zNCM4DH;1Iqmv40Qr&>HuI*|){oSz@6lVjuZSY$lv{K`pm%gYxvkh|MltwnHdSdCB0 z61U6mbJ}}NIR=rh(qBDUo2PP_AG&h1&@%E2YEsK+$R44;3>WAN;-b56Posa4rnGuP zUz(rjywD4Gb#A|Whl&}IS+W`+Wxejj1z)`C?hgj_M8OWqJnI>am>`~3jmXb3+ixM4nV8v!Xa4pZr{7G*A zmfU6n?$muVYBc_UuEl^Ss{M(&mm)I(efsXRdK~QhuaQKX*89hH3p7pXBgK!O6hQwi z@$1=+u`|#f9(YQ=VIqNDVcrg4OcSE$#u)6QYT7Dfx+wq35#i4yV7^GoC=P(Dq_=t#dZu6_VS`h$!dJZw! zhUH@?lPi`LJ(XYfryNYm^0~J;u9K0f%4mGQaXJQ zx48OO#^n!k5sKW7O2wr=&xJQQ@DV+IivP~E7z+q55NrrGS>4rM^?~00Sj4ZiQY+6- z$nQYolG@`$^TpyI;1f&{uGb`!0_m2V>r^Chvpg!J$)+>fLAL5yXVT9dW~v$_e0+ZN z7R{{Ik_0wkQ6|5d_rTWr>U*QT2nb4dx0IxGr-YPr zcXxM#(%p@8N_Ur(NP~2Dch}wg?*G2HFH}C_8E1xb_U!YlwZ3aw-+x@Ae|_L6w}_{X zFLI^bzY1Zwe}(S(lnR`}tZ1%m#u~rsMbuM5N!N^t$I8XSJ%mgF#HLVrEeH+4H8tW8C zznMy(kZxB|3Gy^W3d!di8AH925TJ5$Ker8^ zRFiV(6-n`71el=WUJ#zzvCk}X2eg`9Bqw!>=(|x~h<1G7wR=2fy&Wr_!t)prsgUQ@ z{Nlb4cE|Wg;;h3T;ST+2+zh9@#jWuuhKT{#fi2b!GtwF#cSGIY^DzE9#XT{Aw>l)I zOr>l)7g-}G;BJt)+e;ihTidsZ?&s%qzS4=fzuRwiazoQ-#Mm*dG2ORnphd4a@X?=? zd~AfRrJoRjrsYI)2)14(V!f_x%Jozg31q8(OK zunNgeNeYCNg|m*r*7P`eAy!W$S^{{-^u2wcg5GH$7<@DN2$pxuPGRo_2BMxd4KcdS z&E(BehH{ZBoHARQ!}rE6w2P3Odxwt5uu-%jzs^6K{XcJ;X? z2<*pw?D^JO@6!3i1wC-yU8M$PlDAM&efc+~c52ga1`BSaAE0?nC0TFId8H~(83@nE zj@u7R4(#|fj4 zYu@L5w`JgLLiKv=v0Jm+A^JC!;mo)*@F{*s4c5Z+n^gTGs^qwOt(ZE87Us?GZi539 zo=w$QTS?lHOw1Rv-wL;0D#?hGbYWM#ySwB_dE_i&nxqbP=!4j+e)$bnF}-wMA8zZZ z=iwWAj#d5Qtkh1wxTXbUQ~_@*3tUmb_U7dhB5EA>=$;#{-HPo-K*qjmUY;W=4?;y= zeA0qJ8&7`K+PU7ToRX(|P+^R$S@+Uy4AX&s*mSK}6IF%BEX3#a1wMC)Gg9xp#Zgx)$uqDq`stIKZDSuDYtt2 zXThoI$Mv*&&?kImQlqoBwPnO==H_}a%zC0fI_#yqu@@fhSZIHcILe(HeoUKB9(-^#bft@*Ake}e}O zsn58$zc^hn<*fK=&pkV3!RH=-XpHuL*2ny7A-~$%|-fC0!lLiS?9)l%M><@Jeupd+Y5r-AP z0T;1&N&SxgDLInvPw>*yOpf*PJR3$C1|8+k#D>uum5f|K*oLMGDVfx8K~HvG&+%e$O8JS1P9hb?pyQW ztU+fOg)fd!?oiiVjb#e5nTv?m*DF7AaKdIqO8D}1dVUm;4ZMH(O!$MpQZn=Uk72JG z7lGe4?*eAxB+8-R>rL9KG#=M4t^;p4l~>_tfrekqFI=sv{lPx(>zWOcI62a5em5ai zknQC!Fk#duG?GUkXe5IHj9-(Oa;{s<{_L^`P!D_gew{S#V9!g2h6-%TQ`EN73hJ~a_owmCxCC?1DnXkqqq9(4u}u)+OVP3tmlCFpW} zs3UMaWeHzI!Q~+Jh#Ia@TkCscWzhf7){Iw0qu_T5y;!CyqeBROHYGXT478~VH-{Y( zF+t+9N2`hhte%s%!cx}PVUu*fVC^9KJr$(&-jA-J)FDK#Nhx5@k+urQVBbJ-@^)eF zx>J06eUKYpsn4aqMi#j($RoF_)a<-5%%vzBz87XEFPNcBfN6UodCCtZIO0fgz z7swWgM$|JkLopj|q3{0d9y`JA7K%mm<-DysvEJsbtm;E|M`u@wgV-Gh++v-SZn&>f z-<#;Xkf0Sp&)@9-Y4Sk-YEElcu7I=s z*F^wL$Vq)Qut-#u7&UyimelTP(0CCAe$-9p$~kgpC5FsmY{ZT#$SXMI&P(ATTdK`q zN#c(4MO^ivboes3p}pHaW+nbyFUg;0bcu^dN1h38SUx4Oqe1mhY;B6S6a32 zO}8%3-D7?A*P6twF1n(X3jQhlv6f9JSN*rgpCs?hmFzaTkK3eaz-M`pQ0Dr0h?9z8 zU4*9NO#lSmZs~ZB?muvwk0)<`XN(QE{R^#j=x#<0FzJAIAECDTBVV$JE<;>(tZ`E5eOSD*#!`2CP;10I;9 zy@gs-UgOQpY-*#%9pA0l6n<=eiJJY@)6-*65og~ed1oBi-^x5koNh@8gLL)1$ZBn zHP`|V_^gMA*6rObiSJ_|;u24Gb$Dc?=*#od*{a}#v(6fVvBoHouVQVUkH$&BuKKT< zwov^UPo@W>?)b!w;+CxJh3+oX8yhT+dAX7KgWh>bK55J|@WEIn})-suA>$T{4 z`6lRlhCvnD&vjWVQLS)QZixQxqpQxQ!F0OFnaZM_al)n_Bz zCsd|@tyCEJAdyIGW>oaoK!s0?TC))getCuor|j6{;i%!W=@{_^>Ot>4qbby~_8HE3 z;n0tzr+?YT<+$$OmQJco29M2=Mx4p_NlrJ7<1^&Z4*NH$K1?-{p@*DABBP*W@4rGn zRBKI9fYKfST*3w*GcKX&QQP$jIRY+l4Arw)e_){w0K)vb&&vbR_0dY^}slzwh)4lha&lS$$!c~J;RXg)I@a6x|`d4PjJoM*tQ0Dz9m9iwq zU%@zAe^V4%yr0~EE#k3Rj)~tDKY$;mg`r$8U(HSLkEIn0>^LP2Cp~G)e?aD%1jzy{ zkDp^;`oE*TBf^Uw73woHB_wq65q*j7=P-Yh?1SCOC?cIk*xeC}XNs1`?V%n%=fFDH z7MW((Sub(Pa8SQH>JEVW^jF);_H4a(9)`#hSq#x)F}ivZkht=P(>a+F3Dh)87V#^c z4wNI}y$+c*w0D;?28zV#rf+%F>-MfJQ`P0&h}TvW@<@$CC)Zd-bOPcZ(1R=j#6ieqSg+V}pJJ}vI2Xp4pH#${L!_|H%_tp#6sX^3ddITP<%}?i_Th;RM5}og7$d?eq zF-Ii)Iu2h5;*g#&xPKXw*PqQoY0dp8teq`#%N9M$qywQ95Hk@4M*npl5Kl>MZV{YnnQhmG`J;5Mf(ayB zU*qsRJ!7H|NLJ-P*eHxqC>F_fef-na)n$jS4Y*>{kA!MIw`cuCXMiL33atZ4hb$86 zUK0>o*@tEyWbuRPc)Hd|FFZ8ZPMg3o0kktT?VK;~qcLzPrCy9rhM%geVLW~JZKQpw z=N5y#+shq?iGC8fJVgo)@d*wNCRpB&!|}8T2^zR^b%XO~x;uVjcjs%s&TiLD+oaZHPYyy7@2d5stJmr^8*TNj(lPi)^K0NW#n32a zeji0AJCG>kNo8}Tw9ea4MJA?Fgk$ zY*T0r4%+4(Z}V_D1UwNLTrKsb3eFRmau!ZGQVYUg9hF*1fA>2ctO;a~LKkM-%uQG? z6roQV$ub2asozD@STokH8@7*lm@F*CJ8X zlSyF`$*0yRpe<2k_4b6gLZ-4>yd(6k z5y6{MiwBX4UYC{LC8wX&YR{f>T<^CLfRLt2ChuakhkV3-Qo?X780rv&}X@ zkTs+2Azg>!Z5_ro;5m2Y=XT-AIx1G9)Z4L`y}@rG2LLgrLkt~7oIJ{hzc!m3*!8-9 zs7l5zRA+Z@xP&F(qP3WwrW*hPXM8vhREff$Lz#GNCQp}nOtXCSWqEtNs`~EIj24RJ zl6e9fWmkfHChop6>d&*{iyw&vh^&*yv@`ljnIcfG0$IJw$+WwQbAem#fPUHrEZDUS z9gj>h$!iFA+-8T@0vAr|L5Q{X=X*m5aZ?+t;pNH(@=;0$ObsT7sOI+qrkh#6)&s?U zSY%*2snC3;kuKYcv4%MlsL_8@kxBFH0d4hVj1rdmcCRMWNc77LQQ1nCUqSg(?V7)6Da+OOBj;hO)Pxky_~aX@pG2Ie{q5ZMYgU-f6O3$WeJ2X~ zyW1OS^QT!c>U}34imtn?LpCgSdyuwFUcnQtqc( zo6htYA}1Slh1G@1^tOF!rdo&cVUniIV7M#r`xlfvh$}aQ)aHL76(p43IbOlp5cT@g zcVw%NkS6;$(y5QX{SjTMV)InHCV|{-MpI@ix`={GmRUBR)ea8`sHN(617RZgf-Bw+ zbgw|WnrfN99t8#iJw>hhQ??_qKO5kQZ=h1;Re_{P@9y8xdW6Mb`{&Gn(<#!>UB}0B zBN`<(~@qFwF`BeYu*t^%>Gvt*Wkj@`3Tv1JTCbQMBo4XLf*ZhsKYEK-#7=NQ? z)0KKj$3F~--XDwXlKM%#wR;()20*H)Kd_^2oArk{Zcm#m!ZhmJR#CgBZYFo2sVN|4}Ml-%8%swULedqnm{h{Wc zS>qLYaBZA49;P?0>1;A$#iiiB*NZ?{QKGGt>MctXR`f8IYgrZ6GQnzuV*ysAdUD(m z;Hnm`1@Y&8c40oUN3m{c{ty-km8lJxxsrSDhN+Ns2Zvp?gU57k(1U-<5x z^`<+`t1V{!aokBGG{JQFx`aWvtF_uId@PpPx32cC2hBb<%5X*8yXyUHc*3d}Wxht- zVt)ItkPw=4dyfE@r@+9)jJ*v}38cdxdM8MdB}SKW`Q#&Vm-6%9yQ$DV(LeeP!Nro< zSl_>Cgv|g}QB@?=%qt|HibGaj3<8KRJfU}{_pvJpE*~C<`#h+=Ku|Neul14I80P4x zkOoa-ec!A}a);l1XZH(k8m~=faB_@@)tzX6OHFOA_g_fZjEW^E&u#Y+hIW^6OryBO zNm%K4{B zoZg?2QNMq`uqe{(2V}r$6_z*Z<7??`YWwz(h{=+R8MXW!Y;yN%x*9SxUR%76tKJc) zm3P)McU!9=5sfD}v6;)qizcCr4skBsWq(tPS7EkmnycCRFDOuoitokg0D^02+R@X3 zSFM4NN{diDiGWHKJy}MWlVTvKJ>_TnNyM>Jo9$6z;8BZ~fkvK@h4@Gi?z@J9wb_{n zs+P`tfhzgW*y!2NuIdgtzuPN3`th zvk{z6mv#onvud%NGyd6bw`TOsZ-~d#;+YgNfa!tauv)LZk;^HuY>l%N@7AX`p2CT_ zDE3?TO!D}n@PJ7k{yK7*}~4?|QUxjT+tm$bXsPQWa>LZ{)&jAh0r!@yVAW{;*?b8W<9p%Z~4$%LzFmVT04f%bRs zr1uf4DWM4}fIq6rTK*0+n8rw1!L$nU8uL*E9xIR%mQ?v359p zUU&@r9g3fg{?*O6qNFo+TJ0YM5gE(}^7XAv)qDRGy)t>+vg^H{V92D@X{qa$tkgHl z)yRdx{zxe<^E2eOVOz_Etl6yKyX}Ln7 zaKojb(K~Zoh=S~-xJ4zSC24d3YjG*;F~>8(R6B^@Kk#>=@RmeiwgCBuu<(jjWD%k& ziG+>*Lge{Ug$$&EmsfBem3sX4+GTJ8sg0Q-eYG4p$%IK*Nk&6&q^TnG9_KG1N|nJ@^~2gEw(E?mSI)4l-MOJ=zwvT) zc$6}=&hh|^+acCxxBYlRpWC}0&d<+d$=-a;Ob3i(KE}s&O+@J1CbZ>EcebsDFE%G! ztGQIow`6`_BzD_AI*LalW=rh)XA#Tihr;HZl3-Sbk&1wP4SJpmbiPCkX*RgWi&+-h z{TO|>{SwK092qa!2sgLcw!$#CG-vkkv(4+`PwQDFBTMdzvyso(CyCYLyW~9*hH@B@ zV@&~J?@C=m#;>|NWiyN~n+_A+`GB+xz5WjJ~#+W%<# zLfCRo_iDlurbQZ$tTcf!1WC zio^Pn05gF3P0q;8bPG8;O|fDKlLP`m0eZSxPL9}ow*F}pmyKDz_SDEH$akLz+-(ecg|GxgzQ%FL2k?IlL7kso__Uv zZf6N}*W=|}#SIEfcx|h^sp})TsY?#GBlZ0Ke+nhGzVbHp%KNibyBGZJcxfhKv=v%p zLy0s>!Z10aL|l%F`mIp zxnO~C_=G?`u)vL&@Y^9qw(XePE1Hm|>}xY41E^%)-AS zl5a_hl`FSAV^%hOK?#^lJ&n1PKexGb%@>Vb*c+o3abVY&N^$78W9*-)p%9rsrZ`=jNjUW@ z=1!^>&4e!}GWUvOPh|H@0=uH%E@V4mY_Z~ND{F))hr!kFJa4*-2n>PPv(N?=5nOwS zw_p9&a5w~~v2ytHVz@-HQ#2jneb7Y{Fz@74US7r*0{9%{%Enb7omB5@2dR9PZ{xyq zPUIF(2zj%2{~g`p&|IF&+**B5mk(CQGL_^F6FEUfzbSN=M>R;G7G^phGI*})Rv2{U z#nIrDY=r%2o=`w`c=uQ3%Z;PIGV=y}!uvtHa3hPlABP8CfcRY{AFR z-xCV!?M$M0W=S_Ng7o6fLLp7%KQ5z0GgPG!BR$*D&9vTMQn1$%PdB6b)tSsVIC0Zw z*c#QzC#4}rpV^~ThP0~dxmx3{P8ZU_pBxS&yPux=rK!i5YXx+kqjhOw_`m7YP@vKy zBi=};?5E7Wmch+^c@iolwmiSJBRC&c@l}-Qt{NdAa2VKqQ@@m; z2kr#=ExUR%V~lS1v#OlaiO!s8y9m?B=a(XLpxVb{)~0q&JJCLHDGh3kD=A6xS_=sD z4IyMv#C(d2Pcrt5Fth%tS=)Lg^oOtZhp{KbrQn_Q*>r6!8VuC_j!#PYV%s0`IFFBo z45hzEnPNE{l|oIS3|lu@gK6H9uJvLxnf}4}E9=V>yZ5FxYLH(+ZbLEhzYrko+tK$B zdak4IUCsVqq|h4$IS{MERno%0(?M9^Zm$1Wd=PQ9C;qFvR3HQ5XJuJ5HvtuzR7Sut4v z&ohmzd`hvXe%;2VfDTx%{xsoaUb3gCw6^AaOid&QJ=$qE~$zf`rEw&gFb) zT83dh3b;Xx3jSYUoG6z~#0>!Ub;kvJFB-XvBb)GOu7(oG8c{grV5Fu>r&b_^!)4d) zea3^E!t22Xx&ngt|CL!-p;6uv2>fBj%Xq#)k6_?Wq+m}I&n)KIGkm5o_JG*^L!^4w&$7(k5!+vK(8hmf7`W6|7#o{0B z{!9fCtL1DoxhUC^QvalP16A!H!F-1~{}V1tswi}LRswI*_;lO3uoFgKdHIg^9OFE% z5mp2(f4Z_ybLcVf8thPfQYrt-Rs$LX#sM@+tMl6ajA54iA>ddDb8Io%SKS0Gj0Op4&7^}zg`#cxkwyW(u20E~m{X=s5uR_R7Oeb+?j)~@^M2C@cGFz3J}N6+>4 z%w=yP0A_bhIE)ofo-nWDE%XQEsVrFx97iw(?Op+|=}&z-m@3AUU7Bn<^nP}_X1V;= zC$p(Bl~mdjR64X@u9zj*Rmf*wZu)KCl?_UMM_hOb@iMF153_D%keAZbe2AKbV84a}U7#vxzsyEhnhU@5TLGVX_n1(`)ckvc@?M z$C8VTh7%KjU}yAkfgt-2lrW(Cd^P3fDD6z?gO8_(LaNi5iBS)?8eF2&Zh)F5JZ?pp zgde5y3fF;>;*_p_GJQke=;w1eF80_H7y(Iy_lWbuVRo+*9SNVKQ|g_$%-{62-nlXj z_G`ljfOl5IFpqcG`YiGXed>P2nWgO-z1i^))rcxNW}EFLt~{ev^eXUQ;Ua@tayHFE zeDivw08^CVW#8iV|oAiQsSwVdPH`a-4 zOsGiFTHXw*eM7ztg+Qkxybb7ssnzfO(CqcV=7X#|6$A;#hmAp_Y}jB`#ICpEYj^{C zKQ0dMN-NseQ<(WkZ%xyA++y2r*WrY?()CuG5kkf?e$08KB^;UE%x0i_v!_?14-56P z5#&~;&G)J+{LY0vKXtSrTDeB4a0K&oLh3 zh!q{^9el41{+g|ZujKm>poK*x#`n~6EiJih)>iCV7~5wsD2E)({C2SR&+lbh4d zj7ajdObJh>ebSIL!&R9KgJGrTV-779yJo0><3>OgC&$H`t&-HV?S>3t5-g#ji|z7w zCWFI&znf}%LMp4RdKwrDdgn0MsWDe(PC~1g)$`* z5F(ch6(tiK=Z2T25bQf8b{-1clrQCP&LejiEF-=}Lzm!yUCqTCYsSvMkQB30amjKH z<30_#o9`;EEt!$N|U=$AxmNEIJ*tkoe_gZCnn;I3RCRDtCt(-vz?fYSNn!v^F$n?9V?&xQ=^ z^bgrHCag|>Kh4MxBinCs8BwLjNI27%CHXGq|Iy)VyBr8H{}@KRivBi46tn}GNk>yV zoY5{0NiHxYWXjHWH^|&hTMh=evAfH4@6Mrd6yYvMnNnf`sd-dXN7CkV84jBH@vc3R9R( zs=dJ7xXeiQ1-`>pSzJ`Jwb##bqGr+4)7I7pRI6#W$o(gCtsBMX3$%wC(1fs}z>-fNvh**VT{*(^TEA3W;Og6A~13;A}Cl*rQ-J6Yz? zK=r7Mo&PsyLulW9yszxV+P-BE&QRB5KVd1U$+Ey#N|R!9)@dY(%|%@zfsU2o9TG2! zgtl25Rh*A;C~Q;~oW_Vl3O+_%n=<$Fn+irP_Po!lP6`bu?zBHzncD@_B#p^DZAZ&E zf#PYtQy0%07pg5@@SwVmdAE5v?U@NpxkN`GtPwaz%*USfcF9Q<{-hUd-{A4$(4wQN@V zy?+4{ChCZ+o{6n@zIC9HK*u${+`J&YMo8ul+GhW#v*EFx@@V3reCop7mMKxQ}o&CkHvJ53kfjOz)K=uk;K&HVMyCto0RA_1lJF}=F{;4{rsCGuAJ zw&c#gpIc0O=LNadOG6hmKXrT}g=Dk}{kJ~IH^1F$l)j^cZNz&$h1i8{CO|ttnMFc6 z#QV%N)i}d~tR`6p+#w9aVJ`8Oq{Ez}+`BRuUI>)=O%&~fsx6o0PO`YdY*^@Ki&m-1?b9Ob}Bxb6(g~IZbQ0$2y`Y2f^_lRZB z@nceUdj>bx?7?;@b)e_Isqf!i!n3*AnKl$8fr=09%MzD`nq}+lQiqvPk6o>b&!wPh zC4beKtNN4?iB8WmXVc!7=avc_c+qhkMxU6QhM^%$EBdOxGwcB1mgDCwo~sZg`p>vHN-BT*kW@FvrU6 zxH(4?yi|5iDzGN1ZJw(V)vk~DYi}1s{#Hgenw(p!PuRRCxb@=SX=;5SUFh_Osi;m= z`)tZd_hOe~JdmAUZ?bQ#&S?ovpy3)`!LWb*LQoaU7EyhPv4O$EmHcw7+1TTDJv=L1 z+W(89RX9uXo^vKbLz0`qdgnk)52e<7g6r7>`P_H%dU#$Euh2Tn0dMr|D!^*TY%)Be z*5{h9z3<1)V`9yey_|(PR|oVOF95%;L%yCaei(1d=ojCK!&Zl7wN_SP8`3TZph*J% z0<1+k^2nar`^*g6AA%pTIEH@1C>^L|oKKJR$4*AXke`ic5`BG;t32?lax#5>k{kN$o5B1t6cz`c;Kgne@r z%G!yy;m%C=COO8rsTO7rgC2MD1AUjWrNs_V+neq~PH@N>K zv!+dHN-ChlmxYt-+9!;b$`?0CsF9nA%O4rnHG~OF9FHQ>#+ZD{OtC2boQ0{>s+}@n zT8tXs;ip}{@q#Eh)7avb{Ao_^hf#9xF+z(qx% zFSQrYCJ}_G;La(WH(GIntl;hyKLHn*pdDTAn1>0tiPZT9Bqg9*LadCR=)Z9`Qb;r-6DQZiVMfq@!L2Vw&n){C&BwG0iiF7SYPTo6TiT2 z{m^ZN6%O=m2MTu%z{w_HwQ?sWS^VEhO}`%jzI!-?>jts2VU>mS}#Al8?+K`Qh( zQ;7=f)mhA(%S@^Mgqc7kdg0`zl{Fxxr-X&K0gZ+ump5w$aW&CFWXt8~3v z3MRi!!YY=CX@Ih`{P6;?V(|IV?vUM}?cnURJG{O4`s-;MBO`Smgdc^WR#@N4lkx9E zYBkvhG&romfhGNm-F(3ehdE&G%yYul`$f=hCe1(a5^ z1*T+Q9K3Gat#C35-+?e5QG&*x`k3tvgR_F)tlnak3V3(;5B|hQw+MnB{=_rHm!n0H zGrM@1XqiK^5x}81_t~Z2J4=S_k?8u7*|yq?Z;A%=s^LN?FtrW{bh1%L#)C@B{ybH! z?{4~(?~s!T_~3PYD>q1pU#|d+w*YUnL+7=6_(R~m?z`F4{kj?L4KJLkw71yM>A97q zob~A4Bk#&I8VW$q;i(}4+9wWu0#5rMfPgA;wLiV_74y$3FBF=+;!qSWYv{obFPuoV z*(fA0-)%@#3TNpe#bQ)e_cD|^_Zp0aj%?h3W{kV=b^(c=df_o+#R~0=N>-HpR?j;o z3$e-;*z2>&%+bTe{I*<9WT<+Pw+1tr}I3sdEC;=*3 zJ?7iE6gDf|6KwyZ>Pseqdd}}DdUQ00OW-nJbHRztw9=8)0j{&AXS zzv~w$+<}hNhN4D5CB07k?JPEXis|-zf531Z+h>TPVV})Vgl5&{X1V@5Bxc=^Ck!7N zLWL~a^keJn8%QO}WEB8px-;;N3Ss`?IkN3kz&9Ajf#S;&gb)InHoN{FMX)cg!VJe# zqY-7d|4>K1ArTSG=|RO7x6maVBNW&QDl0n;LeN0y3ygLLa2jObHD&i0n<(tjx zvLxormSLoweGB?U!%b%J;t}OdE`)>s=JvtC5CkG|ieOoT{g#Bk3&Y0- z;Ll=W{yvF;F9a%lg}T(2V4XM*y#e6c$YW7O$4dmZ@{I>}GW=9DY+TR{)?X`Nhnxa# z;ue@7E_A1ENt{?k(5qSuAYy+R<@hb#D-lK>y(m~c>#~)e%rAR)87!SESGw9^%V6)> zeV#oL$4!a^yJV{{G12~LHrby!=wn@jvf8Q-=oWfwXYfw+`0B`w5ECSS=GLL!QG<&g z1LA3eh1@J5dV=}OaSPk>0s#S$_yS?P@&eTnuC)tX_FyeRFa59*7V7+ zOFqWC;vhWZX7Ube+_|Nl|FNq2+x>`E=HJ_>Z<@T?kyk@b>;EmJ1{5J8Yp3WvmRN+A z!M^@l==f%!l-6>xGkhWu*$=wL5E$C0VFmKv?~PQpi70if?ITgseiG8^tA-#$!q?4u z=XCic5Q`#@ukm?-o4VDPtH)Iny1O~HQ231`(WyqK+-pbmAqrzY0#ei)Bir|*4ny*2rituM=h6bPYX=EZ>+))1#MKwo!XH=C|E+TZ;&i5@ zx)#i^$M@pQJmCaAmU`Xqte}?VzK0vVYYZHQ`_a2qG}E~Kh4N($1iKJg@b06zoXim$ zOlCifkACpOIAoLZPGrkCOf^JC{3wAQz3^+|LN45e#Y z_>+m#`9TSD|G3l#bgE8NY|=#RA0Y&Z^tw0@IhDFG?EC%doWJ~p^WqV?X1V++X*DaO z-ysnGmV~Xu#E|?r1}ZuGIb>)ley9rhv7w})p(v7COqyx{sRaj_k-R62rc(7c&0qTR z-LFlv#BcV>fu_=$iC7}|OGKmDk-9(oH$}Iuxvnkvr4D|ls!S35BNRRu0EcycOc)IR z!^uI{P*n`1uyri~+`94r6*G|x9=kPZoXr`oYkL4zEKoH;H#jnl9Go%!9K?M_F{C1m z{LxG(WM~#Ddi`PL(k+MspsTPGg|Q8R!Eb|RNiqkv(+w)DLJ)Qfnzc2%;7cu$xUcM2 zT)q0rBD@k2;o)E}LXw!aixKWtaRM|}T08+R%$N$VDtDG)xQKlY(U}+q3fGZRw3Eb!G2VP`mshn6%*r~f(cPH zOk&kV{Rz~B9MgjY>yTLgbe zh#SH#!z)qe@oNa3njSs`caB8QBWFX$5>izU$0DQ=bbI;{hve{nCwA+bDu_{u1Y037 z0c>p{B9gtId&s_AsDDd=_48Nk?|Po-?aDU9>&l);Sb${A1Dy;S;FMTp5en{7eis-R zu%&AA0fzQa|INGLb6Ax?9$998KTi66(I(*g-oJFFT*iUWI1$)~?DB^3NIxml*W;B#-)3R#b*M(u#elqu0Mw*+* zIK^YpRpS2eS0O|D;$`d_6Jy-sU7ExXEN-dw3)1Sd4<3$5&wMQDAB;bJBd?yRKakPv zgX>CJ(+VjfFzPohk(pCxLc4?lNdi@gD<&l?Ts(x4rPlI-H+) zKTS!WQEQG6#qxwv>pH^zmlfIA^9jcTQ~$ZWs9fGVo_0s6hEZ~rQ3&vF4>&J{Qyme3 zO1fhkt2s_e%+_kU7L)eBO?n74kS0TnK5lXT2a!dFdVNCLwWkBp|GN|T?eELj`kN{C z9RGhufJcC5P6NDCx^a9vB8KGDshhnM=^H-dK;*CLB+y5rZ2NV4 z0L6A4j=)XutCzok_s`l(kt^J9sQMfiqw&;%`1T7Kuc9%U*31mu#+JKOoOuMQBy(|O zD^J6J|I1f+Af}R6K_vBKydtiNi(g;C|MzV@*RPhHTXz3}Sj9_VKe`3J)^5zVdwGO! z_-NRibAiIhd(iD|@E1=jFvNYO6|}`qf)FV~0iDh<`FOB-?1^-66G@72m)@l`Br;odnmARk_K5FrA>AMd7yVn5P6$+EVK$I zVlzbHa|!{Su;5i;bis|P@-?g!22wR5Vw3C0>yx|rr(q)p+~BUSG9zd%=@nb>F`LNC z2EFPF6yspCepGJI3~DE#w?;D3z&1n_bkxI9=MR`pQiUCW_LOJl30tThtx8=80xrkk zxRa$8lLV=k$BS?6Hfj7-nT6sp{*0I-W1ua=3>e=>3?o=ZLG-04VmXBn{u&gv6Vuaa50Y~(r@=k^e1i=U zs6UOsq*MO_q~JnuZ+!|=2eiSztAOJw0RaJ_*$(>cowv@;2$>9F1<60$15L*a^pAQ* za((~){f?*Vr3V{=@;jqoaBwiO)Emq922fw;WAvH7zOJOauBW~@hwHszgDWdPD`Je1 zlh?tA*w56*0jOU8Xtll}>~EYD_E5F)F?q$J5xC~}wR_=yQBhH#$t7)qh%13i3{eJD zHPlQv^YP96mAyJy(E+6(BCiVQ=*8AhAq(01!)xoN(NLliC|q8*==U)_sOWf>zb7lM zHXO_WoZ22WAYhK{fo=~Kju|4G1KtI$*XodE*9`Cu#>D!c3VY~)iCl{oHANPM&Ywbbf-guB!0>IuZNeb73n zN$48SprgGv7&+f+Rx=^rF@Z*!PszRC=?D4m92Aesr>?25K*uvLhk-YTx;6!UfEGw4 zohSC7PgLTYWtlK&K*!~qarS;@Q?(fgp+@mlr~9e^5@KcWNJRb7(a26)e=rgN&$f^J zTPDv3B5&D3nT+H?GuEUDHUgQNr@|wQPj2U%5|Ux{gZ~`9ulZ#cNTp2t$?NSwcHkg9Y&a`OeteLs0ynx6k{8x#S--T~QYlp+-@&ie4_}W;Z8&Z8 zA>^~NmwkosU{pUP`&7Z#~zx`dZX6i^f;J z-dtTsK%3N?rAtwdIOu z`1-ioRdaF=0T1lWcyOR`i7KCzechg~Vpoc`t+Kk)u-9R+Z}py42)v?WCup+! z3rlNgcT|(w*$)={A!pb!U(PhStgQR=WAQYNz^3Djubvn$KR;pGlb$sb<;-*LILOIO zo8LXnz|+|4(wUAgEbX^1^L$uY^rAiOz?aMZ_JQl~?foIM>J9K7o{9552wZszoNNZU zrz!2sn+IQjTfrt=v%7K6=;+Rw$GG30F27&v4!i;B$$fhtj_K=T(&p^lv-ifChi7Iy zocT4SMhV7E`vZLO*Mag;s zPiN7{o|=gf3GAjnoJrWbQ!o9mpz(pPy>tQ{Te3whnx{G$$k z^ zGqduHQ0)D$-WnpzOi`Z#ZyCBe#xApOx^^d^){gsNoA_+2FYaNP=^S}N-`MsZIe5$F zO?^PARLU3e+GF!C^1boe5bVUtVJde;e+U0;p(_h5w&yA@Txqg!T8mVW_uV!3THpHC z_e_~K>&7&;ts#@ue5a(PtO9OAU1N1^-mEooTEOd*SFc)V*A^wY%3oJ?HE7K7>YO%R zF^NNJt5nmsoqfqEaQGJRp5##AhRCqo*Zhl`=JJ`v+$hdHen^6Oh0nqw)`hZXd#0ux zspw&k)-Z3w!w1?=Db#F8D6bZCsy%1Lu`{WuiSs0I*yC8) zsS`6^ZMN}t(wZ%B?DPTVvZBX(8=Xb!q!rTDr=HN};nz^HnRHi3sPhcpZ6yvZ$#jlY zw$ZzVbA$u#XEyIOKB{`&SjnK|MZhIL-kkZ(wgyH`R%?N6=sh}0z$=cIs9sNx^N^3U zo|M+#FfV08uh`0M&(HNpo;qN133x5fwyS(nCLEqlKwl+RaHc3Mk@nZVWP!IFfX<)qFl7R&;GHP~x@rW- zH&6!h&sbz#N{2S(B9wuxrfU;CL6RUf#=ut8`JTmUwa|0$hlc_G*}KAQT~DM}3o!tJ Mr>mdKI;Vst06I}CcK`qY literal 0 HcmV?d00001 diff --git a/neural-networks/types/wavenet/index.md b/neural-networks/types/wavenet/index.md new file mode 100644 index 0000000..5ed98c2 --- /dev/null +++ b/neural-networks/types/wavenet/index.md @@ -0,0 +1,196 @@ +--- +title: wavenet +description: wavenet +order: 998 +--- + +Still need to expand my thoughts here. + +==- Code example + +```python +import torch +import matplotlib.pyplot as plt +%matplotlib inline + +torch.manual_seed(101) + +words = open('names.txt', 'r').read().splitlines() +[training_set, validation_set, test_set] = torch.utils.data.random_split(words, [.8 ,.1 ,.1]) + +TOKEN='.' +chars = list(set(''.join(words)+TOKEN)) +chars.sort() +itos = dict([(i,s) for i, s in enumerate(chars)]) +stoi = dict([s, i] for i, s in enumerate(chars)) +vocab_size = len(chars) +block_size = 8 + +def build_dataset(words): + xs = [] + ys = [] + for word in words: + context = [0]*block_size + for i, ch in enumerate(word+TOKEN): + ix = stoi[ch] + xs.append(context) + ys.append(ix) + context = context[1:]+[ix] + return torch.tensor(xs), torch.tensor(ys) + +Xtr, Ytr = build_dataset(training_set) +Xval, Yval = build_dataset(validation_set) +Xtest, Ytest = build_dataset(test_set) + +n_embd = 24 +n_hidden = 128 + +# Departs from pytorch implementation https://youtu.be/t3YJ5hKiMQ0?si=dIVp0YFYBFH0NpdS&t=2620 +class BatchNorm1d(torch.nn.Module): + def __init__(self, dim, eps=1e-5, momentum=0.1): + super(BatchNorm1d, self).__init__() + self.eps = eps + self.momentum = momentum + self.training = True + # parameters (trained with backprop) + self.gamma = torch.ones(dim) + self.beta = torch.zeros(dim) + # buffers (trained with a running 'momentum update') + self.running_mean = torch.zeros(dim) + self.running_var = torch.ones(dim) + + def __call__(self, x): + # calculate the forward pass + if self.training: + if x.ndim == 2: + dim = 0 + elif x.ndim == 3: + dim = (0,1) + xmean = x.mean(dim, keepdim=True) # batch mean + xvar = x.var(dim, keepdim=True) # batch variance + else: + xmean = self.running_mean + xvar = self.running_var + xhat = (x - xmean) / torch.sqrt(xvar + self.eps) # normalize to unit variance + self.out = self.gamma * xhat + self.beta + # update the buffers + if self.training: + with torch.no_grad(): + self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean + self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar + return self.out + + def parameters(self): + return [self.gamma, self.beta] + + +class FlattenConsecutive(torch.nn.Module): + def __init__(self, n): + super(FlattenConsecutive, self).__init__() + self.n = n + + def __call__(self, x): + num_batches, num_characters, num_embeddings = x.shape + x = x.view(num_batches, num_characters//self.n, num_embeddings*self.n) + if x.shape[1] == 1: + x = x.squeeze(1) + self.out = x + return self.out + +model = torch.nn.Sequential( + torch.nn.Embedding(vocab_size, n_embd), + FlattenConsecutive(2), torch.nn.Linear(n_embd * 2, n_hidden, bias=False), BatchNorm1d(n_hidden), torch.nn.Tanh(), + FlattenConsecutive(2), torch.nn.Linear(n_hidden * 2, n_hidden, bias=False), BatchNorm1d(n_hidden), torch.nn.Tanh(), + FlattenConsecutive(2), torch.nn.Linear(n_hidden * 2, n_hidden, bias=False), BatchNorm1d(n_hidden), torch.nn.Tanh(), + torch.nn.Linear(n_hidden, vocab_size) +) + +with torch.no_grad(): + model[-1].weight.data *= .1 + +print(sum(p.nelement() for p in model.parameters())) +# 75811 + +for p in model.parameters(): + p.requires_grad = True + +max_steps = 25000 +batch_size = 32 +for i in range(max_steps): + ix = torch.randint(0, Xtr.shape[0], (batch_size, )) + Xb, Yb = Xtr[ix], Ytr[ix] + # forward pass + x = model(Xb) + loss = torch.nn.functional.cross_entropy(x, Yb) + + for p in model.parameters(): + p.grad = None + loss.backward() + + lr = 0.1 if i<(max_steps*.75) else .01 + for p in model.parameters(): + p.data += -lr * p.grad + + if i%(max_steps/10) == 0: + print(f"{i:7d}/{max_steps:7d}: loss={loss.item(): .4f}") + +# 0/ 25000: loss= 3.2859 +# 2500/ 25000: loss= 2.5645 +# 5000/ 25000: loss= 2.4274 +# 7500/ 25000: loss= 2.3042 +# 10000/ 25000: loss= 2.1279 +# 12500/ 25000: loss= 2.0867 +# 15000/ 25000: loss= 1.9211 +# 17500/ 25000: loss= 2.0363 +# 20000/ 25000: loss= 2.0193 +# 22500/ 25000: loss= 1.9068 + +model.eval() + +# evaluate the loss +@torch.no_grad() # this decorator disables gradient tracking inside pytorch +def split_loss(split): + x,y = { + 'train': (Xtr, Ytr), + 'val': (Xval, Yval), + 'test': (Xtest, Ytest), + }[split] + logits = model(x) + loss = torch.nn.functional.cross_entropy(logits, y) + print(split, loss.item()) + +split_loss('train') +split_loss('val') +split_loss('test') + +# train 1.9855728149414062 +# val 2.0416152477264404 +# test 2.034022331237793 + +num_examples = 10 +for _ in range(num_examples): + CONTEXT = [0] * block_size + out = [] + while True: + x = model(torch.tensor([CONTEXT])) + probs = torch.softmax(x, 1) + ix = torch.multinomial(probs, num_samples=1, replacement=True).item() + if ix == 0: + break + CONTEXT = CONTEXT[1:] + [ix] + out.append(ix) + print(''.join([itos[c] for c in out])) + +# daorbose +# elinnah +# xaras +# bryston +# ambiya +# raliya +# zahaa +# eyin +# laui +# riyvanna +``` + +===