From 81b2ff252583b79a091a8bdd5f8b2d57165dc1bd Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Wed, 25 Sep 2024 11:14:13 -0700 Subject: [PATCH] Llama3.2 Release --- README.md | 14 ++++ assets/dog.jpg | Bin 0 -> 44644 bytes docs/multimodal.md | 73 ++++++++++++++++++ torchchat/cli/builder.py | 7 +- torchchat/cli/convert_hf_checkpoint.py | 22 ++++++ torchchat/cli/download.py | 18 +++-- torchchat/generate.py | 7 +- torchchat/model_config/models.json | 38 +++++++++ .../model_params/Llama-3.2-11B-Vision.json | 29 +++++++ .../model_params/Llama-Guard-3-1B-INT4.json | 20 +++++ torchchat/model_params/Llama-Guard-3-1B.json | 19 +++++ torchchat/model_params/Meta-Llama-3.2-1B.json | 19 +++++ torchchat/model_params/Meta-Llama-3.2-3B.json | 19 +++++ torchchat/usages/openai_api.py | 8 +- 14 files changed, 272 insertions(+), 21 deletions(-) create mode 100644 assets/dog.jpg create mode 100644 docs/multimodal.md create mode 100644 torchchat/model_params/Llama-3.2-11B-Vision.json create mode 100644 torchchat/model_params/Llama-Guard-3-1B-INT4.json create mode 100644 torchchat/model_params/Llama-Guard-3-1B.json create mode 100644 torchchat/model_params/Meta-Llama-3.2-1B.json create mode 100644 torchchat/model_params/Meta-Llama-3.2-3B.json diff --git a/README.md b/README.md index e426e863c..e4a4f7197 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ torchchat is a small codebase showcasing the ability to run large language models (LLMs) seamlessly. With torchchat, you can run LLMs using Python, within your own (C/C++) application (desktop or server) and on iOS and Android. +> [!IMPORTANT] +> Update September 25, 2024: torchchat has multimodal support for **Llama3.2 11B**!! +> +> To try it out, finish the [Installation](#Installation) section below, then hop +> over to our [multimodal guide](docs/multimodal.md) to learn more. + ## What can you do with torchchat? - [Run models via PyTorch / Python](#running-via-pytorch--python) @@ -18,6 +24,7 @@ torchchat is a small codebase showcasing the ability to run large language model ## Highlights + - Command line interaction with popular LLMs such as Llama 3, Llama 2, Stories, Mistral and more - PyTorch-native execution with performance - Supports popular hardware and OS @@ -514,6 +521,13 @@ aliases. | Model | Mobile Friendly | Notes | |------------------|---|---------------------| +|[meta-llama/Meta-Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct)|✅|Tuned for `chat` . Alias to `llama3.2-3b`.| +|[meta-llama/Meta-Llama-3.2-3B](https://huggingface.co/meta-llama/Llama-3.2-3B)|✅|Best for `generate`. Alias to `llama3.2-3b-base`.| +|[meta-llama/Llama-Guard-3-1B](https://huggingface.co/meta-llama/Llama-Guard-3-1B)|✅|Tuned for classification . Alias to `llama3-1b-guard`.| +|[meta-llama/Meta-Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)|✅|Tuned for `chat` . Alias to `llama3.2-1b`.| +|[meta-llama/Meta-Llama-3.2-1B](https://huggingface.co/meta-llama/Llama-3.2-1B)|✅|Best for `generate`. Alias to `llama3.2-1b-base`.| +|[meta-llama/Llama-3.2-11B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct)||Multimodal (Image + Text). Tuned for `chat` . Alias to `llama3.2-11B`.| +|[meta-llama/Llama-3.2-11B-Vision](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision)||Multimodal (Image + Text). Tuned for `generate` . Alias to `llama3.2-11B-base`.| |[meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct)|✅|Tuned for `chat` . Alias to `llama3.1`.| |[meta-llama/Meta-Llama-3.1-8B](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B)|✅|Best for `generate`. Alias to `llama3.1-base`.| |[meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)|✅|Tuned for `chat` . Alias to `llama3`.| diff --git a/assets/dog.jpg b/assets/dog.jpg new file mode 100644 index 0000000000000000000000000000000000000000..37e2fa03726460ce8699325567ca35fcb1ce361f GIT binary patch literal 44644 zcmcG#cUaTQ)+ihhlrBv`nh<&oy@LvbPUyW!2@nXOg(~P4>7j)ZI+)M}QF;gIy-NqB zH?dGv6ur^2&;H)?e((L`KKHp3lFV;r&FZsOrtoX=*E)b&M?+f!K!7j81i}EouMI#s zI2h&%0BCCq0!RP=z%>9B0U^L1Uz5PUnF)vi1o-b1{F~+PdKSL^yLdP{dvf>%`1?cQ za1JEY&kqLm<8bkINOunZ0E8Rd z5kKCa1BSpqd3oUx1Ufo9B4OSLVLaL2l>hzxYlC>n9f|Z&0D%zxqK+=!PEb*2Z!b`= zqYp@2R15@Ax*P0+=Lm}AaDuwQ5X!($ZCyYPn2R#dLRw!;-$xDV4$}(tgPMdwOr1kL zo#kDCcU8!gf)#?je7vAYM~+}GIKp2cSQ+>SxdOiayBP#jAtP1tb8%HLR@e9&2LGfC z{F|Aepdis8Nl|YT;dVDO{N&xf3ypSx;er@aF{a`;SUvYbn}D$8FV)Q>E+;sXDYx; z8SiS22$(akqnD53?`D7R06%A_vYWTJ8yxyit24ga5$1+a{;e^*S@7cF6|JD|h;-D4 zA@I}lAC>+OW(f3lc60)%>uZY`8XIV7t7@Bq)kVOTYG6ZC6YzawH7zhiQyT)-hNy!r zwIP~74*W!NaaM5lgE}I;{Z#%;7Za%OzbbH;ii;zjcPP}~TM6{r>+voBs{DZg{nwHB zy37AJNB)K2J{5#`V-my81=t|w%-x+Pa8gD;ynZOPmIMs{1c_22z+P``W=IR zeE>W+{CDKzkQ5aI$f;`U6A}H%GD)n2h5*uk9Eg=L7-0YZ*50gy2msUnZsz}50jL4U zZrmWbah;5WgoK=ojGT&znu?N=itQEy4HMTL9&X?rASbVoEQnV?N`Mn6c3)gdUQtCw zg-29fTTMwzR#`>yHxdFea&jt4DpqQ0Rz-dwzvBOK`PBoUAt9(Bbh$=A10bX!xJE

IxN*jFu zaq`&Ldd@5d&yaU)nBp%wK;j=Dwtc zV@{VD2&a>>xJO%VPpfr!CK?^Zq#Mq9-)dcx0ZY3U&O^V$Rn)Vf-YBhzPTLvy^&P|c^Q&yl3M8G)){;WUpsP!JcnbK@CV^m|nk^$Rht0|nZuP@-g&#o5{Z zIvRUz>#rF@=7bn8+>td$NmUi^U%m9MQtKlWx+_X9NwJx0iE#@=h+@teaXX8OkzkJxiE5e#Bfup z9-@8tR)p6nyPcA=4saAZ#BZWg|1GXEL}AVS`%7Nd z7vZ|QwLBSPrRPMpv_(zUC$QHu`4|EvZu40yn=H)G;Ozk-m zlggzwHZcHjCVtnrG|mbH7hCo z->##|)M%Hmar>Ioht<~GV$BGQon@W0|I1RU$fO0mq-8UabE}i_vCEc$sAn}Iwii$b zUhf45i8pC7wFqzdejB+`mnW?`p6KDx%G@eQ+;WTi?t2Mj%k#il5&38{-BV9Kh9QLB zurAI_zT(;Rt8gw{Z(jg7oq8Hm1s+C9z?#O_jz&+VLAs@xjBMvGDni$0OQt8jf8X*P zFnm62qx8djti`@=$i~%ud~{)a_PQeF?Esg3|9jKY@QMe^R^`UzlI5Qa8$4(GUe}M6 zcn$kXeSl*#0VAx8zL#bwmwdyKuP%rB_PQ+cBh-<-ntTDSQv;7;dZwX+Q=io9-f4o` zl^tl;eue?94p+l}7Tzq{_a4nv_B}fZ{N`Xg_QrSS_JHDR&(S4J#;fS32+9C9lk`M| z^d&Jp%UcsB7Ew*2AAsPNaT@oG9AHhMpozbM&W?g&pfh5$;+3iyC1@hl2TjV;Mz%2E zz_lte8j19pALTQfU+oAltVmneCtIbJykr_tu>7Dbxi6gG*_3>#vDD+G)rJR%eqTpegM1R@ZoQ`RFc zencS{Bj&&ihxi;Glk89}i`*#<{Q}RIYD+tTclodrgL=cL4ophxmaDQ=L*xQ5ukFV# zfZ9RB?s=Q#on44TSJzeUyi%OZoWHAJw5YXEUzSFzIeu^gnsEyuGh84Q!9Cvz@WR;ZPlqwNsf zXO4D%vg~=NMwnh}&7NfcH_3}vpZv~V@D&Y!c*7Yo%{EzLKoy~klZX17;UO*`BJ1=E zB0GCrx8CR;Un$M@nP{!956i-E8$GYx)!S!0Y2^lo2&;b(1qF3B^UD(%FSO78@iu`HWb2@CDlOnP`@R5-L5%xzC-HdeDJ7g za68oRv^~^{uMH=C8ZGv5RG-!9F!H@)i91w82=UfrVj0a`YChz(cL@SJ?-yIf2vPaF z3Vwi~8OLi`W{fnzP1xGm)Hg@BS_B$GqU}z`Jf&hzax=1 zXmQi!A{h24Q+_-%m$}GQd$weYvedm@Eg(8>`y*h-d1nMslxxjH%9)8qPQiLWaH){pAMF!6C*wbsE*(Jg`ck-+%z#_v;N*tZ&mqWD1vf^${gd!QYvs;7 z-WKIT$-ysS{d)@s?J8Y`C)L^hkfM28^RbmZ-GMKgm@(4Xmb0 zU`hG77J2UL4fPm&<7t}HJ^2h12t7!je_<{C;Ad zZD=c_DYY3@NxQn;1W$=OU;8ZejVD(+TVt&zYrY;inL64caprl#>we1kDt&XQVt&E6 z{i8Zd=i`T{V&XbkKN})#)$=)+U~_Agw(ntaZpcg5YORlZD>6g*)9>HgRBtNPOYB)! zBQ1Lu@A<%w;h%%YEwh81V{3IwO1)}ewS8yTL!TDr1sjSQeEn;lA zWA>eIfN(yqHL)TqCt3(S_px+X%-*EcEb^$oGI^TmqV(dN*%~l`Mx<*1A$EasFvp6N zDJ9`{nD6S4s$vhDD!Q7RId&1%`cD7U^wMNrI+3i(%`YKD5fIGlENiFeOKmZ?CkjL3!zFnCaLHQsnU4(Cs z?)F|Nyj4IvG83HqKo03+C|=4;)W_+>#t&2_!VRnD+Y#KW;kYS$gfmv2WbhUdLzt?; z^pg^cM5J%rfUU9hiG{zNZ4+oiWQ0;-6?&-id?0che4_FmRouWik_+WLPRFzgFn&L< z9d*x0oyS;+gz!{Yrib5%w&5Bx&lgzESRGIs6Z2}85A<*}aB1?m61ki! zeziE0VhNF3&!dZAnyAfyMovjt4(qb_@b6F?=ZO!p`T(x|_4`Etu~M&!6GZc?P%LC) za_XDvZgQ^6hyXz>-e9=U`{d>QPx>=fx+MITgRFqatt2@ffQ~(9X-$B3)RsbtM4+D3li@7RJ7uBD$g9ywUOu4@sDt4ut>7pR?KC!KC)*?Wf zY2&qYau6wT`Kgs>JhK=o}8c zEL^gN@*NEI5ACNP)_CMhM^ck&st`f-T{V@{tzHFbrM6NUs=mK%b1g~1jz5yV0Cdye zsw|9hl%e@BkJ$z568q*g0GCHVMV+t5ofbu(dIqVb1+zOC6ujRYodqjoQoA-9%2ZvSLtCjM=hbUBEPw7QQWTOz;we%0O@^DhpCnZ|DU1{GoyjOWDSxi#KN*6^cp{|uy%?xf6nQK-GB8y72C31lI~z**~-KJjM|7a zm9`koeJ!Gxx;~P#B1VwLv2u1?N`sxmGg^bR8_=Bre~?F~61t$Qj`PT*&D}7+-10ic ziA5C1-qp}#}Oq4_C~J6AmDl@{A+_@ZXRFCh+58^|Jn=zh-{4oaiQ?>Q{SPOV~exuEY}` z_bSq|;5d*{jyONU!9V&pg;%TIp>HLt2)$$?zJh z^Ctp;21?JK>#53ha%hWegb zF2sInE3EUrYrXg4%Rok%ZE)U!hK20)huh=(@1@R;KYcn4@XWhuJWdiRc#X*EY2>}X z)>#E!%iRE&;OD$t)(Aw-W&Mde9Xg6>1Il0+oFtpu-QKR?lFOc8voN}M&i4XgQ1;nF z#r~(k%qQLXkWZ!K)_XQ%*3jdxmpgpSLAz&dVZQ*HFEKRxmM{*ZC-=7g@+kvw$eBdR zS%s!j{$)s_M|^9F^|_rK2P;9w(KWz&zo0~(17r8Xb#LKTjf<1vf>umI6U%{{jk0aM zr$pVv-u`jO<=X(!`r0L>Nyden&Rr98)gg?2%vnY8KE$MbQ5jWRe*xkN?hzXS&G{qv zWvKFKUds<&H?KaCK>TFSQY2!NmRAhJH@%lyL$V%GhX19g^hhVSe^Mn$L$m4)e@F~MCp&NJ73q_ zDw>L#I1eYfG^Ad0ghZ!JKPRIhW->t&(Z|LMYH8^_;d5g(#g2B!#rZ!WwgeVMaNzq$A}p=L`shx7~PeAJKf(hdHSm|O5h&qjj&E>-#s4je@aAPWQ zd*6MgyP3`(1sSM~+Cf_~_fOdqd}mN{p^F7*U~gKfoLVHm9tZVC6c!-0U;D)9Rz(uM zV+27YD}f*Z2L)XO^VTyO2jP<mVzk#@-7t*E+q%Ec zSQk$%U`N=gf(?TrKjxIVtfZMjL7y z6}+BD^1#Y4At1g)?yPc-dMk;3gNojliUdF%D=!(voGKUh;LTRWa7t!Cq-BYgf3)0s zq!zv7!7~e0N_{flH|QuQPHt@{x@abW-q>QEw8hGJF=D_v{W`!A4CotRBE;*=No=tR^p|`^OXJ3L9TdxclB1+Ol$Mnk?RWe=3(v6Pm8dXQPHJOO;8~2 z1?~N%S2!};RhdtB%io27 zkmedSEhiD3I6VjOz67zDJbL=@3O%E$f>Wkr zMB})Qi{^oMTES@5V{v*THDI6a3KTWQ)0geK5dzZLz*1c5gCzB^d|CLF{gL875uVI?DMQPL64+OKnG*@@g0Lk|7 zFM4g@_TC9l`)rAp z;GwlHk28FW(lAd3ABBoIwM0wJFHiTS7L^~gIXSOq1MA*eq7?!4D1IcDl| z2(MA2WfaBJBy|Cnp^b4#ZnwD7IAz1=bFBz4uaZEvxRg;6CduXPp1O;+Wl>RSBh`(F zLHue1+hHGTB!gCWpp`W&8Blw(i#q={{1aOsHk=bu9L256Qj#$nSwuM_%bJHBddq#H z3-|3fVaBc6vo{)ISQjM)?xl-w_ZQxD3y)43qN=gDOA3$WPg|Il7c8QPD- z%J6EZ(h2Jsm>TvUsZpZ|)rky%EVV?_^*Ez;0di&!%QsY*vSg{i?E~@qBgzcCp^$u< zgHYFeoBO)vrclWjCiz`nzXla#lDEA z4YvBmH}CZ08st}oipq@Gs(sg#P9B1RO|&OQuj0AmX{r0HLbXZsptRsSKB+4-Db!2X zeR7bbM#p{RdxwZyM@@XR@yq5~s&@I;+R@vy z{Ug-INut>OtYS@0*7L&3$EQQn)*vEWa^-ZF>m^r8dTb|mOFSPf(YqQJ6m^Nlh7djV zdSY}h%-XA=F2B?=j;%(}FOzym$6S4xKf#ad1?{Y_%Zv|d%YjcVzUYn9Lxrec0A-`6 zJ9Fn&vO3aiLGA;br=>ITOwAlkv|n=*T+T8@qq&V#XgEODfif{(%&I3uC1vl{FBJ2JU-Z%591d?j-u#LSzDzWe${XJ9%ZZ-Y#~j z@{Ql*vbIvgFW^$@Q{`TI#T_6KHQ#c$SXhx*5gKqh&`G8cg&1@FC^4A@bzTDo#_HuA z6?LKwTZ{U2c@D#!?^Y_Hy`QX1l~qtr%GCHE zOQx|2MT4Fs)D6wYrU9KLX=GX8xWp2QFw*2DpaTEOhiPsjCUid2k~2CJsJhdax0%oa zQjf)X3_a&n&*U;I(&4{z;Si&s(2m$(WL$px3ji53M&~|))RpHZajJlJ%*Ck) zV<-*3$)MDm-JN3Ycxx7iZH7U&JH2(->!thYWi>X1(N5V(VWWJs8F36&D9-k$4F!Z> z+qQ&DJlS$#D3Ba3XI$5S5G`1xZ2-*Ud*atxW#CFxolt=ojp9iz-6PjgHn5@a#6|9~ zUR8z5tSGT)1wS$lMHCHvrLF%8i-m_NZh#Z4%`>@m{1{TLwTuNvWT@!16BFV@ocD(M z2KxHcQpsY5l>)aC3Z!%+cf4r1@{1m(pgA)H4>tV!^9}R!4GbFkN7Q|rsB&5J9wswT z<(3s1VTzL2Oy5EZRBpSwk)$T>ztomw%$_5?gH$CtxVgBv?I83z&QD*N zDW*!7d3Kn>mPd6xdB`JaQ(s#`YrOjly*`Z1kzQNy;Ms1A@xkSrnF9G#2Z~_K#A;xV zqEBxsIio@HU5mGp)zK5kRt~SYq_9`ul6r^%DZhC=@=ED+ck)&;#WRDl z+KL$k1lE~RP`d%+!L-EcH_U%F-9P2H9Jj57(G{N!3DeZmI^ptKYxe`)1Tv(~GvRSS5+#EQI9jN)Iw=JgJLLxl(5OJjk57#^^IFz@Tco zR*4%6kslVBO%E>W-UHLBJIN`(dH>lH$i+SH0WGkOFiPF24a<809P*RWed5QD33rLz z!U5x%Wpc?%c%2~|ELP(4DK24hr3SQ~qHHf&?S^NfBw(h_-rQ|IYt33OfJv*6G2(b) zo}()RN*AMsW!G_>%N8y*cE@iw&q~i64lj)!nz$ug$iEYruw>Mp6rzrKC~H|VGI8+{ z?!fsR^HwImmi3nFC|6|g*X6tMc`@Ev14GU-Y<2ZU%MFWM%$(gBOPr9{UHZP^Uf&g5 z@QR{7TT30T+vKs#@f5l~GYhnMr6U`0HJhZ7e1~nG1@sg6_Yo z9i>Orh;wQt=Dj2z37Z^TNlfi_D^r$drT4;RJ%h6Y#aQ)mas0G>x-6RFTsa9=CbV^p zy5tNp{B-4xnrL7`dCZZ^K&@K+>`LFVdP!)RTCVV_>=l_z)_gp8<V|{hCun&dpm9JbB(c$uo>|FR>7D{~9 zs>j6~;p&U<*-HkUl&g_O!M~>F_6UK_C4z=yk7$f1mce=*V#G3!Glj zgs~L|8ZBu;*&;)cDmh#S)R)|$TvG+H;nbJ%>=@^RQYA^Xr!wLStXUlyxZ9yt(C@VH z6;*KJ&z}abGx8EookBrQ7oY>91}c)^luM*a_RV6=B6X`r{YP#Q{S!6jJZjUAeuPbG z#5K@=qpr7hXh2P#9o3A5P#wILspmF2KLS&-7@2azo0w+cUHdO-7nEKCX{n=*YCJmX zO#}*oyNK)%%5RjXvRZ2ryE=Yqhc3w`;^%bArpcxg-(61w%O8!-j80R}gXt~Kc&ZJy zQFJpZNO!Hz;A7nJ3yhWPmT$*Rfi=alko?^s7o~)@85?c&Ws_Uzo*X0BS@#;t<9Hs) zdbwILMK|{kQ#cUg70-6sg2XqJ))I7oX*>DIn7;^CkGIqkorg@{3f-ouPFy1&;s$~b z|Kc)|W#x>^MX~2L=&joQ0+_34hs>5I>V9D08_4WIsJpp@Da&$&vc`z-p%1x3qeF|{ z)V-{y(%w6961`{4LMkqt$X;#5fSK?Os-u%iEV`Isc)Oc zHo`eqR?;8;CqGReeD#j~U1@BS20VO7C0>VIDJnx6QD(qykw$xd_tehiJ-d}kE<$gn zP;Vx9x1&=JCZR;;3JVkSWzhL_Yz_jc^Rwv3{GOKMMNgLXFhqeg2Xhxw>-fNM>jM~w z*60hSyUM)?UkWN|bhxh9Mp2uN+sR;4@#5QTa4A^ ztdOSl>sm@s^%;B<|6HFJ#MT(Nc+A!+v(R`R`Zn{sPG-&bN2uBEoy7EDx$13iZp{9! z1Z19dxyhi^hw^AyT?^1)S$i?!11KTooLKvv_$zxJd@ z(Hi{%wA$WvzDibE_*6T-U8d>tFKmTaN0feGDQum2&Z%)aFc=j@1v9DhHMb^A{8 zH<`Doqcgk!B_5oI3uOKGf0pklkpjj6x!>=0QY%V@_>A+o>Jar|Gemh=OR$7jo@0{S zrjcpgB)7k4t0K=l-N^2dm0W*nzg*Fqe$p@ErA8a=Ge*NFoJnh0d2UKU&BiiRtOC6a zPX(8R>w}=S0W_mGqJu9>Bab=+q_utoRLgTq_u&?|CI-CX@SdQcsMhno91k{Qbo8*c*s^O=P3V`IObJwIWP{0dIWD%6NVNxv2WOUt8C&{o znsA)iA7GNqeSFOCD6@BJ&Q28@(?5!5dh;26aaOQ>9(m@*tSHy18PszTzM#|urX**} zHBMWMECMEH+nTt^GIPh{CSP9vD9iQq{dN$&=7zNr%%k}1P(f=iNzZuTs`=EM{d*ic zPCGQl3Ocd(u-FDlD!%#kQ&3yts_WCC#+~?X0jGh}?d3XKKj=4V2;^?^EE4g}I@GEK z0kWAL+5RS(PgfEuRcxNi)pjiFmqve+*>AGKhGBbw@9F!Tm7bF&dY>fNK%^I65}hPc zLTzAbzC*YgbTX?bQ(GK1nWM|cB#E%0n@-r$z11Pyn<^$i-kkpfzUDU_LKjd{Xx1}x zRxR~?+gGpVjSgyB&enaqlJQ1#**KMqkWyme(g>SmPZ#tK_fDJ(-(^TzX*$m`nmCo# zq%Nc_Su`?Ua4=eW>B((vD#)9dp|5g9pS0HV(w*LLvCHY#UL{=4MB9}H>x|Yap`tCW zC9o`}(;Id34>v%US@Tc&WO%v#tnbnVo>ikDinb;hm5EoDFIW>!-Wkj`UTS%T%iQmA zKgG@Z6w3(Y;skh`L50=i!kr3kY$_l4iw>vONXv@gHD#?I5_TBNcq|ug2)i#!OQ<;H zVx`P(HK)TfMIK4emYr2}U0bz&aon~;rK=U^KU6eGlz*4YBRzSn!ZZkx=i4vmm{zEN zO8{AmI!jX)KJbx1Ypq{`42syTUMJ~K8_O>K2xeyJg8EopjA%Y7Z!AbkvO1eOx@3$+ z&o~VKEc$H|-F;ai`7=pO$9-2ILz^!wrVb@b$N{D(^y_v6xV z)ZoI>&qp;(?b|=PMo)eLm>%6R7(D@vJYwAJLZiMho<1(tot!ry?+d$w-)fVh0T5gx zxIsune2wVuoi_a4!)rv;v>cq`hIIGE?ioFx2TG_q8vA}G_WA1=TeIKtL! zoP#P7Db(hPIZvG4$$Ae0t4c3eFuGpJKDzEfxzN*98<0~MU2);4H_^O!WHl{gA*@S8H_Jh$wZ1B|_lD80_tqELcvBedu@ zVlnvzNcVhoRq?X;Knx262qX<>I0qoUz3LLV`NBH?g6ggM@qFl_Js;E> zUHpTOhXEO5v~pp_rjSmptYaB2vD9f*{*dps0enNG(z)^nu8VjeRG% z5{vhIgMGfDZh_Q@8<2%mC`gNG~U;N;>5tt{f$1wZGF_QVlhIHF7jwKwt|e z&$wE$WzyG|RgbR~uVp^g8Vyd*&laP>sTmN-Ru6yvuT=-o-ujiRZ zt!I;~tqE6;&uXN?hs@lb%jBrF1~e2MkbVM5gal?AJz5cH_3nAa1=&a^CfuAV!gykD z0C!|>2$=Y5WV61GJBlTsEv~>%M;&>R?cJ}2Snw>eO2@P(@2?#}P{FP=x0wMw;LVWpWOP~7M?=V9rcF&OL zKF~NKFa6n5uTHWb4v zJN!K(rvIc}VstzuMr+371Rl{61a&(lgcfGmG z2_+0SkVdgM!%w`J2U8RKzw}}`iyXm}q`uYUIisH}^8Q5u{C+$W+o0eT#A8`3=2d&@2~QmL zDBXB2icYxQ+}z3vyn~{^-jtECEQ#i%%#A#TCSt^KeWRQ4>Q&&)L7IEjFnZ}D3$pQB@x3&z8xoVet3!HeG{HG|PJ5X7 z)y(10z!r~Rfb2^ti%&_otF%v8k;(|rq5&<_s}_moX*J7g8fRH%znZ$>wOC(%bTgj6 zC3+*LjqjUl)IW`Djieq|OMce;&fTAgTd!%Z?c+>z9The}jbkiN=9#%@lK7qmq#t3a zR0G9pfwX+AmQg)M`w1_6-emj&(8I8WTcQeuQ4QO( zR*kX6;x$XRZ`QfT3Tt6W{qhIw*sCV39i|2Jzj4wE(Iv(sl*J*!^$67ui^JqHmbFu_ zq6V7n82Dl^=9UF6QMoi=DV@bM{KmyVkGzv`?Wk~_1u3Zw7dvh2rkA=oGxK%Wnpforhqk(2dOYyvwh2oCYxH1PuduiEe zxs3>7okli--D@wo)u=Q!bV*ayZqqN7u@0%nI=Xs(%}{@w+7n{wbRuJj4#r9xudiAz zL1Ud67?@s7t1?5`4G2SOpp+~c@xW*!Fau|*#@ivVkMA8ae%;zfWL&d%i0o-ghPPnr zkO9rwc>dU7a@nM2xlPsmRZdRSgu``iBk-;0HFRGLgBt2dV?6EHY@50s=UB^@fD;$0jppmG61KNtpgwjE#KW~QG5`}xoZO?O$vC%bl&r$+ zx$Yi?SEJGmjNXO(2J?hW1$Ko68GR_#^+snWe&&-d-3XjqUv)ooq60XS! z1Ch(d-HKBWCsD z#CvYT;{>4MFM6XGc!e9~SrP}19di>0HYizqol+Vo_s=PNzPz=@ODSx$RKIKlrZ=*m z6|Pir6;4?!4vl%7R62sYU*+8t=8^=9OzjROo8?x3*@JzcXps|96DGEDh zl}xQ=#tyY{^XKa0m`Af+pz$@OQGAIQ9#uW<)Y)1Eq?gc!QT` zXAz9f{m3RBZml{i!=m#o-N=DbSp0K4KdQ=}rN2hc3!}RD%!?n_N&D5Tr(;(liq8F+ zmx8969U6=?i&9N?orYn-x6WT+g5vvc#S2k5#ipbTDk$r6kB=F%MjoX)MCqtrW z$kSH2Ww$ zBmS3(cY%yEUP^2db^p?)gJ7_OZv9KdPt(s`Gw@nV2jt>6zVXl4@Y&A@jm_y67IX_k z;%Ze_*Per^|s!(T?ErW2a`FM$npz5dnms=i(78=!MI#Dk` zN7M6bv%M!?dZ8|9$!WgOYI-9ul=R(-e39AD;jf&uWF>BuZLe{^08ongxnBSj{W3CE zW5IPGSa=oD-j2KDrOn)_NuzE@b3lo)swiuYj)3&l;?KtBO@1c-q$qM`D7@g#T#@!j zbpMdDs}Y{!)d|T&|5VrfcIT4o+uc$=XEoJ{#_EY=d$$T8^Nc}_&cccZtRne1_NO?d zG^>%-Lg7i3y_D+@k`r!Y!9!X)%4{TAf;ll-3=TxY6B_QMmVA2(@~(VH;LGTI%j1W z)my`rcecPakCcqqxVxqTi#&uDA0J3rd@sW%bw&I+J%05-M0$|%+(4n!$T&fTJrZ*&=Qfs<>{))hT{`=Ve`dz|bfCgA;^mM?T?#ITR zpNPlCOonO9BtLqYUNE*GGUh&Le;+ijyv#iMY#-mbY{h{6fwmI~sf|BsX4|=-vynZM zY1PTM{vgaEx__D8Quat-c)`R1c&h`2@w!uAf=s`xEUc?s|HPuqRr1mwDsPXtVtt>@ zCJbXy<}u>$(mK3QeDNcHzEdmai(>K>xU%Bb-RF(X)Z^8B6|b9@-nuI7O>gP3a;@gq z3%sDw_f(fIX3vQP?^0}=-m8s(D$YPf$`q?U9qkw`}s=# zrLz;6o2oNh+S5qn=gAz0#{I1)JSa3o;BMb{#)I51G9G;h^Q*^ym7Do;=H60Nf;B+$ zw>t*y(@kWCZEV|TS&lm^b|;r+=A@eKizQ(`T*d7N^>I$Xqy{iv*Hx8=0^s(6tNbs} z%?JAna{*W2!KVv|7=+qT-%bYnOA<|AL#j}i%on%%F1N?L&vQ(2?C0v24kWBr{flVX z_=v|Y?P!BG!jdnQrI#52fMk6IoP&b3f?%x?_U8@fSchIlu2DM7qcy8I7b`i5=o;1z0&$Y~#>Y5&i*nzfdA5-*x z&qTgVtJ*%q&QA1}>;wycg9CFIKYn`9y(Zq=YBs1?+omm{`ZmIC*g@A`p!?B$$lbf9 zr)vcR&+pJ{=GxeNy#HZk^02F5@~^VlNr%5`o?fS55_29eL=Eif9pw8w=uYNIzEsGJ z>rPx{`bpgV^ak={{%otWEtB}J(7=PiEQ6>`-*=?%Z|U0NU5rC<_YbQw-lxqZ4W_}p{xaAlfF z|FS5Ncj4h0ZLWRnFTjDg-+ttas?Rb9nn5)#7fq|($-qQ;wB7XFOZOyhn!6P5`7fLU z{3E%W50qT|pH>=dw-$Fj_~PUIm{%`*7XOxTv*)8r$_q2~tIa`S8ar7mwb`eeRRgIAdRCz87#ojjo&Y3)7$o_BAXgiWyZi9SI`t^3D7>&VPpf6Dd$e{UdzO#(7a(zX8kP%7Y2qaci?TF5aTTN4wn=0<53P@+{`PbRwl^5RP^K*zNatMfbqf`sdCszzuPo z?-=)CcQYD0@eVM`W9jv}-A}47>z`0nUnGfm<1=0aKTUu5!u8yq3Ks_5v zkv8Wy;|(pp0K!|xC8IXl+LEiW^0Xfx`{`OfZR6KYJuPEnpIfSn1Acl70k0cfktgW_ z1@&?6#t8^`Cq5zD*WtcRhpRX&yua9$BrM=9`)ydyu^a6V!EEjbK;Mu z?pCB;PSj0i-)bvDd&x_(MJ2%UCFHsjCC&!oM$2@~zADaK>CXDgFb`JX@4eq~0!r0= zMto&f2+YOo6G2`m9!#VZ=H=Fgv+D9??^i%gF)P83y)8!YSF|{6J!?IA~Da&cu z@>V{r_f6hGL_kg9K$e7qt^_al`ML1BVb7G<9|xpZd1%5BHYht}VJZ;RSKHL`@FBX4 zxD;{(GReLTB46y4=+y1u^(s~{QRlluIQoUII?BI|P`d$WoK-N=RlW@AbGG#!71k}z z7w%H>dj&uL-FMC?#LfLe_mxHWf$wNO+ij0Sb?GxB)&bQN4Jpb|k>i z@K`LWa$(CMk;WVa{++A&sqr`IelNc9%kRJcW0-&jpQ`U0;Zrmk%TXiNJ^gv(`0vC_ zaF?@38VzK(Y*(OrB4fgEgYNZ0yz-sTuxwIv^sv{1z;s*cJ}Fn*6sM4%#N4(NHn*Rs z*_0C0@}G*SGSyLpO+vfXkNans{nDQ9LREcl(mdxFau-!!ACjA%`U;P(70^G{?Czu_ z*Q}ASjo60DxFe|b@^41B(^lz`9_xP|$}{s3AZlewi^#FPP5in@Ec&!ZZ$z9^Op4uh zAduTLG1h2<6s09f$9z+ik0y)6^ZJj;Yews1_VjuSPvR1Y{1Uho5@Wq1e7Z4-6k!Q` zsn_Xvzc$eo-xljb-Om+Lv!NUQUqroSP#j&fwoQWj;4`@E;O-8CYl6#Q!7aG!0Kwhe zl0XO!!3TGT3=$YD0RjmSAb4J$b54CW00f<}9-*<1!}f`(hHLsHw(9AR1ns$k9@l#j|ft;=meZ+pO?u{vQgTCU3exxse(s8pf?({t5B?J^N~DkP!EG znSI$>r5ovm%pG3VD6ew^8}cdqoVX?o(oW&Yy?ML$_BQ2>QFcP&5@PaXAo!vCcu$t1 zB4BwWL8Uxk+?a)uErTpt6lt>>mW+@60tNdy}hV;q&XWFDKYwW zqw?YMW9q&yQ9nY{tyg$v7Gh6E?z*3rT%$iPz`6r6l(%*fdWtspDXz|1c8L46BLp1uwaa5|G1%ZRDmVy zMlwbQUM@P9!vNp8Iz$@xjGf8sOp;~uyH3!4(lHwW**G7p*(Q zri~Kv^b7*!0@!IlV^4D8(nXkF$iD=wvp*Zv2`c#GhiwkGlo`OSi4h|>zr4(yW-Qa-}EEo7wLWz7{T3;mOPxKk6XhmF+CMJ);SpH3Q$?X@WeJrkQn}Sv5za8${7}bVm)z z;uY0Jf++IRR&hDe_?F;=?64T1=j7r0axJlweCt;EJZR6k2iAj~ZP(TGP;N4Q2Ap9~ z_Jw-#k8`qCCNl-({LP9LFRJk>6Qz8zdJ})_KsK3(3Z{@?c2kx<84g|s83=oilyxJ~ zTJ`0pBs!v90h6FJRXbaWOVE>tD9Y&hlvw7D$O1It;_;YLEi3$z29$tm~N z=K8r`i75YxYv+++$al%I{nGfBu<7EveSVlqu?Gj>U2U;S;okm*+*oAGKqGPh9FMQZ zUW$sD?&wY>kncGu?WHS^IUGfmRHGRJ(5f7VW98NVA(3C?XCOazY#=_U@UYPbH3gHp zrh=7W4pOKv*ARRm8N3X|d#oMA%i%+}>K|y$yf4|cPjC5h08{d=OG8@K}{ z2}RK_AcuK-3AU-;J=*zrscI^1j%8;|j-m*#n!qIS4CB`kWAZ@;() zb>P>4(Fd|^YpEDr2afpJ(`+BGs?-x<>qBEmFVhsZTy(&Q}(ZNyHA| zi97JMOjk%b*mvwh3&1V}u&>jmMAqmh#qs&DBLuJ{7Wg1iO)&|4!iY5@ib(4a5Pi9d zwmsB4uZxU#V~ zM>}L=yFX{M9mcB zi$Xx67Q{GWtBAahy&U5ZF`U?>B0x0JO4ufuqy^G!p;l4I&Jz%$tRaj@FDU}HYeDK9 zzw5#x?dy0lq;kou>eH~BxC8$^O4`u&L{Vi`E`;1jNSyqV=0hn_;a{raX^cj#` zbzsyzwj3jy_?jV$oR9!3O*3&QnxA4^9g6=I;&?4@oP;a(&haX9zKch`7xkCwymSk# zI^%^>h2a9nSqxHbV5X3y1w(aVz|t1fAXX$p-e zVg6!Q%mG`V8C4#!hGU}abXFQM@ZNCEzxii;DsfD@N+ET1+y#vdl}oDPi+E#-^Jv{l z#&idwg!F4H_IfoJ8xBpDE^=cvr_DA4v`+~D(l3S~tKtpRGJ;s+W97NjX1Ob42`bni4 z3tTPAvY&(O)}IM5QX6oVCJI8WW8e}D${zs~XiwO0l}zR4$37=&Fmc_~k|Yd0@fAVK z2dk2mF7w=pZ4C3E1H+vmcE|Mga!Z$d7m&<>+<_j|va6ExfK>TIVzK$+?U#g#0W_qp zzU6@mdo7!J{<$~2bB94xgv&RjFFGPSq8=#riqTT~BgNAl%jG^Lcn+8V8C>%O?RiAC z0DUAPSH2RTt;1SUqYmtg-&rsLDL#hk?2n>>89K&VA1_%+k$_J)0 zfVecBjV-6uFLKYSpCD=i+OZ4P@eqcHN~{o3aA@v>XUr}>;Ep09@bFD}3G5C8VtY59 zKEKIgsnl%X=~G)7YXBo3&W^VOF_sVF-YA4KZ@@k508=?4qno-TKuv|OH|U(4{7-mz zu@I;IuwjfG(JqqcSae+WB`T*}j3_JzR94(B*24KBbIF7(r6;cZ2QNxo7{{W}fgP)A z)V}7x4B$=_(2HR7$ zMM%Son=JaHSXzxL+c2L+7Xh%4+8clwHEDBv9i!^I^$KNj-TWQ~L1kCr1TixrMvAj2 zB?c^Q$vwHXm7!38msE<6yF&=oz`H0H27~0d3dzVJYrKHn79``OJcuY zO^q8Zr;FINNS);!zp9~UOJtm}pY@aw{4y}k>b3exUI$nez01HrUis}i zDni{tJB<)Kqv1>Zofy{nV0juev^(Ns;w~Dr$Ji`;hS-X*>kANDlE+B6QKoAV%iy9BCFC) zg`eh+C^{6SG%X6X9$?cHZb6R529LvqHE*T;{SON7;c z_*zI;cxH0V%k*uAu}VGBiJ;v=3snJUv!bGnVK?1k)*Wu-F3=`G(FHr_wB(w#=j>0( zuudzhYw8PzT!=L@p~>N1K%^k+1c#a*i(Aq*vwa2Vu;Hd2;!a^7Ro=;_w7SUM`f9;N zWL!$QFD^1?FU#pjrCuOVe|PdR-dx9l==Rb<*pF zkH<~vG83G%fyDDpT29CY%I-I4+O00%=M6$fjV`--}^bwZmY?{&2RMc3FpG-=DDGFL@2ANYdV1M z971;j&jDTnj*h0G;O)rMaQ4_bWzi~g;gFR=xYULJYOMEz3R%|Ewr&{4?9sb zaN2@UgX5>SBb);;c`BMgS4%i!9dQYU$pW*8t9$@ppd{v)X`Z!^7o8}E72KPrV7qEu za7V7*m^UOX^lE-{JkZgt>}*xJkS{g>qbYK}z8m<clDrHUgdTkw%PFz4_cf77UiS?3|+sUpng**sGsHx~zvnE*+=bVLy|l zak6P}B>UvuE;=@--ra1NIZ*pwOb?}k0P%WcK0l2h+lc_!j;jltxj(}0_BRwqq#9jtfyLw^*nRG#pZGcixsqM5gw7S+npvtoX$ zo=IzzYxZ_g9mo9gR@S={&AXO`!0VCPX~s^u$SIc#f^EMNJLM02Ofy_r3#(L{*xnnE zfQZ`~5h?HmClzb=g=FfMdsw!7+6v!JHw|eqQ9|f6P;IHD{ z$Ys|wfX^&;DUuD<^FgNXX^8Z|>DX(53H4o(P@@To3RxS!W^uhiL zO=zEMlOQUmQS1@)YdHz>ZB2+x?_nYDmU0hG)`yEC-9Ea2tupyLBCVqF0Fy(-$!e#U zYIbyu9*4qi4fF5qx_guFfb%v=MUHyO>QZm^!xsF4_nmlB<{wnc6YTnYB!cDlKZ@!F zOKE`N(1$FKAp7_EJ8Q94QHv&bOa^KfiVhQ*tmXeu@Yn#2O~%QsXgc?-y!JuiO+r6t z2yo|Q3{Z+LHiL(sG^{r;mvP{o9$DQsyvW!dyxR0i5g#QU&F%O*WT8i;(1*5aB4O?6 zr-OtVsU_^DRUyvTgzm_v;RU>kBkRM1j6kyfLu6b0MSjvaNv>!Mn<3w42WLtA{K5y= zx@K*~Lw7~%VK}p}XKf`#QNNUP@dNkI?vjL0@-Tez!;5( znQA|+(trh|tyNy1;JZI7Ix%?93}-yT#Q>djPWr9UXBsTw2DY@%H`~u9xm}F!TQ*&A z?W@FLe0B&|DPeCJy{m5%Hlr~8*pmakbR)So{$%aWV})9udY$I-rZwt>Ql!MG*~nNy zhb{-VYl8TIzs;%lC_x0_@#JR#`SqV{cYK+#I~th} z))QM@*&rg@sQ`|KocmYUR`c<*nmNr&mntEysa`G9>$YN z$74;r+(<7g)T_Q(q*5a})ybpc#xALiJqEhC+TKa2vt~(SD0a>=7A#Sz%}aXtR^#=& zx}@=OmS5C4RC{ob@tWtg9jkmV+Qfm#xsd*iQA1mK)^SpGe@n%Ic3dBGYR^Bq@j+qa zwNb_Tq=Vmtto`605KKH~yV=gIdNC^i%qmW+U~7iC_K5DMlKet8!}n%}{SW?o|Dsed z8`Xr)vc$2}Ou>cw7X3ck_IyKCc%%^fl!xYL#eXOYyOx=rJ_UCbt@Ga)c!Ls5RpXl2 zrot9_5?3O9mrs#eIgg?dzr55UE7zIiBU(*`o|i4<`DfO2>WvSDMJZxeKWu)g;gurP ztS-8eJ1L`iQr2#jLv)J21eN4Lj+w*4o1 z-c0{JZ_5M_J-?DVPceO3!!;Z(v1v8%c}hYfidQu0Q62!>SMu*>&W62fhm zP=o%V>`)rG`~~fwA0`hMNaGSXJeU^_`J?}!MfLn#!^Y#kbYj~U6u?OIkiKhZ>?lqP z>H%@~qqIoN+M2|lw_I?8z9c&)ncg|5%K$SLwMz0b(GTD;pH!pXSs5+-5D$_W8jg#k zC3_(0d{QM#T;qlN(c;<5htdtaQn@&KWy!fTpD`Uk5suocI#I>df7q8!&QU4DJEmxI zIH|PAX?U-qu(ACH(r;rd%c*vE!@y*JDOh1yUyuHy<_S08A-avtBOnkAs#<&`F8{UY z8lUAML$r^X*1nVd zD}a_vwuzJcUb7?_x0CaMkyYV>9jgb3wQwS`h{!iG4{*?C39*AhFGzZS&rhBD6j3#| z4(4uFic%USWm!(=*qpop-0^h|*{HEL>jzADv)nZEFb^!rrfn4T#dfBy{ryB%L)4Lp zaIX4O{=@pg_hgCEMj7BFlxWe`TrYY@SsWLsQ=Q3zv>jiXna{agI0-?gkx+f4^kr5t zs1FJzcN1Ctj@CnBTCGB6d~i$bbX!Bs0!5Zyd+;lj@8?#?E7RD>28N6V%z4 z+DT-NuK_`9?bb6L;CeD?I7pA@XP2b2^zZ%g4+jp?spb-*hW#KSk%G*>xK2(-`P|b1 zYm4HK*XzWyi~A*@}2wY?8huZ!Y&RvicsA&v)0WK|M~UvkEA) ziXKyfwBkRWJPXexq@&xV zdw73>tNiI%82oX1-sfhgOKX=XS>-9G;#HCy@p^!7y|$t32-YYMcS398+)d;RUWy?vBi{MGEKjA7&Q4CQS0cUR09T|d?Ut4E8JD3 zKOmh$M5-z_C}Q{tl%>TVj>B6zaKxC9ECbF~zSj|c;-qI{%6r*~MoRS}At8lIY29JmO zs_1W-y2fRVYfQ7*%Nc?)n-GzKqDq?$?R?N}dzrY0z!##}FkWC&1rOSf6Jz%Ys#oQF z;&_DMff&4|3YD^RTXpPVi&pR(tvF`N1$t#3njY`ud`9|4Y!bdC6F8dX2Rq+rGAFi% z`UT}cSgFGk{W@z%W{zgUOoF-`0P269h=JcX2{sx2>PF>1T zKlSimo|2D98tD9u%IdhLBHrhgzCAFUUgF%AfC8^q`^$!c9+Z?4YXQ7V#5_d{y$>Yg z(lf35Rr3&k*lZI>_*NU7j(VDHFlOOWPMI-Tky8BdWx)T37RBCV-uu{)^lt-|nQInFnUs}uBybAT?wvxZJ=#1Oe(a4dM}wJ7-8Je*m25trujkfEQ2{+or* zstQ%^-EvcsnwJBt&vu5pB>tUUhox59dcHCbf1PF`M6D|!A;?6M{sdmN5mbU;%oa>q z!uJ1QLZ_*KzTkX5`c^UY73Fbg;48eIEtB<6 zbYQWN>i9gIhpFlEXf>8$&p;@m#Nx3CcKeUU77k3pIk%HzUJBJ8waz<7n5jkkzkzdl z($UVzZ9Sga15PNRXU zpjLHppXc;>*aq01UQ1x0eM(uryQGP`e=(xKt##c$FfZNBy z_iDdCbWKyQL3bCjRNFgRPc7HHFHv_N>h1hED6<@L=2z|PXm**%rR(rjUVjBLNUDD(+@3-T&R%qA7qu>4G4wQT(|v%VT;1R^mTzSerQy`k-^k{AA2c$?&67wh}Db&nz=Ppz&V9mI1~ zl24x>+IRma^+#_Z8qN!11|JL!*D!q5SG*rtM7}{zGl{2MGt|L2+e`QYFVHgEyEDCj zroW#|4tXdcZHFVoI&ImZt~Y=0#53eci${JU;&AY>=tXPo|N10uKW+GSbwR^wt0GA= zBv4x;BZ;+Y=x65|O<(x`eQbcEzTPKk|9T7m z4}WFbR(yUm*QI$w=~7b9z<2W%`>S#B)yysV6ts19PYUNDp#)N=*F?FT zkgpRZE5O7qPx;dV_lpIkKKDc*$eq-b(hPW&B2*SSD~4Ago^O~=SPVz!{VQFN#eH8p4SDYZB<@X^6Chj6+fg+Eszq5U#qE!>z1SPd-89rpRPvAF{=QUDK6+~E}%g=~mH|>1lJov17 zZD;BS?~B?nRiF^6COLV3*~9OUsXS=x{rT{ge<^zC%2ebxGE7Bs-0o9X>H9Wwq>Yyjkf#q1(zJIvlgJ|;e4AeQ;S~j#WPXBoz zL@=}Zj8Z)OD9Hw^?@u`0Hhe#bmYhOs2kOTgs-Z|c{8e2)p*-n^9y|+SOLeuO+y~bj z%_a^F{$hP6@5;6mhKjIl-#G*&IWv?iKpeK!30oA0j(U0)cc4Ch%@mwOC|wm5&0Gc3 zPrOD4X(>J)b{V~*P@FQhP?F--4WgtBL#s&}McC^wZbBRrQmXu^3X=)-k7QzI4tu!q zodb-cCJ%c&6TX{F46C)-+;a!_sDXu7-+h3HZaX zjDeG=(Kyx5`QVVTn9+Oe{0fIu(rUn!{w1LTJ0jngy&GERu2g=GvAD7iWmDEmdLaUMoBD2|Iy$0F>=JBjhjAh`TWAuKVxv-kkl zeOUwWCUPfH!FHN~)}4H0lJ4c96gM`$LUKY@pwjFP79&0uKbmM1kiSuV92(^e^_tS2F>G4c76))If$5)Da#!)2wO^jd-_m zoOo~fa%_O3Sw>otrb3q6cRxq~sS@x!=H%IaqwnL+%;A~uvzXg%?7E9ZD$heJMtP|J z{Hn>9K2gyT4s{C=(V({-=Oy3+?3j34VtDWK5BFae_&~7~{3xoio=cQ1=fZiffe2TKB=f0_rUv4ZKwKJ+N8=Dqho#S zw{+^weX$FI5`v9pDN^@=RP&Ed!zD&jwe(IQv_o&31b4Qj6DruwZLaQ5ypz@if!Xj6 zRKwrAh4h+sk`sQ3bm|~k*|g*h62o~h*85R1c22tqJbEan?r*K2bz31{ ze=5VQe8;A3klMrCO&_UnHWq2ClJolRi?qjtE8j;eQc+RJ4iuh}Q4V@;e7WS0wf+$h z^m&q`WOgp|ZthE0=0X2Tm~x7lvK_Ra(Vh7(WqQpMet6}q>(f7!VYZV3#_12q)MfW! z7(%RjKl)Oebri)#X=Gl^8mrEVbRCFbJqTKrD{+yD!lSM()9a9v2>>}A4jIbC3 zGyz6hq7j}+e;5{AyJwY+uC9rLdB3q;>E#C3%CN0HVKLVsmvoL1`GulWzL%OBlfe@= zc!j-IvO^H>ipRSwf1*{=^9S1{&mw8LLLA^`&7KhvA(M{YXNnhth)OC-!}G&k!ywF) zx)UCz&p$6>_P^#1Jz%pxyI3?%SX!UsH4e+=hf$gS5Lx6FZA?Sk7l9(5Xbe~`o;H1R z4;A+s`9ne|gfnj$lyJiTS|ZUSika`Dh?q>vsb#`~=|G+Yn4Syh)=bI*896jl=7u3B z1t#A1CFtDZ7s4p5TVp}<#2ag5K*VAQTZ_V@_3xA$@K%u9>X^Igu%Cl$3Ht}_db zsaomdA+Ie&cpAlwS2X|BSUJ@B+yQO!nr#*R=)*$}g?px>_uY1O)*Q^NNPOkhA4J;H zQeY;tjp|Ji5^4-C$MDdS(@NQnhV1O@94h|6#R@!Qg74uOE7JbA9bqH-Pj35v+S~ux zk-R`50xLbUdcK~GJoX15{=Ci5|H zDlW|#Fvq~HZKn(cb$GKL~j7Tg-C z19o6yZz4B|8Hj1CvEmHA-OsuwZ$}4Ad7Ig|4;T7CJ7CcKon+YT3nyvQIPPkivc-Ux znwc(si(au;11rho^%F>Mkn2v7h`B?qea=`ymb9^P`m6QeLsd zUUD7AU;6uS2;i630IGvHn*js{Lu^TP8cv!@3QEaLf{kfXM3yFLO>79-` z4*b&k+)dH6uSQRMKke>$U?*c)KfJ%D``HEZ=x7MG4>DoeYo$$WcJHi9B=1{xN~^6! zd(VHM-dr13*>o08CWuiE2^iM^Nh0IZ#(bg@j znJa%C`|880r7R}QFm+Q@?7(KWEcCR<@u0%1Ss7No;#~K#BK+QIXv*2RiU=FOgcEiZTF<>cI$|MGP@6-X+<-^hBY&Av5S9yb(m_SE>=5!?XE69 z9}?mAGijP=QIPWld2Gr3D_zplYv(EN24$5~(%^&jL&Vmi6p7jxN<`LFRl7FZ%bs=B z*WVaz)9qoNbyPk`O%e10V|5zSKqz6I3f_^}c@X)WdkThNXYL2M!xWY`DNU-?OZ{N- zk@k%-uVWh3{v|@VJK>j|ICQ+ec(!qiHh-GE{4^|izVvRfjdrXzeW39}woWn8Mu;7v zrCZ+bc*vIM(_xwj+VT+E#@0>}Hm2SdJ%Zup6AgX8Ca+u<`ct)KJ#(XN=!GSd+h2Nj z_!3ia4vkK?ZOvaAhWhq1tf00Y^0lPAd^S%_O0mRcfC|0mS-Wkzte=y#`Wklw+qoPuP!z)Y@fsHWSe)S4~Nq)JOG@==W**_A_Pc+;RVJBdgtYMDbV%cJt~L z&D(c98YYK1Ljxn#g&+8j4^;#C@kBER=FDqdBXe>FJ(gX%#eHJ~dsJ;#O6xpaTe}A8 zK{Rv5W{x6TBECx$cjdH0F7#}ckY2#oltFOiU8wIo=+!RL>S`Scl2}i zz2uue-*fp`;A-MUp=qWapCqZpf7p6)^=)OX!`Cs+c2+)fFU>+pp8uD|v!W9r+3Q~3 z820%=K|_F2mFZ0=B8Nl_n5Wnrb3FJ*0?u|ki&tg5(;=OH<sMFie z)W&{s{c!z^jb4r4Hz!9J`-d28OTG^Q|CZuQV9Gz_mdM=2R%79nlx(_jUOdi{5W7v; zJjk3!y%U?61UWg%3FWWVq0_g9fKs245(m{Q{a1waO?~%`wiyxk%`NLR+VyV)L0}}J zM6OE%uvf=CWxLTkKRyM|;a{Y!w^`)gY3fMW#I2?CDdnxxEiY)MF+DRdKRszIq^(#b zxZ)O8)nd^Xd@8|S@yk$qXL*&xDm~~|8|?E!G(!`y_Do*5X|7u?G7I`5SyB)v<o;5T$~*ppg^kKc%(O`}W@PiPuSmuB{zNB5(PQuOY?-+WKj1fNB6zkc=*s;f zGQl$&2#^%d#J*AcQ@t66#%AoucK`#pla{Z+7?K^mA zrMbr!qO!{S9&dDRjQ5ljDD_STGkX>^MFwXl3_lySiU$IaNhWsvq|LIZeHj0-4SvW* zZrKUe4N}au zTLu;U1H32p%tpv9-SEBQE(ql6Ce*J@TN6>?{h-xf|JTm6-9SV`ak+Q*J{n^>jOoXa z+s*^y1QTw9z!VsnrZxm_iH!@`U|nAT7ed>aM_qQdKbB``A}af zSbXc&!|&F<@qD}1=42qK7*X;x_f72DGROtsXk&@MyHCB1*>M}13|+QO6}%W?kmM@|@UEtfoHLsX5(mrF9otk@VofQzt4 z(mx|S`E-|-aH>@5KTJgNO-nBKC+%O8{C%+v87Bt}3-l+g8#Lntwf8ZM5qr=l;COc~ z<^wT>Egk^KoPl5>1{*io49@NLlswl$;d{`5KNBT#Oy2w$Mm={+5xwyk+a?ebGwyq$ z`rjV#*xBAWhy&gDZD_bX5V8e}l_+U|RS)K-O5>i3rOALiB)VJ;D&Udfj~%jpMZe)@ z0qd7@om_He{siN z&+}K{wNy9fE~A*{NoE^c8cu1r`DQQBuTc+r&~(y=M#H(uhrT1il(7KGQJ4?sRFCWH zp^&#`Z-0gNK5|>!avj@>lU{^>NfWaz?LXMNQ9F}8$x(E_j8AB+l`jn=7&#JuTzhg{sPqWLLs-dhf$&b zP@0LhjFD-6vr==asr~zI{n27sn~Rqkk9>W^G8KHtC+)YTg8Suvf^KlQB&Cv=piTPPC=xD;4fFUVAR-5b9o>nwJIMj zga~i7GV4pjAm5r65AR%p?)#=TgWSgbg5v(3U;nvCnH%nUo?i2xYGt0NTz`Ls-?S9B zuU#w>*T|V@H$Fnx^Us}DG!%oal{%loAN{#8I?Q0F5Ke}jXaO-ZFUKQ_yRSkbgB zIiA1XepQoq9;n&UF@eTBNA8gk2gz9jnT^pm|D9VF9}0Pw&?T1460OK0VK0_nCFh{9 zBPzhGUpdI;KxAOQ^})NEK7l@*K{COvnTGy7h!>>JLPZXM4vNrEz?{%a<0IcXCE#pj z9IGpS_u^Y_dXK^qE0=a6VcTm>um2=aezTtcV!H8}!J63gE2quCLOROwiBD?A8@2ve zp@8!~uBY-#Rg>guCW=B~eK zB7;{}g2m~@sL>&g_nNtqmnB#WBF6Wn*;&UCo}>)QbQ|s_vNOF*SSCX&!($_;K7CK@ z)xadt4XWNZf1o($7R%jCw%Reniuf{F%&UiCPDrsUW|AXbXq_X(`1=gYCmru`nqK{G z5TlG&Uq6zoN|}Gxh`ckmDdH`c=PA<+maS%Y{7@BbCJ~Tf*Mos#9g~6xBEetBc@q_R z&MCSoYeV}H5Mjy5_p%^ofXCd~gfVOSG9n3BF24pd-o`_Z^yrg*s#fZkUqvB4qSm~PLX?RIq~z+v7>p;e!}*6Pn!WL0TFSI? z4d9>RgRB_x?1>(_>@3ba6u2iQc*MJTb&PK#dR-dEjg07KgsDDXN0(EwV3ChRoxy@bDud7f|RXL{Ct*w+@{_aw>ozWW% zk+M6Z0$s>3>8I4Y;vay)1WXrk%VIBAnG{0k!}4_o{ERjwckf1|BTeyDeo|ZyW3`QV z2GZ`yPQJbwW}%s)R3|^@>G0ykp8jABAi-WT<6@vZW~fLL0ve(OOf`8w~O(z7f?!FTEo{2peOPM1NppMowV;nSr&ft8JM^zG`66TRq zk$*d_@XIV>i0@l^MyGV;UR?Hl#0O3$!~P>FUCAGfd^bgjpO;x*h!>*fIUAvlg=Y>? zM7=`&m02_b->2~x25;AEyUX)3M$dOv)m-*r3M5gHo$s+ltPOvsj={zx%z&$AX8yri z-riSz(8v(4-X0lC>d|vM@RsG%AN0JuvFDQajw9}VPo1$FMQ;gMuT~iWY=HQ1C1E({ zeB{()c8a>r}-jWEsCutko~07&d(g>`gduCK@wVh=*kHE-e7!jNEf}hP$%N zK-FOb^pL4YqUBRLW|a5VU%~`B+XcAI8M%!F+*KRqCoht<8LKDoeSX}?0x0WbmLUD5 zcy%X*Z7Ui&o2)OlY6J$PqK)A%F%=&RvlV|6o)NFuj(@pN=JRrVA`-6LC+xg?+~c@o z!;wQLez^a_qulb02a4I$wh>pSi^pkMOR2`=6tRG{4gKvWqEuZ8@8qpntACtYd^P9v z@nY^p;ue+Ktj#+X=~vi#-|b$F)yCIRhGfLIy#zp4cU2{JP0VswJ{lvwMBx|m%R8RT z%+eQdbe>ZxML1V*B{kQhRplBjOQ?(y@hmdFg5#kK(HrH<6Css@7z5SmjD7qm{0!T% zIZ}@6R{5ywD7q`;mHDMgSjH>-T&Tl}-ofs7%UXvM5MeOW$FN2)0! z!lf&emFL6cveacP%X0jWF)nC|d26qqSXQG7BgK>EEt+Z|C(8Z_dPD=UEbPFWDSQV5 zjBDXsj%-haiMI$^bWhCa4@H0<5&9I#D#!f;v|gx)E5hA`l`p=H_66rF(&6UjUzkMM zn&K!G?0Fj$l!X>L7K=?=iN_1 zW|}Hts^zb6Nbxjm2rXBdU8SJ5A|{$`~*yCHMA^c z3<0<~q=J;q)-qmvv@FHe&Brp|V0&M5At2?CW`i8{xyDKLd>ddMeMN@GRU9WS-EZ(3 zDQkJ)lv%2#TY{0b30kZ6u#BK#Iq&eMaj@e4dlC7CfCuZ3ZxnG8sDWjbWZZ6r-km{d z+{y|nxk}C?o3I_t7K7gKdKv99ke4%?ZkGJ?esyUl)iuN)q%4h2`Ix%MC5xL)w4+A{ zZRfiCEPKa&E2Z+z9TTG04eKM3B*z+N@`><{9{!`jpr3`d%!i-OxVYk_udCtC#XU}l zX8OR#f>)y8k~;#iB;E_mdbG7XSpQ(SHT?dKbrkE&=)ARbwI&cPKBSs~JNqr*O?>?h zDUjLdlv;lCN4y&u&L*}h^m7Ot(jt%T`dp5v^32@_LSk+ zZ1Kv{SZY0{Q-U)m5#aQWK16&h5lxQ~XqN!j6B%fQa{Iv)0`)9}w}>i(VkQ*}42OI~ z3xkk^*y-sGsVi8^?bLs5#=H=&VP;^AbavKxKk}xq-jDl+U#$UF#LEH7;VJ_SeW6l~ zmF3H?fL21uEiV-rl{mHR&A(IWj|C@h@@eu!tn^SsJH>0=V_Z+AlgLESK+(=}sx1iS zc|17kok16Knhc~Vur=yw={ck!%VJlgB`kp2g)8o1Vq4hEYHqS{NxE85QQD*%!lPiY zO|s3+ln`IMQksTf3(7M_;2BenIW$^mJAf;L1qP$Y5R_RmEA0~rX$#TwAHs^NF;}fi z<^UE4THhUzAWCx+Rxw9e#+XEVCbr?!y72<^_=PjT8%Mx zr3%073S$kg<}6PHw}<}#Vhbr^o+4U{Kd{Z9^0Te-CVsZZfOSg#uYTkOKk$-Ac)DnwO9(v<0RB20`=?3^G;f3u-7ZnrNORHUQ}_ zSpv~B=HnYJtA^fET(cZk5I9&&8Bs;Mgo-K(SM*@9YM908-w?{R`#{LWFhxKTu2Toc z<06&JJ5{)X1w(47!5xfObBH3>RW&Gvk(XUTrJ|^&2skz)$A|iXa!*2c^BO>R$n_$R z$;1Of7SDS|Vf zSqh4jh$`4MRL>>Nj7fg3IZK3*WKG8^AlICUR&L-`4|K^v433Xj2#-EudzZm2LrUV} z@f4RRcZMW6T^NTj2O}=qTR5n)Unek94phWy8nHb=uhgoibu&ZLueo9uT&Ku}71;xz z`tdJc)OUTLHIH$i3AFISzaDF*1VD#K&snolgU`uPx;mdn$ItM9LOCker>t9NS$-!? zWVbHOr!_60Phf~>xo#7;$k1#|eTZLmh0IF~ClITMA>VIGr5Zlhck*I(I%3jU+o<-_7Q)EN+Sg7U9ktbtik^P;3{t7!#WCtqct!C`>(ho)yh^? zOAiJMNB-qvy(#!&xn&5`B@3Zyxr70w5Do_hQ3G;3j4etPK~?_%WO0FEl=guf8ipqT zd>i(|CgcfG0HaXO6$+(@1Q@DYkI;oQ^B9Wr1R)Cy7&k;|WFb1p4>PPyK>#$^*kpHC zQB8)4Qz9*vu(zhOsHb++ykV(YC^(A(psm6Xbf`R+0aWu0l^SSdRMbJ(9$;Z*X?YA| ze5uY#5Q-x&B*AUK)ugjJu1QR`ueb>wV&d^1Ls8~1D;5x!shA*u%5xWbWE~H}9wPMc z8Gmo;CGg6eq%s*CG2g-D`0N9806%d#S>-2~)c1lUGTYSD^U_kaj)IaEz$K!5>%56t> zyVzZ+>x13u5!R=T4ECrMK+Xk%lrrMQk#lg2I!=d}Xb5b%>Sf99;k)&6*wpr%uTiUZmQhH{KdHZs{wJ7K8zv1%Kc21*rHSq**W2s2VJXgkd^?oWu&)&iq71x8Vmx zBLKE7s=9^+$%?Ar^;{Hbla<`!3!P^%aBWIqs}$)L-;om%))_;NBSMUhE?0W7M7G!F zAp|d}nyB*#LcQZHG{9RMn=N1v=b2;XC6kGKQvjp@pThb7n=U#JuuKi@=%Fi0)*$H5Gvz1l%+drm@Wz)as9jkOl*_h@=#Up$axj z`j@zMFmWhRoRQ*fmBuq;(r}+SGU_}LYA(Z1ykT8w(Bb(%lKVDu0jX~IjF-uJ+l$)~ zw!oY`PR+9PTu&8cR`AfsSThH0(+DX%BKsCVc0G!t16NTj=_3@R_lH2M1Vsk;lp}^& zbsMhPs`Khr7f({_)nwTdSjf|I=$!hj<|f&4ki-v6OPz4_DL`COq465glL(Ql2!PIG zfT&>t6Lk`lh1I(?oq?2;>uMpRh^x%Vyp^XYrO2F=J7lbtk=}wHOO&=xX;7R8x8K42 zm*7sPnhyk{sjF8|RwoWjFLR;sIV>ReF4h*X?KqSx`Hb@whGx)2Rdn>g;v{vVuZiYRUB_6MLM<%VEpV#`3V`0Bd$C)Iu~4?9(hRX^%E4hE zh;|G72um4spH)5~!7vHPJ|+i=k*E%mlsMUWr;a@)4>JfVH*oxc^Qd|~KXVRNUlT-{ z&S#_usjMi@n4E)}THF>QGRw^2_*Art!sW$(`7E|u3aR3TEi8ip9~F^sc3H|hTqwNF zxkcSXYt#oGV-6q~6G=me7DHtjItazmF{E6?B(K0&Y>syh0R2Z6(Tf^>VQWzJheakl zGqim~nh(_E>^gXAHhKlTyh-&$le`w zaLVoxx#At>B;btfaHt1JS&dvnE*mc8#3O`a)J#$gT|&o(f$2RoyQF+rK9KRId?3(R zQdxYNMIg$lf!s2_VzY1;^(tP(*AexHn6_G{P*V4>!mXM^r2hcOEg*cVh55gTUall6 zFBQ3dBCgT$CHzK_PNH8CJR)qP11oR|E#e469D)~8g(MXKA*Rg~P1sE#1&UR`Ff1^E zDk|-rNOwS^34R}ie#0-+3-79WD;{uHJ6f&=zS7tB0x`o^Z= z(m-2W%gm{fHVG0p2VaV){4&P-krR!6Vg|X)GE)BlNd2Ev{=}*NpMw*htg(yg0Ah8C z;zfBUr7tv%@Z2!xxfRFowJuqrUyO+A8_@*dfG}wsTszoWR3V!7gtv)7DbyhEF&ss- z2v?b7M^P9cx*|&nV~8vG1_axi_!)3z%gL;<)tG+f6b@$zEWh2+Jy!8!6%>L0Qze2qCP!#ud{1O+>L#E}524 zharAp96$;q8mW+pcOD?@xPAr_zZW!znP42HzGGIWI*OuQ^C)im$Dt}%jAUCkRX7N; zB6dnxvf-lTRQ3E8T(Vqc)M9=TIF#`b;!)s;@dC6z2#c|Eif3E;pZI^rI7p@-)>EnA zj8i{|FEGYxC?ctra(FR!FPW!e{G)=9^ek+*!*GDkdK%*DpS$-2T$%MDld{b7!y%(O@IiZ@VbMm%N+9p z;RoO`nR6^c2q+S}9Q@0z-KH3O8gK?gWYx(jdwUP8e5JRpVfl@0Et0p zR29J;tMIQ0#cdWJl!{K4e)*oFk|RNCj- zTNeCIETA>diG=KbFq{pr-RCk?*=1q|j^h|C0%f5InE014Rj~x7SHCgXNSrVN!l}qq zLxlHY2jCdMz?^<4NMxcp30stk4u_b!8&dmUh^i`8D`uan4=LE{32#x*@iN5~La^dk z>&yk+^$gh;oOJ~ryZ9_vgndYQlhsC!j7ZV~48+T=eyD~%sJBP7xX+?C=g|xP7^b8h zztmQT>j~DLgz_k>m+A@*R9FC{xUx2ocQ|L2JjBdiWHx6Jm%hH5ROxQ_L@z-L@vEWI7No2(cHGhk_3ffN(?_LK8$s zh|{Q!Dp`pPgrKhf01Jam^|{Il>~#h%$B9cV*MNz1v5GNhSkG3nqEeQ==V;UjT(chi z=aE&;&`NJHkz5Jmki*g*$!GgD}r$oz(;^a93RG2 z#BQS;YM&9yO1&ZoE9S6G-{gm9=@Q02W9D*z{L7W7+cZVEb0voA<`6>NO586i!6iJy5VY`hq00@bkR?F6@%iDxR`|$OyjWg@RhLI^trOU5Q7O4Hj6f zn^j6Uid=(ga)>X{c>UB21=6LZ1@t7u4p2n5GNG{CTM2R8qd;F8z==)7BK`B?g*8uM9_?B?l3_ zv10{*H@NH4QA?F>84IUsS7vCtjl+ON;AbS#7eLBunR+<&mA39W`URc_kJ&9UC4PB6 zTt^)8>KQm>{i`!)yM>*7?D!mUPZCyvv{owJ#>j8H8fnfpco+%5E7$aR{1k6(}EaZjsl8RiMu42>? z*;*lYi%KunQmd9K$B1ZAfzIF*ts+qxa^#|`mPLqw1HQ41xiA`+i%V61;fyyV%pPlo zP>gs(=TDnGWuS6YKTF@VEg-l5Qst%%EI$hjG)G1!Qby z(k4t;WulY{umZd8WC^COSl@Ui3@zXuVx_?qDmkgytyU_=j8I1SA@c^(GL@!RA+x0X zK*~RH)bG{CBU12=7LWpRW-DZAjUiU4QY&Lxl9r^VhMHMo?;aySt@}p7Mv(~uHOv;% zSYd3zc$;y|A4NpGI|>P!x^om5B`~|`oy86YT%*<0vVfi@m#(1|Sa>*=M+mP4Eryq9 zGUEqVMyNawFa%-Y)t$k`R2LfdT7AMuZr3~8KX4~e6dnQd%z4E>N|DR&NRM zaEpCy_KH@ca7A0M66nh9zd&;ZdPpbLq5%gj6%^k-i)_1c&wG}gSExXnEcTpp#+*Yo z$Ef|HuSk4~FbgKvf>v2Lf>~>HjJ4fD)lqtgpiqk1%6VY4IXp`>aHU%|DlK>FCRUls zsQSc#sZW@dZ##x32boa7l-8b%00z5mT&$j1pgZdoV~DUL#7(fyWfe!My`b#^cai=s z{6diiH7M7ZjHy#fIeB1Jcj*Fy;D%PQlBupGHfhyC)up~A3!y)7 zH$L|(G?I{5PcaRm!rroL5NNYEn6`2`3p1bEtBQj>Eo#^Z#NkvmLvIQeIcEvnJQ!b! z%xZetBFpsgE(HS9(2LOyIz4ZP)LmE*FJb=xq!^ewU%I#M3~=C&*irQXxD>j2;L6Zd zg85Q?!vM~J56FNLOviC2!08_rLN`$C#Y;G)T#ON60?h}P;k}yGIFlmCJ&Cs?qY#%TnHu)gN z2MCtey_L0H480b&Au!cD6)C3yVB(-7hyk|(003NoMVwvO$6SMv3-DoWB}4+PTSw#2 zjJ>}=BXk<^{(=J;H$t^u5)@f^0hOgF*Z4&rAvXU2LYPYE`WXi8WFWGqHfb~) zd&Cd|?}v$7K~vUU#+_7K6$1b|)GCq#N!H#dG@~_1&Y)#9amDk@^Ihew%G0d9=Ag24?Awj337a!LhiMnoow ze=@j>N|)I)^&dTJ2Jhk$UhM`wh`Cs)*tbX3}$2raxF5a@D)h(!yBq-xFC)j?hi?mrN1b==E`PGC~Pw$>#9 z^)4uQ#VEQahR=Y=n9;>Lh>Zd39R{EF!V75S3K_XJdd}4+n}@aCaWy&_PzhZyS76m_ z2Gaml;mxrY<#7x~qNRK;!Y~7Yj=%(S42XGHYmKZ1Tx_YMaesFz23_qW89Slymb0jH ztAeE{-!Igz&1Hey-Z8e};WXTQ`97W+j4_lc*>9@mGVyurH@htVeL#UGAKba!8fFDtF|fu zW5gPXJ#O+JN?Ff05Dx+F2Exvybr|pcr2-2$7Y6Aom1=xu~2!L|3Az?!3!LdVkt=nWbOKg5|sA z(C|P`S#F=)l)$CoPakoqhCu%SJ;nrzu?l_tAs)?$n|@`B>bRQRumn-t@P&{qSiU@D zw)lWh9w@+sp?K&&Qr#zl$rlcE7!Y){4k5lV=axE!Jw=eU&c>iH>?L5Ls{Nc__E22D zDl614J>?LNJUj2+{9^hlC?m= z&~q$+m|AKWTExO2Zs52g5EoYsKt1FWp#yY2BRAndH23oXi$KlB<9#3b`z6 z#Bis?6@{XKvhowc1^1LtDDWBY7E)c5EA0O{e-JuMLhb{7w8=C#3l=SjOE7qsBGKSbp!)P zEI5WJUo}m~iY&l5#^^7=*scq5#i_v%y%b(Ha6~rIUDKU@GGFMR;OW<_Ei&pf`EzCdUiRH_+Tbm<=@hiVS9)-;0ep z*D4ApCUU@LFyA>QY8vlr;pPJpq^g;5$xI z0=lnOG(P5g_Bc`L8(*!WapcX3AX}4w}^%j!Oz+;^bN7f85 zrlA_+C8vO@E8O7N#i@D(Dc!*gzZ;#(ihM;^mu@3;w1{dCZ@;9euf27(#<+SDN?_0} zc)sNTis-m|9zXIGxEv~*^lA!Tc!#mgW&Z%)X9$i3k-jQpq!r+P-T8=0A={(Y-oE7| z`$`_TpYsH{vI%GG)&Br};LxeEul7`|(wl61`q=6x6rl0vkEq(4R53~b08yaP@pl}n z7kLMN;}SL&yo1BpIueTRS%Y_o*>J9dsdQzzoN2%-XuNvo`5qregch=h1TyN)@VC42 z#pOrJ+`tj!IhC}}1sEi-uej2z`cfgREr&l)uI$Q>V)4{-M~^a?0KS)b3L?SkiFAWzbpHTy*Xicl>n!dZ8{>(%sK5O^B_vQQ>Wa3w*pyJ_X0hW5O?Mkg^2h5VgP^V>MpBqcsq<6#n3x4c#_Yf<|3Z#4EaIh7it?w0rpJPb);vE8<*C(}0#2hv=YZWwt=SNYp z92SOrK#oI;H6FO~@%PG_RZ&BK-&hC$v6w&e3u&rYSLMAks4;q%viR%L;Vd?7Abs6< zn_3$bY3o+==2=iSK~Jll{X+98T2Fp~`C|RJ>!daq$FZyRg&V7c?stucNSmr~sz8Yts6RaglomnxobN zg5(^z0zmR!zPbHGJZ{{YEKkj40Cn4Xb~Qo~4v=jsWGfE7`pd8!}r5jxG6Dd&q} zRvY{lGl7~7_IuTRLdr)`&^(FMw&`8VjH3W5-B*YK2sAE!sv1P%jSBw&vID?^fL|(J z5WVI>kzOeXqD2I{r@;F64GU=PvGK%TI8e3BEfq6D!TX6d152p(r+-lIQ;J^e)V704 zE?@gVEvHr@z4zt|d#ElSOCq~?QQPG*8z@roe@O1QZAj_tirXz{0X%mYxap#kzd{*6 z6A}17%mFxYs=nXx95gvBKI3apCun-B<|!!iQ28J&52yBUOgQB{7vF9StqvUCLR2WK z)k1%n>5Kx4^!J)LpaspHFo9LW;_G2rDvH_Ybt@DwvHPfy^D4k!aF$xR)|7pAJfZ48 z?gE;C5ogS9ki+Hqfl6N-l|dVeaI{D6<8V|wjyK9fL{(B*=&54D3Mov#y1al*aUFDg z{{X18t5~X^ZoTD$JqN>PXz&Lf>M!n6Qh79gj8k2+lP**Wk4hN@4Y#{x_zxG4#msUy zygrhxx;RTxatJ*XUyf%YAU%)qh-6?ZU&On`9B)Upe9J2ni(0+LTAEz~J~;{(^>IC|E~)d5ttiu25{r-IabsELd(r9%p-)Qeuyn(2t%?0JnY zHdc!5vB)K;76Q}e{KY&JvX2>+_Y#0syBzgD&Bbws0HOJR_7OZl3-k|Jr%CWd^)wsI zw%Qto{R000)Iy3*1UY%Q+}+&m>tX$t-O{<=JO1Dn@+1vD%-~4jIsT@Qrw%Q@F=#DT zqki2)3#QGQ`y78GQHYk@59(Km1=ladLdbN~S%p!(0kPWu0Bg6!UVB7cvX$cb_?D{R zpfC5bnXtZ$^f;C_9^5nTX1m7IZ>+dfHQ~c=lHjxlo9S)=lq87y9Lj24xTnYwPP>JV z^J8c)Xo|Giv>g8ci9lfSPmf5u6r*nHHYjiub2H^Kna`~$17qLbY5=0%ANdg1WodKf zS`-r&EQN$w%Jcl0r~?arsxdYJM{{5!${7$sr8lwYLcmShb^ict%4>8zf9!1rRc0Jf zym>_%_!s+PhKMNo6%)=m8z^_A;%GTIeq$9euq`#7S;oNEuzkfOE?Wg3C<}ryIPU#n zP`W&ZJ;vo>K2hv1%rG2WPxLZujuxP2`}dbps<0V*#z6x^mbHFvXjw`(k5oJuPO!P2OkkNfuT>4++dw- zMLhwz$C%EUnlLcKsSD$;_Z~&%5Td`w?h(5lLZ4ETKri47_hnv!`8xJE$bzQy@9I`% z?v$^fl@Ow^gNK4syg&d%u+-PIXGv)w%#g%=A zM=Y~7`c*(3hXqs`QqHQc+ysG@A$8V5i@2FeJo~|;KwDpEbu7HD9-d-UgNUotVy>zc zO*;7MUYRBULvPO!E=nbBUhl3LQ5MG9g*^H_6LJWOQ@rP^4g1DAI=No`eTZGkPJ4-tyOyML8L#czvk_{a&ctye?*$760uOZb+Mpp~zz zs9iIbt6vZ|9|3FH964(dzpIuMA=1G4*e?QW5g&6XPWKDHa-yWV06+MJrk<#C#bFwU zp|}2}hz(+^x9d)X$;LrFyC;h&Y{%=Y9r|^gDN1y%F z{cJhI{{TNp`u_mopY?p7fWPnckNaMe^k2`y{>1+Pzp*(U^ZR}m?@nKH{QeyNKivNS zx3dl&%>EPlr~I!Be`0d`r~Q1-_ma?`rqL{eV_4qPxyUj3H*No [!NOTE] +> While the commands refer to the model as some variant of "Llama 3.2 11B Vision", +> the underlying checkpoint used is based off the "Instruct" variant of the model. + +**Llama3.2 11B Vision** is available via both [Hugging Face](https://huggingface.co/meta-llama) and [directly from Meta](https://www.llama.com/). + +While we strongly encourage you to use the Hugging Face checkpoint (which is the default for torchchat when utilizing the commands with the argument `llama3.2-11B`), we also provide support for manually providing the checkpoint. This can be done by replacing the `llama3.2-11B` argument in the commands below with the following: + +``` +--checkpoint-path --tokenizer-path --params-path torchchat/model_params/Llama-3.2-11B-Vision.json +``` + +## Generation + +**We are currently debugging Multimodal Inference on MPS and will have updates soon. In the meantime, when testing on Mac, please set `--device cpu`** + +This generates text output based on a text prompt and (optional) image prompt. + +``` +python torchchat.py generate llama3.2-11B --prompt "What's in this image?" --image-prompt assets/dog.jpg +``` + +## Server +This mode exposes a REST API for interacting with a model. +The server follows the [OpenAI API specification](https://platform.openai.com/docs/api-reference/chat) for chat completions. + +To test out the REST API, **you'll need 2 terminals**: one to host the server, and one to send the request. +In one terminal, start the server + +[skip default]: begin + +```bash +python3 torchchat.py server llama3.2-11B +``` +[skip default]: end + +In another terminal, query the server using `curl`. This query might take a few minutes to respond. + +**We are currently debugging the server integration and will have updated examples shortly.** + +## Browser + +This command opens a basic browser interface for local chat by querying a local server. + +First, follow the steps in the Server section above to start a local server. Then, in another terminal, launch the interface. Running the following will open a tab in your browser. + +[skip default]: begin + +``` +streamlit run torchchat/usages/browser.py +``` + +**We are currently debugging the browser integration and will have updated examples shortly.** + +--- + +# Future Work + +One of the goals of torchchat is to support various execution modes for every model. The following are execution modes that will be supported for **Llama3.2 11B Vision** in the near future: + +- **[torch.compile](https://pytorch.org/docs/stable/torch.compiler.html)**: Optimize inference via JIT Compilation +- **[AOTI](https://pytorch.org/blog/pytorch2-2/)**: Enable pre-compiled and C++ inference +- **[ExecuTorch](https://github.com/pytorch/executorch)**: On-device (Edge) inference + +In addition, we are in the process of integrating with [lm_evaluation_harness](https://github.com/EleutherAI/lm-evaluation-harness) for multimodal model evaluation. diff --git a/torchchat/cli/builder.py b/torchchat/cli/builder.py index 1049b346f..1f5a2dd5b 100644 --- a/torchchat/cli/builder.py +++ b/torchchat/cli/builder.py @@ -16,10 +16,7 @@ import torch._inductor.config import torch.nn as nn -try: - from _torchchat_test_script import flamingo_meta_to_tune -except ImportError: - pass +from torchtune.models.llama3_2_vision._convert_weights import llama3_vision_meta_to_tune from distributed import launch_distributed, ParallelDims, parallelize_llama @@ -404,7 +401,7 @@ def _load_model_default(builder_args: BuilderArgs) -> Model: for submodule in model.modules(): if isinstance(submodule, Llama3ScaledRoPE): submodule.__init__(head_dim, max_seq_len, rope_base) - state_dict = flamingo_meta_to_tune(checkpoint) + state_dict = llama3_vision_meta_to_tune(checkpoint) model.model.load_state_dict(state_dict, assign=True, strict=False) else: checkpoint = {"model." + k: v for k, v in checkpoint.items()} diff --git a/torchchat/cli/convert_hf_checkpoint.py b/torchchat/cli/convert_hf_checkpoint.py index f90b59c25..1e5d3eaf7 100644 --- a/torchchat/cli/convert_hf_checkpoint.py +++ b/torchchat/cli/convert_hf_checkpoint.py @@ -34,6 +34,8 @@ def convert_hf_checkpoint( if model_name is None: model_name = model_dir.name + # TODO: This is an incongruent way of resolving config_args + # See https://github.com/pytorch/torchchat/issues/1179 config_args = ModelArgs.from_name(model_name).transformer_args['text'] config = TransformerArgs.from_params(config_args) print(f"Model config {config.__dict__}") @@ -132,6 +134,26 @@ def permute(w, n_heads): os.remove(file) +@torch.inference_mode() +def convert_hf_checkpoint_to_tune( + *, + model_dir: Optional[Path] = None, + model_name: str, +) -> None: + assert model_dir is not None + + consolidated_pth = model_dir / "original" / "consolidated.pth" + tokenizer_pth = model_dir / "original" / "tokenizer.model" + if consolidated_pth.is_file() and tokenizer_pth.is_file(): + print(f"Moving checkpoint to {model_dir / 'model.pth'}.") + os.rename(consolidated_pth, model_dir / "model.pth") + print(f"Moving tokenizer to {model_dir / 'tokenizer.model'}.") + os.rename(tokenizer_pth, model_dir / "tokenizer.model") + print("Done.") + else: + raise RuntimeError(f"Could not find {consolidated_pth}") + + if __name__ == "__main__": import argparse diff --git a/torchchat/cli/download.py b/torchchat/cli/download.py index 4a8f43515..6ac3e8d9d 100644 --- a/torchchat/cli/download.py +++ b/torchchat/cli/download.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Optional -from torchchat.cli.convert_hf_checkpoint import convert_hf_checkpoint +from torchchat.cli.convert_hf_checkpoint import convert_hf_checkpoint, convert_hf_checkpoint_to_tune from torchchat.model_config.model_config import ( load_model_configs, ModelConfig, @@ -50,11 +50,17 @@ def _download_hf_snapshot( else: raise e - # Convert the model to the torchchat format. - print(f"Converting {model_config.name} to torchchat format...", file=sys.stderr) - convert_hf_checkpoint( - model_dir=artifact_dir, model_name=model_config.name, remove_bin_files=True - ) + # Convert the Multimodal Llama model to the torchtune format. + if model_config.name in {"meta-llama/Llama-3.2-11B-Vision-Instruct", "meta-llama/Llama-3.2-11B-Vision"}: + print(f"Converting {model_config.name} to torchtune format...", file=sys.stderr) + convert_hf_checkpoint_to_tune( model_dir=artifact_dir, model_name=model_config.name) + + else: + # Convert the model to the torchchat format. + print(f"Converting {model_config.name} to torchchat format...", file=sys.stderr) + convert_hf_checkpoint( + model_dir=artifact_dir, model_name=model_config.name, remove_bin_files=True + ) def _download_direct( diff --git a/torchchat/generate.py b/torchchat/generate.py index 6b7dc1432..d69989161 100644 --- a/torchchat/generate.py +++ b/torchchat/generate.py @@ -20,10 +20,7 @@ import torch._dynamo.config import torch._inductor.config -try: - from _torchchat_test_script import flamingo_transform -except ImportError: - pass +from torchtune.models.llama3_2_vision._model_builders import llama3_2_vision_transform from PIL import Image @@ -753,7 +750,7 @@ def chat( Message(role="assistant", content=""), ] - transform = flamingo_transform(str(self.tokenizer_args.tokenizer_path)) + transform = llama3_2_vision_transform(str(self.tokenizer_args.tokenizer_path)) with torch.device(device=self.builder_args.device), set_default_dtype(self.dtype): data = transform({"messages": messages}, inference=True) diff --git a/torchchat/model_config/models.json b/torchchat/model_config/models.json index ca8c5acdf..2d3dfcbeb 100644 --- a/torchchat/model_config/models.json +++ b/torchchat/model_config/models.json @@ -69,6 +69,44 @@ "distribution_path": "meta-llama/Meta-Llama-3.1-70B-Instruct", "transformer_params_key": "Meta-Llama-3.1-70B-Tune" }, + "meta-llama/Meta-Llama-3.2-1B": { + "aliases": ["llama3.2-1b-base"], + "distribution_channel": "HuggingFaceSnapshot", + "distribution_path": "meta-llama/Llama-3.2-1B" + }, + "meta-llama/Meta-Llama-3.2-1B-Instruct": { + "aliases": ["llama3.2-1b", "llama3.2-1b-chat", "llama3.2-1b-instruct"], + "distribution_channel": "HuggingFaceSnapshot", + "distribution_path": "meta-llama/Llama-3.2-1B-Instruct", + "transformer_params_key": "Meta-Llama-3.2-1B" + }, + "meta-llama/Llama-Guard-3-1B": { + "aliases": ["llama3-1b-guard", "llama3.2-1b-guard"], + "distribution_channel": "HuggingFaceSnapshot", + "distribution_path": "meta-llama/Llama-Guard-3-1B" + }, + "meta-llama/Meta-Llama-3.2-3B": { + "aliases": ["llama3.2-3b-base"], + "distribution_channel": "HuggingFaceSnapshot", + "distribution_path": "meta-llama/Llama-3.2-3B" + }, + "meta-llama/Meta-Llama-3.2-3B-Instruct": { + "aliases": ["llama3.2-3b", "llama3.2-3b-chat", "llama3.2-3b-instruct"], + "distribution_channel": "HuggingFaceSnapshot", + "distribution_path": "meta-llama/Llama-3.2-3B-Instruct", + "transformer_params_key": "Meta-Llama-3.2-3B" + }, + "meta-llama/Llama-3.2-11B-Vision": { + "aliases": ["llama3.2-11B-base", "Llama-3.2-11B-Vision-base"], + "distribution_channel": "HuggingFaceSnapshot", + "distribution_path": "meta-llama/Llama-3.2-11B-Vision" + }, + "meta-llama/Llama-3.2-11B-Vision-Instruct": { + "aliases": ["llama3.2-11B", "Llama-3.2-11B-Vision", "Llama-3.2-mm"], + "distribution_channel": "HuggingFaceSnapshot", + "distribution_path": "meta-llama/Llama-3.2-11B-Vision-Instruct", + "transformer_params_key": "Llama-3.2-11B-Vision" + }, "meta-llama/CodeLlama-7b-Python-hf": { "aliases": ["codellama", "codellama-7b"], "distribution_channel": "HuggingFaceSnapshot", diff --git a/torchchat/model_params/Llama-3.2-11B-Vision.json b/torchchat/model_params/Llama-3.2-11B-Vision.json new file mode 100644 index 000000000..5232e3512 --- /dev/null +++ b/torchchat/model_params/Llama-3.2-11B-Vision.json @@ -0,0 +1,29 @@ +{ + "model_type": "flamingo", + "use_tiktoken": true, + "encoder": { + "patch_size": 14, + "num_heads": 16, + "clip_embed_dim": 1280, + "clip_num_layers": 32, + "clip_hidden_states": [3, 7, 15, 23, 30], + "decoder_embed_dim": 4096, + "num_layers_projection": 8, + "tile_size": 560, + "max_num_tiles": 4, + "in_channels": 3 + }, + "decoder": { + "vocab_size": 128256, + "num_layers": 32, + "fusion_interval": 4, + "num_special_tokens": 8, + "num_heads": 32, + "num_kv_heads": 8, + "embed_dim": 4096, + "max_seq_len": 131072, + "encoder_max_seq_len": 128080, + "rope_base": 500000.0, + "intermediate_dim": 14336 + } +} diff --git a/torchchat/model_params/Llama-Guard-3-1B-INT4.json b/torchchat/model_params/Llama-Guard-3-1B-INT4.json new file mode 100644 index 000000000..df26ab399 --- /dev/null +++ b/torchchat/model_params/Llama-Guard-3-1B-INT4.json @@ -0,0 +1,20 @@ +{ + "block_size": 131072, + "dim": 2048, + "hidden_dim": 6400, + "n_layers": 12, + "n_heads": 32, + "n_kv_heads": 8, + "vocab_size": 128256, + "ffn_dim_multiplier": 1.5, + "multiple_of": 256, + "norm_eps": 1e-05, + "rope_theta": 500000.0, + "rope_scaling": { + "factor": 32.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192 + }, + "use_tiktoken": true +} diff --git a/torchchat/model_params/Llama-Guard-3-1B.json b/torchchat/model_params/Llama-Guard-3-1B.json new file mode 100644 index 000000000..a3994854d --- /dev/null +++ b/torchchat/model_params/Llama-Guard-3-1B.json @@ -0,0 +1,19 @@ +{ + "block_size": 131072, + "dim": 2048, + "n_layers": 16, + "n_heads": 32, + "n_kv_heads": 8, + "vocab_size": 128256, + "ffn_dim_multiplier": 1.5, + "multiple_of": 256, + "norm_eps": 1e-05, + "rope_theta": 500000.0, + "rope_scaling": { + "factor": 32.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192 + }, + "use_tiktoken": true +} diff --git a/torchchat/model_params/Meta-Llama-3.2-1B.json b/torchchat/model_params/Meta-Llama-3.2-1B.json new file mode 100644 index 000000000..a3994854d --- /dev/null +++ b/torchchat/model_params/Meta-Llama-3.2-1B.json @@ -0,0 +1,19 @@ +{ + "block_size": 131072, + "dim": 2048, + "n_layers": 16, + "n_heads": 32, + "n_kv_heads": 8, + "vocab_size": 128256, + "ffn_dim_multiplier": 1.5, + "multiple_of": 256, + "norm_eps": 1e-05, + "rope_theta": 500000.0, + "rope_scaling": { + "factor": 32.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192 + }, + "use_tiktoken": true +} diff --git a/torchchat/model_params/Meta-Llama-3.2-3B.json b/torchchat/model_params/Meta-Llama-3.2-3B.json new file mode 100644 index 000000000..87fec12b3 --- /dev/null +++ b/torchchat/model_params/Meta-Llama-3.2-3B.json @@ -0,0 +1,19 @@ +{ + "block_size": 131072, + "dim": 3072, + "n_layers": 28, + "n_heads": 24, + "n_kv_heads": 8, + "vocab_size": 128256, + "ffn_dim_multiplier": 1.0, + "multiple_of": 256, + "norm_eps": 1e-05, + "rope_theta": 500000.0, + "rope_scaling": { + "factor": 32.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192 + }, + "use_tiktoken": true +} diff --git a/torchchat/usages/openai_api.py b/torchchat/usages/openai_api.py index 6381e8112..9490af2ba 100644 --- a/torchchat/usages/openai_api.py +++ b/torchchat/usages/openai_api.py @@ -17,10 +17,8 @@ import torch -try: - from _torchchat_test_script import flamingo_transform, padded_collate -except ImportError: - pass +from torchtune.models.llama3_2_vision._convert_weights import padded_collate +from torchtune.models.llama3_2_vision._model_builders import llama3_2_vision_transform from PIL import Image from torchtune.data import Message @@ -376,7 +374,7 @@ def chunked_completion(self, completion_request: CompletionRequest): images.append(Image.open(BytesIO(base64_decoded))) print("images:", len(images), flush=True) if len(images) > 0: - transform = flamingo_transform(str(self.tokenizer_args.tokenizer_path)) + transform = llama3_2_vision_transform(str(self.tokenizer_args.tokenizer_path)) torchtune_messages = self._openai_messages_to_torchtune( completion_request.messages )