From 1213f2079f296f12f39ac4cfdca964575d221074 Mon Sep 17 00:00:00 2001 From: Artur Niederfahrenhorst Date: Wed, 19 Jul 2023 22:16:00 +0200 Subject: [PATCH] [RLlib] Early improvements to Catalogs and RL Modules docs + Catalogs improvements (#37245) Signed-off-by: Artur Niederfahrenhorst --- doc/source/rllib/doc_code/catalog_guide.py | 27 +- doc/source/rllib/doc_code/rlmodule_guide.py | 47 ++++ .../rllib-concepts-rlmodules-sketch.png | Bin 0 -> 69705 bytes doc/source/rllib/key-concepts.rst | 49 ++-- doc/source/rllib/package_ref/catalogs.rst | 13 +- doc/source/rllib/rllib-catalogs.rst | 260 +++++++++++++----- doc/source/rllib/rllib-connector.rst | 2 +- doc/source/rllib/rllib-rlmodule.rst | 70 +++-- rllib/BUILD | 26 ++ rllib/algorithms/ppo/ppo_catalog.py | 26 +- rllib/core/models/catalog.py | 128 +++++---- rllib/core/models/tests/test_catalog.py | 8 +- .../catalog/custom_action_distribution.py | 10 +- .../examples/catalog/mobilenet_v2_encoder.py | 80 ++++++ rllib/examples/models/mobilenet_v2_encoder.py | 47 ++++ rllib/examples/rl_module/mobilenet_rlm.py | 82 ++++++ 16 files changed, 660 insertions(+), 215 deletions(-) create mode 100644 doc/source/rllib/images/rllib-concepts-rlmodules-sketch.png create mode 100644 rllib/examples/catalog/mobilenet_v2_encoder.py create mode 100644 rllib/examples/models/mobilenet_v2_encoder.py create mode 100644 rllib/examples/rl_module/mobilenet_rlm.py diff --git a/doc/source/rllib/doc_code/catalog_guide.py b/doc/source/rllib/doc_code/catalog_guide.py index 4c212dbc77f30..afeecdca68bf6 100644 --- a/doc/source/rllib/doc_code/catalog_guide.py +++ b/doc/source/rllib/doc_code/catalog_guide.py @@ -30,9 +30,9 @@ import gymnasium as gym import torch -from ray.rllib.core.models.base import STATE_IN, ENCODER_OUT +# ENCODER_OUT is a constant we use to enumerate Encoder I/O. +from ray.rllib.core.models.base import ENCODER_OUT from ray.rllib.core.models.catalog import Catalog -from ray.rllib.core.models.configs import MLPHeadConfig from ray.rllib.policy.sample_batch import SampleBatch env = gym.make("CartPole-v1") @@ -40,24 +40,17 @@ catalog = Catalog(env.observation_space, env.action_space, model_config_dict={}) # We expect a categorical distribution for CartPole. action_dist_class = catalog.get_action_dist_cls(framework="torch") -# Therefore, we need `env.action_space.n` action distribution inputs. -expected_action_dist_input_dims = (env.action_space.n,) + # Build an encoder that fits CartPole's observation space. encoder = catalog.build_encoder(framework="torch") # Build a suitable head model for the action distribution. -head_config = MLPHeadConfig( - input_dims=catalog.latent_dims, hidden_layer_dims=expected_action_dist_input_dims -) -head = head_config.build(framework="torch") +# We need `env.action_space.n` action distribution inputs. +head = torch.nn.Linear(catalog.latent_dims[0], env.action_space.n) # Now we are ready to interact with the environment obs, info = env.reset() # Encoders check for state and sequence lengths for recurrent models. # We don't need either in this case because default encoders are not recurrent. -input_batch = { - SampleBatch.OBS: torch.Tensor([obs]), - STATE_IN: None, - SampleBatch.SEQ_LENS: None, -} +input_batch = {SampleBatch.OBS: torch.Tensor([obs])} # Pass the batch through our models and the action distribution. encoding = encoder(input_batch)[ENCODER_OUT] action_dist_inputs = head(encoding) @@ -75,6 +68,8 @@ import torch from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog + +# STATE_IN, STATE_OUT and ENCODER_OUT are constants we use to enumerate Encoder I/O. from ray.rllib.core.models.base import STATE_IN, ENCODER_OUT, ACTOR from ray.rllib.policy.sample_batch import SampleBatch @@ -91,11 +86,7 @@ obs, info = env.reset() # Encoders check for state and sequence lengths for recurrent models. # We don't need either in this case because default encoders are not recurrent. -input_batch = { - SampleBatch.OBS: torch.Tensor([obs]), - STATE_IN: None, - SampleBatch.SEQ_LENS: None, -} +input_batch = {SampleBatch.OBS: torch.Tensor([obs])} # Pass the batch through our models and the action distribution. encoding = encoder(input_batch)[ENCODER_OUT][ACTOR] action_dist_inputs = policy_head(encoding) diff --git a/doc/source/rllib/doc_code/rlmodule_guide.py b/doc/source/rllib/doc_code/rlmodule_guide.py index 2cad788058117..fa97168d12abf 100644 --- a/doc/source/rllib/doc_code/rlmodule_guide.py +++ b/doc/source/rllib/doc_code/rlmodule_guide.py @@ -399,3 +399,50 @@ def setup(self): module = spec.build() # __pass-custom-marlmodule-shared-enc-end__ + + +# __checkpointing-begin__ +import gymnasium as gym +import shutil +import tempfile +from ray.rllib.algorithms.ppo import PPOConfig +from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog +from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import PPOTorchRLModule +from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec + +config = PPOConfig().environment("CartPole-v1") +env = gym.make("CartPole-v1") +# Create an RL Module that we would like to checkpoint +module_spec = SingleAgentRLModuleSpec( + module_class=PPOTorchRLModule, + observation_space=env.observation_space, + action_space=env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + catalog_class=PPOCatalog, +) +module = module_spec.build() + +# Create the checkpoint +module_ckpt_path = tempfile.mkdtemp() +module.save_to_checkpoint(module_ckpt_path) + +# Create a new RL Module from the checkpoint +module_to_load_spec = SingleAgentRLModuleSpec( + module_class=PPOTorchRLModule, + observation_space=env.observation_space, + action_space=env.action_space, + model_config_dict={"fcnet_hiddens": [32]}, + catalog_class=PPOCatalog, + load_state_path=module_ckpt_path, +) + +# Train with the checkpointed RL Module +config.rl_module( + rl_module_spec=module_to_load_spec, + _enable_rl_module_api=True, +) +algo = config.build() +algo.train() +# __checkpointing-end__ +algo.stop() +shutil.rmtree(module_ckpt_path) diff --git a/doc/source/rllib/images/rllib-concepts-rlmodules-sketch.png b/doc/source/rllib/images/rllib-concepts-rlmodules-sketch.png new file mode 100644 index 0000000000000000000000000000000000000000..9e3155a2876b1478188b6072b45ec8fba59ac4e3 GIT binary patch literal 69705 zcmdqIWn5L!x;IKnDJ23*cb9Y{-6;stARyA+NJvRZmvl;ZH%Nzsba(fn!FMe8K4CQ>`C2O*{IV$VXU&oG(iVQLjW~k(n-*#HlD@ zMMKyy-yZB_`B~th&HPfwLSz{BBX`FAxrB#CKhUUI&S0Y|GVQhLaqi`?-S+4E&1jN? z*-;C;&zxO^bZk%|{5KXxoZcsQ6ESM4yE*wjFi(A{%V9at>qNT4BG8e_pVoY4IWC14 zpVO;I`1DZqrw#WRahnes4E8g-CuKe<7^iPwPUuUeb&z3H22vF0aAequMdY5se{s#k zG3&RB!!heS)k{+FWJFtYhKYU4LuiK!QbHcqq|2GUdP4&v3 zWEJDQUCoO_sW{X>=`pWR*vYd=r@mbK`_NPc;pcvk6UE-REP$=^L3DWHNb0;{2iHl; z<-;SLi1)H|h5a?+8_VET)HnI4sI#Y-`=ZU?^fB@C%V!zWaEeX>xITQK9Ri|Qp`!^1CxVuaegCE?dC9js2~OxqWZrZyACG-4`&J{wF`F@8Nv z7O{!u|FC(hsglP-t6``gug}V%Z|P9$L-eEl4y|fkY~!Y5u7ObN-j;cWs`XnEYfcx7 zG&)H#S|kZ3Z94yD7>j(c+KwmYi`(9T-ZdN*z3NV4?#*pPIXAUnUDU5@IL4>lqV=Sk zx2|~wtbBUJGdHSZ~gy*vs3klAf$9-0YwmE-!Et@Dyab{OIJ0WCvw-Y^b0xH7G zcw}NR$rFrYLd^!&B>~Ochd*f~Ms4i!eP0;x`@cyw&V8d|`-U&_lTe7Un(w|IPlwl=1XocqLr=chhW}Ei=TuQ9f212g{`CyvkJHe1&4yP`HR<^C!Oo@t zJ+pWV%3YX=jxj~LLDJIr7eWBsboC`Q?DkB~h**>h*q z))iYaGCAE4CF&5qe>vCc;Zn7f75OPj|LE%udsv6nto z!#W}L(j%+mXt1qRlPii7@RIFMHM?$jV7XTi??p(F5miWub9_)(Z!PObej1=A!fyGg zDx=5y{x%StL8$dhHNcp`c-4Vr3-|pAUOScTvjyLxe57WWhZUmhFj8ccOmYr!3^qwd zO#W^ZLsB7WQtXIsayn_g=m2Cal-KY{L9rAJ-ynUsIZtMTI4EeoIY{H#qq+H9ke5X% z(FPV~ye}a86cQo%lJ@yCvGuXjkGOmd*)`(^Qw`xcI<9VRbq z*MK@vu%}&96DR(wiII^du127`!Ea5XDY*3W7sUkoT|#T(-vcM?JnH?#P7uC_D)pGH zhMq#M6`NsgddpTYPQ334d}UclXRriNEJa_Fg|Y@H%NCGY(h!n=_w)D>bP^#=xrbR2 zd}Pa#hv^VJ5fU0g5De*F?`B@3Sc3@>AE%g&YL-!0_Pgh}0U3FW&C5Yo%Jgxuleea@#Q9$lb8oz&(?&$GBma@Hdh; z&FfaPr*)vSpmT~$5+N2SsZ7axE2TFJo%^cF*)Wx zUKaFEd@pNIKhN2scqPf6UGY(8PGF8}j_!RVuF9ZnXh~eQdTEVRky??yw%WV|zlukp zbMB?ep*4|wuWC?9On%i{v$heZR3(b1Qmj%ZQjD)uxYy2tuNokkI(NuP(Pt2 zqsES5qE4kwS16HxE-xp)uW+2CGK6oMWLh*dH*`B3K1?xWmN=cfTuZ9W%2>%XrTM+W z)yCW;=|e+%<=9zLTW%ii58sSjJ}@Ixwuas^@8@f(u6u*L-QmJx;EQKZ}o;HKW3ieI%hR zp%fuRC>9nH@gvL)o1fZQ@*s)l{X%!WuHEt4!+Ju`vP`++H_B?NLaIEfDauwQRO$Co zlc@rHZl-4eI1e~)WYy^DIN6*$_S);)?TUIKISqMndAcMvoZfra4$m)WHWYaHRIJF& z)_fgB-BoW(FIc|QDsq~U`lqVIjg6YvP21|9+`AUJws_h+qCQT@9}m4q8m(-u(lCXX z-kHApaD}4j2l0EyOU9dGzqBp6dDKTVBsWkr`ffmJ)CTtxszbnGOc=F7>LgX=^SG~# zG#-k2f(`nF)Ju9kIAJUw=_mb!&xo)s`dOlpB!nd5BuMfo6?s|JjSFo*^>WOchvo?~ zq0>KF@d>5!w^#-Chc&Ph4}BO}c#W16$5v!jJ9}_b<*|h@I?J2zn%m{*m@0ybL3U|G zE_wR(vhCydn@Dez$K2YtHE(s?jeV#66wpos`~p%#zvGo?UzCKsb8cOzXwJGHG&)*4 zZ;QBbIxO>-d!69O?4;dj_tGxOb{4YPI_OgROH-wJZEPSo?pe`J(SSyauAm*H0X5;P zf(5MwrA6a(V42l>vsUZbMwb1mY5k&$vf{GW1@GTNyy0!SHp9IM_PJhK(=T6`R~eiS zGj(XcDr72RsKu_sweqTDvYSs@#&7#=IlZjc`mm3_8gZ0b!9Q=YVDZWM<-z;+p;NM- zJY4b*TdQddl+;qa7USJhZ_io`%03*Qd+pwp9%IZeJNwvW9A*6MC={y}GkGT6RUP=) zbI@~?7n@hQ^=qpZw=7;PhzR|d$H+eCtl)Aix0kkei-f?E$(o3VnQzI3e7`%pHpI$B zkvhG3j(B3pjs9SAcH&LGQkr3!x!}InWgqXT!|?XYZ332h-H$uAhddWe`^Q0ii+lvk zjrx|28+Jc(Ei9)i23r%9bWrN;Z|U#$0xY7Y8%t?)+T17(C9f2ghRy2EtXmp&9q-Ol zd0kLk29JI$7uDO?w%uZ+2=6s5Hbx2|yZ$<5Jz7nbT99HP#uIoPXa9o&k@=~%p3gb@ zJ&kDh*Zo$Q~<>#Dt6$e{n{n}^?H=?lx}viF}(AZy(2HqOmXZu4B2 zg0_EjADc#x2ZeULW^STRPlxhzl$3;(ZbWY@FN=n!<82I=kI=U;(7*B+MI5|?@qaNq zgs=iZhbf?g`L%Kr_FB>VG3JH1w`m;NNqYHx7%hV}%w)%(t3;t|7YBHsmid)aw7sHS zSWE>((k1wek&!N^onmVKcd}?$Y4BXVp2K~kFn{v;M!ySpKRBS}6f&=DU+T*SdwMK$ zNI*bD@zq3A##BK8h8|oa!yv%EfD%F^0_&sC88?+89{4ZY1mPWpFqae?AWW5+wiV=xEEw!s6oM!tBD%Y-4Z6!p6(X%fkAa<@IYOFoMa!&Dzn> zmC4$H;vbXz*F54T4#xK8wvOgD)}+vR4UKG^90keAp$q-@zklr0#MS&?D_J}I^H|`3 zEYNRQ*qB*a{(Ellr~vdXpR&2DiKUjfxfP%pSVM@D6$em;S!ISVgv_!e6_NtLf z2s!nC1bn%h-CQbD4yb28oDXd~aMP)^#SF8I4)L>7y!Ul*q!fYs&!0F_RzJ}e?8A1^ zZ!jqT`$t6DM>G`!?mzCspu7!+32+a1yH}u7oz2uwLU#-47tuh_frfwh&Rozw`Ymupk%uJ7oS;+5ojlaYD~0Pz;m)# zKAKkzIs3@_1isVej$ezqPI;08&lH!e7UXCWO7)Y^a!|4-HvX}G`w3Qt zGD$0!NjQ{o?M}}F7LV>Qo%eZ8epXDo_fO|_IDn}gCyd*FsP&4f28oPAvKk`s**e1s zxJL{NF^Qn3;7A3MIgt)Cl@1lLSMnJ{5&myN(w}hKXhNFj>z6Oa9bF+QQdm}L7x%Y( zaM7LAx>9Z`X7X5~i56uowZZhP3Q$oicmazFdC!T5CigF;IYyi3Amj^q%u)1Lv>V?@ zq6))3j%Fz}{Ghu$7){+34dkkw6%!mGP~!v9zho~%1dcq53N7(R*X(|Me9$hRp9?M|C;D zqUE*~JcpN|=QO2&7x?}RW#-Bccb+!9#b&fBUb7AUf^9s?CAcMx6C+mTOJi)^ z?@uNQfvYlrfd(JJych~U22op@< z1x`yF%plJ!=e(Fv)h8H9ldISBdbpeDz2z<{yGmAycP*d@aK|-z3fRGV02Q>B04-tY zezDq-cJ(|C)f=_)kByg65%uwxYqBz&v5{Im-fLbDY=o|(^;%H=+U2cldoDNFQ?t$G z*|vSzP`iEUVt9O&@)_&2BTTdfxl%p`@S_yWygz5p2>2ngiL%l)apn6oDk5Ox1VbD1 z_t?#s;1}$ztg6$BSCsNvt!3e!Zu!q>>`FAF_qFbe-ArYgg3H}%ioD_*<2bEa&AXfi z%ck#Y1rHl?EYeeRY_YxZJs(=pO2JY7QU*MHX634p0@l`qyZ+T3MDwNI z(@{3!*1x(`RCh1BZdpe9rdvDzgjT!m&MhH*_jiH_4co#l@uvlG!&Dc!{j02IcDl*) zyrByqaC_oDqAiLgKP!PKP6h%!uu7({NxX8xaV#_;PDOCb83Q5D@;>#)iLC@qB8J$Q zv;JgjRleplVIb`HNv|aULEWEcD3`h1Ra^k3y`cIfQ(#O?jeuWEoMl_^5x!Cr+|31s zm*oXFI|a}ZSn%L04(w2AI7($46v**U?ZIK0lDO0JTDlZ z%uO1q1jf^YYgIdTcChFxpfzQ0C@O-%^biV|3Fp~Q$ka!3V;8y;D}QZ6z3r)jqwRn& z>#C8x=WB4(WO;DZ{*uu~umX-GxMr=EPek~nWN>#_=kiX+E@0k%kZDmv+e-cJc0&a= zqG0jwkcMH5F=BZb?NlmyiS`seH+(4FYcaY0?k4mR#4#O6?j8RSNRg zyOqH~hRML7@3p21KpA8y0VRQ{B4kez@8%^T1sDs{4(3}pjkJgWRUskDnIbH8pSTmh$93Rt>A5 zTrS3>JkU<2+a5V_R1);uf81?xx9yGy-C1(aTUNPsof$z;W^Pt#Prsl%UvQqcdiQ>n z#B~Lhd)f0`;^wgVe8l?C!&T%+iaTK7WV~4mpi4BcWi(N%PEg;l6Z4PN&78y1qG9C2 zG5B^eG{7VdM%>;}(oSxM0R8|8I0jm|9rG9L zAYx&jnhtZvl;XV&8;|k2*-RiB3e8XcR5*UF7_%yh#0=2z-Hj!WV?md38JdPNZ%EOP zbu$!A$mM`)g0*2J{}InssQx`U?$h%wAfoa9RK4V(jMyefRW6Bb+>0s&+z%)Q4}eT_ z$Ot6g{)sdZF_HD^neHc;@yF205IYt1iIdkwkOLrh7x}`AeB0k08mAaAnM&rZn@U~; z|132;M`kq{h%NUV=UQvK7?-o^M?#Bm>yrIW`Nw1VPH(@i?ZJvcbFqnOxEc@c6n~H( z)4@F36!jh0*sRKJ#?$7$kfS7;YD3p zKzP21EF-MxfAga0{U-?h;`4YCYjK9fS1c-s4+%%fG>#P1C|uh5v&r%~Bp{%kX)e`$ z??;7z1mnZ;=RX#MeE3dRfpI+M_X^yL)lR)cF4eh?_dc9^sh-JmEQ^X&$!73*)=ZJgMwl21a!015X8L1j<}I;D{o{6w-P&&Vib4!M zT2z8A_iC(LT|cVLGxK0*cwQm09Lxb_!e{6OQmd?~=OnAy#*&-5_trZ;4oa#sYG5|F zRq}x65`iPZ{6&%mXOb9j`}>_f(R0$Xik2}+0vU+0kbE~)*fnOSM1#k1OC6K&MyQI( z;Un07b-b?gtmaILO>4s@fs<4-)XZ|)t#wbKG8AKmQaE9gi2k|VFPqk)BHoWWvhiFe z2wA;tP=Q?T2v_yKTk=Y^vvff!wKHPyV=PWPk#| z4Fw-jL7Oc3Na&V@NHc^55C>S%AO@jTi7-)-R@XMor-7uwp2+Wrp7gek9ny-XBeB}m z-mDT5J(=rxyBH|Pu*J&q@JkHikZ0WctevC4}QQM})Njp)1`E5(fSwXfU zg*kl}vByEC{|oOhMWP2!LsVp41|vA!&LADIvUEm2@GdC}x5fNG;xZAEsVD1}z3+D5 zwS}sG_BQJrmb<1bR0&B|pF87Y)!p>n$TF1R435xDVsXheU|tqvN66wF07oXlsN`ke zM;6pQ@IKbuF>YS&yl?;dY>*|=F7A@W*a|^62$*FijR=D#F0ZDF-f?1>j7S2sFE3~S z_!9uWu16RS0oV;?9N#7^v1z?AWzuXM!!FJ%F zgYA;rcCI{Gx`FeXt{@U`p7bvkXyW?{`W`a+e0`R%HjAGI(9d-_gu0u{87ew-Sww ztnN2rl;V@TOwOY(MPMOzCTK%`QBX6`0T_h#WM~7}tz@}&n#rpLhtU%XoJUsR{XCd& z>*FIW_m^GQ!wuu~T~`Al4DEzjuG---9EZ?ME>N8K(}{?l*HR=EVm zyAE<_^SP;n(53~Vq0I@Db6b-qQ3h|da86HRQhmI84(2kc+f4-*ExHVXA#l>J$y<(3 zrmOQ^jz0?RZ}%axv7 z9*H!u{&gqbdnsbK5Aw7(UmA%{r382;-;Dc3M%jn?-D!O3t#vf+jj&=jkrF7zJa%9W zncm%=HP1XJ_~6kI4TDyY3gW_&<%c5|rlF4B0%yI$()}NiK-FgOwQU1$KF?ZTExJU; ziD@V2(4lqu&yUNo5D97Th<7^p(-FGjo-Ry}^*cmX?yvJr1(*O}LFy|2{Nz8O);FIZ zu>VFP-*^nG!>-BO(_WHDp-zaPS6JM{xh)cjr>(XgX0UU_Nd*7VOnmB#J$$SnxjAm4 z>5AM}k5mCse4?5+TU7I7pOGuF7D(sNoO(RHCiOKnNj#FpEKOgOt$e9LF2^L*aJh1_ zl}}{NWCFp%b;UKQ#?Q8$tkCyL!Viw6!{q=M^$bK&orbcs&hUqeI&?dn&smh2$kbvR z=AzSv36+YkY9d3uIz9~07FZFpcwpRyNq^yFqCdC3DW4LOyuRR#;2#c+FLzmVdbf;$ ze3=)2(cQblwXmC?PTW*C$lO%DS+|vJ(bH-0ftXq0hDta+GD_)S`^aIvH!^b5trLaj z+7IGYa&YNq&RZ*Tek$Df&Iv zSGCOQ?8JISpK_BG?K(m0b!0Z(g!S;K&I2UVzR(EzTH+;jbWP2`QpP7EtmlOPzrM3wTg(z8cDcTY{Td8^rZxt=hqhfj2z?*N}F*xGPk zAj}8DHTxk3pM6*<0%}5n>f7~8J4giGtMRglP9E~I69ib9;P#(u*6kG*Ng95T z4S1G4T7fQ#8^wH+?z$Gjmoz{k*Px>8rE}v~h{mjAF?ji-&TpVAwcLYsLMn6<^} zk?7J(lRLp;iHx#rT~GAvF+ReGV39n zonOh!{*v+4r92s(&Xd_ErQGD)*;i$W6(hlD)EW9~1D{CvhK|JU<&Bn}}S z6As)L@jr)cZSWW5SXywr-VXM1A|wp4)ov|dOL{VXH9?$Y0j4f(OyojlZjHogQl;Xe zSXp?*2j1w)&PXD|TFjE=k2q_ou1`nXv;&b}A)}zBWzdqPi8bxS zH&W%Y>A00Y5H9N?R!#9VqTSf>ec4D_H7OaN_W`m{Fi$7n=`Q?qnLeaqZgHtXt}Iy0F9xApX!#=L+1 zo+SvBOrD#0173!c{X)pJey7}v#8u1uJV-EWGkg9PQyu?hfadfS>)K4He}A(gQZN2$ zwb4t8OTLB7AmX|B)Ws{pP96TlF=KI!e2QL7)z+xDMEWG!SRvO&d1YOjb~>gWvm@xM z>jY%eP97lM({8=lz|Rb-nnH?ZmW&G^;8QDY*hyeW&l`bRhKQlg3r| zbEWEGxu6+u;Gc|T3_na-R|N7I=-;8SG_$_qm7wtpM!d#M6}k!%z9+*~?3H=1vXEz7 zf=mkgf|i|Yw<9ukI!0|PG!{O#K^cU&>Bx6G6R|h1`235){B9)x zbiHT3%9r6E_Z2gMbPE5NF{FuME+XtwHLP27MyspqdIhu7-fkm#6dL*BVt?(RNp(*- z;jHTWUwuOW`bJ*x`fuI~8F?mBN#rGU<`F?hPeGa!KH2Z=;4`dLp0!P?^8!6n0>aHMEda#!9mpR*%JSz{@#8CzB4!3 zKLDB@qlUX&f9^zpo0$emxcrA4HqV8Ce`gXjyE&*Djp#JAE_oeRWI@`YLvtIF3iTXzl_~yU}a&~RAjx!s& zpIRY(VTq*Ioj8^~kxV@;%>j87QeSEi;Mv&kR~-S41YrLnNsk^YM|K*TuYrRz(%wRr z^QytX`Vu9z{5YbqVH9EiDkxyhlGmL#D2Y|F=jZHAtF*&FES>%QzOmy%FQOe8tygw` z?Y)8Km-D!yoy$s5!6G`YOOkh9LGy88PZ3M++0rC$Ri1gxBycXynFb`tj2x+aDC zGlyfuAxBNeN8y3)(sTHSb?Bj6{a&XP?*#IKcSe{S%)E9jD|a<+blMo`CW{tS83_Q6 zN&=o{FlV?KX#XoI5uvQywJy?0HWrP&T(4#C4b@}oM--axm~!mqiuH=UrAwuQ(T>GZ z>{L%BM@y=RkdXP!$0-ShJkXyzF!fN91dAtXSeR|+Qyc&fl;?lpB~WnLDe&nBzvpC| znpDJFoOrY#LVIaO=Cv;S&WM6zkUzm@+6q0HxXD&1UrAZyLbu9DwZRf_WYxL?Fxg{! zP%cdd?dPr;ynJyJ00Tx&z2|fq@F%C9nf`E_;_oq!q0t#y8(Hj+2*o+HV<}V!UoGh% z2+5THT`#VEd_1eszBQsb73VQIRSF$J_K3Z&S8E-;D1Jph0%)_0yA+QE_k`gW1*7~> zu4ijA>s(o?4Ow+4bE7oUu4|Zp=lK-bP<3LbpL^}Xj=axKC;3MxM5%5kdi_P&12`XT z=|Az>*53CwsQ};bbQ`@>L$aAw#DrDk+s&!9rw9kZeOZ7ahHdZ1HXHXZ8&pEL`Bfv% zxFN_1wgBT=Gycg<(0gzFtVAhOs9It0D}copZ$L5)!!=A6YWEhA`;eNxRBhY0Fh2@j zEj)jYrJ4?cxZGp<8MjuQ4Lz&#fdgPnB#7RqIo-_UiNSojo@Q zpXE5)7b@KH7=iBITI;oP%w(P84Y7q>tUzh_>|)FVOdo$smW^X=am>Roi8})<-@^$~ zcQ(;tsbo&HNo`z5|FYwLwkg>5N66*N=)20A6x*QdIFFNxwKkq$=jDP%^ke?Z%@%-b zq${IrCVGs%&aJ+z-JOr1B=W|OpqmWJe4&K|)TOi5?~Jn*B#G1{heWwOyiT-CQSr*%dcc%J+w=6`>5v@@-@}6c{=M&So=X|JdzRja;ezVuw7&E@9LUB~24OX+_ zO`l+}o#0$y0AHs_LBB&(K1LKWcflnpJH{jb<%}`=GzLSrarfd^ogDM+L>Hq#j_8Fn zkE53i3D1HQVV7}`Q%UfTE2dW-QBZ@ zOPILXky=Ex-Z9RYpMR^r3#amL7;zVe+@Fs|E{?kUMIAzs?a;I{Dk)R(j@58Qu4T(@ zCHZZ}oE*Do5L()cW$!Ch;jq~3A&~Y7ikNc_+qPN{=eD*GlRaxn|OWT05U|8y?#^*A?0cuMo~j3e7x=0 zL^?BEDpMa>JLv6XGxb7D4ak6Hqn9*57v1%wf@f|TYU(Gw!TUwj) z+eS2mIyA}XMUDyib)m=8$fnWHkLQZsBrkil8{Q2t*fRC zwJt=oX^H6#0|0qkI-EULkKdGrkT44E_$iXO?B*r}{yBLXGXEh)L8n|f?YqVR2A`dd z!7An>S{@nC4Ic8fY+MY~eUtjW1{#|nli~(6mLtLk83PsKs6aAA;sWgHN6J1dG(U%- zM7E==wm&WWAeFd4UsSy2r?M-bHL~>xqSOY1J4U1mBrJ5h=V66&1_H;T8EPI*Ap`Te z$O6|Z{?&N!k&7LjBitL%q>yI*fPdcPV>x`AY%qpx{ISFl}q zS#^rT2(QD?YFCb5-0Whn&}Bt|u26GVb~VC)Wm&b!t9V=AhDL3W3}57&)|qgLQxt@} zt$V!Pze-iaf~LCP#F<0uIM90LqV~5mcJMIY<%OiSw`s|9IMw^~D-QHclNz!AJZ0f82RK3)5Mums(`Dc4Nz(2x`Rz z8bVS26Il?-WLzilf>5R$Ko^orT+;{X3{N=CZ^sqA#11huO6YhSDNa*biy*0TXqsYG zdxjPx7J;+JEAjywcos1!Fu_i8>dzVn9Ockwy5q$3t@BxCW2l zTAdOUJ$`B!CV_G^5ecwaNR!-*?C(!XUJ}thdSbWwC^&gk0q$Z1& zJ~2Y8r2mFf_Cx_UnVEjtLn#~$D9n@+9sm*;v{B$Xq&(>b7;Xhp8Pj<`G?jrg@`0LH znQ>9Y5SZQsTK|gSI*0BY4KSp!8V4FBD9Zs-*I07lCIeE7gKMfzQw=b@29$yHQu2)6 z03q6w76Aa_Hp1b)>$?*l)Q*tC;+3wG9ZMc%kI?u*GnPv3%x|8^odjo!+n#e z2I?Nk(7MNDyw4I5d|W#(-eR$pL*)vXo)?-nMxl)P+lWB|i|R!3Lhmo4!~in%xzA4u zjG-h22rU2s)Ni4N3!D&EIf)W*P8n*p3%5R)fW_>gHKFIy)M8M-F9bKO>!SOi2SrU0 z+2wwn(20ZzI_XDpa9rwIzI}j>euaa1xJ8@)mM6SGd~^&?oDHbS;Ro(hBqzmzSg8-_ zj35Q;6Ck>zzKlTndO!74KEslltWy4<32qrenLJc7*a*cPK%!dS*@yj2F4FtV+tYd;R`n2N;$8M<(Ax$4P>$4*4nix8 z{}aH$1n`j{LdO8~CECJHvP1<=L=A?U*792@fZ^o6SQL_98VrBCL=+?-r)hy*O+rt@ zlpGQOPV?`Du5+*x?y_MmV{nQKK-`HcBWZBC1?XU-s9lu@HVzL6DnxoR6c5A!7Ywl& zlV=1(7lEdQ<(`Zj0GcNQC1`Q88hHg+n*;csmi9snO2QzpsKZ<`C)gl9G#LWxYgP+p ztbrE)LQRdx04ZbvS@d4pnnJ6{0zlY?EgYsm;gwVn48JJGj|PkKf`w8hri$qRr+tuv zl@kuMOX3N~|1zTo8gATEh`@kpiNWK6k684vpc?CM|bHP?VZcLgV}{$}K%Vjz_$ z{}mL5caZz|RV9D%W(hee4532#E&^zP^f3(Rvk06Pa-UQ|_iAwCw||tq2vz>s;Ufh0 zC`8{Z2_3`>^luc4ItJ(;J|Em!#76M#FHR-~u>W63;PgT(Z1m7obI3?ZGO^<+_C)=J zpnuzn|FiSw|F*SgiaoLE>S9cAG7@mgJD=VHup;HRz;*zSb}vEZkd?NMP*yC=5XwGr zgYIA~LnlKk$3>^!2-YMVXxay=agEd_0Z@$t z>`a3fr+x!HR4SmCOIDW$xKsd=uyjG=0|6ZP|2_|^CxUguX1or))A=y>cC?~!$b$QR ziJ$~DmAwcplKsq8RtgC51siz#aVMk7`9}B6UcbpRG z+tR!@#*}W@oPMFbdzn)0_kPAPlxEc(a(JG}uQ)y;p)`;pDj4##3|fKy_dI<-J4jA$ z$LrYlvzC$EuOJ-(Ibjme4~F?#(8I9 z>QWM+0TZO|oDl!jYmp3kQ6lwSr`W1#*OoPcDV0e*w~d!9MYM;U$IacJ^mZ@aWAqG4 z!W^TfODN^PrJdv-TNMPNlLs`|=F?ymw6`G_3qMv_3h_=iyn#b{C>WRsTIBXmNK^c; zA=Kfv`N||;^USEA+5!w-8oM&}U;hvf6VOQ23GD_mr20lVdH9x4_cN-Kf`2cVkn%(b zctIf)=%q3%FbICI>~3J>0A{)9V%i<5Z01pRrv>=--}Q_(z{fKiCUzK5S2&#Z`?WhhONAMIN z``|=E^w-rwyJPZd1@@tii5~U_vjrF*B6+@0$I5}%X8?dib4--d`$F40WMZfro^da* z-oN{8k8a#h%&Kpe>O@$peZcWdYAF)%`&ztQ5$w1HQ2jWLa2;&to5&NmGcwuCG}u@E z(IOZKX&|bYX?*(mew}2|D3obROg0R>zJ&_WD_}#1a=IGoF3dU^py&R^@B0P?qNq*2 zsV74%4flc+G{BGr5w1 zAE)k@D4ImlVawH$b#^${*AX{mMzEUE&1=wK@jsm?B`a-a1iez4fSV7e*^~ms$ySO@ zqgCDFRLk{RsCAsWhMn+DztRFo!0EmU1SyRXudAlz@UwZtAfg4XdCTXi=b*7lE1+>P zVmJ*?`+c4w)Y$%}0?u=>*03MYJ}|f)bT_^{!qB-SbRx}BCdoqj@kITmi{x>|dD-i# zAjc#nO?%>ssdlCl+8Lta<_h-8Few#5_|i1-g1EAC;s0%308}wKsf)&=+CdxACUe6^ zH0=;^W4B=D%$kMP;pX!@_^=j}na|?5aR3O-AU?`#Vw}vxhM$l5kZ`&=<4-&Ag@J zd7bo8MAnaGOG(XI0;gb|pX&U7B|30G=zoO&+(h2(y=$9uHugotwbd-~29G*G)3A#) zfao9DT5H>zmFeI>sq=5JYivTVU9kN5&%+J`$K@QL_^h);W zUvRkyfT@dFHzrh=esKf7JActot_iw687NT6CPX0Vz5^NB$JKD+NKidZwWa(sz)Nt>(1i z!G*Tu=^;F6*t<()GhP4=lL6^K>GN&Oyyy8adjcARds*|@z>$>jt;(qB8EXZ!B$L$~ zitUh}?%gSqOh6T_Uw1cL)YmwMl-cOy`A72Jl}IlxT6dw*<4lFxhct=2#ZNU0pq{$}nyn5r zxp4N_2nGIz^O+O?4Ld3fKq22-C?IN9XC1wShVTCPFOc5Ut@xoe0`g9*7MuZ9*dq2k z=}63oFQ3wP9Fthg3?kkWx@s7m)n0N)95*IlvBQk+6IQwcauNusFxUJ108y+h9=~Eu znEP7M7$T4S0omkm0Cz8OxUF>xd0L)S=C!G_dwm8FN-X9I7+MMSX0X!*Kn#WBEW5I{ z$HuP5^X*iZv?fph&f>X*@xxsEoEd;ySvNNVk-h@5vTYEUUgw`wIlAE<&N+q5|H9H6 z=xyWQRCK~;(1k7T5N*s&?{35|qAbsd?ArB=J62;2)Or^l*6`Yb&+gVq{_NmtTUP9V zO4-Z{*a-p82+>8UfN5a6qzzob<}sACE2)7py`W~5&RjD0-XZ<=03Jf#PS|grII4L7 zNwuc)QDNbQ9)*44u-GRu?xdz`;gbKTUQEC6qGD7P#sc*_AW?4I9q4XyhW5n7lJL2I(wer{Zpsmk|H~dDczMDMW zGrD?l3G@%GM1WHExGemB7)bFm4`57TmnRpb%me$e_|oiql;%NJ+I;s@wWW+*4_W1> zf5Qk}DB_0go`;&DZ%DpxP@Q5wpKbKJy`@OtI0&w#{94kImt^{iS7B4E+}M|^>ET^8 zFaK6g*VFf-O1)fm6H3Bfh4+;Rw>P7oj~?&0g*~!0(fmL&z>rnNPu!()O-CDmBrcbV z#(#niM7r;-27_Np#>xx5wa{_#5|7Bx)fmgadZ&I*ZCzvVAo{{L5I$0^IAL=@e|)*{ z&gCB=JT)|&6lJK20DFX|VMWeFkGTTvQ@q6 zKX;oTq({`6z_R=e4*Gm9k;g&Bm|GfB>&;xW(AI>Qk1+#ReM#-y0JP3v&%b4yc19y~ z8ZuKup9UHf*_XUppluFypnoh*V*|7n4U;L^eSX;2U$m?mrp~W9fy#=lHhU7PuiGQE z=5K70uN=9$f&zsJPlI~8FXG-Hu#(UN7n=XwV=nhq*Mt5T8po<>w9w`Q-yO(Ec~N(R9W-3mi~C^hhI zB~m~xoob(&kFYtx#y^|W8ie&KD=@a42(mit76h%aQJa1``&(TXg#5?`V{LS5c|44J zSD-m@cJ)Tf?&k}&vfN)x>5NwpMZ{pK%(MXFzpo;QJOBZoE*sV~0+=cp3qMC0i9WF{ zL9lSPB&1mR&)jq?E~<5L&yn^pk?3wr`u&GQ#_oKUuniQuVBTeCz1Q;V2|D&ND7arg zL;IJCfBLXQLR*SgQ7;!IQ$SmmGxvUR)opaJF0`Fj)Gp+(?}E^i2w8jDaIhZuKd$rE zQ-GO)w*9-DH6XtQ?UF#aq7Gi>&Z3oA| zUBzTZHTqv2z7a8s^DtS9<$d!YiMZ(S(j(+!zN*1zr+KzrFu7Yo}MEBcd=E$;zFQZ20SmoM*_s3;AAa)z)=o znPDcG6DAst@hv;VQ+u-*9$b2ZH^EAGZyJRNr1j}?*67UYiJ4dLrT^yx$plR zC({=r zhROd3CZ{CKaM%V#6S?~Rwr0>96v&W#cf9rax3lb9r_Zk7eT4ZmI%qNs`uYI-d}__# z>})$}z-=cQ%Yu_f4z`N8R>%D<@E>1C@N^x=eto|N{q(33rMex0s2|Z;D?{Su$wv6X zt)VEgLSo(E*6aSXH)ZmS_a|)?=a1fCwmCSDru`AP6beZn6&j)~(87WG4yMk~iwKVE zDQ@-Mh)e4_>pG^-Av~6kx zmlqkC&W42L-s9|PePm;VT2V?6H7S{P@&9#9&?w9~%-MtMrHYjFLGJ^=Gef{dTODHg zGhiBN{vgVmzNTT=^ML=+DbUIZ9puk{g9)*Bsg5??Y?eB;GvU~7NU zffR(+(eXituoweO!p$4*c|h_RbWTfbU#{$AM@9_4>_z6?dMZY{|GSGS4cFV%lzZ+Y zMZ=4m0<K(n%ni#-H3 zgv?y{y2$7n`d*No*|E_$bc$XXNO$XTpB(@4r30Rh?VIV}4^FRIi0_=AoM=72NcK)U zrJX#_>8}Q|{O|8~#6pwxMC`au6Gj;Gv4%~r?eO1p_g(OAf2T+wCo#g>n!`VpUoURp zSQFD!1pQp*S!SsID(`{?4oO`>r`Pcbwg#pWJ@*%;JSLI*!}|3I73`U_r$zJHNGhKv zRitJISDP(|^aHg#qPvXDF{kE23M9JbV6$OHmsOi&h;Kx(Hj+1y@u=jZKzQA}%SSKIx zVdH^z!tbDAGU0CylLMM`pLqWr0#b~$sXm^rP;FFFq*zW)UVqXpbeXS2LF^UUW*Cb*fl$2DqhY@zkOaA^g}xz*ig`5oN$cPNVpJhKtX99bzBh zT)VZFND+Z$f5;l$bT+^smECHuBq?2i_zI!Txf}+D;?OgDO-tbx%GaJ^-10xypaxC+ z7}FeGs~K~bsv&o!1wSM4Fg~R88N5|NI6hu9Vr=4N^@L?}ELCLZga-tG8yXwN4E&Bp zyBAq0bQr+$rQ2@a$9CMII)nE~f;{T?{|{Mj9TsKOuJO{{O2<%A(t^MM1JX!{g)|I^ zfV41lN(&At-5{Ve64EV7Hwe<*9g=5_-*@)j=Q@AMCBw`+YrSi&Cw}++Dh#>@^uPZq zL2&Z53aov7h)1QH4?)Bnk;tzm|Ske$zxhJ(6(-fNN) z(^-CHs@gO(#0kZ7+JHeno1VL==Ck764IxQ4VuU{P0Y#R3Fu>=e+#TQ#4s{@fotv$o?)wzv#-bk`35;J%Qbq@@iQSP z5_?!sY#JI-Vyfs+-Eyz{zjdK6Ch&|svW z;G_s~6eg26vZZ(mwYA3)A{u!|-udid$I$BSe8sq)^{*X==!eo2B+rjPA!8tHdNU1> zwj_jux@+IGWoc}S{ZG(PiXeK?O5q{KyNM+JBm{|@IlO5MR7rqJw5OBJcf0=cZG$F&ybVm;|~7UTl)JA$zF73FpP(;#TE45)k8@G~ zn>ik=S8}{CWcu%h3b%bMyf=2%B!GA^h^T*1HAbYliLa~{?Z_+C$+2n3%(!V%iHM<2 zLa-y%P5K^M8Y4}uveDvee+da{87q3}#MuE5o|o?|+t5wg^&{VbqRV(W_Hh*H>Q#>P zPJ=TyCY0l~sCO-nF(IN1?at2N(}xoVP8(|FwYz2STAZf$-T&7}SynQMl4E>tzx*>AZU`-I zNYyug&aph5L$4px$_xD%pwdcZHx)>Cz}#EL^-uQ1fjM#n@Mhk4c5S|p&>u|GlW&RS zY`%WUBWRcHfR8!ZkmujRkLj~rkP@p7_lql^aVe3b!<+v0+5Z?$h^^nP>+xQ^lMywd z4%-Zo_&o);7PS@f`!rFmT(?`ckI-HhVe(kda*%PTACPP_02&aMX7|-3Wo0XUqCJr7 zlD=A=_{)l+lq)n$)Pq~JT%tU6qHRip&m`^P%uAYT@`C=dS01mui2nY%JeTV`L7CeH zQePUvY>hd`c7mz33}4uC9@|TVvtIYQzIfT=W(#gkFY-;D2-4}@)-F%3U%xKNNAL%~ zm=44)9zg$PJ*bu=+#Ws-6elM0Sf?OXoirLS2g0?N1sM))_$1+uR{3jITU)AKDDBBK z;6K`Gr=Y!b)?7WkxD`QY$3 zNCY->s`uq|&U5r^-QHFqMfC8Fuy3BNIXeeLuMz7nhP2Vv6I;MuaacEVL)>PM(u`He z;bL?&xUApx(<=HZzlH}*sLaut5cyA^q8-({Kc9*dMRmYwz?VP8X)QQIMLzXI;W{2b z{np+&b^H^m)NMXg+XdnaP&_p)_d10B2DN2|8~B3QR$X6B&4TyZWDWggXLvi;G02^G z^Rps{;M#QF`N5jthW+Jx(O$wG#F;?xFScFnGuZunsz zmC*Q0X)^Kb;KBNwjSoR>1s$OijY!qL%EyiIUZq+07!8gVR0I?}uhG^88PT0d-(Sg#05(Y(WPTF@-2I04@lqVk#s!V$(I zQo(!!N!M`Stt6F4>=#UEsv#Qj;+*E47K43|9BV3DO9uZ8MqqT~M}I_OZG=8D z;eCkv+|zMf4-?T@P&fM&?WoaFLozpfY^890`vtkPLFR*k56J(4QASbKG)Dc8DE&nN zrT-(s6F^;Xrr!U+_oe99UZyNauWyNiSUVUD#;==gYf+*pt(He#@E##vojk@ve~DAj zJjfI{LlB{d4e3QN*?(UMCSfEdD-rahzO(Nmyl*>#^XQ)@;a<3KDu9|6RD@S6IKx~L zH`s_1$E+9L-s#}Fw5#8FBcQw{7^J)wG9%1Su;1DWv{B^!qFq)opU|N?nojx%MckU0`_=9yqeCHrjgJh5tKY3V`z84# zpSItn+IEi&Iw=coMb*^Gn2pJIk4K)kg3MYi&UyI5*;7Aspw%TMgM9nm9pgg+xV`e9y7$V5{1ObLY?JG4JnEN(kAqT&Ef>X8Dmw_7j95A}4huol zB+5IsU4ix|?s25=>6>+JQY_pT+#f09K?70kg2}i5R^{5Kq(u>tiht+JR9LIUi?VaH zN=*yP)WMmGKw&Xm+Ga_p6cCQ`(Pwg*{i@pjUIZ;N-rne>vJwN%3 zQ%*AC9@q)Rt7EirbKbryZO;(TrbCFv2ziMB@s=%Zd^m_f?7+dcHEzlaGzzjictjwb z3~`dV^q3&IAfu#VyA>?&Vz9>(iCErjTns7r`uP<$6|OE-)z%ByB!VGs^*g)2E`z7P ziDp&lDub#6D;Roe)dVY4jYU?rg#fWlX|SxVRY8-uZOjYlzq{$hpxW{T?dVVWSlaC6 z45Z?kXWY9*+k27@g8s?wZ^hatGt(Ut$bVy&Pj718_oQ)B*T-hT`Q^Aqqg_uPf6et}dBAr-km~<@T`2IuO_O5+TK7af zV(y2_9q$ftP^aWEs`HX-U8I)SScwU7u7Jj|}SNy-U$z2rDW749Y4qp7fqW&n^K5zq0G@$ExfyvzjA5`i? zUmggC=`R^s7XOzt5H^qy>T*c%mW8>pQ%yI37Hd1U*opfyDI-q*Iv20waj-K7se%ZjXTg!J^pAk5%T>ldBqQ+NnXY zR4%)URo~29Y{a;@3-*ww&Kdj0pgEO&DP9KOcIOp*f};t}(s_**l;H=_{|O{y!K4!6 zU$m>pLgNn$^qb|p0czrYm+cRKbp}a*0YR%m0~m-#9X9~>fvIL=_EkSnrfr-%571eK zZ5D)iNx9O`beE2+hk+pp!&)|EL+qXQlP?5BOD8uGie6k(M3LJa2w@=2q%6)a37cPW zrtMiRT^_cnxHd1+9GrxDOdU=-T)WU@GR%iVH|jRcQaA@j0P^vqD=Z5&_L&x7f~0fI z6u{6Xga0qp4R~4hoy>>zu;U32M~B!=*qulGYp`Zx_y-Y0y7->9W+3;}Z($Z0&%WgC zvuU{B@sQ)tFRO>99|R^k_U<EH_q>Ru_aLf`M`cR8y&Hz`WEeQt5VRq>2mm zu7ZY@>m3et2atz50&ycBk98i3J9YrqL+p!oQRFksRgMQG_prjME#-0#bvCZkXpC1X z$cN`lV-NU=PnY{`Y0%Z?Yb9{{|4XXxD#!*xJ0v{Pxkm{x`d)*T_TNbnjKZ)yvf!(% zpyDXhb9TMjk|#&|r{lpXUdO_J0%iVrz|)v6ixW73wX~A7lTi{n`IO(OF<| z!S43`d<>>G7sdKu43l9WeDsrdIAKdvIL3TVHx%5!^|SC&YvW-@`IFwB(7HYMZoLPp z_pKXvg7g1ppO;6;#tM>SKY>Qtx~vHZ!^`Up)cJ$WEHz`WC`+stQk8>>2j!E0V&1NU zPRo@3)SrnT-#JZ<+jI%qS;x6YZF>$=QFqDzK`69zQ43t@N1xc-Ht@ihej!cCtD4Cs z^d~?zn+>qHvPC_cXRLoNt2Vj5U8NmnYwan7oS~+j85a!me;LmI@@Si1qF~WTYlpk* zz|};|1cPj|>FGH-b>Z4J_o$~*7s}IvJqjsH8CX|@CG&Q)z#;Iy;V)VMgZW`qkb`wt z&haFjH^c}68TW|R@Z9~_5&oHCAKy8YCG-eRq4-(O{0(~)3d18%A^Hc2U4yQOZw6C! zXg!FF?>yyi*l)8vGl6Y6tvh6bxo0{ zz06=g%p!5*C%ODiy%Q7ISb8UOqJn*8$l7COvHW>9JFHvkB@Qo%!)V~}t(eta-!*F+bxIBa(qwGGJ1=p{waU!3Z zQ6G;ASs%)Z1DE@Vow;3`E8lE@F-#IuSQ(i*0yaIabnu1;5lNU}Z)?X7)v~YB z(}Yk5DX!*hJr%0e9%n!-hcwKat*r6XYp$L@+xUbJHWPkJQ$Op>1 zEbU)ehi~(1;NtFPyvlWdv5GQrusP+@V$)*#d=Zh`HMXuqk5f@%7 zU#vmZ&l%S@&$eV926@~PCH*@KoY^xwYhk-IUm{arS;*yO%({u_ib_R_EF|eQ*Z?IX zw4KwfdVV-;;OB0*sbnPg%v5A&Xc8IPxma4#L71KcWHmcXStksFyk9R?ev+bV@p}eM zjxp`|ONalZZ3^%$9ZMp^zlsETJbj(SkME_ztfq{U*XX>Zvpx=wy4OsqmUN?{zW15o zzk7s6Mhh;>XL&7Fm#y4853K)+SFgp8{0E1PL4)S+-+*w5GVn5!G3X*w-|sIt-iSoe zFNc0(Wsx|!7VRTReZ7vY?_6cs*=QrNaAcWPTc)>ITRw-LHW^T7{1=wA#PA1&srwxL zh0I{hrR*=4|7+WUzPDx;tSlswHWolPWV`2_`aI7HEd07M!Mc!tl=;>)P0pK(>Ap6g ze>1;gCG(Dc*6_HVPdG=#0^KZVYBc1Vje2c^$Bn7gru^V`!Rbn>+X@;)a0OV0zr zot?IYc*+Bl1HWAXoh*R_F(=eQ;6Rr)X1vER+K+p8bIc2-!vnTz7M4MUERrK#-5th0 zl`I%L>4QQc6u=tN6sG$K{5}q=NT(}1(_E6^q$2DA%3(=~ydXwvVmj|5o#@vxmomA= z>8}Gpl!2%i;5CXF#;3+ajh7JmU!PbW5>rPB0z}r6e|lG7o+se(+ru)du!&Tz;0!uY z<(Fp?11|Fav|c~5nyeKj*kZ_3$~De4?r~@&-`&j$iQMvG==QI?a8%w)^oFsEKtxYE@<4o zWTO&XP-ynd?Yr|{DPc!)I9(at9qg|t+&Ht@K5MW0T}e^IVJ~jB0ihGIz}}J%ERP$s zVflT%jx2NBx_|di@;e&2AFk)yRyjl_l+9Wk{tWv>nV!-ugwf@nPmgndz*|(rqwQHH0!Ds}C3Ed-&WlzwfD#p^YSoZ=VKMg}kd{Qu zMhLQ@Y`(O(&_faA+g8wZ0zP$QSdQjP+Nb$i1Lg3f7d#k116Vntk8TH`0Xxu7(WFf9 z@8)DC)MQ;VqKb{xl;d_!7TE^#tfu!bIWQT-s?_9@FL-5#^k8lok{&u0ICmgj`|R0! z<_PNU%*hC9a)Kmu4g3QOH-Ro1K`{Ifs&>wToyWSZPRBM1#sXUikUh z_~JrJ@Yyznvq_I$B{zli%}(C#f)>zZ=tUT3`lJq38qW=AQ9~Cifj3X}ETAbgTM6s) zl}A03SaW-<{O-ijY+8AJleDZds?b zw_?^TvlwA`4@3KV8bB&v@)((SQxpY4@DX@_B=;Jen9b#94@(o~N^I@H7gl+Y*Z%^l z2#9JnGg%5Q6e9ZNdlD^Ja3gp+S^R5?{D0UWJ*RB7MG~7aGO%;czmxV;4}^P9g2>1g zaBopv)z!^YgBoDaBK%_mpl(|d#XhW?)tf6dg}#?6Jxu{#=6A|}k|Q8#rlt6v0I-yJ zfS_1MeNO2z&A)8s85tRz1ftoW!a5q@egJR$Yy1HJ3QQ#ue-LGH*-VNJqN3 z6@RKtvKd{*AP;TR0Ae5*)w$_S!>R)dVNWUa3>8;pl47v-7BY|817uB$sHkdO&wI4M z^95~fm$FQdhpSOL!(v4be>nai*w)~Qh0{(mfG5c46le6kI*`sEd!j&a;$FxDKr6yPqgWipBj=<{v0~wWr*ev36%^dtRK-wbo0lhcnL4U4i^i zQJl)q`DqPsqCz>p^k0Lnz6)wydIBcl57>Ox-XvRboRvp<_x1s-%?`l+i04nF5qC@H z0;1WJl~j)_%mNzB*vT(RVz#CdK6ROoIeM0~ABr1ew9$Pbxb<=}c?Ae|QC)n{^qklIS9$5=-40{7hCwG0S7H>yt;oc~T@Y#D@0)~JTg2ghNy=``dM zz$~UdKJXES3M4nJeW;EJb5!^jl19raWR{U5Xhls)4UMn4A^=N1&yuN({yKN+P7y`I zvLV+>oYT1m4e%=f){`5_s?21YLg61!*f6k3D2x_l^1X6A160)*3Jlx@c#Cr+?4khW z8rC*+Xym`#C?E~igKe0{j}c$Bzl}x%4_|)?Ns)!idH=eX;OvIpf<1$gC9{6JuY3Tx zJrB7A_4M_;44;cf)FG%)>R2TcM6|5EDeR*Yx)zMo5kG>8)v{)nyb%hQU`U;K?SD*mZlD&`i`f@1!cCqCj{8yge*5&%!6=2DUxYN&>m?j-O+lhk{OS+ zdFc2ktDpRY)yE{Uo*}WguKD(QZx;-{EBM!U#sYSfv=34E1K_q@z<(vPR`kcM0!iD} zDL`AfNT59yiSXd{25wx`+1&1`B4GCqj(xTGPxA>7f%e)B$lssJ9;#oh83;~FLWB8b z4R7K}7~KFB^^A)LUhx!a|)`4gy4DqQMa1 z#P5f`Kt|b){T<*Ld}p5pynmq(sSM)blPkawBXo|pL9_QA0CHs*hly5qp?LEzUm1>} zs_k~5+p?Lg`4k=H1y_q=I?N7rY>1wd_*{%+tkzBHbg4^=0bo|kwS!z)!IcR(VED%? z69@w~`wZZ=f%$Hll{(RaGgnqN3EE+|S|Sp!Hp^-<%t1&r9%>14Rk(p?Snavj`_mD2)h=VxF4 z^K?q82Kw1?u;DKeZ`IT22v)vNV=%Q^hjt5{Mp4%QB*C-Yj9g;587#N1-95n9P*lAz z?j~ub=G>!E?E>4jRrmWh=QFLpos1cbdLm^hn|~h_GCpSo=A3@I9=njUZzQ$EW@?z03AhYE&1&Hb|E0yP3n0)3t0={l29k$5FRf;kUh zGiY1Zab$A0836>yHSeEZRSUL0G~~;j_+p@X#t)pis*fGe8SzmlR3gV=AlgBOA!xDSEj&uJaU)wOt5adksYX52&-CuWNAToeOHZ)G0xN5n^_H6^7dYeBEzkf{r)7YRrQp-U&EY3S`TnRsf>4KnX-`9%O^fTl0##zvSAzs(3C7FryNy zPb;)u=(Q*&HMOVGghz+P2K_R5-vG3(KUW^goXc-wIF7m)@|*aa0SW7BS<_=G>QESU z1kI1^%*H3Mzsr+VCeBcilt{zlR1=8pc0Tc0kboD~X>zofYMIrE8Q0quGvI8d zPyTDRdy(ck`^KxK8uacfj?wSrNog z<>>%NtVfz^M=RG+Z*HB8v%V*$Mn0}8UD&%btM9OX&}5U`zDroarf3p0TDkeHNt&$X z3?t8^Z9$B!Ig2S~li_oyksq_Nmr%&<2NfCX?ihb^S5AI&x1FN8$MGWVJdV5yUC zY$`Lm2`TM_F7edOV>88*!%|bJb&Y7p4wKTlLv?i$-{0JStKf|aAU;A(pV%d4x=sh^ zslW9eBMvlFi)^~DU7CXT7%qe};wgDSa`e{mFhT`t<*iQqa}fhh|A~I*Mew&1rF2r6 zD+hW~eT%;zFS-*u2-Q(22m#fJHTL~KH7F;-f~FXsLB~ZkVHZ9fS7s`KNeLF0=At_d z547y&>e)1CdGkd!LJ@ISnmg{vUbK0x7#Dk%FN;h(OEpdtALS#C8PPxbajQ84A*`QQ z&qC71aZQdsbY_>@?qaqW&HPW7lqlb5^YxB>LNwn|fuO&MW6tD*h0FO}Z7*juR4l@m z*q?K>281g+R6%D>9GNnLAMQ3?-_iIAVvuZSrwrI*!qdQ^CQVcLIbtKR zsr-w3f36FB_~1tFk>5NdFh>i5b<(=Z?HjwF7O|8k;ap+$kxYkHlo)+$8Wa6*b&l2^ zdiqC-?F^<%avToxhj;_@N@#L&&554w9Q)BlBQ52<$~H9p{x4ton)vxrrZoUXy=s#A zF;_$4k#9TCC`&|T;sK-C=s0JA5a%Q`B6m`Rxf0^F|A@XasB?VVvX&`J2@B6GWj0!w z>34km-h2BF21fG5?NHcZFqirk8C4pAKdp;CElU$-thxQO#3!~}chwmWgHH@c$4SZz zxY~b{BQjLFOl|ArRX0N;UAA~V17`sL+Q2B-YKLyy;82{BqvZ<6#TEZ_fZQHu(x@{I zA=zV{#@za_ZeMFM^)&jo&<+cWpS`%d-F`n$x2P!+x*EaS=zZk`a3n ze|Zv#j(>DoQ29raRXm(GmHg|Ij*&7q-#vj1WqWN3%1W0|sc!~QCYxo{yC{(A$216* zz?)Eg_Qwn)RCxCNv*tF=69P7^6Zg$lWi1H?P)h^Nv1_PHaXKC^`ind1?w-9b+~fh zmb|vOSP*0pH&C0Gbi9xAOX~J##6V-UqhtZ=q|?Q09KI!1L>l!*#u-elBxOm2`-G0~ zk5|-&Ats}f*ztE48V+7+>)v~q57|oK1z^L|eUlIx>IggKwRe7}tzIrtoafVi2oj7t zZhKH%9Zxj<7Q{Pn$d>8@HadqHFVf8%m^;9(=vO^mbGgGCJlSoMowp^2KuWE6alYey z5N=HLZGBu@G2V>KO3WJx9Y4n7wPia#XX-fpsLL=bIZ1K^JkuOUG{n-{tK$H*>l4pT zQ~d_(n5xxZ%k@0(eh1|cW~L|Yduls!-Pe!rtOlSrtV7fpHfBlSD3Fqsocy^Pvod(8;E$!b*n0k=VCSQL^Q&T6E z{?FpUF74M3KNhXgV2|;y|FKqk(0Ap0{A0SI{5rwKtsjv%AD}g-iSPo{#*blH4-Z12 zlu&zToo+H-=QjbK11FX)wE9T6PgC5`ni-|X6)_n#Zxl6QBsb2~hQw*pah>6}fiz8h z$n+Zdjx^px=|;T{_iNQIWajiywuFk$p%??~dP0EFSz=g}YHM|U^ zWR$$O)tF!NSe`De_pn?#wQnT;006>=t2qLj5|`$@Q(=lrHWDigd-lEc6aC4l@|GVw zm!p*_Mq)%H{}y-(3UmHK9Ag%K)G;#2ST6I+tkf8e3(EaIQ({3r*;J`eKuFy;Ld`UR zx{)|Ldyb%o9g@1fUR!$@2Fi(W=a;widLm9&!Or?iG?+r`+lm(EW#Ot@sL+`ZQxs&H zw$>m3ux50d_g}#H^z5N}WVw~eHS639x-j|XS*+vw?4Pdk3v!HGOiF!@+{J)q)|E5ZQxN4Nc*L<}iaMr-e0gKuvUzvu8@3k!j{6!fj z7tNeJh&rLkSx(Hj0Is%Gl$3=^@@1t>7n>{@EesWF(jn6nr7hT_dpDPRq-%5XQYPq) zU`^*!+L$k=<>m9M*7nc~R6O%n(?ZgW%n!TP{D@D*~IwYAk`Ry2hix~|*>o}9>)Ad%QqJLT=bsC?_MS`$Po-aLEy zmyLc~7v6TbyaU5$Y@_HYaSDe~hd z+&BF0lE=xnp!{bI2KcbNK?IlK6`Ti62_FgFETDo_>%vhSr^pH^rj_0<91Tntu|44_b7d?--q zmWSnyVCwKrQN7p23HoiJ)jCCYF+uTkDt;iiZUK>D>&O2Nz z0Zq-!!6irEoWaa)W!b1m@U4yvL~*?OZr1Fc8WG1wvhU~QnQMe?I)8`iV1j?7IuEm*xn__-o?)@L%l5I;noE+Y;h5?tLgz&k2k|}yBe`t?JL5| z;@8H|%avt7Q1rW~J;1XSk7(I)W-dE?H#w`G&(}qUa4TID3kJ{SsQ;sO*l9C`ykPYz zOEh0)oz}#1mYu)$i$*kDwy33iRjYL%X1td;z&}wr_bYHVkCRp0Y2mrjd7r({i1rV= zY5T>r{4F-dU+2XeTZ+zyTW^0`VC2<||3+XA`p4yiKuhz%hIGW*y**c}v*BZi>0}Ue z*1n9a`_Q%Gxek#LZ`1kW!W_9;2336g&;9f*i`<`jR2-uRL;X zq>sWPGeC8ygpDl)2PS^ucy8Gi*)*b(!&jIJ0kyoOZ^1v?PT?`Z10kv&Wij?$l{*IB z#?TA8)y0N`AM^`&{!Pdw-W*WYv0QSc!)^~@fIi>%Sk;X7(C;N|o2ivQ8K*|XCXBky7;}~i2uivkbgV@y z(s&SFN9kWI5rGMU7ik~PSu0nXPT`DqioBR$drdA|F2v>|x9p`Nul}4Vuevtzibt?zump7M5xSznGCVI-p zz7zMO%N3R2RaZ3&X|!b`S*0$+>qT@C$fJ-!IeL9F zIE)mRJI{zV3Q1ncp7!wy1do>=#~Tu|OOThj1XN zj?&?=)&4rWs|s98IrJp%tm$!z!#u-X1=4{EMskc>?3O}7Bw|TU=^gF-Tp{OVjZKXm z-r?={K2PsJ6=VV{`R((~u0FPSy|MmE`V{AYu!qL@%I>6QMFHwEI-XEsfVl*uzHJ;M z?@c}lPIVmxS!hUF!+bzjZF9O7q92Y^@6jY6Kb^&v##~l8cp~o!q}=zL*x%LSJn%?? zxV0|N@?(WM%+K^acVzin(dg|?xL-xG#ti(ZtBXgWoF>MNGFraLa zI({LVHI|{6X9T&>4wSDBmminWS<4)&mcK@N{G{>>$DJQycUaCLa9$+f+7T;KH`Yaf zU}XB6ZCLy=Y`@*xD{(PnnR{(2VPDl}R*n$?kWzY97AJr{Zoa+8WY!%=B~!-J>iGw$ z&Z@506qfXb{+xb^er62SDm1@mO1O7#e*!*O?sr+;AjrMf>7SyZw#L~-7VK8~mjT)W zr@RNRima@XzdrVkct(AFaDR1gJx--QlYYD+`rYCl%O6m$r8kWABs*|faWkCrLGs=I z_25_6=AOG`QZ;Lum+nZy`Lj2v`HqK7k94j`9Mex2lQ=Re;aWJ$0xL0xx$es6i>$EO z46DU_J~N|*LxAJyD7bv3#-VF@S`)rUb2QZX*V(V*5&hB`|Jvf2wflW{*F!7#?=a1( zy#-g^fyYC8U*y2!ih2d-G$>**Fy$zm`k2=Toj4|nOV^&rf1kc|37nbjk^ncn|I8t- z6OQ+SBeD6noKLzm>t#CkW)nyiC>3~M)q-Dbx86Hll4Ch3r%>tS)M!aIm(73X{Ae9a+K#(*9rS`cgAL(E_WZBWnIp{ld+^C?yb~+-|exFf`@8FS_b;d zyK7N#=lkF|L~P*&_@2u?CIGQWq%Yom$foo8BQe_!Py-r!=TH+o+mtFZ*L%vjaDl z&*+<pRXI~Y3f49+_;|NI zV-Rj({Yx(^pV9|dwZ$UsKi=UCuMCZx#B%?mAp7RMKDLD$XrM#Fs^-+U?b!rCBZUP!x_#tZ+3 zhe+e^4)uNZhTB4Hu1geM1?#5w0wn93GPLbR;4pY*erAZ ztIH%D*Er6gWML+9hF4270u7#0=EK;*JzIM2xnFAHuRZ-Mg~QvStrFdtf&JRB72%9F zuiG-`XLP9gJl>~$eviDP*5zD!-9jEYbb8+J*BO9tn-y%t$g8!yAI1jL;gl%4$F|@F zQ#wW+S-I=p#`#)jY{!k%2ea)bBK+Ji=tJC{<4c&Ww{y~gXZ0Lg+RYmk@Oeg7LGyT&sIk+9-)?i>WHGGgc($YgN z9q_P-vAE1e&=?lpu7(S#nv~RE3nI^D5Aleg6Xl^|IvHl0ck{o{?p6Nl#gSl%nI?lA z>L0$J{31Sg>tV4`5VTr&*eYvx@qn|tBQDQF&I|7Qq@2IP&Pwl~g_sEs z@7z%&Go$&~g(rTc2`7iJ6)Q^O{rXBUlf4BiI2O9@I`jG|8tPcog!FYekiyULw(iT@ z3TlR|9}g}Z{Ao>{(>vdAb7y>B;K~u^n&M5p$S^TdCpD~+@+#S(4yL>5U)fU@8n_a( zr%&$dD@cK~SDN#?yW3vD@o|KR?1}GDewYB<63)#YJl(aY(B!+n{cn}Ts&|hdSK-do zkRS{anD&h(U0S>Y8HjrVA=KK=Qj4K%*-u~NW6s&hV@HRj+gtPJEX~t2Mx=QR*e)>v zf7f!TiU-Y?PGvL?*3!DouUcNe$?XEMF?aTrjfcsaC#cDkjM$0Kb<}x(X(S4=P*E-H zK+yI_HWu8z^*vO3T=A)LMX!{@@8KTGhCyggQ_SW`g^@{d!5ZC)|4DFh04zj>`nf@` z*w~p@IJ)Vj%-n;|OOMLf3|^)&y$mradLK)Qc6~)SlHHx4U67+b&u$*~E-cdXJU@-3 zF!G*eOk~-)#mI;i>7w{no4uawZ?lNX5n@jN-0R;v4+eJ_-m_`s3-WR>3lAp6xx5f0 zJ!Bny8flb@j1+N5|MANc6MkUSCNWF>(#0*|w<~l1(;E?zg^REZjz|%>GmxAyZp53D zhmmNx!zv?N*=&d2&wXf$G}9T&bF5}VOU|TLChRdP7!x4z;XEu_ID4L37@-xS67x7b z^63UI$$O2(wk|G};ZUgd`%j%UHcWz?H93PS1Icl0Oo2437aLyQUFQd^Gnyp6iS_ut z##0x`jNFQhC*BP0MUB)UPZ_{nxuj^DMtS^mgp+7z5kcc$^Cl$bp@*%l+ZSpm=djng zir)TH>M#MHQl}Wfj2FXxkt7gDMd=BZ=GcN@T57CN@gl3to=_&s3`~onzRdOu@Qks{ ztqnD6vI(xyx%^qx0`Q!dtA%mfCYMeKgO}F@Y}~=HtF=}-qX%U{Zys`5s5zG7QQ$g8cL1b3d=L3�$EhtfUD%G#huCFeM`P;|kHaq@Sd#+Bz zSLyldQ5SA3oNBWTcMn!s+Auf21ny|@z}T}lrB=DWIJqPEJ*R{dzzEO4MD9rZkc}Kz z(0|_6(q&>uXR+3F#xrUKPYOPD2#j7MYR+Ia{2b&ll`e?>TG>FTbV^KlhLi3d@d4aU z;3Kn7zPsoo9f^P*-|Ktnr(%;P0UZ+CWWS*{<7T}zfnjwnI%BhW0o2B&Bp6TfQRYpZ zF>P{=;}z&^^`}c6{ly`8HQ~J*fpCI?qpGg7Q}c<9pXE;qIQwdqb?(5&hHJ8e0C{N!hclj;jQYFhLJ9iTWY&>(z z|9$NK|_4b8m%kX)C#GK9SPjZKte(1EL)ugyK{8rRd3P{Uq+)uV-5qZ043Td>iRXKO1@2zqisDB4Xo}`+T|nxvN9Iw+tBRFFhPm<&rOA+KeCXC4d8ii{?*%p&4vkt&q-oZ8HvXaj>{iPJbA2UT*Gz@3&#(f z*h0%}s(ygQ5nab%Dc}gpO7C*A4XWvv`E{i&6H9hv#o*Bg-$Y_ZI5VBngg2~XZ`w?` z*$s|Fd?B{dj;lWSmbz789`@>m&RmZ{x6X@nn%V%B>OF|0F5M~utc)iW`m^AX zHHt#NlKE#X)^pn-aCNb^If^0^O=S!@v^`~=v4uU}qO&qHv|fALuFAmDfYKlrft9g? z6pm3*fd;YV8G*gaBGdZkbl(#aMoh(RlTRV8gbF8Y9C8`OOX^$15*N$;(UFxOg z@rUabc#%dU$DILWA=s=S)tPVQPKFPLM z&j?P$DI3lXG~=2v4`IEmQ_;a6Bb^=i)Sw43odd^n`R8e;--P*CPtY(oZCpv-3K-hI zvgvv^dn30Ki?VoVZ-Udl+J;{@&C^&(j9*FTr-$fbar{xt((NY$|F9 z7H)P{bfGA-r>|piF7m#BI;>+<&^_zt^&q}m5u7abxs}#n^xC^vwCyP0=gG|-C!#2~ z#rpoHTQ9L!mB?Gst#blV~}iZc{6CM)ARo!QQ1pRtDmD_h<`e z)5Qf1(`tkrvu9IwCgkO`(ye&MK=3=sGejjCq9^v8l{J#jZ2D2E^&=MlsFqVTX1|Mz z-KY2tq}WsUnARtS9$m8Ht8b8pUDkPLRJwKWzxCayGuAh%z~#pEtJyiY!?XZTs>APu zk*YZpokja2%@b(ib{CrRG%!Z_3fAwimvEe-YcaIPJW7TO_lCZsW2ZL ze_m+G@LfO%gkTKg&76fnfmb7!MUyV+Brh@hXiK4eK(RIB?G^yWtV(vcB%57ktNq0T z*s9YyRPVS`KPY)n@Xb=ptJ#OKR<{DdYA>Zeu6l_xir&a@)7)KK@@Y9Nt8W3n6o2@j ztnBsHzEb*TdEK9y5Gn2E!gRDv?QU{Xe4839Uad2z9N(J6#8IT?pg+kxAzlM!uJ2o^ z4z;XvNB*1r8Q1Jmqf*qMQ9uR=Yh$X32K27))l*y&d79cps>lQs-~K{9V#D%_S+T%l zR4@I_t<|O*c+e=qipunf$K{NGG%;z9uOC!cfz*vuE#6|2i2_F}kywN^_+t5Ut z;?Vl_cWwHy`wzcH=^S#H#MTMnB~S`r8eCDMOOEx;D_B*I9x0@M-=7GD#V->;WL-ZB zs(2ZCuX&%f)eSw+TCifgwcg@;s&BkEeE>Q<11CudUw_so>gtwRU`iGr%J={}qyAPj zW&3YJe7B_^eks;xZX4Q;rz!FHLOL&~fBec*4UQFU7;Vy<#)|kf5L~ByztEYs8~Tb1 zjSo`~Cw-I5SFXx%;xWtL$Ls+zejcjOXex?HP5kv*KH}BiR5j~3$DN=@J+VQ4j8&%; zvZ82E7bNa3K8JIy;hD#Lv2EThu^}HO%#fb(x`EfY%ItBZ3P)2J`j`v7=~?+1Nwste zv_x(;J*Z}z=3JP3ab_v$&;DJe_*>k~Klk_GR?QCv5U_CSQ33yA%uNyMnnQu+s|x)b z15er7*6+(TGvFG-uq!oUkNjHJE#SBd-$D@(^4H%bI;siXJd6njOJ~; z55C273Ah-X)&Gc;$oWOGa`D$}J z49EM0XY}7oG_9R_-0z@2#i8mjd^$dT_z>VwO#ym)D(P$Hn#CAhD6B4{jTlEmBZ*N}c=UpZn0s1z ze15NodO{9^#eXNp&p8C==t3{VXvAL=EfFH=Qlwg%Ih{L~>svtRlj^iH7}_*q|4aTI zv$t-H6yKA@!ofJEXs<6nr(K%_Qq3L+3-)5sy6gfkqu+lzf#ok;b9LWLfa%YbF|hIa z&FNv=$oq&l+UC(pl{?8y`|wgJ%53ce=OQ+K(P=IfO?*tzu&&bYvUbsLd0xUFF&(mJ zhSg50au?c&LQP!onR|ce(ZtIut+>sjiQQz4w*RIOzI#Fm{glne51Og=nxcmGr*_Y6qJH}R=z;%~ID6s@9Q-7Vs54~H_WC|}wVMO|dujp7d7z(2zYzHvYx|Ez~MyoaH^EF0|ty9)?B}=Qa;$?>A!2tKE z$qv|BMTRRDWBw_WVky~Qhi$~hXqMN7oEO*3PHp6Yj#0sVYs6ai#n}qf8$4PmSU&WM z!Kmpm2#?m%rr0Q+NHtqT?Y-Jy*hkLjV~%Z7Cz^)x4Jg2Ehgqa}*DS`0Q?H~u-jHr4 zv}mH8-0<7n@XV(p%9+>`;(|5WKTVN}JM6mR*O-ScW%t^^<~_*^vNf``(Y{=TP-Jrz zMLZoZ%7JUGli|Pm2HItr;_q!%mY1$LuRZwoG-W=~SB`%%i?T_6ZSzZqk5=>NJEEmu zr!RjTW_#Gq2#2cKvfFZfA(sDti2CZVD8FcHK~xawZc#$%4r!2X>6R9dmKqpBN*a~! zX6TeIDM{%R7-<+vy5XMr-TOV?e?Ium`<~e6oPG9Qd#yp~yI66#_pbP(u0oI14pQS~ z;c{Y^=kj}$(18HacbCKRwzs?!vaM2d-$jjvc(K>DUL{A(trUs8NT%#Ewo5kBX}A9p zYC~iC{m;!00r&dbYEZ)e-;#89DX6U{$;`k8y(2UH?brKb93c1-k5v4OsPwk;N;#(W z;RQ+7^(%4K!5VjlQUgzEZ$S$8^@w4CFjE?`=Ka&n#Ua~JZqMr&rhJf8ES%N6usGk2 zGjZzKk-|TYv>Jd{QNLSzer)BGQe7xBa}MbC=#kJ&;1j^NTMncK9IO}p$S&6;iay<4 zZhG9W0$IJv}-&%Bdy(*bho1+ugttpBAx~YGj;+`JWEqfBju~~)NqeY2 zUhG53p16{8UvE8M5}L44*kP_`p4k#>nj4Tft39ciT0DukdkK);Iwr@vkbzWKzVOcC z912ppY>axVgoc7;1X8#w@@Mf_7?xSdsSqc^F;S#Z`0k8Oc@C3%T{R%Sno539sM@1$ z;49`m!}|&o03O^1sG?U^{RAfR_nR#(xk<8NUS=$oxG4@m4f+${OMBf&zQHb*zgwJn z^!{z)Ca5%!^Adl7SN=WL@~m3P>oY=*_wsml8zc(0KX$w%p&+OZwQK@33e5`(H7$%Y zhVoa>_?{L6hYn)IN74SR{ZVE$g+~p3HP-K)agC*2W)UD#!zZ{x8_-YJF}M#bAhU#> zol_pP>cVhdG^3BBm`vsJJM^`mBuD?KzVu^K2d*U_C~w=d;s>3aA&U0v$vXQn&h%Igg&rz{0IYr}{?S+jh=j}?P2md0jB-{IH!|-dcHyoEYkFuL0f?B_M>VTfKfvEgdFt z3iL`jkXWaDGw;;LU+w{B_?@vz%F#?n%5|~`OS3##iQT!QpStB9!;Pf$a_vuc`T@>msP~aoWyp_;$HyEeCH0$G#^CgUZT5} zd$80ZD`71+cekwm-KcFtys;%R0R=&7OPSxK$g_Wiuw;*5Vc zcJx+14skC4x8?+>9&S0)+naW`e*-V3PVkgpQ?(pv3P?;p8DimlAxnoVe)~nPjvr+- zga4h&zsioGmQzXVN;va4FrLO%te6CcA*|NHo;+s&5-y(=OYEr~Py zxws=nT;jMb7oHQ_loZ)BrATEtudvnO_eK{MFqFUj){84{Q7-219-MOKleI{qPRMnj zo@NJr)IdGOzc{r)9TA0+%VU@H;1H`dO*^E{Y*x)oqJHvuZ8Y=p8AhoADnF?Hs2A2f zT^m`<2VhaPLOgBI;?@^Qa@ZI_XxiU9kkmD0j`$>2eBMp?va`diL~v0}k)m{cEard}r}?Go{)wEct8z@(>PiUa zM>XWGb&LN`{I>3oLu!EdAIPxwAD#f|)Vgv{d76X=wFvh0a~s?Cld9i6U7>M>Du?P@ z$EWEPSJBx~l*9`Y*jS@yQ&hlD(U9}cNBcNw{n~Fw6bb9+dky>tG&Oc~`fo@lW{SQe z<0(>XIdW{q#4jDm<}`Q7CuIt3tsSV8Whg`Nm?*Vo()}~ZMW~-0)wld7s}J1x z0^d>e{c{qWjf4f)+>T!(f)SBncE4@;I{lv(RzkroNbYf*~x?c~P2JmjbXnlA> z{q9Y+^7o-#x1MFwVm>4Jnf ze$M=LN65&UOQBvfp%|vmvJ&z40%hjB>yrV#gM9ySa3GWFk{S+@io|U{Twm}evz#=K zYRtaJM0{KB&3e#0DG3<6l&uLnQ=4X~JQO@WHSg?=(u;%#p?M1#JmsQYxzWoq@1iDC2|V;k64<*3Fc$M4>A} zZbYq?xKXB;S;FFX_~q%z_Jru5MRV7*e{0(##jf4IHRQGxiAUWX{&Ymkb&BHRP%^6Q zYQ(DOL$}kh(5=;Zp^nX|#-)cX-k$6d^iC?GOSHJo(OO@c9#D?z9^`7x6Ih;Wpw7|) z^vQmyE2C0N2;PXs9*+;dk2BNW3;BibsyV7_Vxl$iRySKjea1+u# z%DA4--fi&3B|%m(IYAgg^3@~qTwO8!>m~>NKfo+Rf5y}~cmS~ddkjwQdidjdC$#=o zZ1DaM-pKW*juH9jcTeFIX;I;pB+kCg6<_A>OUhhWMJlihvZ{jO3 zE0`a7GeUmq*d6%MPei;vQ7$bMZJIeiJF84vu6?WJM@!_A{S(9kyMUbpkANkLZ=qOa z8FkyJ$9!dkB%U+P1##BuDNE98H3u;1^%aQRdoqk0a7Ww*yzkq9@qUmHUd*K-BJ{nf zEmz6i^wX!3JB+a;d#%KTUcpJV|=EVtBfpf|Wm1hNr1eW2wJV zizkZ9&02@W1({RYpDfaF-KnyaQ$?A4G;VOkt(}pw0|~p&2?%^gHXbsmzPdb-yiE1x z4@L~<6Lw;(xC=g-D=&Lu%j9c;c$RhftXRmYU!Bd#&y4yy)hp>s4=R{q#aY17v7r#u;uYKIbq~%}XJLG3BFScL{835l63M^H1DKYl%v@{-t=#)! zYiq77Uvz%fD^jE}8>5o4Q^r2a+h2pzcrQ|@41^wXPlt%S+n7dY$}ieai1i%cI=od5_5?H?)e_4p9ZrrcJ*G4<6j#b7wFM z{y(O=y>yHldyU0=o&0BCY4fQSx-`NSn6kyn6$2lCb#f%c8c6fbmB333e{s%-gRWfu zc$jItHN>z!h(Wo4Ip)h-Ig5_l2RuM~KFWm}LClI!90NH1#s$H|t9ro5#FCi$uTfN0rX?2s^v}7FRpPJf+Rb6( zUwSoGtVj%SyHEc&dZ;l{XgsVnniO690je~9<&m_dYUzzU=HP(!qK;mXUA(y#S#ZAC zE%uh9X0A0Wt2>XM>}k!=CQI}iv*!;2cF|6Z{+zGQ98_P&4zCBUePnw63=RBU8#jsH zF+&Ie%U#e6el5sm+%h|yCRmeLSJXK*oW%LEF`>CSs#BY?;HgVtrKOpBe-!84<+f&( zsQ1}Ne){o6r27o`l8-Y+jyFdq@7UIqS>hDVc!1oYVR=0H!$w;?bH;hM-Psm+Kk}~V>i0@rvg306>uWz znIas5hu}q7yl4ea`;D0k=@rV-uTq1Oi}UCY54-A(6N{c2o2qk(xy5D6iu~*l1CL6g zn#Mq;v&t3{yyN4WwCD76a$Tke3?rJOtxRS-Ll2Kush3vCQE}xM|N7m(VISm}&S$B< zz(hgUROw*ybSqxdqMv+Zu+1N+#*a@)`yikgy;cvFihkZxb~_!Xr_RoOa8nychV3hr zWsio7upd+(2Q7?K$=Z=zd=ja4ZsUXvUL}!bFWDNkval5R=FV7uK|nHVA3W?fLc8(= z1${DFoQj!R>kM{H_WA$+fK_twp0&!@I)1%4t-j~Nru3L6uJ?0~7*9}f0rM{V6d_b2{Rg8%vEA1_oVVxZk# z7NE7SEIvM%8o%^mKeA@F3&Im3JY|H6e!9OQ@;dYZnhf3hoQ7TE-@;=x0!8mEdY@jr z(1G%|?|mS$3TnbzMhAnU+%b^eA}PYc2wgUIPisbqNw$^x)p9W5^!Jl=-8uVp_tFK*6XEkCf=W28h~9>y`d!i?8d{2kI6Nwn>U+fJ zd)i6N$Kf8}1#9TKma3eUPvL83#|N?IL@GoG^i%oGXUkGxArpwhNR(@Z?pTeV-60JqV~H0Pp#P9N_IbFu)Qp zRW4a{3NYUh4l>57YeZJs4u_Gt8F*BLA9xEVoQ0`ECG02m}gxI^NVM+ubx@N&C%et)c2x+vEAt zUh#vO|Lx1yrO~`B5Y1d0iX;fWIBbU7mKg%W6m)docs7@ZerwQM28WpTLW5fU(+ zL;_BfbcTc{81P!_VnqyEBKha4x#9^SVY*TQ&`silB#LI8!d#h-t!651GXpVO)ptDXBb+?0jA$3YGuGulUW<@Ut(C=o=i_UI{f5ULPEoRle*tHzWNuiV9D zyIbJGdelx{kVz81*J}EPZ`utrvx6wU2H ze+>U7F4!WP#2;iY1sehsx_?JEdJvJCbl%f!2O=x2ty|V_L&ijxIgqh9hE#$I*Gnpb zr){&;var{YNnyds3?IxKhDRjMhp>MAM3W(!uxueQHSFB^`PydF$op4R@BnHJ1!K>V z<8-NB7>Vf|cOSem?JW!^r$^+!9^oR4`fbKBz}y6h>!N7UeOzwlI`^)B>Z+^9=dCpm zL5SEmcoi!ZfJ-owo%#5cxGFxa~o?rCHi27 z#Or5+@HutiGiP^fuG|M9zF!yiulujN>nE6=~z9hmjzl$T}zxbI{7W7*#{dHVF83Z z1XX7}eGPc8#w(mxJCAWmgZ&}xr|HC6)Vh~B4bLZK`|&b>KauiQ{8VsieSi!7w7PGF z9eJL=Ml~Wre9<-bL!Zp_$8hWUG*@kDE=!s%$|ordrk-4~gZQy1P>1*~{dw>UpJeN^ z;tMb#;(x`^7+5EXu-Z`_h_-Veqc@RtWf|f667%P&u3+vwQQOqmSDJquNpt4QwqD#Y z&C6n7-I{{$`JjEY$je}I{bXSud&`~o?ETP0Jc8fg1sGr}fl!;K6Hhfgb<--A(%5jC;wXJ%{KruPu$M${GpOLc z`YgU0+@!y?F`SxM*(XHzx45KSK^voQrWX0|U-4QT>J z^s|dVo&maz)HFc;YsOk9;ZK`*1L(7@Y6m+9RKhUyODK_APydR6)pEHLpylyZB^N>D z!~}@nVtkT-m54$>pjUP9S$vl?sY&H^R8JceZ<%mUR$a2SRK3jWhBXE2PU1Z&O3AN& zqawG)BJJ9DLz;|AziU#ytBN+Br-?ZJLVAnPrMUGvc3d_vwDSZ>OmyDsy_4t} z`aLr%8;OD1q6p*U4Inpj<8pgxWoxq0?Eoscj>~?D^80oD>pdgIA(uyLC>XbazOagc zkQN2mefe8RF~7)boRdWrtUz|UIqiLzQlD+7%Qyn6m&rb!KCt;uwVnA&LwIjwMPTcL z!*KhHRl#zTUc*D38yQYyn-aw~1xL1T?*=N0@G8|S23#nxo++ZoDT%`)q1RUl16<`S zG3yoh*`HZs06~Zb1cU#9JCdc~x`k=b-sHI- z>9#gGO9pnWDZu`%133@g=O@@eJ~~($5@YfR|6g#^Q8Bvtx{5|H#P~7O$leIav9Hf zD%oy%XEYXB=2Q7G%Kin4<*wE2^*w)s2vnQ^NqC8oODP_T-&S*uR(CBCv&T`~LsLPU zuIfU>LjEXu-MH5U1-`L$RNMgMc5c)={kr!i9slAtAa}9Mh zYOF`xHFD-IMGWe;492cucIzKbfA?Q3(-Yt95C$L>>O>B_Mt1TqK`EgdvMAsn<@*&b zy7y$ng?s8~ZaJ5(pG_~MWAE;H{f5kXBblGN;N0c;P*n6rg*$7NYjcIQX6Ur~`HA&m z3O>fh$FV-UU4MJPHon6I>`73xnQP0Z!3Z0Pt^M`7bCwP<%GmxcfsA--x=-T$$LEy_WBv z%z+DGzcWM6Aw=j3N4YvVON%<<|GwGq%##{57k`xh13ac}wOZ-52qOLmml9qlnAAmv ztPFIDAKY4VTj?{w3X71*vd!i%BHy9C)86dOFCgna?ShWPD8sIRoTbeBtM5eW(%RDX z()n4rU=dCLm-poHkO$77JVm^q)@2z^DJ`e^;!Gs`npNmNqmk@XfF5@tj)U~{aLF}% zqCw>Ie`DatE|G@3fl2#_CFdB(V2W#%J+thl5(C#LD?ByD4NcR;8=|p6wND6EU*Cn& z$jMeN9D#O;JHe{JH+m-9-=-K7APb)_|CNe zaW7LES(W|1fZ!Lm`2l~T=(tPjTDW-wk5ix659KeW5q=WDhF=LuYs==^aVdY+pe5ZF z*|5q8TsIa}U+Kax;fiBL^%s}UHC)w0okrUnhr2k|(lC&NU8^Lq3PD zD}tS!5k*oCHU@0`f0znH32r>vcqu_@F=0nZMtBRQ>i*QHFFx@7i2+J$rK+$@pQHX^ z6*c)PKcGmlxO(y}vBl!@_8e~VPHuZCM6&hES3W*j5s}GlkNNrBau*ZSS>Gp0L4SY& zGUnzwX6imZyMx~I_bhCP6a**F&@4!0*`XC*xhoXSUBz3|qy2YzeunC+mm97|6{R(L zqY&TIyeshQt=CXFx&!*@i&u4BI8lE&stQ&UK>@BiUFly`Uy$D z%7OI=8G=DUCsgj}E+0r@A&MVu)hf`OMP%F(QK^=o_4{#y2&kjCRh+p}+^=1kLm{8X z+0^*k`=>yCWsy;w9k9dhcap{)0spEwX2M$RdJd(0uu-c(ZDiv83|e+M{As(({*3FM zafV^|CDE~LJ4tmt9AC`vnre{1VQV@j^j?Ul2^sJJEz>k8P^M&V2iorN7-Y)&!xGAg|0qg_Y4pM8P!EV1h-P&eg`?<6uX~W{4}Aw z547~0Vm!NFE&x$Rr+TEm7YHIRmQ{8>ZQbk@#5h-{IsenT_DykN5<(;&_ZyJi*I#a= zT8A&ScpZP^Qd`d)kYp}l)brC6wg95Nu-!U%NJ~4$k)m(to&Nk@TXZ(XTz`6G4|^l& zai(u!T*ixAN^S*VY~-oll>TCS<{&)!lZ|9+F&ZHj+C4BQE8KsBNiFe$7?xy03eT@+ z`g>yiv;}JhQUv+Eo9jHU9(gIj40<$2`eR<8c_Sy3r97AVcA)Www7y!*R@k$g8m(X> zuQ?D$w1Fh~KdEg%Km|ESA=m_cD}OtRS{+*Ztl0_%Xv!*;k3>SITcvzS69syA!Mft% zvN@vH1$c-Ab4uyI6z~g~ab0tslUw0s4Asg(6{`D_La|09S=D}p-)#w!z-NJX#&Fda z$PY2JuIGLhI2n~%oD4tA69oa$Bej$CHao$q6nHhut`^ZG|{y;yGj^Wp58ux(TOI_|8FOx*q zDvsb_Z=u~**Kxc8>L<%XzG|h)m4i_Sq80CD!CiNu#XF_Bl-8Y`4GJpA zn~<`T`S4tfTZ>SYmuIGUz#( z>_2RN1Y|FQTgs2g+j9!7-KfY*cytzXGYUUb_j<+cv;)4PepEw{?u_vs(8yS!KC28} zKidFl77zWCU8Js3GZ4Q^yGhgU7fm*}E!Jrrum}8?`#*7fogDlkAb-J@m5dY=ehew> z>D~C9zB4JHIcaA8@S#%UC34S&p5)T(Ew)}EN*Kv$BKkm1eiun_x%TcLEKC9vqQPuf z7}Gy}y9Trg87vod;|k13weZnx3M6&RBG^H;GWe0+;Uc^5VZ_pD<#%i0RWkWj`A#38 zu1&Vl5_>I;>6WUW+qeGuNjNcLnv;;A-|{4-xFJosw^ z6k*V^U55?c3a==jpS!wS{mmSNmj{$@u@Jl@Y6SmQJAeFJi-}+Cx_O(2^RgR?&J*ZA zfOow=cvr5tr%vLas5FORBQuS!2$rxNS~a>`W*Cz{*V)VWGhyqb_e$Mxz+o(Za14Z2 zzMectxg|9g+G*q$3nxTmf(0QKS%w*#UGLFJVkg0Gh1kPG78V{b!Mx+;2z1{d4|&@donb?hbfE? zKE)Inn4ux9-c^qjEG?v64vbtF(CyLQv?@F{El|oXU+&hA9ExOBDp=C!;U9*X-SqOe zaqhWuwHt^W>gir(z^tdw!uY%~9=AWyJJYMDn-HC!S2d8yMftQhBCSoyYx|rB(Fg|; zVBbvd@Rtx{G0F9%%NHcC4jkPjO1K`CoS7#pD2V7pt1;R<8 zzDaO!yI24lhhu#K)s);(skMk*N~@ls)6YlyZWZ3?Mpkc%omT*R2Gtv_7at#t5@KR( zGAWNqLeF;RXe5Utd>Feu2aIfo!rjXmYnodie~{6p6bpeEk1AyPjWo78AWwb`_Z z<_*>E3ed7iP**L4DDc+m`2}Y1SBWZhB)*=UTD=97ZS9G|81FHvkg*Xm+ofi_G`W*- zub~AGI5f<&hFTXMbpt&uF-g~z?=E_>$PWl|_IT9WK!v+9!2rIgs_)VFTiM)AKzQbl zBxC;5TzKOJ_(3xb?fM!Zrc${wD6KkY*xPL(R++&4l*cJ&@1);~7vnM&rB5Kc+lJ18 z_i;7c)D!sO@kTks>azX$kkOFoP&J|d5HWYQ*X8*^)Gwt{ah1M6A5+|BVqo9oqLXC7 z(eL_?Xa#SM6s8W@ip^Q4^GzCj-YIj0+GQH-1@E0h z>^vL9W^}PIo_f!&xRR4og?p7H-Hj)<@nEQ;a!{t+mV2}g0{M4e=;0wdPq<{)JpDY8 zDr-uCF@#s)1I**YiAoEg8B!$|qZ2#?RGVM`t zl7e>+s15~fIyjnV)mz~y%=`S1zI2(_!XS8%diJFN|AS^QdB@Nz8eaBAocqNC-}*n+ znb0BmlUL3Fg81ir-)7yH_pO%A00{SyakD5hwQD-0lJXr=h+3boyHn;?{~i#clFB{k zl-bP-wzO={I31L(Nu*&|kh*UKsf84I@%7*~=VYcf=O73-oWKl(rJp^|3fIz?9--Di zu`5ubb&Rj$3U11mlD++6fwy-Lh*cL;b9ZadZ@I*jJ0i4nomn+tigb!BI9!OiNv9n( zHM_QojoYgYplzvI$9cR*r-GCfYz8WdMQuMO%`(^a$3fFwZ!oir%?0=HAjpxJ(XnANb2$WtQ`tqg?AXxX|n1epv8UJgt+JzcVZyei8327$>UqrNn^PB;-cvg44 zXiIX0$>L4QsAk@fuP$1d*#{A3_p;5KV~IQ6Y-;WK(kB(ArWy+7uV#DB6-}<*ana4j z)PP;K+iRS${w_hOiLF}JXaxaCb`$E=Uw*v(v0jU1kF44*QT|nSbe_v2ev(w%I3G2KW1DXP(7tW(T zI~~_+?=UyySxN#r`E8_*1( zQmJT-qNBB2UwgD)8&&(8Y=s`l9?zX-e9%GJGp`7V*Y(N?!H%*bwdkC>u|Mg`vJP(} zIfKP8PXZ9ztuf&f3^#(Vw!c4~XsIOM!q#&y9otzq>Z!Cg^co!;_Lj187xWi8;mr5O zx=T1mfSbZ!+!b9nU4^rJMMxILzq$Cl0QbWq6fN7+MC}0bA7%+8(I$;`7fCdCA~lLa zDScDWl8Z}shJ-HE){of^hgvu}5U9O`p#5BdX)`{wj>|(I!b(*fJKuP@Lo>Lmp7`nt z$ir_vnGs5CNKn%U%v|`*PN#0CyH`>RKmMk?oflv*q*F0S)x&AHOl9~Cu5xD7XJ;jh z*R2Oo$B|@qPs^gcHlBN-n6agQDXM!Eewdta?6DNjx&~=}BsN)=5M@nf>r4JL%>~6+gdM^s;y2*~be}Y<(Tt;TY$=5_|*%4!XEs z;xh&%X!jbB6)6-WBLITB=c;!GQfpb&L*97@ns+yNm3?g7cNsiaS3LDPVYV z=W0IRo2$(|bt>|h>x?rPma1FQQ;djCkg4JJd$O*aEGa@c{_#*9r-GHm+8yxsigP_% zg#6_Fbpd$cn-*0<7!|)n+nODBIf6Ty#SSZc#PsFUNrIBcUn&y~E|(o=)-jDQE6Up3 z;ZP4TSoY?fwvCJM2i-fc|0RD(cIs=O`EsN$G8h;sW^H2d`|>&w2jVd4tkjU%Bn{NzrhJjyB29D$M}oM<;}38{;>^EhYIn2!Qw`;*naETA6m>R+nv0~ZP8o; z&bmpMiXMmG5#=Bu!d7mg`EQ~SF@HQG*K{f?|C@T3UZ$?9@X7%O=8{eqrV|rOM8kUV zilt5Eg`U70ChW7{-TT)Lsh`SunC%rN8lI)bIjlSl;6c5sFL&&$XwMhnE{8G)H;E%u zaZjx`S36fO9SBCf3ArJ^dU=b{lf=2&$2)7Tm(>qR>qq}F)9FoEw)Xk@RDW@Ctx)if z>~zREJnsK}+rrZ-2YX51Aq^&N7)=plsh1pWcNIfTQG&AY+2{y=4ue2M1waYAd*+lKe)A$oQ{ ztCB9^_qQg68QZ#n0UNV4E`7AEze>OMu$UQi&9&B=3%ppKx2s6!`=vh;e){tX%|bmY zbV!vGH$tM<(QxeP8J~;!DEI1uLDMIjYE1Dh*-}nfhe@@S3fU|u0>p?$7-DwPM+u*D zlSh~LQt!!EWL!wdl+@X^yJPILY=GS-g`r_0B;)!L*7KN`H5Yvq(=F=R7<#;VluX43 zJdwj6c*1Qafdf|(l8e3+k8jJ9;7qyW1Fyx{^hSu>Nu--Dbj`y&qYdi&1}HpmVcaNF zVZ{z{C1q;$onuc8DXIyrvb5+d#bCJgn?qtop}tq=Ln6Q22{w2@fUu+Vgk#yR$e@bujtmo3WHzg^+vtlv*0GU)OPcrH#LPen;4IU;Wn^g zJdQH=^4s37%Z@u+IR2ljTLEyLCGbFGPYvwhyW*RW540d_@INc=@;Dl`=C%of11Kr zWxg3!KH0`SPW!V1H{q#4xkqePfQ__d>C5AMjq5?hq#!^x2f$pdtdq6)Es&ZYz5&|d ziR${9XSIcm;P&#f=v*`=jQ6b(Ah1?BJRCOiSe&Y#P485%fZ|yhW;5Lgrhd0V!aa34Z&A6tem|gx^{W*i>&C`l7FSx_ILtBFe8}>#x#dhJGFUPRM0*y?t z7>w)vvV~yu%h>hy+gWQ2=3!-h=@-M$p|&rcL3d1>OYD-HDQ?)Ntf}*IG<5L^-szt*Ov!GDS80S5Oo>xz@iTRp>xby)hOogq@dX z5pJw+jNSL6x?R59rBl2YVb|Qszx|YooOdy4#rAI zLgNH_k0jiubIWg=Xb!)~3Y^Z`I18SlA>N#&(cIAmLp~lH)Y5JdZ#vyom5rBE_XT3V z=}V)RW!(V!7Ivp|Q7+jT>R1>!-czBM`Nr2y8BKM88V_@{?Q~Sj94DLoddaLWw53M6 zLL#zplmykxzjS9~wc?PNaEsNdrOBWRpJe0kmaLKTVQ*1+1J&c%&4ND69$Lx-Q5Gfp zUr2WTlN{6I``Zln#mS6Q{nRWLg6enrH;*3vh|LLV6swl#I+-#s3L;~e>{0jQstU0u zaJ&-1qL`M<79Fafi!Jgyn&>3oEall&ljCll?3*KHWgb9b$#n@9ENrf>FfB-Ibq zAUhrAqcK~R!d2I%886wQp8d)}*9~U7u_{H?bYEvo%XMfS1h@YPA&`{5`X*>g#$Eb( zi9{|s!YP&I^l=Hvr1n9y7ZKYlM*bsNEW%~V9wv$XtcVP z3mZt5NT)xNx=W_GHmWH0IazhI{ISbIeFqid_Xzh+m2MhO{juCRi}Yf`>D9cmjX%27 zTy8}0ru>WlRO2ck3@Fm6}{OaZly_#*3M?lW4epZ6z~1+5ALy` zN-HwV7O7)tW~W!Srrhn;@6OF#`}Z1KWKT;=toe2qPjA2CPWU*Lb+T8{qp)(W;?!$* zeO@UMX}-O%6=>4kIzKp*Dwg>1b!62WC7b6L_$*_vr{x21Qi*uHt^*hKzltvy~$ zD?Y{-a>F$mW3VM-6OS=?(p_zNAIIn$s62-?$l4|A@|=SYCFp7vy~Zo+^WxLhv4uXs|2V}$CiCIk`9QLgdjvs}O!sPnnecD;o`7j9B zrj@=aU*2%BJ}#5OE2Q=w_NM9i5PS$E)Q4=?+&>U9@|J(B9g|ruYqKlUtvHuAo8|`} zaa^gNTLh_R4Apj1@q@lf`^DW$^_Y}__Zq_07uo8#`XFgXU-uym&F7 zALDQ3XY=^X9Pd|z8A_$7Wdg7#>MI>}c8Zkuo0&aYiC#=OmY8m0(Rw3O&8&x2AO?O$ z{dIKmp9emdc@L{1d%JDyy5Kb?IbTJ_rBkyQZ-Nc)agrc_ zX+T7VtFFyhzB)9^bT35O7CP#<{yi1_u%^(k*&5Fn-X1ootBb#4_Uay}J6v1L`ey7c zA=oCB^O)0_D4H#9;E(c~x8S90Rdi*zeTomVGZIPnX+dUl?fLuowCbR%%J5p^^mp_I zt&RFo29idS!v_*IYcPC-PVcd1-^XRnORcq0j|c;G*{~sy!-LKwJEmBiy1$Z^!MW?Zs< zt$0jkLT7gpur!hGKL2`-X>@@YdO|VR*(*d)Xqv>t`0;JV?^8ze9n-Drnb<2nbhr4> zmFKyi+?DefF*2{Lu?Cy4{RW4ykfyUI9Ua2LaRpIY3X{w0t0sDe=v~mXGH5 zLY-DX3&e87cEk526Rp3*__SC;rC~-5)#kcpQ#txfWae`~SIDrPLp}L79$U$hwd<8v zj!AlpIQR%Ap_`+wfefDZKd&ass&WQj7+u=8nrTC(yo#yeYvuf_rooJ2HeHFT?=^JnD_REP`HDa1JbQFVNp^N;>HQDgCKmS_f zC5@bQ*b5*^Kc?cv65lB4%>M`xv-5KR<*p#i2@i4vWwm*3~QGzTcnvlZ^O z)?OGi6gD1c8>EF5ZD2Z9{(Ky4AG03%(+i66!RR6De&HU+UyXos+^nfGK#6k0j5o9<# zP0OKELAmsWtpoHU!;qV*9~Y-Hn@8~)VuW8Cul_OzWFJ)S8#nflNr(Q-GjkKWvLD}i zrl*a)_k2SQ-4gs|IzGG|00~XT^ZNE%*12lsh$9>LWz$&-&HxKD%^E`qOyzil-Qeb! z=e$isaDwQR?Ons(_2EO~CYd2+*D;WQ>` z*PCsr&Len;Rs1zCbWPQcpeEvk$%P~Js7r1HAw@`AjDj!M$Xurj`f2|^#z_}+7-h8z z{#5IUuj>5sNyhYtK>)v8n`aRWU`PlUg~2##peZF8WIN70{s|e< zT7F$JZhrhrRu2^O3+{d(@@+KVy5`Vd2o)DWpY@1PtC01@#RNZspk~YIJgZeR({rm9 ze*G40BQzwGcF#3>~V}gIqGSl@yTeATg|__yb3(T_unX;ps)JMzaXhV$!@7_iU&zw8;i&T~= z+OD@&?XOF#;Y>ByAEtj@mAd)*@m|_c8kv&DbNaN$VFF=q!+xo;q%`OZDF5Y8loaHw z?-?`mISvxoS92;keul}d++eV@buh>a3N{5O+GWivP``9INO*<{RW`ZpZ~?H zq`|xuhoRc`>Q|S{%C;(GXVfzedm^3cV#fEV2DC9ulVv_tB9rThWjX%PJv|jA zLFb`Hg}lAbyxH9S7Gxe)O_}V9gazx!3V{+1*I)sV0V|`;qQSUz3KsJ{%bnNG&iL_p zDJ8PpMv^Xv!M=JoS7ckjozsV4Ho-jz?6SHHD7Op)vz^B7>AF@og>a2z^$ca|lRNot zdE98*xab!_?{#JSC^Ab?=VpWN3?BA2*r;ddE?t>Me$#PVU>jD|AOc3l^ormyNFwQ> zUO5-1s-1hch5shqQFjbiqoK3-A?Zua-B-cmC*#F1T){RUGl3qvsR!lOQ=%5DxzH&{ z%?WBc&A46jseh>K#f)oOZEKxG70+Iz$a*_;^lpGaAk|a?XB?*#|AVE-~Fh~Xg364Yw z0}kQVVfVMM``-H(+)sN}=A7=XuCA`Gex9lg721EDJ8Rg}dOs2ZWd|5vth%bXS)P-N z*9f!xuCZRxS_^ybcjHo~e#bd=MPvHYGPkj1k>>6Nv*MP!2k1HJuJ&>tV*6WBGD(9L zha-AZEK>>3d)~cIDfMRAmTeoq6PGVOx*^${NTR%rifJh{diSi^yxMiT&o*L?U2!Uc zwdFV{%yF=W=d3TQ!8+koWiAXh)_hYOX1_ZWS_Q>FhVi#U4IuphQum_}T$ zrVjjCZuDT1LFIbbN0j9qs+voO8{u+q2%^K{81Tc%2$bsu7YADRP{ck2Tb4B+0+9VA zk4zqw@G%ZxhQ|r4S^Gqwmh_o;_-+(ALD+lNE9xbF#=XAj0mjc;?v-K4RPENA!F&cc zu>*aHOD+2?Y%gCa2zYazetX0Fyz8ppF9?4Lg`) z5Ua152(+feO+p}U*~$Qg#GbIYmNi;e>=$wpd`@td{Cg?Oy4`g6ob}+`ZR7WE!45ME zseu61`neD;-4=fH85Bn~CYoDeqQf0I&a>W{Q+IrFPNAOr^A2D|J(9t+BNRROo;m?q z50&jZpT<_QW=bfG2b2@qt4&@>k;xYcwp6TH9>e?gn#Xd@XS-PJLFG0O)>?MN3%Fm8 z1m~l@<`%$<KWcV89;I(ZvumI(6Il^yC z$!1{M5!|NL#+Tb(4+Ac&3Md3f2TH>Ys3w_F_4lZr%C(55G;E7$iA`v`qVD^TL zWxYk0UT-_`5X`Ix9)zUb$Y%+*emK5GIKO|gewvQqUD9KGNB6C*qYC}=<(aMiml{cV z(touf1!}!JQv`K-6ni~M5XkN=(&In%5|(p|7Z&xHloH^s?HGPI2$jSp6#ScGc(=)r z<|l53ypWa-I=EkT>%)U5bt8olgjh>#MA-B2hpb)I>)fp8c8-UkV0l>(0@Ptd7*EV%A*tL6BXis0vnLEJdlu-?WUAd zPi8n(8d)WK$$4aZ$$To`b&HO&K5Xc@%_{CeLD4(U*|{-a*}@V>we{uRGs11FY_otlS)29)|i{;DN}|w>}Htfy&g@CLyU;FuHZT{=@ef)xAiQ^bF>*Ob7mcsb5lbj0#%<;lEMSH?yIrQTaQ zF^9}#a*J7ey{z%!yN^Y(svpWz5WXOE)ac}%8P6Id`-kBgEbj&y*H+rfg$PpbXi8#a zO>^#;h53c3(RUW;;Rieoe{l6Nt+0o0pZn}Momfv^xQwPyJjP|I=IYFqdD**$%J(4; z+*9oBRHo8$eY*wsSf|;$ZtI;&hvO^AvSRKN0nrxEiT?-#4kZ_9fyc+C&+)0DwLIMq zc{=a3A3p@q*b|<02&QGDGV5V!k!smFB!Y)8_o;xw13N zoLX}}tFf;rDKn_!?W1Of#6K$$BOgN+-^-acV0j(G>tR9r5+Qnimh)ST`{Dhy>*LUS z=00;6%Jw|}MOZiXz_Bw=ie;vSSDgrVX2#`8oI<0{Bg-`rR=>unSS+Hcq?q?7kpG^d zKsz3PVAR3epxyV00bQzfp~24o_^%jvc$Kg=DJBot%8naR<7f1&2z9L^36y z>Y@0LBZs1jI3Y5`Zs=ZyF?lWLiRN09ogL)Wnfu+P^RcKKAco&r**c{aKk4*;j1hGo-wo zdjrY25N*(56$nN6c{%4mCY~K9XOO|C!{7%;G-jBgD7%GY#-r^Mk%Uf%2i)Ld>7mw- zX#MwKE;{F8R*^9#ntS-6xv;D|&H0fxTU)P_9mHeW!MI(*P-#PQgD<|zs*)NF zIM&giuX4RJ39GWJHHWf}l_Pft7_qB{$$|r{WTk8VU-ma5uNNX3;sNy9dWsQER#0O# z@OQSKy1Z!`H&77mb}E1}rZMzdVnAtJ19zGmGlA2`JV))gT|=#zpyM{-aX9 zS^kz)J8k7snq|A&?r%$Edun;WJJ;Z5+;CsQ!RLv>c^&A}E7+IC_}e3a6xk<^=ShnB zCq02Jy%g)XCf=Ph^E^?pyuk&r4tkbqsC*GlN-C8urcl4wGhQ(LDImVOXSf#(F#VzO z4578;lA47PGdxA7FL`h^M(8i?GfLHDm%mmIp1aT^Cy?}}U)WjNS62Jhuoz_!W~#E( z<=!LQS{74f5MwRx25qi6r`a9?>A41`tJRdP2b#zuKr5rh*1YvGGy+~G=2jiz>guu&lw>d5=U66ju~RRCQ5IjRNsB|=Xc~Kc<7@g4subqPBA;z2($qIsS#Dk0h&(jPD^M~q3R%uHk znz-?o>V^pL=!hR1sKm|$J+&dl7;#Sn8Q_0S5XAIb3#v>xi^!405BW7cNyfT7QS=BL zUY!t5JI8<4VlIPauzN}YwxAEz?ba!4ay>AF>BuQ?X^(B2bP?~kL+d+p&rm;_dzr#8 z@t&Mu0^CJ;il!vclpLPLsbLD!DO`vG}9-cRBs2lOLF}2(J{ppJY8>7pF#8vqANFTfnwOpfCBrxX#YXb~5f-vH_)d3e(j-X!i6dJg>&PWrn)e^H(B%*Zz1YSz<# znc@o4bcElGh)v$#UB=<1DLxvAxvg{)5&PiAW#eVNvEcc!?B66oTOMv`xa&b^+k{9V&8b z)P27x6rzj55ouG?-#|+t4sBk1>5;?n=P^m=e|rFKNpB*5E12CV_b%&#AZ;SsA+s(G zUaqd(bST9*6h$>oS*#FXtTYefIGgLtE{^Qhsb`_L*3}d|u%IvDc~XOxtqyk%Kl(li z88TJ?iXSLgsmc^kJ_c!S{Dz)U!hL`_pU!<`lLQqH{auaki)@nerx2IF(r5gcg!?%7 zyA!bCJ?mNM$;@D(68}LS5(F676~K4?O&`&gTK;ankv>#SWpFe1GCovFjq#t{$QmMA z{e!G&1?D+sF3JVb&yTR#GNmy0BD+e5d`^!-J%*+Dg6s5)Q~0t7xREVh{5eh1n~1Ck z#VPEViyRj4p#?Nd+R^Zy;8U&!1Ppv0+=&3w2htE2lV?5=B6-G>pOK?wZ}aSh;j3K# z;&nB4;TF~ydw^|h^fud$n(TyRsixuU0T<$M9xBJrXn? zEg)venpsyYx7haw?Y233Zq0d^o{$+^gXHx>T zbj&x+_hxA7lC&jXLvx=74K0={0eNgL1X&9k{THOBM(XBY+=MACsXg8+$mS{V?*8sC zD~i9RLW?-bIbE!8K>qH8JIQ@&#lM~1(U4*RDULP{5E^lQchO0o*%vIgaLZNR2#9?G zC<22f%Yo^)`RtXyL2@!TI>)zHEuOuh#vnDIGbb6YbU~VtIeIe8oNJpQGR8b)jOy_V zM`t5sjLv$3R)K!psIJvvJ)D)snp$uX;4m(6)hq#Ny5R-aA!N++1bkYs)WOc|#PzIS zPfP=?J@>GUhZ99m7v+dC@>P}g=L$*C&O`q%G3IERLkRhacbk-aHVxdJ`^4%+ofTg1 zu;WkDg_O7d9)Og2=XDrYKzUDhvqDA-`6Mt98Ba`Fsleal3(1R@{ETFMVetL_SbHI9 z6t)sHu{D{({%N_LkR!ZqXP0WxV*PBqRBcNA(j7LEq#;rq{9iH6T_dZNPK$#wsin6q z<7`a39oAXffCKF}Pc5a0(kyy(5XvR}p$L#Ym$)h_2PnhE8rI_i+-{ucNC@1_ldZcf zs4$JX|Ml~R(-nYVPbF3y>ygDddbbdZRUVK`lR{e-Xd%@UI?|h+z!b?vs z+)WO2z#Wkvz_hux9R{rWTTIm@&(#ziIirD>^~(qatgz=wog&IcB}@^~A_{?%^Qd zILHQK3U;JjU@vPWUvB@0cFS6fD3YhiqoX36wLp(W80qc=+{TUC6rIG^fiw%Ub&?jL z#LUqJ)cAntAmWYLMr2X7+JM_`#{FaZJ10G_ALU7XVL0w`>vINmy@p?Yv~b{vkWCN0 znD@#!YdsT&1__!qBSEuE;DW{95&r}SI8Mc!IE8(&#$Bz;jpJPDo+ZmjrQv{Cx~XfG zZW|HBxWG);WGNRUN8B6Zmt^ox3Lvz;t|)ieYP^7G7o=IZ@+q$F_t8oFs?d_lI#djd_-{myV! z#}+N`*?0AHjh5(0Ytg$NNo_WXa-ZzRlzN#wr=Qv~TQ}E z_K-%r1ZF8P%UfBq|P(0zYir~9$D+Gy;?mg|86;GMnqor^anPQ`}){@RE7HDP0g zJkE5-@^sww?B8fRHQm}xnUB;l1C&H@d<1ksoBMV&4_G0i*54rx`CHN;;q{$$VdPY) zi9gDXZeesfky})6VCtq^*X8l#))1c4^tOdv%ZP}Wc}vdzbjv8GGw{(qQJOQVx&ZBO zLSo?=MF2bf)tuxy*qRzqCj7ND6jple6y<21@97Psv`%ra^tTEw)4JtmnO#Y@%GKY@ zaJFI-5T^S6Nhk20egCEzy<00;0CwQ(wMF+Q8_lrk#`#D*)OvPhGl0@w8sbRhNF=7a zMq?B@IZBGk(-pL8!x}dHN?t*@l1vMF`^h_BCnu1so!LR9S+-G6hU!;5WxPiPmItyA zJXUKvl7{a)ZCJD@Wpo`sO;5`3rqCZ)jA173-6EdG4tZfcR=f+13o@&2F20>6 zJTW-r)A>T(dq8s33J)Fgyq=JbE3(|XTXyz#BY-5v0R>@`z^+%E+ugXbn@>>`yZ<&qrm%Mf*d({6gdCl?DuIl%RdaD> z29m>S%%3IsrLj>d5`+VYi#MIid$%{&MQjU8KhnO&{=BDHu-Th>>9(=Qo0U8IEuD~@ zVJt@Vf?~aYmZ8r*0Ay5qzn^8OKS?E$DL-)L)P=3;#%W@gMR%x4zt;+mfnR{vtT~K^ zqhkXZO*F*N1>-qn5w*U?(Pxm$xXnEF`5XDoKoV?H)+Rd=AUMimLwoV;Q@?K_ilFG4-JU%jd0o%LNyQ z=nf~n#J4HlpQgMR2NS6)h86*yF5e%F5KFzWRF@*K z{nD7>(lBDDC)Z6jgxiQVe9yXJe5R%O!rTFXG%{asIfh2>zO-Y>-tDsa?Q90Suyawx zSIj#)EMBgxz}2Jki!MDJJly!!2@$K$6H-gCg}=x4d#j)wWK#Btn5L5qfQGqC?FK{o*41m zf{nJB(Y&akM1MyS(DfT~h`*(Qzc!GyT^_Pg*kTi5=1?2iSmv9B)hDL67r#(Wobzir z_DYYd&ixsR2<1C7*nKAgsgd1-Et(1p9vNZB8zlLsT1~w*79pBH_ zU-`a+jN{nuOHWTkHCX;&sI=cbJXk&C93Y67m`@EMK5rQ+Y-l0#+kW`GHS8G{9Ancz z%r*^44aY@j?nf|qU1KzS_@I9=z7+alAbi(=|+jIv6a{@3kIn-sIRXl154<27rP^LJDmH@md4?{@$vt7_6MU1~l~oclhkX&U!Wea# zMPQQzteP9OObXU6vR1V-`foOW`2-%|3TzlDb_$ifdY_f6lQUK|^YABkd%Pw2Q$?fo zTEfG9+EEh}M*Pr_Ybh1`Nzh6&V0yA#C-qqfE*u7T}TLk3lpcaqq_2H$P)^Ym1h95zUV#swZw0*z%Ng8Mu5`+w_VjZ z5dhqSA2Ff*I~c>p+Fs{zB9lgi14=#7?{OqD7~iOAIJMYHfj*D9r*6W6tzd@rT0A|3 zY~cy(MCNM`ti{)Xf14UP?s^iN)$&bACd0VKY{pS{_0{~`@0gEvi&!K_LsONNz+fl+ zxz~mXr3OkvlPiI@ubzMI7o5yF)g2+b$sz^;ufEqYA_SrFq9fK6SlBCvUZ+^BCa#v$ z=%CIAlCCY%wvh)MC>t@x3SWWNuI6I^V_roUplbuN7%@@{Z_sC4%_GM*%*PD7w9e6fQMOAu+EPuoKh&Y;RGlCmo##kB<1>6vbmDoZkBS}LD6K+pRE4w^tR*IJb}XsCaCtZ zx-3w?`InjY6>=-`30l7*x!ae!b8X0+oj4u~l$w|C;-(H)vcO(!`|?gZnUsdLtm5dk z&FG=9_9q2b+RM%=c_ioL_FU_h7;Y1VZ8>Xah88?1h`0~g_p}uxzMfbmDNR95!J5Cn zn&|#gxmm>VS`7j_9BdO&|Dc(!8#GimnJoM{Rf=Jdj;Y5NyE<00ZGHG9A<#NyWve78 zpltY7uN6i~=HvFt&T8`N$DV=y*8x*hPhG6WdINxOPhVTxxT>nZFpVpA6^Wh(cfxyG z4>B>vQ<5Nphy~z{EVV2ngzT6fWd;-K5%M2O*OAQD@fo8Ws^181musa}@?HTQ#IkQ> zH*Qyyr0K76m!hUeG3#Ac&C~sU=FRtBCL4VI_a_#v$4p$|J9dxdgI@M{S-;`b*J04!&{+TT-|Knri znEWYv(U@|Dg~a#TJsC(ZXZNA45|4<^&QZ9nJmLhA(?mY@BNJ85k8_ac-!)r<-`2)t zi#R=r|FB%W{*duA+tyo$C*dyTi48zY{vP5Y{ASuM!T*_y$hy0QIPUSMTJGFK&8O~i z?^$Wr%R+zDSuWP;JpAvyzzwCQ)|zcS!pEY13*B{e_E^etq1m=_a6$ap{e~ok9*+4ZVhxX;v>S&^G8q&vm!atxX(=+( z$#Y&Gzl(mGBQvi^aZY3?ASz&4CtF$iBX!WgBR4Gy0_Cd8#Ot#kkZRm2Y(}@VHx3mo z1tUsp7A)Dc9Cv<~%fI47-vn1kb#tQj+XJFk{M-vTIa|gW+ACHvU8Cf>an!Vu#g;P| zwIRx5#O~O*U&{XFW7r>4yc?#(XK9;@rC^Yc=ECL zDSlG*O1(IIq%eHz;M#D^+yi8*ZpBZT&kO%(4eQ@9JL=!M9Tn^;w9kqSX{0 zR9rRu^07C_4;}L=CJ6CRenCue!R8|c@~pxt1l3(vbBpnxe903C;^+{GmAuQYiOxG> z){Qzj9lq4BGc@xYw8IJ;N|?^o#-6O`Z>^e{6ST4+Lz}M|6CMD3&dF*E;cCF4T2P~B zZs=XsR-x=A&P&TBUIG!9av|51%)0Pgwu(AR6&>q`YoQuxThmMMdm`~eil-O-P^Kfa z;Ae!te?h4{u7~8Z<--T#ui~0@U8U_ndrN~;WJ^InFn%)3`tpJ9> zsq%$QZF)F=1Q2uEIiDVJ5RST83^a#_&*o|uh_+Zihnt4zNPNZRt6X;IKyRdTh9Z}= zefr?c7tc%ofF;kvg^b>%`&BEA-h~vOK~jgQR=>D?9EM>C&n`WKfp)9dUU|(mqtvz9 zV^ru@U*r3s^g4!*8kWTAM1rNo%Jb-1Ceh2GV`w`f(UDzFD7-yDT^uUdDDrI5!rG;8 z`!0D4t+Dg-;h9%9d(($AWxx|3S+;&G+Q?<9*xd_f0<_askNE7_{F93c6m&(*Vh6Uw zMW@#WO9Dmi3mYj+n}`jzIU5sdLjN#mx528X>3LHkf9bUP>m!%qbC;>!BhPZV(P-_0q;mccHMFy%8-z z96Q3R!xl2q&$0y?MW#%iS90h1_Ls8sr7vWPHU{1trp*#`bLRZ&Pfui6A3Y<~6`4EV zio1@79?s4354-b`I`y4{)GMdF{9G);nVIssm&-KjEU+|Z;8NNcoYnkA*JaK|TySD0 zN4k7)vs+u$0u>XMx}#EGq~p^}ai4dg*K>&bw3gwcDOzG?OI&%r4FB}=@F+<1QMEc! zCb*f!tCg$wTZCs8lnW#5l{+?EK`nu9j+hk{XGY)lIJ(>yl7Q*_RLW%yfrs1pj<)#* znOR~k=9hycD4FiCUd&UKsZ_0+pR&K{LG2Ee7dT9+!#2Khbd_^Y*cTK%^~uX!;L7r` z%1$w^Fl9CNcXa3}4jqO=#JAQsrDp~*L)2~u*?x^2_CqEvGF1R^XScdR($W1cdw)sx za2e`?6uUYU*Ss>Dm*fdZPgAT$SC!KAkw9507NEpdlIZ)12=GIt{-S>wE_ptt>71uJOWB@q(K7q+bJ7mqT25U2;Cs%7$g;;x)FKCm|O;IB-nov8wsm`f1@ zCrsu1u;KE!^l*=SLa}ZwK%_4xE#hSQpE43(8De!J@2OO+nHbP}p!0mBwfGgc?PKvP z)8=cR{p(No#?k8U=^Z1_OUL_0wC;)%i(~Xbh)G21#02CU4SKU-^O-|mT4$vAvObe& z;dUT&F;`rR;mKEFB=#nuGyI-V!!U0%8W8G7d*N@`O*n)iE{_vJP;rqtM2*OEOwkO_ zti?2j6oc84UCep-`<^>x>5HP*!c!BgMus{y{E}FR{d_{w-~>!ldX_@xCHC#6tNPAn zxG>cwm=##6;M#!GGUJ+6<`+?6SS@JY9q=?yseb#EVek73{7bL+C(CI0hhOOt*;5MT zO$N4gW)~FKcNZ-f#N^qDY!_NV`*uTbW~Tk~-Wz0Pe4)P=bnboS%ypOg3tQI;5t9jfpcpv-L`{u^N{_b3j^cH>le9joP zbJmr7mTmQ*!YZ`)Lbk()DkLbRokt>^KQ4CAFx(KZWZE)O(U+mcur?8j`=GHGF@P^j zv-j;G$u?7lm%!tqFND$gN|m(ag)A!Z6~oWMM>7aj9k`1y`dDwuzCFj0U6liS`wj|+-03TkKYX2;W>(P+N9;T4eQDol@SCKB8%YL`q2zTr~j zmjFajo&4}qgoecaRV?3T@tO5#od-Ct1}>GhYi}61iT^BomzZTu?7pp>x_Yx^b1ou$ z5Z65Do#(z)2G!}Nb3Ip$6{iPx$eb?>xvynY@Eg2(On%ue?<~6M9(eX#zInO3_Q!y* z!7>8D&CdL-vp*-kymj89?X>my$Wu_5`0Qu4$_3lvx?cI!(pO@sYGVf4ap!1XWpf;s zXN7FPA8Ph9Zutf|#A=gt1v-&n!6yQI5zSCG?p8+_n&vOfg%&8r*VpXxvBN^sW%%86 zfCb87T`8{mb^p!X4*D&g*A&!nhh+mcQ*z;)8|Z_vaud(7g)%hX=o=&wu;i#B=K+?p1$kz&f$_zth|u`Z?D(=eC~87(=>ce_I*9IGOxp;6 z%B-BccN1!FK6@E7d}*mD1^2F3cFYO5K(7<*D?EtFQhZW|QM8~l_Ui?o9W2Fz z$i%n)a$pp_D~(4!VSHPjw?F?vlqYd!ib>JKZwrb$vH0(3Lu&8-3&=rdLJ)36LTL|T zTwh63U+8Ab1qHfP*WuSz+AE?AarUnlEEJbS+D&8zFrT)t(Iujy%V)IM&i&%E7vtU% zrLuaIAv(Dc=0`@PHGjvNEgqohDfBCFwOk4JFI6HoY7jjGpg3H!rHd7+ODNQPl@O58 zCI!#eI26=b##FRD_=3#|n=I!yrGfZmOpSIryekQwz>tJ@BprA^Ja~{)9&>ew{VA{RsiA zLtqUB3w_%+hOF+KCn!%2R;u$Gs@Te=o*UTPXqQ=WbB0oPpXzN*OfDGJ{)+i=75*Yn{PCnSr31ObJIyhz^DqQ{m!DjGGy$K!2n~XM$ zR+7&V0wr$NogqWQi3$)-B>E7^g}Aig6hgVq0tMq){FL+46>U*SY7niX9MGc-3gL`f z&J8DCz~wuN2G0HUza9>QtHjTlQ$HZ`exQpz44}tAsO>gthku>`)WWLQL6RTzB@p_u z)k$FmS?|lAz#PB4pi#!$hX>iR4ez}%w@sA5@cTqvgUt1S)Cu*=6<Q`jX-Qb^Cf126gVPq*}PVA`FQxQBMi7+AqTOte;I; zdMKPpk~np_a-CaD7rN4B5iqcwFv7ynOa&H3fqCPD+*OLe;{*`aKrz5O%VnVi5Y}+w zT+2BvgGzb-hC1opnQH2|>vJZjawj`aX@lqvN#9N|dqn9jRDkT5g^Un)>KB*Ckvql7 zg07fL46X6&8GI=4P90j=8&72`D+QLsFM%23vAB|$vZ3xqfC+~@($Xit{R z7{5{33m0x)1)1C7(`Nz5ack~DVl03;J|2NNp2BMtUDT|AJS$v z>0rR3+SNzd+fTwX)f2)K$mS3Pg7l?=k=DWz8WORIgeBCc48s(dmqf}QaH=#_GXI7D zhW5AfhIR_S$=8$>VVtZgf-lH~)|>wUhu$VX$n3H~7(~yOpcX_zn+DUVEzq`o|sGgn*zw z6Q}WXp7KKzeUg=iw~Ue7+~-~F7vZgaLbMvnBy6wiO0NfmZGHEr$>91b#{=&`8jv+V zstI8NAOs6T<)L*(W`jFReo!IcU~gBOYIWE^12}FQG)ydT`H!I*s*JtsQ6wkw`L&xs zH#2&mH{kI*mtlOlVVvza0S{*CK-*e8x2a@>U{%!P@j@Wp2J{;~6K|ih7TS ze+I5iG+h^{X69?~YYd#l@*MW>TrzOJF%lV}+nac$8b)6G2iUU}+0V01Dgk1Qi!|C* zxnpTboG+2HRx5U_f+0diZt)=JvG~I*x&=nSy8B534IM!?bA${31&wvuXc8c{!w(-awv(Ss zcX)Fq$Z;KIiS>Ux52ZmCqns>)iDDY^r18deF?GUZ0#X$7Uc%l<;>II6JN%~AZxgxu z^2`bE9^t0{xRBo`V;A_^|>rid# z^*@x(7IYgTpC_N=?3*pdIMK!U58gSUP(Qg!fE<+GKv<(BOM(SV2m`0?OI;G)(MusS zE-jRuPu-O6^1Z)#zF~s9cc<4Kjsf4f-a3wy*^%6_Tg=$*J5Nopb?MtWH|U5usEff} z-Y8gO81)>UT!9a~WX|rIPUr8N)!ZN}xmjXKHT_#Lr+}aYBh>uynVLxD&aR2b#-po) zzCzkQRxh1~6&~u6L>+;UbN-`y`b3`7PlGQiNbf3eYlwnf6N`B#E9Tp>=wIDJWAsZb zCRKBTtGmJWyX8T|1F82+<}?i10yvvEfBof`+6jIsQ=ReMNSGIU-nO(*GaEG zO zcCZ`WthcEC3iW&UD{dIHV~%XCv(0y7USb}V(oUO=pxO{zAzOAN2QmdAE(|?vrWHY_orFjJ4kcvV z!ZA&Hstj}GZ!X1fi2N(DP(NSC^$Q>f1V?veBkm@kiilyqcX6F3^MnsjLKA6y|_eI#Yroy4w8MT#7?$ce7{2ON1SKNN)^p19#$T7N!% z2WrTaN5G?uubMh&ML04!?0J!#pa&OVD0fhVzzMP?fJep4(Y7$5%PcS|9rHwT=;(PE zik0{b089HA;c@#r$3-x`w8$OTqgd8-=;$37N^&kIvc<8$Bk?Eh=K!4ivqdWyRm^mx zApF=L9>b3DxXh@m zHAV3vBORZ)o($oG2LEqNY6UZR5>CCGZug;b21zU(o*#2o&m`Zw(j4wa8>s!^bwEj# zQp)z=^h-pT;nWEr`EW#-!n2X0(4;B&XmU^oWdf2!+b5mJ*V``4S!b6jP^+ci%4D3Y zc^^lf7kWwU>78&Vphhc+ST*Ns&g{Tc{^`pbkUgqS5)ylCBZE=@o}>OF$G-?q$?6{2 zTP*jG;spj0y1B<+GA8S?1_Qdx(tgAO|E})+9eGBq5#9M;J83Dogr2+0!MY6Ww4 zS@$};3m&Un#^_<=Mj{ILYyz-H2y0s+m~4}Ki5{?)o`GcgCQZek=Q;R;wWu=QQFxpnz^vuRaZl}(&+-$fKUW$dJJzLKD#gdxBDm=!b!MQcI!pdD`8yW1HR?mvi^|9W%ft<<#);2 zS%teAEwN*2X*ZB7{@^BNSV&hrs*k7~`0z}gGrx8@Cbf5llfR=d?u=b;{zw9Cer;4! z;f!e9LziqWC*SekmPoh_5qm0B3i{2CqJZ_=XNN2p3!$0ZffBqZWWNAKf#2xCN4cr_ z5Vb+xK-~1dACq8ZXP@;}K$si<{i70?o4lKEk?Xzx{z3D9qFev-ouf~Z0`e@u>I3kwo reward -> next state -> train -> repeat, until the end state, is called an **episode**, or in RLlib, a **rollout**. +The most common API to define environments is the `Farama-Foundation Gymnasium `__ API, which we also use in most of our examples. .. _algorithms: @@ -115,40 +116,32 @@ You can `configure the parallelism `__ Check out our `scaling guide `__ for more details here. -Policies --------- - -`Policies `__ are a core concept in RLlib. In a nutshell, policies are -Python classes that define how an agent acts in an environment. -`Rollout workers `__ query the policy to determine agent actions. -In a `Farama-Foundation Gymnasium `__ environment, there is a single agent and policy. -In `vector envs `__, policy inference is for multiple agents at once, -and in `multi-agent `__, there may be multiple policies, -each controlling one or more agents: +RL Modules +---------- -.. image:: images/multi-flat.svg +`RLModules `__ are framework-specific neural network containers. +In a nutshell, they carry the neural networks and define how to use them during three phases that occur in +reinforcement learning: Exploration, inference and training. +A minimal RL Module can contain a single neural network and define its exploration-, inference- and +training logic to only map observations to actions. Since RL Modules can map observations to actions, they naturally +implement reinforcement learning policies in RLlib and can therefore be found in the :py:class:`~ray.rllib.evaluation.rollout_worker.RolloutWorker`, +where their exploration and inference logic is used to sample from an environment. +The second place in RLlib where RL Modules commonly occur is the :py:class:`~ray.rllib.core.learner.learner.Learner`, +where their training logic is used in training the neural network. +RL Modules extend to the multi-agent case, where a single :py:class:`~ray.rllib.core.rl_module.marl_module.MultiAgentRLModule` +contains multiple RL Modules. The following figure is a rough sketch of how the above can look in practice: -Policies can be implemented using `any framework `__. -However, for TensorFlow and PyTorch, RLlib has -`build_tf_policy `__ and -`build_torch_policy `__ helper functions that let you -define a trainable policy with a functional-style API, for example: +.. image:: images/rllib-concepts-rlmodules-sketch.png -.. TODO: test this code snippet -.. code-block:: python - - def policy_gradient_loss(policy, model, dist_class, train_batch): - logits, _ = model.from_batch(train_batch) - action_dist = dist_class(logits, model) - return -tf.reduce_mean( - action_dist.logp(train_batch["actions"]) * train_batch["rewards"]) +.. note:: - # - MyTFPolicy = build_tf_policy( - name="MyTFPolicy", - loss_fn=policy_gradient_loss) + RL Modules are currently in alpha stage. They are wrapped in legacy :py:class:`~ray.rllib.policy.Policy` objects + to be used in :py:class:`~ray.rllib.evaluation.rollout_worker.RolloutWorker` for sampling. + This should be transparent to the user, but the following + `Policy Evaluation `__ section still refers to these legacy Policy objects. +.. policy-evaluation: Policy Evaluation ----------------- diff --git a/doc/source/rllib/package_ref/catalogs.rst b/doc/source/rllib/package_ref/catalogs.rst index 3b4bc2f1e607c..155512194a520 100644 --- a/doc/source/rllib/package_ref/catalogs.rst +++ b/doc/source/rllib/package_ref/catalogs.rst @@ -11,8 +11,7 @@ Basic usage ----------- Use the following basic API to get a default ``encoder`` or ``action distribution`` -out of Catalog. You can inherit from Catalog and modify the following methods to -directly inject custom components into a given RLModule. +out of Catalog. To change the catalog behavior, modify the following methods. Algorithm-specific implementations of Catalog have additional methods, for example, for building ``heads``. @@ -24,18 +23,18 @@ for example, for building ``heads``. Catalog Catalog.build_encoder Catalog.get_action_dist_cls - Catalog.get_preprocessor + Catalog.get_tokenizer_config Advanced usage -------------- -The following methods are used internally by the Catalog to build the default models. +The following methods and attributes are used internally by the Catalog to build the default models. Only override them when you need more granular control. .. autosummary:: :toctree: doc/ Catalog.latent_dims - Catalog.__post_init__ - Catalog.get_encoder_config - Catalog.get_tokenizer_config + Catalog._determine_components_hook + Catalog._get_encoder_config + Catalog._get_dist_cls_from_action_space diff --git a/doc/source/rllib/rllib-catalogs.rst b/doc/source/rllib/rllib-catalogs.rst index 5491f2fa20bbe..6d6b0a75291ae 100644 --- a/doc/source/rllib/rllib-catalogs.rst +++ b/doc/source/rllib/rllib-catalogs.rst @@ -4,30 +4,35 @@ .. include:: /_includes/rllib/rlmodules_rollout.rst -.. note:: Interacting with Catalogs mainly covers advanced use cases. Catalog (Alpha) =============== -Catalogs are where `RLModules `__ primarily get their models and action distributions from. -Each :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule` has its own default + +Catalog is a utility abstraction that modularizes the construction of components for `RLModules `__. +It includes information such how input observation spaces should be encoded, +what action distributions should be used, and so on. :py:class:`~ray.rllib.core.models.catalog.Catalog`. For example, :py:class:`~ray.rllib.algorithms.ppo.ppo_torch_rl_module.PPOTorchRLModule` has the :py:class:`~ray.rllib.algorithms.ppo.ppo_catalog.PPOCatalog`. -You can override Catalogs’ methods to alter the behavior of existing RLModules. -This makes Catalogs a means of configuration for RLModules. -You interact with Catalogs when making deeper customization to what :py:class:`~ray.rllib.core.models.Model` and :py:class:`~ray.rllib.models.distributions.Distribution` RLlib creates by default. +To customize existing RLModules either change the RLModule directly by inheriting the class and changing the +:py:meth:`~ray.rllib.core.rl_module.rl_module.RLModule.setup` method or, alternatively, extend the Catalog class +attributed to that `RLModule`. Use Catalogs only if your customizations fits the abstractions provided by Catalog. + +.. note:: + Modifying Catalogs signifies advanced use cases so you should only consider this if modifying an RLModule or writing one does not cover your use case. + We recommend to modify Catalogs only when making deeper customizations to the decision trees that determine what :py:class:`~ray.rllib.core.models.base.Model` and :py:class:`~ray.rllib.models.distributions.Distribution` RLlib creates by default. .. note:: - If you simply want to modify a :py:class:`~ray.rllib.core.models.Model` by changing its default values, - have a look at the ``model config dict``: + If you simply want to modify a Model by changing its default values, + have a look at the model config dict: - .. dropdown:: **``MODEL_DEFAULTS`` dict** + .. dropdown:: ``MODEL_DEFAULTS`` :animate: fade-in-slide-down This dict (or an overriding sub-set) is part of :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig` and therefore also part of any algorithm-specific config. - You can override its values and pass it to an :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig` + To change the behavior RLlib's default models, override it and pass it to an AlgorithmConfig. to change the behavior RLlib's default models. .. literalinclude:: ../../../rllib/models/catalog.py @@ -35,22 +40,107 @@ You interact with Catalogs when making deeper customization to what :py:class:`~ :start-after: __sphinx_doc_begin__ :end-before: __sphinx_doc_end__ -While Catalogs have a base class :py:class:`~ray.rllib.core.models.catalog.Catalog`, you mostly interact with +While Catalogs have a base class Catalog, you mostly interact with Algorithm-specific Catalogs. Therefore, this doc also includes examples around PPO from which you can extrapolate to other algorithms. Prerequisites for this user guide is a rough understanding of `RLModules `__. This user guide covers the following topics: -- Basic usage - What are Catalogs +- Catalog design and ideas +- Catalog and AlgorithmConfig +- Basic usage - Inject your custom models into RLModules - Inject your custom action distributions into RLModules +- Write a Catalog from scratch + +What are Catalogs +~~~~~~~~~~~~~~~~~ + +Catalogs have two primary roles: Choosing the right :py:class:`~ray.rllib.core.models.base.Model` and choosing the right :py:class:`~ray.rllib.models.distributions.Distribution`. +By default, all catalogs implement decision trees that decide model architecture based on a combination of input configurations. +These mainly include the ``observation space`` and ``action space`` of the :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule`, the ``model config dict`` and the ``deep learning framework backend``. + +The following diagram shows the break down of the information flow towards ``models`` and ``distributions`` within an RLModule. +An RLModule creates an instance of the Catalog class they receive as part of their constructor. +It then create its internal ``models`` and ``distributions`` with the help of this Catalog. + +.. note:: + You can also modify Model or Distribution in an RLModule directly by overriding the RLModule's constructor! + +.. image:: images/catalog/catalog_and_rlm_diagram.svg + :align: center -.. - Extend RLlib’s selection of Models and Distributions with your own -.. - Write a Catalog from scratch +The following diagram shows a concrete case in more detail. + +.. dropdown:: **Example of catalog in a PPORLModule** + :animate: fade-in-slide-down + + The :py:class:`~ray.rllib.algorithms.ppo.ppo_catalog.PPOCatalog` is fed an ``observation space``, ``action space``, + a ``model config dict`` and the ``view requirements`` of the :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule`. + The ``model config dicts`` and the ``view requirements`` are only of interest in special cases, such as + recurrent networks or attention networks. A PPORLModule has four components that are created by the PPOCatalog: + ``Encoder``, ``value function head``, ``policy head``, and ``action distribution``. + + .. image:: images/catalog/ppo_catalog_and_rlm_diagram.svg + :align: center + + +Catalog design and ideas +~~~~~~~~~~~~~~~~~~~~~~~~ + +Since the main use cases for this component involve deep modifications of it, we explain the design and ideas behind Catalogs in this section. + +What problems do Catalogs solve? +-------------------------------- + +RL algorithms need neural network ``models`` and ``distributions``. +Within an algorithm, many different architectures for such sub-components are valid. +Moreover, models and distributions vary with environments. +However, most algorithms require models that have similarities. +The problem is finding sensible sub-components for a wide range of use cases while sharing this functionality +across algorithms. + +How do Catalogs solve this? +---------------------------- + +As states above, Catalogs implement decision-trees for sub-components of `RLModules`. +Models and distributions from a Catalog object are meant to fit together. +Since we mostly build RLModules out of :py:class:`~ray.rllib.core.models.base.Encoder` s, Heads and :py:class:`~ray.rllib.models.distributions.Distribution` s, Catalogs also generally reflect this. +For example, the PPOCatalog will output Encoders that output a latent vector and two Heads that take this latent vector as input. +(That's why Catalogs have a ``latent_dims`` attribute). Heads and distributions behave accordingly. +Whenever you create a Catalog, the decision tree is executed to find suitable configs for models and classes for distributions. +By default this happens in :py:meth:`~ray.rllib.core.models.catalog.Catalog.get_encoder_config` and :py:meth:`~ray.rllib.core.models.catalog.Catalog._get_dist_cls_from_action_space`. +Whenever you build a model, the config is turned into a model. +Distributions are instantiated per forward pass of an `RLModule` and are therefore not built. + +API philosophy +-------------- + +Catalogs attempt to encapsulate most complexity around models inside the :py:class:`~ray.rllib.core.models.base.Encoder`. +This means that recurrency, attention and other special cases are fully handles inside the Encoder and are transparent +to other components. +Encoders are the only components that the Catalog base class builds. +This is because many algorithms require custom heads and distributions but most of them can use the same encoders. +The Catalog API is designed such that interaction usually happens in two stages: + +- Instantiate a Catalog. This executes the decision tree. +- Generate arbitrary number of decided components through Catalog methods. + +The two default methods to access components on the base class are... + +- :py:meth:`~ray.rllib.core.models.catalog.Catalog.build_encoder` +- :py:meth:`~ray.rllib.core.models.catalog.Catalog.get_action_dist_cls` + +You can override these to quickly hack what models RLModules build. +Other methods are private and should only be overridden to make deep changes to the decision tree to enhance the capabilities of Catalogs. +Additionally, :py:meth:`~ray.rllib.core.models.catalog.Catalog.get_tokenizer_config` is a method that can be used when tokenization +is required. Tokenization means single-step-embedding. Encoding also means embedding but can span multiple timesteps. +In fact, RLlib's tokenizers used in its recurrent Encoders (e.g. :py:class:`~ray.rllib.core.models.torch.encoder.TorchLSTMEncoder`), +are instances of non-recurrent Encoder classes. Catalog and AlgorithmConfig -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since Catalogs effectively control what ``models`` and ``distributions`` RLlib uses under the hood, they are also part of RLlib’s configurations. As the primary entry point for configuring RLlib, @@ -71,10 +161,14 @@ created by PPO. :start-after: __sphinx_doc_algo_configs_begin__ :end-before: __sphinx_doc_algo_configs_end__ + Basic usage ~~~~~~~~~~~ -The following three examples illustrate three basic usage patterns of Catalogs. +In the following three examples, we play with Catalogs to illustrate their API. + +High-level API +-------------- The first example showcases the general API for interacting with Catalogs. @@ -83,24 +177,12 @@ The first example showcases the general API for interacting with Catalogs. :start-after: __sphinx_doc_basic_interaction_begin__ :end-before: __sphinx_doc_basic_interaction_end__ -The second example showcases how to use the :py:class:`~ray.rllib.algorithms.ppo.ppo_catalog.PPOCatalog` -to create a ``model`` and an ``action distribution``. -This is more similar to what RLlib does internally. - -.. dropdown:: **Use catalog-generated models** - :animate: fade-in-slide-down - - .. literalinclude:: doc_code/catalog_guide.py - :language: python - :start-after: __sphinx_doc_ppo_models_begin__ - :end-before: __sphinx_doc_ppo_models_end__ +Creating models and distributions +--------------------------------- -The third example showcases how to use the base :py:class:`~ray.rllib.core.models.catalog.Catalog` -to create an ``encoder`` and an ``action distribution``. -Besides these, we create a ``head network`` that fits these two by hand to show how you can combine RLlib's -:py:class:`~ray.rllib.core.models.base.ModelConfig` API and Catalog. -Extending Catalog to also build this head is how :py:class:`~ray.rllib.core.models.catalog.Catalog` is meant to be -extended, which we cover later in this guide. +The second example showcases how to use the base :py:class:`~ray.rllib.core.models.catalog.Catalog` +to create an ``model`` and an ``action distribution``. +Besides these, we create a ``head network`` by hand that fits these two by hand. .. dropdown:: **Customize a policy head** :animate: fade-in-slide-down @@ -110,40 +192,28 @@ extended, which we cover later in this guide. :start-after: __sphinx_doc_modelsworkflow_begin__ :end-before: __sphinx_doc_modelsworkflow_end__ -What are Catalogs -~~~~~~~~~~~~~~~~~ - -Catalogs have two primary roles: Choosing the right :py:class:`~ray.rllib.core.models.Model` and choosing the right :py:class:`~ray.rllib.models.distributions.Distribution`. -By default, all catalogs implement decision trees that decide model architecture based on a combination of input configurations. -These mainly include the ``observation space`` and ``action space`` of the :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule`, the ``model config dict`` and the ``deep learning framework backend``. - -The following diagram shows the break down of the information flow towards ``models`` and ``distributions`` within an :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule`. -An :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule` creates an instance of the Catalog class they receive as part of their constructor. -It then create its internal ``models`` and ``distributions`` with the help of this Catalog. - -.. note:: - You can also modify :py:class:`~ray.rllib.core.models.Model` or :py:class:`~ray.rllib.models.distributions.Distribution` in an :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule` directly by overriding the RLModule's constructor! - -.. image:: images/catalog/catalog_and_rlm_diagram.svg - :align: center +Creating models and distributions for PPO +----------------------------------------- -The following diagram shows a concrete case in more detail. +The third example showcases how to use the :py:class:`~ray.rllib.algorithms.ppo.ppo_catalog.PPOCatalog` +to create a ``encoder`` and an ``action distribution``. +This is more similar to what RLlib does internally. -.. dropdown:: **Example of catalog in a PPORLModule** +.. dropdown:: **Use catalog-generated models** :animate: fade-in-slide-down - The :py:class:`~ray.rllib.algorithms.ppo.ppo_catalog.PPOCatalog` is fed an ``observation space``, ``action space``, - a ``model config dict`` and the ``view requirements`` of the :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule`. - The model config dicts and the view requirements are only of interest in special cases, such as - recurrent networks or attention networks. A PPORLModule has four components that are created by the - :py:class:`~ray.rllib.algorithms.ppo.ppo_catalog.PPOCatalog`: - ``Encoder``, ``value function head``, ``policy head``, and ``action distribution``. + .. literalinclude:: doc_code/catalog_guide.py + :language: python + :start-after: __sphinx_doc_ppo_models_begin__ + :end-before: __sphinx_doc_ppo_models_end__ - .. image:: images/catalog/ppo_catalog_and_rlm_diagram.svg - :align: center +Note that the above two examples illustrate in principle what it takes to implement a Catalog. +In this case, we see the difference between `Catalog` and `PPOCatalog`. +In most cases, we can reuse the capabilities of the base :py:class:`~ray.rllib.core.models.catalog.Catalog` base class +and only need to add methods to build head networks that we can then use in the appropriate `RLModule`. -Inject your custom model or action distributions into RLModules +Inject your custom model or action distributions into Catalogs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can make a :py:class:`~ray.rllib.core.models.catalog.Catalog` build custom ``models`` by overriding the Catalog’s methods used by RLModules to build ``models``. @@ -154,28 +224,68 @@ Have a look at these lines from the constructor of the :py:class:`~ray.rllib.alg :start-after: __sphinx_doc_begin__ :end-before: __sphinx_doc_end__ +Note that what happens inside the constructor of PPOTorchRLModule is similar to the earlier example `Creating models and distributions for PPO `__. + Consequently, in order to build a custom :py:class:`~ray.rllib.core.models.Model` compatible with a PPORLModule, you can override methods by inheriting from :py:class:`~ray.rllib.algorithms.ppo.ppo_catalog.PPOCatalog` or write a :py:class:`~ray.rllib.core.models.catalog.Catalog` that implements them from scratch. -The following showcases such modifications. +The following examples showcase such modifications: -This example shows two modifications: -- How to write a custom :py:class:`~ray.rllib.models.distributions.Distribution` -- How to inject a custom action distribution into a :py:class:`~ray.rllib.core.models.catalog.Catalog` +.. tab-set:: -.. literalinclude:: ../../../rllib/examples/catalog/custom_action_distribution.py - :language: python - :start-after: __sphinx_doc_begin__ - :end-before: __sphinx_doc_end__ + .. tab-item:: Adding a custom Encoder + + This example shows two modifications: + + - How to write a custom :py:class:`~ray.rllib.models.base.Encoder` + - How to inject the custom Encoder into a :py:class:`~ray.rllib.core.models.catalog.Catalog` + + Note that, if you only want to inject your Encoder into a single :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule`, the recommended workflow is to inherit + from an existing RL Module and place the Encoder there. + + .. literalinclude:: ../../../rllib/examples/catalog/mobilenet_v2_encoder.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ + + + .. tab-item:: Adding a custom action distribution + + This example shows two modifications: + + - How to write a custom :py:class:`~ray.rllib.models.distributions.Distribution` + - How to inject the custom action distribution into a :py:class:`~ray.rllib.core.models.catalog.Catalog` + + .. literalinclude:: ../../../rllib/examples/catalog/custom_action_distribution.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ + +These examples target PPO but the workflows apply to all RLlib algorithms. +Note that PPO adds the :py:class:`from ray.rllib.core.models.base.ActorCriticEncoder` and two heads (policy- and value-head) to the base class. +You can override these similarly to the above. +Other algorithms may add different sub-components or override default ones. + +Write a Catalog from scratch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You only need this when you want to write a new Algorithm under RLlib. +Note that writing an Algorithm does not strictly require writing a new Catalog but you can use Catalogs as a tool to create +the fitting default sub-components, such as models or distributions. +The following are typical requirements and steps for writing a new Catalog: +- Does the Algorithm need a special Encoder? Overwrite :py:meth:`~ray.rllib.core.models.catalog.Catalog._get_encoder_config`. +- Does the Algorithm need an additional network? Write a method to build it. You can use RLlib's model configurations to build models from dimensions. +- Does the Algorithm need a custom distribution? Overwrite :py:meth:`~ray.rllib.core.models.catalog.Catalog._get_dist_cls_from_action_space`. +- Does the Algorithm need a special tokenizer? Overwrite :py:meth:`~ray.rllib.core.models.catalog.Catalog.get_tokenizer_config`. +- Does the Algorithm not need an Encoder at all? Overwrite :py:meth:`~ray.rllib.core.models.catalog.Catalog._determine_components_hook`. +The following example shows our implementation of a Catalog for PPO that follows the above steps: -Notable TODOs -------------- +.. dropdown:: **Catalog for PPORLModules** -- Add cross references to Model and Distribution API docs -- Add example that shows how to inject own model -- Add more instructions on how to write a catalog from scratch -- Add section "Extend RLlib’s selection of Models and Distributions with your own" -- Add section "Write a Catalog from scratch" \ No newline at end of file + .. literalinclude:: ../../../rllib/algorithms/ppo/ppo_catalog.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ \ No newline at end of file diff --git a/doc/source/rllib/rllib-connector.rst b/doc/source/rllib/rllib-connector.rst index 3717a499af413..0c609f985e8a6 100644 --- a/doc/source/rllib/rllib-connector.rst +++ b/doc/source/rllib/rllib-connector.rst @@ -2,7 +2,7 @@ .. include:: /_includes/rllib/we_are_hiring.rst -Connectors (Alpha) +Connectors (Beta) ================== Connector are components that handle transformations on inputs and outputs of a given RL policy, with the goal of improving diff --git a/doc/source/rllib/rllib-rlmodule.rst b/doc/source/rllib/rllib-rlmodule.rst index 91473cc98452a..f1eec46db53ed 100644 --- a/doc/source/rllib/rllib-rlmodule.rst +++ b/doc/source/rllib/rllib-rlmodule.rst @@ -302,7 +302,6 @@ In :py:class:`~ray.rllib.core.rl_module.rl_module.RLModule` you can enforce the To learn more, see the `SpecType` documentation. - Writing Custom Multi-Agent RL Modules (Advanced) ------------------------------------------------ @@ -313,14 +312,11 @@ The :py:class:`~ray.rllib.core.rl_module.marl_module.MultiAgentRLModule` offers The following example creates a custom multi-agent RL module with underlying modules. The modules share an encoder, which gets applied to the global part of the observations space. The local part passes through a separate encoder, specific to each policy. -.. tab-set:: - - .. tab-item:: Multi agent with shared encoder (Torch) - .. literalinclude:: doc_code/rlmodule_guide.py - :language: python - :start-after: __write-custom-marlmodule-shared-enc-begin__ - :end-before: __write-custom-marlmodule-shared-enc-end__ +.. literalinclude:: doc_code/rlmodule_guide.py + :language: python + :start-after: __write-custom-marlmodule-shared-enc-begin__ + :end-before: __write-custom-marlmodule-shared-enc-end__ To construct this custom multi-agent RL module, pass the class to the :py:class:`~ray.rllib.core.rl_module.marl_module.MultiAgentRLModuleSpec` constructor. Also, pass the :py:class:`~ray.rllib.core.rl_module.rl_module.SingleAgentRLModuleSpec` for each agent because RLlib requires the observation, action spaces, and model hyper-parameters for each agent. @@ -334,7 +330,11 @@ To construct this custom multi-agent RL module, pass the class to the :py:class: Extending Existing RLlib RL Modules ----------------------------------- -RLlib provides a number of RL Modules for different frameworks (e.g., PyTorch, TensorFlow, etc.). Extend these modules by inheriting from them and overriding the methods you need to customize. For example, extend :py:class:`~ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module.PPOTorchRLModule` and augment it with your own customization. Then pass the new customized class into the algorithm configuration. +RLlib provides a number of RL Modules for different frameworks (e.g., PyTorch, TensorFlow, etc.). +To customize existing RLModules you can change the RLModule directly by inheriting the class and changing the +:py:meth:`~ray.rllib.core.rl_module.rl_module.RLModule.setup` or other methods. +For example, extend :py:class:`~ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module.PPOTorchRLModule` and augment it with your own customization. +Then pass the new customized class into the appropriate :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig`. There are two possible ways to extend existing RL Modules: @@ -342,7 +342,10 @@ There are two possible ways to extend existing RL Modules: .. tab-item:: Inheriting existing RL Modules - One way to extend existing RL Modules is to inherit from them and override the methods you need to customize. For example, extend :py:class:`~ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module.PPOTorchRLModule` and augment it with your own customization. Then pass the new customized class into the algorithm configuration to use the PPO algorithm to optimize your custom RL Module. + The default way to extend existing RL Modules is to inherit from them and override the methods you need to customize. + Then pass the new customized class into the :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig` to optimize your custom RL Module. + This is the preferred approach. With it, we can define our own models explicitly within a given RL Module + and don't need to interact with a Catalog, so you don't need to learn about Catalog. .. code-block:: python @@ -357,19 +360,39 @@ There are two possible ways to extend existing RL Modules: rl_module_spec=SingleAgentRLModuleSpec(module_class=MyPPORLModule) ) + A concrete example: If you want to replace the default encoder that RLlib builds for torch, PPO and a given observation space, + you can override :py:class:`~ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module.PPOTorchRLModule`'s + :py:meth:`~ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module.PPOTorchRLModule.__init__` to create your custom + encoder instead of the default one. We do this in the following example. + + .. literalinclude:: ../../../rllib/examples/rl_module/mobilenet_rlm.py + :language: python + :start-after: __sphinx_doc_begin__ + :end-before: __sphinx_doc_end__ + .. tab-item:: Extending RL Module Catalog - Another way to customize your module is by extending its :py:class:`~ray.rllib.core.models.catalog.Catalog`. The :py:class:`~ray.rllib.core.models.catalog.Catalog` is a component that defines the default architecture and behavior of a model based on factors such as ``observation_space``, ``action_space``, etc. To modify sub-components of an existing RL Module, extend the corresponding Catalog class. + An advanced way to customize your module is by extending its :py:class:`~ray.rllib.core.models.catalog.Catalog`. + The Catalog is a component that defines the default models and other sub-components for RL Modules based on factors such as ``observation_space``, ``action_space``, etc. + For more information on the :py:class:`~ray.rllib.core.models.catalog.Catalog` class, refer to the `Catalog user guide `__. + By modifying the Catalog, you can alter what sub-components are being built for existing RL Modules. + This approach is useful mostly if you want your custom component to integrate with the decision trees that the Catalogs represent. + The following use cases are examples of what may require you to extend the Catalogs: - For instance, to adapt the existing ``PPORLModule`` for a custom graph observation space not supported by RLlib out-of-the-box, extend the :py:class:`~ray.rllib.core.models.catalog.Catalog` class used to create the ``PPORLModule`` and override the method responsible for returning the encoder component to ensure that your custom encoder replaces the default one initially provided by RLlib. For more information on the :py:class:`~ray.rllib.core.models.catalog.Catalog` class, refer to the `Catalog user guide `__. + - Choosing a custom model only for a certain observation space. + - Using a custom action distribution in multiple distinct Algorithms. + - Reusing your custom component in many distinct RL Modules. + For instance, to adapt existing ``PPORLModules`` for a custom graph observation space not supported by RLlib out-of-the-box, + extend the :py:class:`~ray.rllib.core.models.catalog.Catalog` class used to create the ``PPORLModule`` + and override the method responsible for returning the encoder component to ensure that your custom encoder replaces the default one initially provided by RLlib. .. code-block:: python class MyAwesomeCatalog(PPOCatalog): - def get_actor_critic_encoder_config(): + def build_actor_critic_encoder(): # create your awesome graph encoder here and return it pass @@ -380,6 +403,17 @@ There are two possible ways to extend existing RL Modules: ) +Checkpointing RL Modules +------------------------ + +RL Modules can be checkpointed with their two methods :py:meth:`~ray.rllib.core.rl_module.rl_module.RLModule.save_to_checkpoint` and :py:meth:`~ray.rllib.core.rl_module.rl_module.RLModule.from_checkpoint`. +The following example shows how these methods can be used outside of, or in conjunction with, an RLlib Algorithm. + +.. literalinclude:: doc_code/rlmodule_guide.py + :language: python + :start-after: __checkpointing-begin__ + :end-before: __checkpointing-end__ + Migrating from Custom Policies and Models to RL Modules ------------------------------------------------------- @@ -541,12 +575,4 @@ See `Writing Custom Single Agent RL Modules`_ for more details on how to impleme ... def _forward_exploration(self, batch): - ... - - -Notable TODOs -------------- - -- [] Add support for RNNs. -- [] Checkpointing. -- [] End to end example for custom RL Modules extending PPORLModule (e.g. LLM) \ No newline at end of file + ... \ No newline at end of file diff --git a/rllib/BUILD b/rllib/BUILD index 8a31f265b32b8..971c99031806b 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -3143,6 +3143,32 @@ py_test( args = ["--run=PPO", "--as-test", "--framework=torch", "--stop-reward=28", "--num-cpus=4", "--use-prev-action", "--use-prev-reward"] ) +py_test( + name = "examples/catalog/custom_action_distribution", + main = "examples/catalog/custom_action_distribution.py", + tags = ["team:rllib", "examples", "no_main"], + size = "small", + srcs = ["examples/catalog/custom_action_distribution.py"], +) + + +py_test( + name = "examples/catalog/mobilenet_v2_encoder", + main = "examples/catalog/mobilenet_v2_encoder.py", + tags = ["team:rllib", "examples", "no_main"], + size = "small", + srcs = ["examples/catalog/mobilenet_v2_encoder.py"], +) + + +py_test( + name = "examples/rl_module/mobilenet_rlm", + main = "examples/rl_module/mobilenet_rlm.py", + tags = ["team:rllib", "examples", "no_main"], + size = "small", + srcs = ["examples/rl_module/mobilenet_rlm.py"], +) + py_test( name = "examples/centralized_critic_tf", main = "examples/centralized_critic.py", diff --git a/rllib/algorithms/ppo/ppo_catalog.py b/rllib/algorithms/ppo/ppo_catalog.py index 2a6df6710324c..474491e3d4a30 100644 --- a/rllib/algorithms/ppo/ppo_catalog.py +++ b/rllib/algorithms/ppo/ppo_catalog.py @@ -1,3 +1,4 @@ +# __sphinx_doc_begin__ import gymnasium as gym from ray.rllib.core.models.catalog import Catalog @@ -8,6 +9,7 @@ ) from ray.rllib.core.models.base import Encoder, ActorCriticEncoder, Model from ray.rllib.utils import override +from ray.rllib.utils.annotations import OverrideToImplementCustomLogic def _check_if_diag_gaussian(action_distribution_cls, framework): @@ -72,12 +74,14 @@ def __init__( # Replace EncoderConfig by ActorCriticEncoderConfig self.actor_critic_encoder_config = ActorCriticEncoderConfig( - base_encoder_config=self.encoder_config, - shared=self.model_config_dict["vf_share_layers"], + base_encoder_config=self._encoder_config, + shared=self._model_config_dict["vf_share_layers"], ) - self.pi_and_vf_head_hiddens = self.model_config_dict["post_fcnet_hiddens"] - self.pi_and_vf_head_activation = self.model_config_dict["post_fcnet_activation"] + self.pi_and_vf_head_hiddens = self._model_config_dict["post_fcnet_hiddens"] + self.pi_and_vf_head_activation = self._model_config_dict[ + "post_fcnet_activation" + ] # We don't have the exact (framework specific) action dist class yet and thus # cannot determine the exact number of output nodes (action space) required. @@ -92,6 +96,7 @@ def __init__( output_layer_dim=1, ) + @OverrideToImplementCustomLogic def build_actor_critic_encoder(self, framework: str) -> ActorCriticEncoder: """Builds the ActorCriticEncoder. @@ -114,9 +119,10 @@ def build_encoder(self, framework: str) -> Encoder: Since PPO uses an ActorCriticEncoder, this method should not be implemented. """ raise NotImplementedError( - "Use PPOCatalog.build_actor_critic_encoder() instead." + "Use PPOCatalog.build_actor_critic_encoder() instead for PPO." ) + @OverrideToImplementCustomLogic def build_pi_head(self, framework: str) -> Model: """Builds the policy head. @@ -132,18 +138,18 @@ def build_pi_head(self, framework: str) -> Model: """ # Get action_distribution_cls to find out about the output dimension for pi_head action_distribution_cls = self.get_action_dist_cls(framework=framework) - if self.model_config_dict["free_log_std"]: + if self._model_config_dict["free_log_std"]: _check_if_diag_gaussian( action_distribution_cls=action_distribution_cls, framework=framework ) required_output_dim = action_distribution_cls.required_input_dim( - space=self.action_space, model_config=self.model_config_dict + space=self.action_space, model_config=self._model_config_dict ) # Now that we have the action dist class and number of outputs, we can define # our pi-config and build the pi head. pi_head_config_class = ( FreeLogStdMLPHeadConfig - if self.model_config_dict["free_log_std"] + if self._model_config_dict["free_log_std"] else MLPHeadConfig ) self.pi_head_config = pi_head_config_class( @@ -156,6 +162,7 @@ def build_pi_head(self, framework: str) -> Model: return self.pi_head_config.build(framework=framework) + @OverrideToImplementCustomLogic def build_vf_head(self, framework: str) -> Model: """Builds the value function head. @@ -170,3 +177,6 @@ def build_vf_head(self, framework: str) -> Model: The value function head. """ return self.vf_head_config.build(framework=framework) + + +# __sphinx_doc_end__ diff --git a/rllib/core/models/catalog.py b/rllib/core/models/catalog.py index 48089700267d9..b956343babae7 100644 --- a/rllib/core/models/catalog.py +++ b/rllib/core/models/catalog.py @@ -24,16 +24,25 @@ from ray.rllib.utils.spaces.space_utils import flatten_space from ray.rllib.utils.spaces.space_utils import get_base_struct_from_space from ray.rllib.utils.typing import ViewRequirementsDict +from ray.rllib.utils.annotations import ( + OverrideToImplementCustomLogic, + OverrideToImplementCustomLogic_CallToSuperRecommended, +) class Catalog: - """Describes the sub-modules architectures to be used in RLModules. + """Describes the sub-module-architectures to be used in RLModules. RLlib's native RLModules get their Models from a Catalog object. By default, that Catalog builds the configs it has as attributes. - You can modify a Catalog so that it builds different Models by subclassing and - overriding the build_* methods. Alternatively, you can customize the configs - inside RLlib's Catalogs to customize what is being built by RLlib. + This component was build to be hackable and extensible. You can inject custom + components into RL Modules by overriding the `build_xxx` methods of this class. + Note that it is recommended to write a custom RL Module for a single use-case. + Modifications to Catalogs mostly make sense if you want to reuse the same + Catalog for different RL Modules. For example if you have written a custom + encoder and want to inject it into different RL Modules (e.g. for PPO, DQN, etc.). + You can influence the decision tree that determines the sub-components by modifying + `Catalog._determine_components_hook`. Usage example: @@ -77,8 +86,6 @@ def __init__( self, observation_space: gym.Space, action_space: gym.Space, - # TODO (Artur): Turn model_config into model_config_dict to distinguish - # between ModelConfig and a model_config_dict dict library-wide. model_config_dict: dict, view_requirements: dict = None, ): @@ -97,62 +104,78 @@ def __init__( self.action_space = action_space # TODO (Artur): Make model defaults a dataclass - self.model_config_dict = {**MODEL_DEFAULTS, **model_config_dict} - self.view_requirements = view_requirements - + self._model_config_dict = {**MODEL_DEFAULTS, **model_config_dict} + self._view_requirements = view_requirements self._latent_dims = None - # Overwrite this post-init hook in subclasses - self.__post_init__() - - @property - def latent_dims(self): - """Returns the latent dimensions of the encoder. - - This establishes an agreement between encoder and heads about the latent - dimensions. Encoders can be built to output a latent tensor with - `latent_dims` dimensions, and heads can be built with tensors of - `latent_dims` dimensions as inputs. This can be safely ignored if this - agreement is not needed in case of modifications to the Catalog. + self._determine_components_hook() - Returns: - The latent dimensions of the encoder. - """ - return self._latent_dims + @OverrideToImplementCustomLogic_CallToSuperRecommended + def _determine_components_hook(self): + """Decision tree hook for subclasses to override. - @latent_dims.setter - def latent_dims(self, value): - self._latent_dims = value + By default, this method executes the decision tree that determines the + components that a Catalog builds. You can extend the components by overriding + this or by adding to the constructor of your subclass. - def __post_init__(self): - """Post-init hook for subclasses to override. + Override this method if you don't want to use the default components + determined here. If you want to use them but add additional components, you + should call `super()._determine_components()` at the beginning of your + implementation. This makes it so that subclasses are not forced to create an encoder config if the rest of their catalog is not dependent on it or if it breaks. - At the end of Catalog initialization, an attribute `Catalog.latent_dims` + At the end of this method, an attribute `Catalog.latent_dims` should be set so that heads can be built using that information. """ - self.encoder_config = self.get_encoder_config( + self._encoder_config = self._get_encoder_config( observation_space=self.observation_space, action_space=self.action_space, - model_config_dict=self.model_config_dict, - view_requirements=self.view_requirements, + model_config_dict=self._model_config_dict, + view_requirements=self._view_requirements, ) # Create a function that can be called when framework is known to retrieve the # class type for action distributions self._action_dist_class_fn = functools.partial( - self.get_dist_cls_from_action_space, action_space=self.action_space + self._get_dist_cls_from_action_space, action_space=self.action_space ) # The dimensions of the latent vector that is output by the encoder and fed # to the heads. - self.latent_dims = self.encoder_config.output_dims + self.latent_dims = self._encoder_config.output_dims + @property + def latent_dims(self): + """Returns the latent dimensions of the encoder. + + This establishes an agreement between encoder and heads about the latent + dimensions. Encoders can be built to output a latent tensor with + `latent_dims` dimensions, and heads can be built with tensors of + `latent_dims` dimensions as inputs. This can be safely ignored if this + agreement is not needed in case of modifications to the Catalog. + + Returns: + The latent dimensions of the encoder. + """ + return self._latent_dims + + @latent_dims.setter + def latent_dims(self, value): + self._latent_dims = value + + @OverrideToImplementCustomLogic def build_encoder(self, framework: str) -> Encoder: """Builds the encoder. - By default this method builds an encoder instance from Catalog.encoder_config. + By default, this method builds an encoder instance from Catalog._encoder_config. + + You should override this if you want to use RLlib's default RL Modules but + only want to change the encoder. For example, if you want to use a custom + encoder, but want to use RLlib's default heads, action distribution and how + tensors are routed between them. If you want to have full control over the + RL Module, we recommend writing your own RL Module by inheriting from one of + RLlib's RL Modules instead. Args: framework: The framework to use. Either "torch" or "tf2". @@ -160,20 +183,24 @@ def build_encoder(self, framework: str) -> Encoder: Returns: The encoder. """ - assert hasattr(self, "encoder_config"), ( - "You must define a `Catalog.encoder_config` attribute in your Catalog " + assert hasattr(self, "_encoder_config"), ( + "You must define a `Catalog._encoder_config` attribute in your Catalog " "subclass or override the `Catalog.build_encoder` method. By default, " "an encoder_config is created in the __post_init__ method." ) - return self.encoder_config.build(framework=framework) + return self._encoder_config.build(framework=framework) + @OverrideToImplementCustomLogic def get_action_dist_cls(self, framework: str): """Get the action distribution class. The default behavior is to get the action distribution from the - `Catalog.action_dist_class_fn`. This can be overridden to build a custom action - distribution as a means of configuring the behavior of a RLModule - implementation. + `Catalog._action_dist_class_fn`. + + You should override this to have RLlib build your custom action + distribution instead of the default one. For example, if you don't want to + use RLlib's default RLModules with their default models, but only want to + change the distribution that Catalog returns. Args: framework: The framework to use. Either "torch" or "tf2". @@ -190,7 +217,7 @@ def get_action_dist_cls(self, framework: str): return self._action_dist_class_fn(framework=framework) @classmethod - def get_encoder_config( + def _get_encoder_config( cls, observation_space: gym.Space, model_config_dict: dict, @@ -312,6 +339,7 @@ def get_encoder_config( return encoder_config @classmethod + @OverrideToImplementCustomLogic def get_tokenizer_config( cls, observation_space: gym.Space, @@ -324,13 +352,19 @@ def get_tokenizer_config( inputs. By default, RLlib uses the models supported by Catalog out of the box to tokenize. + You should override this method if you want to change the custom tokenizer + inside current encoders that Catalog returns without providing the recurrent + network as a whole. For example, if you want to define some custom CNN layers + as a tokenizer for a recurrent encoder that already includes the recurrent + layers and handles the state. + Args: observation_space: The observation space to use. model_config_dict: The model config to use. view_requirements: The view requirements to use if anything else than observation_space is to be encoded. This signifies an advanced use case. """ - return cls.get_encoder_config( + return cls._get_encoder_config( observation_space=observation_space, # Use model_config_dict without flags that would end up in complex models model_config_dict={ @@ -341,7 +375,7 @@ def get_tokenizer_config( ) @classmethod - def get_dist_cls_from_action_space( + def _get_dist_cls_from_action_space( cls, action_space: gym.Space, *, @@ -534,7 +568,7 @@ def _multi_action_dist_partial_helper( action_space_struct = get_base_struct_from_space(action_space) flat_action_space = flatten_space(action_space) child_distribution_cls_struct = tree.map_structure( - lambda s: catalog_cls.get_dist_cls_from_action_space( + lambda s: catalog_cls._get_dist_cls_from_action_space( action_space=s, framework=framework, ), diff --git a/rllib/core/models/tests/test_catalog.py b/rllib/core/models/tests/test_catalog.py index 3d38d7d5e064f..60959bd5118da 100644 --- a/rllib/core/models/tests/test_catalog.py +++ b/rllib/core/models/tests/test_catalog.py @@ -199,7 +199,7 @@ def test_get_encoder_config(self): view_requirements=None, ) - model_config = catalog.get_encoder_config( + model_config = catalog._get_encoder_config( observation_space=input_space, model_config_dict=model_config_dict ) self.assertEqual(type(model_config), model_config_type) @@ -328,7 +328,7 @@ def test_get_dist_cls_from_action_space(self): if framework == "tf2": framework = "tf2" - dist_cls = catalog.get_dist_cls_from_action_space( + dist_cls = catalog._get_dist_cls_from_action_space( action_space=action_space, framework=framework, ) @@ -450,9 +450,9 @@ def _forward(self, input_dict, **kwargs): } class MyCustomCatalog(PPOCatalog): - def __post_init__(self): + def _determine_components(self): self._action_dist_class_fn = functools.partial( - self.get_dist_cls_from_action_space, action_space=self.action_space + self._get_dist_cls_from_action_space, action_space=self.action_space ) self.latent_dims = (10,) self.encoder_config = MyCostumTorchEncoderConfig( diff --git a/rllib/examples/catalog/custom_action_distribution.py b/rllib/examples/catalog/custom_action_distribution.py index 42263f086a21e..979bee581bd98 100644 --- a/rllib/examples/catalog/custom_action_distribution.py +++ b/rllib/examples/catalog/custom_action_distribution.py @@ -19,23 +19,23 @@ class CustomTorchCategorical(Distribution): def __init__(self, logits): self.torch_dist = torch.distributions.categorical.Categorical(logits=logits) - def sample(self, sample_shape=torch.Size()): + def sample(self, sample_shape=torch.Size(), **kwargs): return self.torch_dist.sample(sample_shape) - def rsample(self, sample_shape=torch.Size()): + def rsample(self, sample_shape=torch.Size(), **kwargs): return self._dist.rsample(sample_shape) - def logp(self, value): + def logp(self, value, **kwargs): return self.torch_dist.log_prob(value) def entropy(self): return self.torch_dist.entropy() - def kl(self, other): + def kl(self, other, **kwargs): return torch.distributions.kl.kl_divergence(self.torch_dist, other.torch_dist) @staticmethod - def required_input_dim(space): + def required_input_dim(space, **kwargs): return int(space.n) @classmethod diff --git a/rllib/examples/catalog/mobilenet_v2_encoder.py b/rllib/examples/catalog/mobilenet_v2_encoder.py new file mode 100644 index 0000000000000..3d22fe92f0607 --- /dev/null +++ b/rllib/examples/catalog/mobilenet_v2_encoder.py @@ -0,0 +1,80 @@ +""" +This example shows two modifications: +- How to write a custom Encoder (using MobileNet v2) +- How to enhance Catalogs with this custom Encoder + +With the pattern shown in this example, we can enhance Catalogs such that they extend +to new observation- or action spaces while retaining their original functionality. +""" +# __sphinx_doc_begin__ +import gymnasium as gym +import numpy as np + +from ray.rllib.algorithms.ppo.ppo import PPOConfig +from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog +from ray.rllib.examples.models.mobilenet_v2_encoder import ( + MobileNetV2EncoderConfig, + MOBILENET_INPUT_SHAPE, +) +from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec +from ray.rllib.examples.env.random_env import RandomEnv + + +# Define a PPO Catalog that we can use to inject our MobileNetV2 Encoder into RLlib's +# decision tree of what model to choose +class MobileNetEnhancedPPOCatalog(PPOCatalog): + @classmethod + def _get_encoder_config( + cls, + observation_space: gym.Space, + **kwargs, + ): + if ( + isinstance(observation_space, gym.spaces.Box) + and observation_space.shape == MOBILENET_INPUT_SHAPE + ): + # Inject our custom encoder here, only if the observation space fits it + return MobileNetV2EncoderConfig() + else: + return super()._get_encoder_config(observation_space, **kwargs) + + +# Create a generic config with our enhanced Catalog +ppo_config = ( + PPOConfig() + .rl_module( + rl_module_spec=SingleAgentRLModuleSpec( + catalog_class=MobileNetEnhancedPPOCatalog + ) + ) + .rollouts(num_rollout_workers=0) + # The following training settings make it so that a training iteration is very + # quick. This is just for the sake of this example. PPO will not learn properly + # with these settings! + .training(train_batch_size=32, sgd_minibatch_size=16, num_sgd_iter=1) +) + +# CartPole's observation space is not compatible with our MobileNetV2 Encoder, so +# this will use the default behaviour of Catalogs +ppo_config.environment("CartPole-v1") +results = ppo_config.build().train() +print(results) + +# For this training, we use a RandomEnv with observations of shape +# MOBILENET_INPUT_SHAPE. This will use our custom Encoder. +ppo_config.environment( + RandomEnv, + env_config={ + "action_space": gym.spaces.Discrete(2), + # Test a simple Image observation space. + "observation_space": gym.spaces.Box( + 0.0, + 1.0, + shape=MOBILENET_INPUT_SHAPE, + dtype=np.float32, + ), + }, +) +results = ppo_config.build().train() +print(results) +# __sphinx_doc_end__ diff --git a/rllib/examples/models/mobilenet_v2_encoder.py b/rllib/examples/models/mobilenet_v2_encoder.py new file mode 100644 index 0000000000000..6a3482f547b0f --- /dev/null +++ b/rllib/examples/models/mobilenet_v2_encoder.py @@ -0,0 +1,47 @@ +""" +This file implements a MobileNet v2 Encoder. +It uses MobileNet v2 to encode images into a latent space of 1000 dimensions. + +Depending on the experiment, the MobileNet v2 encoder layers can be frozen or +unfrozen. This is controlled by the `freeze` parameter in the config. + +This is an example of how a pre-trained neural network can be used as an encoder +in RLlib. You can modify this example to accommodate your own encoder network or +other pre-trained networks. +""" + +from ray.rllib.core.models.base import Encoder, ENCODER_OUT +from ray.rllib.core.models.configs import ModelConfig +from ray.rllib.core.models.torch.base import TorchModel +from ray.rllib.utils.framework import try_import_torch + +torch, nn = try_import_torch() + +MOBILENET_INPUT_SHAPE = (3, 224, 224) + + +class MobileNetV2EncoderConfig(ModelConfig): + # MobileNet v2 has a flat output with a length of 1000. + output_dims = (1000,) + freeze = True + + def build(self, framework): + assert framework == "torch", "Unsupported framework `{}`!".format(framework) + return MobileNetV2Encoder(self) + + +class MobileNetV2Encoder(TorchModel, Encoder): + """A MobileNet v2 encoder for RLlib.""" + + def __init__(self, config): + super().__init__(config) + self.net = torch.hub.load( + "pytorch/vision:v0.6.0", "mobilenet_v2", pretrained=True + ) + if config.freeze: + # We don't want to train this encoder, so freeze its parameters! + for p in self.net.parameters(): + p.requires_grad = False + + def _forward(self, input_dict, **kwargs): + return {ENCODER_OUT: (self.net(input_dict["obs"]))} diff --git a/rllib/examples/rl_module/mobilenet_rlm.py b/rllib/examples/rl_module/mobilenet_rlm.py new file mode 100644 index 0000000000000..906b655024a81 --- /dev/null +++ b/rllib/examples/rl_module/mobilenet_rlm.py @@ -0,0 +1,82 @@ +""" +This example shows how to take full control over what models and action distribution +are being built inside an RL Module. With this pattern, we can bypass a Catalog and +explicitly define our own models within a given RL Module. +""" +# __sphinx_doc_begin__ +import gymnasium as gym +import numpy as np + +from ray.rllib.algorithms.ppo.ppo import PPOConfig +from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import PPOTorchRLModule +from ray.rllib.core.models.configs import MLPHeadConfig +from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec +from ray.rllib.examples.env.random_env import RandomEnv +from ray.rllib.models.torch.torch_distributions import TorchCategorical +from ray.rllib.examples.models.mobilenet_v2_encoder import ( + MobileNetV2EncoderConfig, + MOBILENET_INPUT_SHAPE, +) +from ray.rllib.core.models.configs import ActorCriticEncoderConfig + + +class MobileNetTorchPPORLModule(PPOTorchRLModule): + """A PPORLModules with mobilenet v2 as an encoder. + + The idea behind this model is to demonstrate how we can bypass catalog to + take full control over what models and action distribution are being built. + In this example, we do this to modify an existing RLModule with a custom encoder. + """ + + def setup(self): + mobilenet_v2_config = MobileNetV2EncoderConfig() + # Since we want to use PPO, which is an actor-critic algorithm, we need to + # use an ActorCriticEncoderConfig to wrap the base encoder config. + actor_critic_encoder_config = ActorCriticEncoderConfig( + base_encoder_config=mobilenet_v2_config + ) + + self.encoder = actor_critic_encoder_config.build(framework="torch") + mobilenet_v2_output_dims = mobilenet_v2_config.output_dims + + pi_config = MLPHeadConfig( + input_dims=mobilenet_v2_output_dims, + output_layer_dim=2, + ) + + vf_config = MLPHeadConfig( + input_dims=mobilenet_v2_output_dims, output_layer_dim=1 + ) + + self.pi = pi_config.build(framework="torch") + self.vf = vf_config.build(framework="torch") + + self.action_dist_cls = TorchCategorical + + +config = ( + PPOConfig() + .rl_module( + rl_module_spec=SingleAgentRLModuleSpec(module_class=MobileNetTorchPPORLModule) + ) + .environment( + RandomEnv, + env_config={ + "action_space": gym.spaces.Discrete(2), + # Test a simple Image observation space. + "observation_space": gym.spaces.Box( + 0.0, + 1.0, + shape=MOBILENET_INPUT_SHAPE, + dtype=np.float32, + ), + }, + ) + # The following training settings make it so that a training iteration is very + # quick. This is just for the sake of this example. PPO will not learn properly + # with these settings! + .training(train_batch_size=32, sgd_minibatch_size=16, num_sgd_iter=1) +) + +config.build().train() +# __sphinx_doc_end__