From 13aef3361df3d20617f28a90bccfa35ded26a9ee Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Sun, 25 Jan 2026 11:59:01 +0000 Subject: [PATCH 1/2] fix: fix examples/ps error. Signed-off-by: Klaus Ma --- common/src/lib.rs | 1 + compose.yaml | 1 + examples/ps/README.md | 166 ++++++++++++++++++++++++ examples/ps/dist/ps-example.tar.gz | Bin 29435 -> 0 bytes examples/ps/main.py | 2 +- examples/ps/pyproject.toml | 3 +- executor_manager/src/shims/host_shim.rs | 98 ++++++++++---- sdk/python/src/flamepy/__init__.py | 18 +-- sdk/python/src/flamepy/core/__init__.py | 3 +- sdk/python/src/flamepy/core/service.py | 4 + sdk/python/src/flamepy/core/types.py | 28 ++++ sdk/python/src/flamepy/rl/runner.py | 38 +++--- sdk/python/src/flamepy/rl/runpy.py | 78 ++++++----- 13 files changed, 350 insertions(+), 90 deletions(-) delete mode 100644 examples/ps/dist/ps-example.tar.gz diff --git a/common/src/lib.rs b/common/src/lib.rs index 80931aab..3643c64d 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -226,6 +226,7 @@ pub fn default_applications() -> HashMap { "-m".to_string(), "flamepy.rl.runpy".to_string(), ], + working_directory: Some("/opt/flame/work".to_string()), ..ApplicationAttributes::default() }, ), diff --git a/compose.yaml b/compose.yaml index 766c134d..7bb1dac7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -53,6 +53,7 @@ services: - ./examples:/opt/examples - ./e2e:/opt/e2e - flame-packages:/opt/flame/packages + - ./work:/opt/flame/work flame-console: image: xflops/flame-console:latest diff --git a/examples/ps/README.md b/examples/ps/README.md index e69de29b..eb438f34 100644 --- a/examples/ps/README.md +++ b/examples/ps/README.md @@ -0,0 +1,166 @@ +# Parameter Server Example + +This example demonstrates distributed training using the **Parameter Server** pattern with Flame's Python SDK. It trains a simple convolutional neural network on the MNIST dataset using synchronous gradient updates. + +## Overview + +The parameter server pattern is a classic distributed training architecture where: +- A **Parameter Server** maintains the global model weights and applies gradient updates +- Multiple **Data Workers** compute gradients on different data batches in parallel +- Workers fetch the latest weights, compute gradients, and send them back to the parameter server + +This example uses Flame's `flamepy.rl.Runner` to orchestrate the distributed services and handle inter-service communication. + +## Architecture + +``` +┌─────────────────────┐ +│ Parameter Server │ - Stores model weights +│ │ - Applies aggregated gradients +└──────────┬──────────┘ + │ + ┌──────┴──────┐ + │ │ +┌───▼────┐ ┌───▼────┐ +│Worker 1│ │Worker 2│ - Fetch weights +│ │ │ │ - Compute gradients on data batches +└────────┘ └────────┘ +``` + +### Components + +1. **ConvNet**: A small convolutional neural network for MNIST digit classification +2. **ParameterServer**: Maintains model state and applies gradient updates using SGD +3. **DataWorker**: Computes gradients on mini-batches from the training dataset +4. **Main Training Loop**: Coordinates synchronous training iterations + +## Files + +- `main.py`: Entry point that sets up the runner and training loop +- `ps.py`: Implementation of the model, parameter server, and data workers +- `pyproject.toml`: Project dependencies and configuration + +## Requirements + +- Python >= 3.12 +- PyTorch and torchvision +- Flame Python SDK (`flamepy`) +- NumPy +- filelock + +## How to Run + +1. **Build the Flame cluster** (if not already running): + ```bash + docker compose build + docker compose up -d + ``` + +2. **Navigate to the example directory**: + ```bash + cd examples/ps + ``` + +3. **Run the example**: + ```bash + python main.py + ``` + +The script will automatically download the MNIST dataset on first run and begin training. + +## Expected Output + +``` +100.0% +100.0% +100.0% +100.0% +Running synchronous parameter server training. +Iter 0: accuracy is 16.5 +Iter 10: accuracy is 32.9 +Final accuracy is 32.9. +``` + +The accuracy should improve from around 10% (random guessing) to ~85% after 20 training iterations. + +## Key Concepts + +### 1. Service Creation with Runner + +```python +with Runner("ps-example") as rr: + ps_svc = rr.service(ParameterServer(1e-2)) + workers_svc = [rr.service(DataWorker) for _ in range(2)] +``` + +The `Runner` creates and manages distributed services. Services can be instantiated from any Python class. + +### 2. Asynchronous Remote Calls + +```python +gradients = [worker.compute_gradients(current_weights) for worker in workers_svc] +current_weights = ps_svc.apply_gradients(*gradients).get() +``` + +Method calls on services return futures. Use `.get()` to block and retrieve the result. + +### 3. Synchronous Training + +The training loop ensures all workers compute gradients before the parameter server applies updates: + +```python +for i in range(20): + # Start all gradient computations in parallel + gradients = [worker.compute_gradients(current_weights) for worker in workers_svc] + # Wait for all gradients and apply update + current_weights = ps_svc.apply_gradients(*gradients).get() +``` + +This is a **synchronous** parameter server where each iteration waits for all workers. + +## Customization + +### Adjust Number of Workers + +Modify the worker count in `main.py`: + +```python +workers_svc = [rr.service(DataWorker) for _ in range(4)] # Use 4 workers +``` + +### Change Learning Rate + +Pass a different learning rate to the ParameterServer: + +```python +ps_svc = rr.service(ParameterServer(1e-3)) # Lower learning rate +``` + +### Increase Training Iterations + +Modify the range in the training loop: + +```python +for i in range(50): # Train for 50 iterations +``` + +## Notes + +- The example uses **filelock** to safely download MNIST data when multiple workers start simultaneously +- Evaluation is limited to 1024 samples for faster iteration during development +- The model is intentionally small to demonstrate the pattern rather than achieve state-of-the-art accuracy + +## Related Examples + +- For reinforcement learning examples, see the `examples/rl/` directory +- For more complex distributed patterns, check other examples in `examples/` + +## Troubleshooting + +If you encounter issues: + +1. **Services not starting**: Ensure the Flame cluster is running with `docker compose ps` +2. **Import errors**: Rebuild containers after modifying `sdk/python`: `docker compose build` +3. **Test timeouts**: Check logs with `docker logs flame-executor-manager` and `docker logs flame-session-manager` + +For more information, see the [Flame documentation](../../docs/). diff --git a/examples/ps/dist/ps-example.tar.gz b/examples/ps/dist/ps-example.tar.gz deleted file mode 100644 index 23ad3ea423d7752c1aa37af1103159f8da09917e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29435 zcmV(^K-Iq=iwFoRfpcjB|8R3HWq4t2aBO8RbYXG;?7eGm9Z7aA*q`+)2we;yd8Eu8 z?+5{9%)oZf7_f~CcL5ieo&pii65SNZ;i2l`M$d1bwes+BjwBwYbgHU7w?&uL))v0lXAg|pJGxX>e=tU{`}AX^DduVOjkeus`sE#duw6W##yVk zwr>2L&1=#2cgiVay;0T#+P_%&-HH5Hvj28=^*`hMkL$~KCs&)3vv*gQ*Ec8saCz~= zzxA6}C*Sv*)0S`Y>G>rC!>g13p0D$}e$%i2(64{s3;&3B|HtL^pZI(KA)mj`H~rai zxp9t9Ui~~>pWj{f-``(c^y_EOo}G=8)6)yQeR_KG%{M2{Pfw|Ur>D=~JUhY9yUW(k zPrkVu#*3HBt2cdfb2|>a_rb_sy!@m3Zu!{N_1VSEi|4%8*~RxKo1ZW0+w04V%lDg; ztGmIUY?jfV++5?f_|)|L<+J6>Kc3yZy&dL@=U1E8{ipoy>byUHd6G9L*Vo(OEqB?R zZhoj3$MyBZi=EZ};se>dQ2n)j`SSiNKQ2@6+BD9)0M1b#?yp&Jh3Z{?|JZ>|F;scXsl>oG78j$-WG> zBmJja^*y)y3(fPj6q|?8yB?34QdHT=9`yf5sEs8On?0GOu=3`{VE3D)c{7 zYJdLAo0I?h=Ke}&o0C7knbi2p^9l3uF5kR3dv$-GmpcNuy~gj)F7o-w$6ufBzrXY4 zuTk-D=V$tg|G(n@d;A}QGGP*{qQtLm;BVpQ>gTKL%m3*0X1ck2cm7ww|Kfev;{U)u z6h5b{3t#d7Z}IcT+v)%A8BsY9$}aeQ1~9uhyS(7bGO0#Lt{-fpk;_urUFY2Mz|%K1Bm`(d4G1^ zUT=Qh-1K+f-QSs)ehwV>{_5uP@_h66-*`UYtCQzHzU}?|`FGE@Un{x(3B%z4A3i)S z@15ew^K;yQuT2l{e)sI^=hpyu@6T`Va<$=g$#kB7clYu4KTMm;_t&-KumPAFq7n|6l$8SO5Q8`+uFZo?NoYgs=YpZ{g=^^J`~+Jn8>~G4|H~YcY<# zAUM4J)&KtnKeq`Qmz!s|zb#9vdw)z97fgbwb{G$ z52kBu`pxDQ^DZtn+;QIAri$Dy)ZTsa_xSr?@b}%VmQ~&5?pM#AJ!^e@oc^)h(0~5? z`5*G==Rcpc%O5ZJ-e$S&bn@^0=Kb}>=04?P{vn^A;Vq62gIYedz1D<_Y|~IK&Nerv z4`V&~W@q5jKU`vLoBrZ{kNuB3Z+?6EZNJ!DVqCiqJN+9s+0W1Zv%h#DlQI%s;eGsS zB&t_0Uv5(;Zny3K+sXgvC%LtIVVu0}*ZrjQn%{5w$-DRGH)mJp{Vw(5mZ_hR-@AA(l+pApu(64{IKD+6glg-&h?YCcKLdw5vGi~tk zi%VSdE!XciknQx4D04Tm7ng7!`R47^f4ag=aN+A0&;Q?f5&W|L{&FYhWSM}6Jf5u< zy}vm--%QIF9`5~OFBRfWd!~Q=w}1Y_w=X_QleoR%^H!&W4RG}9;n#dHUtzgdDPq4$Sl^_nlxx8dnQ-|N_ z*L<#zALt=%=v$f;{=2+@+o3DDxr9T*e7wDr3!h}<%?*6Zhbc+-_dof&lk4+^`}1d3U=1@WaFLjj7^e3If9*_zm4&s8=WE)d`&KML*x#lKJ2}qi( z-MQF(Zr$EMA5s867%JabwqC#6v-zw0ao#6{ZeDyMD-z%Qx66x8GrgLY7hvVfD;?hV z_O-1WU*s??Fy+O|k1n)fu42DvFZjntld*haUpV_u^y(o|@&oVn{j2T6{O8%#i~G;K z`uN#CcsM-J_jiGv&-1aPxqaR{cR`SLx_!FuU*4PPhxx9Tg((EL_xi{VU!9y^e>lIN zPV~Kh`|=8$6n6FyIhR+aKl~s6_~POFdr!){e1jW3$n(QA(d}6N?o%R#zk1i(>0UWE?j*=VABUG^1-Md z6C#G*uU>qlemmFQOC$aqXYlwnZdK(W!}$p<`AiL2C;%J`g7hV|gulM{I5k|p+j*Ri zlba9vppP?sKg#gEx&HahN9gSiNiZVxpKe}!Jl=;3^q*?Kx;gp7&E?fUFK+@dyjdu~ z+kUaYqkeT+fhR5+%^MfK0ZzK;KW<;&M-`tdm}3UMBcV^}>=t6&g`!U(l=I8YW{G?k z=jW&V;}(|OM{=0ttm-)XYxJj%t>FfEwzlops zKP(G1zs>sZr|e%_Z^ho{|8CF!DfOGL|6X5S-W>YeQ}6%(^yeYNS3SG#x68KQoR~ui zy3hXj$18;6{C)rKA#L~}<#s1&*i2d`^ZeO1({;&jz3$(iZGfBj_S>7AtIeBvzWVv< zY`VPue%_oddB-PzdA4cz)bfS**XO&R<$X4{S!-`Efo$9I&D#NOX4Piz=GwzhC5Un% z8dHX;B|24{^Vx)Eoej0;QI(51r($d|K14mV@yXA3kMepu{0Ry&egB`&U!A6t!=shV{L$6-dF{IZKhK%S_GQNgtL?u;<)M3RB=s0|AY!Z#_3RmP}0%f0U zbgxd{UlE_Zra$Iwv}pfY)N4uKs_@3SH{vF%)&E8LEJ?}_Jms^+RNQR*90p`G#Fima zs<*y2SBiBIN^h+w*D*2^vW_K=qKnN=+ca_wK|T}{6ve#00I7VPFMghWe0%<}z-*D$ za0v*_C<)QWG^!TYs~xJUGD35{ju?w?HCgTPx;D}=F5T*$z&z%5g~04&jHSR{Et%k- z(%a4Es@FZOli>a#Rb=07_gV*3`9A) zgz_*Am_=OW;5@iDTD>uDa$f%;iYscaX-=V5XlYm8#9yDvf|g<_s}JS0LUC&e9yXy&)jGOticO_8H>z_*wNQh^hU-VAeep7MFA;__y22ex ztMx%*T2wS~g~Tk6VI9=m|2Vt&kE}oNw`%fVqPCwrbMfbsyIZ}>*ME9{#Z92w0e!ie zkC`(NS{CC(J6okwfO?p2OzlH-7aMg{qtmPQ>V_EIY|hnb4f)R7=YG9B>!Ug@%D}ma zOG%n>L+RkMi=n3sEr*wBIcha3{BTlPM!}?DVoJAWc;9^6nKA2>`lNGquf@C;_pKIq z<~O1z8{#jb<6h5M&8GP%wG8N)h5J=5m#UfzDh4=+QB8Nrmg1r=uvBk*mXzSM@w(p= z(h3ES@WI{UNrz3kNrk4nhIvsYBPv9dqt7FEZ?YS}X$ktChZ0PfZ+iH!IL2~ z#fO=QVw8DgBGPLezE$y!32&?bRQO*+Z(W{)o|T1ltE9V$-niVdmM&R$m7QX6*QSBq zr`ltTW(092hE%mm383UdIlZn|c=FqS-{pF_`RP>2sWPWOMStq;r`5SRqY_3DbgcsX z1(wlT3cxOH0J|jHd`5gQVghVNXW*PHF-4PGBH-9jXXPI?{(B+%H%6xDZh^)Zi7w?i zDkz%m5&=cC8Yw!h5H0{ALmB}g&8Gnbt21WYix1wb5vv}~*bJwzPIT&2PV@D<#{>u8 zqTBE(+tz*VxzRtW6akhMJfsvztsuDeYH|lAiy_KTa2yfd8s1<>aFCI#5S*G)P)7(( z^;y>$`af*2y2C9dZ%hW(w;4JJXem*5JEDS4Sm5&LU5mj-ZL4hGu(iQl1!GOm9qtyKh>D5 zwF5Zas~Dm75eyU55-5hYx>C%Lol&RPXPf%N2O>4;{^d`~G}-EhfGKJ?aFdjUx`I9; zB1UJNRUVAGbr&&WfG-^e$gx7qRJGZreBP0%N`8&*x(SZ@2%&;A=rJ4PORS;zVd3Gx za^T!lf*XUYwOoo)CE%UtoQo}jd&cY|WHywlPYCt>=6owX1^FE`{DQ!zNUgw~a%k|I zk%|dF-h3kUDp}c4QZg14F9X8D2iuHjYHQ4Phkj`j{2CwYCy`@(tkJVIvk!@rSrDXB z)B~|dGG{Xt1Xfsife(YXvOuXhk>wAUN$h*q<#U90Q1htas!2yEBr4So9te0{bu%P5 zFy|a3O8_mj;IjB*0GsIw04+gBKXx_G>9k6CaJxqcFQ&Q6Y!P~(McsjJmrQd7y#rp1 zE_DQ@ZUmyN8V}-5RXNHGTj|0?vctwo-j} zqd}_B!?;#6tBRF6R;BnxMh?b?Ppn?Q2 z@`Ul@{f@`dH8_bGzXMA>5pjMzyFgU@bl8ZoSS#IxtUP3ZfR=3yRhI;S2-fA{&K%Uu zimwF`9GG_0F}T<$D(r*}WdYDv2rEv(A7iCC&#ug+b73Sw%Zjea`kHkxrjJx@%}G_D zVKNgi43$iWF%>tsToSu_T3Dy~U3*NsG$3_|z>Q!?6$-6l9gtRIDCIg0ev2O1tG85f zk>*4)m=t0NuxwE~Yb44kS)*NP3P-uM?&hIpt^BC4$^jP!lm!=@?P$@29t8I25L_X< zQitgQk;5lFN(O0wNX!a>J&v`>jT(qMf(*QvNGHFu7Tzls;3%yD|3`sRw<3y|794Y6 zB%-oG#2UMt8!qZ;fr07ycuWvAkrtUGl!PD(Kc=JU9&r+PN)8b~nI_^YAQW5&%tZw^ zQt3^H3UCm{__V<8{$0)QI*23)9?%6aG?yxHE90vL13^5fjuAp>8^m{fG-iM(qcJRC zstmH}_V}G%fj_L8LI}t3yE?Zt!_1p0WeUhi^$B_L8aw4yXW~h$(J(uR*cA zRfHJ`)C>g(ILP6^_uw7C@8F9NjziHy2u*d!2;JTcQ)?yQ_60_`$M52bKyBRwO*=x) zVa!3zJPPFnpq_+&s91bNq*kr~6jm^*;vCq3>b}Frji4QzZ7Qlnkgb(+{&IrnK-jZZ z6^d9w4#W@(?;NgI*nO#5!0|KaeTayW+;@SlxCA1TxHzy!@X{Kg0%ogYV0P8Bjx$$9 zs~!PhIybQ$@wEgRQ3)$P`cD&U8qs_Qu-4@T~F|`#U(;oVmI_rQh-@= zNfc&3=vx7|4e+$djr(@qDnNi-btk3IL+uf~xW@8qeh7XVXKekT&WnBjSa3h1lvJp zJOI!hzf)@jrYC2P(XJu0$@2iqgkuPG^r1~13BvSFRuAY!IPvZ@7)e)i?c+~5L~iVBTiTi5(sa%NATQ=DB$cA z)iK-{F(7qF5H1C**G3g}?OFkHw4OotvMI$fr%9@S!bfYoj4t3QCMHlaQ&!6E35I77 zaY_I^>aGHTSf{ga*Q1Bxa>K0l9IGo`HLzuFi%G!;1_oO+J{R^Fo?o%5;M@fNeM|sF z1n@9Z836J*D5##dfR++_&Y@Uf&ZeOOUV%4rwKqsE5vXJ&piZz6c~Ww}62*h^F&n7C z?Pc9_gI#lH2EaK4=@x!4Ks&oZeDKR|Jj@(4C3R&}F1u_RS2HWvwBus*F%&QPxmU0} z@aD)Ao7CX(Dm+pF^mEM(m_DE#f+C{E7#YaE<h>scq&mhsp6<}0&1Y^NOQ=~NxaS@)(=(U3Wa+1gGTAcym0kspzO7_f? zio}8*)L18wPQ1;wh=FXL$+Ot8jB*gwfj=ntLif0@W7> zWhJjYEa@H8J*BIMb>dzrBF+q1l8)g8Te03TpUDjd zf(9I}hX>#WXJ;x5*r!f?cP~nZr!6`#-z*##;Nx7~T*cHd5ak`jH!3#~H6l7J8ByMs z1Z|9P{2_}E70OV3g77f;aIK#uy@T!$7FntRD#UFu5D#uLe1Px9RH`ma8SvztojX6R z#1vZr7VpCqkkQ?^gU{&|!c&tE#~3bRGavD({06Yx@ zOXa`_f!sjv0sKnU<_X~)V0vJ5Qi9;#BEciE%4GgWBm6QSx3nXBk0Pquc zsh$>DAu%wQqZVPLGxM;UTA?_D12)R{)Ik@(>VgZ_r93eI$uaRO2mFJ-fNy}vO;Bd6 zk=Wy?J2zbi!VC{eAQSc}+v9p+Mb@U+ zDe0s3^2{9b9J|#Z@dgS677|+P%q~_KA%dE(CNmovPh?@JVd18uG!J(Mn(=9g+1r2D z@8!|&rv%s;R0|pkj+?@vS;$HeFO%OD!^Qigq#0n=j))36Q-Xlh@8@c+i3EN!_Q(}K zZ|C4a;9#K{4Bu4PBy=qc@QBF{c&hdeYE@d)MJZ_7C^|H2!2o>i>qPei)2lExkYvTw zUzn-}^K5nq6UDoX1g+yVK^2pZ0?h+WVIq71Q0XGtf975u1Q#ZK6ew6+R2sx2Y}K)Z@u zHWY{+-rLnm&lJUBn8798z37;Y4Z&lMTMXfIMEy(gJb>ox6Z`@Av$s`TP62`+mI*jE zdZ$}~yfaKpk}z!+(8~Hy5~#^YVQ&Td=^T>qMhODXacdV^X6`v6Fy?5Thk>VPLItYe zhIz2w=q#XFFjKIs5o$Kwa{srs6#H5+e>uzRAaE5hs@9P~lL15lfOKqiKvS7koU5rA zu;%V*6uNsZUTDA|htKTgzlvRlNPAao-axn0a$N))jIQE<^%ScwHb!1<(~d_boi| z4}Ql)$H3LBU4@KVtq`7`>@f-%8Y8=P@bVo(!+7y4>;`-R?4F07Qcx;#hZIgNI_G(a z2wzf<{`931AnX7ao-Y zok4V!&k^3i^>08@hyY@NW3gfA!t+^hY7HP#lv?bD7$M=JEic#ZnA;`vk5DN zJX z>JHd`;68qIsBRe%(PvQMc?{BYuwuxR`*=Ml*DbEbiqmSEg7og$pJiu>9fU4PhD4nMMMT?h8 zBpL4kNudWAJfMf+*-Raul-#eCtXZ5cJap3#S4#q(^bj~tx`7ozH^J_^X8i+Z ziz_gYlt7^X{dl&-8@soY7gnrzX_-LGj@b%F=!jKCTfs({}`t6kb|i9>xofOI!pe3V>R zQYArcvhUv8;bd6uII z)``(6nLW6u*Em_NC*_ZksvC$-vnp_7h?zV;08lc3TtkEaiwDpJ-BKnvr@%&y>!Vl0 zJSar~5(nsMT?YWs}%kXefTp~%D>wc_?qHJR+lEA7uS-9R1B0?^6vWji00QLbC z#m|!7Vci2Hiv>Z5Cr?|_Oce$d0n(6RRE9^=3spvJf?>pL2gn=`Hk#T61YvI{&#VAP z+D`hYyf9_w8fI<+7$wR{DuBF!E8eYY03=v#b^+we7n`|JADU6o1>o@Bm(nDkCA~xL z(6Q{4Qne%!Czje6;Ks3ZF@Ty{at2<6+iNNeVV+Q7_G{?$hXJvFZznIT*qhRB(np=Q zAB8nsTxsCMvEms9B6?HJ_`X!BFs3{1VRZU-QwjzPw47ScC0O4}ek$%9AbPB{1-g_C z&zVcA9Y$I~Q2-}v14#2K)&+xsDKl?5u`~+EfCWs7Fx*`n9@lIYXg}$rRuoj`E{j;< zk$#o+P#$cg*oZh>F!qeePzJ6}q&hf28Q{%Y;E1RY)!O*9v<^`{J2H$;Z0Hgy%pOzL zMTHL9!WC4DaM4&~21vF=-$8C%3}vK{x(li9Qe$WJiZvJ=CUaCCe-JLJ0%XAyz0$;C z05(wl#4YxwhP6!j0HxvCUe*{;4TPbBpovLf4R(#h4&={)im2vu95qGm6=PapaWPcf zAOe#(@g9N5Fq2Ch^w~x5rNqq&pqjj^W6G|PSe!sr4;R9@dY5nB?iImNllhGcl<^RA zn88K#$SlnUoS~F8(J+}jC2xDLMY8~2gXiRCQ?0B`0$AjVFc)~3g*~l$kSP3UhI;WxlPq`Xn(>oEw8LvK#sQ1Ms zX;SI1!d~tg`(Y7J%sZeRw$BJeiPbb$?* z=tF~vMU2=~7gQ)EI{L!l3lurq59G8$ZyU6fN9A>gwQ!>p#_XKS(lk>cP18Bs05Me~ z9qeXsS;40Q^gT@QyN6YTBD5XQLV zfius&^WcrjT|7KWY9KqUK&Zp@$OI~Hfdct@*1Ppgz4I}~r$nfEi2u0Ek4dDB;uX)u_F`>_ z5reZt_1qKo+=@V?bk=8pJ6Kg+s=1}ON@@qUQdz00Vg^FW!i?C&8g@oZ$qhGYrf1EZ zHmA8$-RrU?&Em}<8-lR+AU4nH11_x)8w1TT3+W*dMml1Y!LxgK4i|y~bb*mFu{aUy zEhT4L8L(WC--ywL76j0vfuTPxww<-wFIy}ZVdWT>Do$P+m_CaDdkg%)cmj@Ow1MaI z2=K6D!_BZh$Z8`1lGxo!FBad8t3(R4b5v4EOU#W(HRcc%;u!dKU>PBY$d_s+yHf?u zXBz_Hky2*`M5t_Jxt9(#UMbSgUlb3y!C?%LsajgIqPa+$cwz`5^hK#aALzh#{9sWI z5X;ya!cH31Bi!u95fi{WR|wSMIpmlG((LD^=LmumdD_i69)ksHuEQ#?E2Es(84tvR zZ<@ZHspxQAfRPaS2;p&ou3x*=iN%8p;t9B(RAv-__JAT9$6}cP^gIM2xq&4%pxEYA zGc2@Wb=J!7iB3hp7_~xlcJk?%6K`JTR%QS%_#E$9sVu;5a%=;m=24nHobT1bSPTp7 z0*^;#Avxn9**sKPDY}DoGq_L4wC-lOknGd&3eq8$=k(-|Eh{Nd6%9C13DQ9%2ZA>< z1;*~uP}B)TU$2mypVCoL*r?goS=m|;Ek8QJ# zqYbWQn8%}%J9Jjuv;0$1O82x)m?CR5%Jabz+&VV{(_%7X44ui?#5dZ)v*~ioqvzlyEgYl&9<-&7;KA0zo zg;@@$=wrfzPduXn1RpVMx(fh13;)`gXay9-P7b40&E;8m>jI9SMl38dTzR%!#lzxy z9M%ayVq^kA=h<|gUyK?s4~na`;gY&I%RtK{=?0+LTVuim^EiBtduSK|gAQedxXh8~ zc?2941dN5*3*xD&A|XKALC-ZEYe#&-%o}+5j`)>9zlsMcwb}MEq!0U%xIViGSPLm2 z=5(M&Zn`$l`z%Rc+!@x2v-91)Z+}Rj^WXjMP+^O`09R3R@?@ELduy zFH>A8#2}bsuazyGi#rY&U6JHKpop2p7Pm@j2RAqY!ZOcjgLA2nfnB(*iFlNV3{7Pg zKad+H3qi*K{2I)hg~I~@1V`(8?284_SBT9`{-{ElvCi4fAt1;P#HG&cRSVi^K(Ro@ zKw@B@1Hbx?nC4XI2B^%&uG#`YbUq-qquE!LkxRBPM#bdr!D>GVl1q(80tG$BPcJLZ7JjnWgQ#Xrg7$*gs%a z+~KSH?2CK)Pm4~gDIS%n4p>~`EJ;c%;N2za9IY)y>oMYz{$X4#3jDi(W_wVIl)F+M z3Q~Ed$C{O*J4pF~dpl$Vjs=g`iVedIKVZ8&<-~Gzkvl;#6^sd#AC9pM8)9dr`ntDt z49sdxTmh1`nbJ|UIDJ@_-L)<}En+|!AToik1|AJe63;|4ImKZ?Rhjb%U2X@vPh;`%f4U3pr4@&A_ znJ&PR4iyR-o}b6Wg{#q#O_j*!od%^bB_W2flvgp_PBWti2Uy5+jC=4-f?mVEbeaH8 zk69)Lov1TB6c4wTQMlVUSno-3Ux{_uf-;^tf`#?Q8uQ#>wqT5zU61fshsE_ctkdA0 z-m>N!8$VS@s2V>=RfOlySi>WD=MZ6iwiV#KSnVV_*AVREUNogrD!sz_8#RTaPP1z7GbbtMx)fHc3O8070STTBBb80R!^Ym1xAl=|t z7$295Lg8sm+YEaGITkTxSv|mDLRP?M_@7i`3+-vSJwYSj(%Z5|nQSCmNij2%rMS|& ztvu$MSf(9<^&NK{fOvVhs9Wc$lHJpoKL>u5sd{TgM8psN`s_S)PI}|VBMd7M&QvopvgRLVa7C@O%1>w7y z2)x$Vl?-&bD;{YD#rKG(x+aj09UkGyhU^Nt9a~k^3F~bo?x+HS1E8}aOlXmn?AUp3 z1lBiz0ke#l#oGzd9kU8;V0~Sc#v!>KO(YPuD?k`mGTTCHaYK`X8Vza*Qq5WxI)#)m zJ~6fz9ujYc_03A$UjVk&Ynb;6nWhPV|Cog*GdBf>Haun3d#@d z9H~kGC^CAmMxf-twH6Ymuv(^{&m^2qfFh1HQF#1cvahPL$h1{TqVYyx`vno#L!>|^ zfwd-+MouozJep_)Q6TuprweyK*v&m zib0FGdy=v^KrTcSXATJTGxl)|_yGWkwh$RY7*G*US_|{GSj!@(0P@NLz}1)-1V~^E zZ@9SZDDz#>F$-Umc4K5IvlMZJHtOBT@C-8?{x%H+4D2y#kJ%$Gt3rhhJ z>w8qf0LR4@zR~*0AJNW1;u1iO+y(?Y9yW<)8X!C0vW%R|4Ma-AmvxKKn%QZ?T5kU$l&HXR{X6Hp zyhNBqTp_A>WNSd>`ClGZ8xx8%AgkRo$P)PiMKC=N?v5oyE9}@p-BR2xTJ_}c{$Wua zA`>3)E?Z`@n#?R~1XL~2iUXl=JfqM0kUY%Mt4GLXUG6Ny!*8en`YL-Uk7uh_$jix* zXJ5f2Mx0>;uJ9z3j~E=YzzY7&g*o*OWT${_f!L3x0+Yxv|Fs&HafpYk@~JXVA#$6P z-C32&9sHqrO0Za-<;2yYN?uqjsx<{&)193vN2}S#roGx@69~R>#o8RpB!{EUC={04 zfQP~m*!iKebgAu_Uf?N+%BiMh^d+O&GwXomEe!jW7=U}%{l_rn6GfohK_6z=3MH%F zLG^i#1bT)D(BaBaCsdN z7cRA%nib3EbeC%ooeUc#aQh)PmAJ(N5E!^aJbDs%{>EckPYUh{7S@@V3sM&vI3IKh z7+t#QI>7wcZJ@|t0SQ5X0wZI6FZTPgbX`8BYW6effI`>UXE?SMpP7T51J40FJ46r} zmGlNPC?&Y0>aZ{kBS8R71z0n)@QiXGWrjKRK0PV6CrDT~1t!Q!LY9Zht(FFC7?kyu z)wu)&)mQ~q%OY_q`_e_jqdE_d%HwQ%nREbmD|WzKn&h}*=3y>j7Kg~E*aaT33SkPt zz4z>DiK8V2v3mAp38T@u{FGqW8o=zARh2VM2Qc0D$+F5tDtX__fEqgWIc#O6~ zf*IrO6@ynjbe6=*9i+QAcDcw9a%~61b~FWZ@UIc3r&ZwE3F~8KK-ZE0#u?k`3|X)T z4llXnO=J0q+5+JU*IV}}n1X4n5h**%AESxAP^t!@3!Da%4g5PmZLPphThGje7`tLy z0&YKKsEGwjNMYFA*uSRawIcm|2BvvDtP7KLK)VpPU<<6G;vvAHSyyzOLML$Nfg2(_ivU(YR_JNOy}S!1&kyzHNa)yjxG^2FzY?)8Z~TK zR_L6P39;+Ag1jluOuQ=AgK@!_p=%0cm4`${8wUu%hMSqmO&}DAEIR4L4@dS6csvLn ztke`p6f_)I>)e{CZ1AHkuzKGX{A0a;cobC;)H3S=Cm3S&vJ_w2UBwf8R_hg#GDj9Q z34Jc`c-|?vJT4i-Q!Az{Xh66sjV%^xj~31broh^~E;A9;fiJ}lP^At@>Ja;4(V+m; z(*`pzK?vaZ$%Hj4J^LyGZDxK+fgcVA{sRx1xl}xsQ_yXXeVOReiW54@O>$Jb+n$!~ z37A-O+^>_RpaZ1CLJk-k0-UR0V_{kDiPbu=Lf9q%Lt`0@91n}@Kt0r+YFAd}WkJZW zK2E7HJB2YQ_GNY4G(!A=k+HqEtxYe40*IjLwwu@_2(~J&5m%VZQO&+vU;1BUW+{Oi zEXC^$0UnDJjHJ|3-Ea&>e|TbUB=Q(jV=^l14X5&$xK?lFAZ!~&AO}JmGPC&s7aI^7 zY_Pyyb!r~Qs+fijIt}#)jj?RdlqK(Fw{j~dhG197Elu{Q`XL!QRzO-)1I8G@aez_Y z0`CIGgGe(CCVMu(0`WGL)kCuJ%7}t?L+^5TPs{BI7LkJyxX<9&V$ML5Sad*xX+_0s zP2h1A!To_bNWWO+Jhhms@i8DuW!c-x;o9NTV$)_)M|E&5jVaX4@T^<}rBALJh^WLm z!r(DB;XZNdh(d$u=ei{=tiB76%qm{FuM^ur8c|qBy|U!9ZDK*QKo5jFmX2GAjyXa+ zBo{@%9jf!)_nb7(~aLDr+_b z7$}z(gYi1603yjHC5SWx)0#dcx1)K4WfDm!%t->+7#Nr!SX9;F0$JAzbjHD?A$M8& zv3s`fgpM**vG-*5w{ln-ZiP(!6po4%O1SP+nK6fe#7tV`v9E1Mo~l9jj&fC0jYg4X@a~TASjS5vF5IJZsk=((D>b$3Wtt!}X{fc|xGZ zR1MsAwjXTeK=Ny>n@M5Eq*07GXERIPo_H_Lu&^qSx(dLeVcCRGjtd(WEXyVY>04&49;kiU;d*w1DNR7q}H*C*wzhOgfhk5cLDyCJ<*}3H6*s z?eFCT`*co)U2-ap+RO2UZ5JFXc?c^dgH9BTatvE_5=2CdE+mcDpnM>UmbLvufQFI> zso-U;><$*g4+Znqbu741Kx_fmO0hfW+pTKc!ajd*!J3t-cUs{e^q!NcOdwV%d zj9OvH#!g-xRUysHO3xrVW9fhb$z)*;u?s534wPzG+Do{(0vHX^IUi~+OO05bmv!oK z(H$TYuEEWLkO0Z`(v@SmuaU- zX23NBQsAFvlN$Ij!sKC=u&*_-s#NHq`+`Zb(D@o$W!oL^P1)Nf)ShLF!U}n@x9U+X zsT=eDd(EaLuOW%fRh%I#^a_t9c%N4P4&7a1zM(1=81@P{d^b`c%O5kbrUXOY9 zl?-bNWpCI!tXQ_W_oA5|S90Q%?S^rOdAB}2TEl%QkR$3g7HPL~790P$x;cAfCX@U% zCK5P8^sIrra#k$=9LrKWp7m>tO9+6v14fv}&bcftp9edFI<6qq43C>1CbID#JmAh` zCi%o%K3!$>k?t|Vjhqf7rkV9#z=&CgqjA2Op3Xf=U&+aIgC4?4U6GnvVaw>a>($uT z*FQbK!VL8LJ}4{Z_u_}^oBG)Nn!m>UiX+@H?2{h2c{nu8AtAaX{KNk3hE1e}8MlrI ztyLMN0lu}MMsP6Qz4=`paCd&qr{`BbIlqSiQ4WD8<6?BW4*>8Gh$gFNzam00#CQu+ zh0_7oQqTav??{~6&GIx4^BW_$=Hq>C?&nu?$9WDQvCFnwBzpEUa41Z3nR#M&1VC=8{XfH^aA&_$%2k@E8JitEB;*4GT0I zw6M3dG>_2Pj7fs1O3w|v-4H%(%xu8XEPFuiETkl7gLh9`Fpmo>w{nIrExG%1-h z+`yMiZtGY5qV9!2%bd^I~+P(C~+SU zs)uK7J~y6d!v`9G>W)%eTx_-BUJ_pvpA##pXuu{HMrJL9!0t-I0s~?+gN;+nOabvS zp=p+`1rJlq_yr)_J$n_z2N$h>tbGH3mj70(H*VQ=a!(boyPJbCZa!hyEY4@9Xnwuf zzxd%l&Tb&YKmC#lpvJ7s+`H`}!2aC_V14#uaX`7kT9;+%y_Hy$a5x~}25@FN{0J93 z*lGD*0U(%LcW@_+U(O*|%yiSLS1FOHzQFVvw8R3=8R~<>TGX?A;@Mk=y2? zsA0E33$g@Q!M$8eQe|Z=&(&qNN#HV&2OD6C=%RU&3K%59?d+Bn_AEM@f`4innIS%z zwbQt{Jje8X(s2hqxBPZt`;iZAuJZM!Kk>0&B?v?}+c>AJOw$P=Q%r}C)ggrDdUK(0 z-U`e9)f$rH>e)cJOkf0xeSvm^z*f%pabPRkFVZM$f@jWb(;h5Rmj>^UK{%^87FqNX z0yL@`qC?<=M7I^}NTWdA>|S38wv>U7dD7)CeTj5ft5Q1?_5sLuUrglwiddlPc{qT< z2(#r0u2NY{TdK9ecz9&zq*fzb056g2U8Q3pxIhmaR`8*=qXvs??%9DmzyZG~W&jAd z2CTw%ss(5OZY3gk^bCy{%#1_uu(ih?xcrBq0;^iX;v1zXv)d~?BE);^pL92oBrt^Z zIl6KF38aY7^VeC6bDl*6m1gf;)}*pL6v`a8!qgkb78_oQYKjfVBtvK;c}@7&5l&eC z;FVodb0v%tJh5$S<78vo+1MM~wsB(Hwryu)+qP}vet+SQ}}Sp=sm_z`eFqb(T)u|3K%e3 zKDK)4bYGJ>vyb&hx!)h|`=2DbxH%iC@jj7JvbP1Eucsbq# zfd*4 ztv}SIx%}HO^({V$L3`%}=J%8w5gXgoGJF^UFelKY&2^QFsrLukuWzXA1rG}%H@WwkRYDdrYPg|xa&X4r+2n$b#fC*u!A%{ovwt8?NnqpGBp zAoa{Wp>Y$?5bIz4s(IO}xotj@RGXxhaWy6XQ>>=kXgcrpSi}x(sQRKt>f{HmOIv2S zYRuFEqB(^~a;X!G1Ti75DD==ZGtjjC`AAz`(9nZP%U`Kk3{l!JfV~@42~{#(MO>RQ za5UK;N*kW`*9s)hlLmUA(}#uhc#OYWW_s2VQR{qZ?d|O-mPoGfJ(?$#bn0ht&S>!mj1AU&emszo;KdF?zn(#Rp|M+-xo{6~c8Os}Y@ zCxx7=j+=%10u>RQ;yAM*#{-8jBF6q2lxIjBe(e#&4hUr`Kn6f7HP(~UQ&SOOiELlj zlx~8MYhZYKNH?%S|2&Pu4m61*eKwGzdpSXn??!~Xtu7mHi%K*{}ZttqMU(YwENmP5kn~ea&a|1$Mzl-%dI2?tyD` zn5Ik*@=xm%vUae_1&yX^i~Q7MB7_e^La5F(`~BJpM^hQXEqMmw;tIC?Y^=;#+cOjr zmqz*^2z1m)XpNiEMt7W>V|H?0-9t7cmDLankVk4ghiaX&JTBDkBi=YGZ}&S^E!}oh zwh7hWLMbsxS-302++e498F33rDG{{2m;$Tgu*6AV(o zisK?qT9(A3R@=Y_Mc5Hn@_Ad|yU#y5I4%@~2NqTy6@H2kajY8i>|~@2GzG_f6FkI{ zX(IcKs64{J;3$Et%ie%pe2FPKPCaesm2e^qUws>d5CvY~oek6_tU)5VW6EjfL7xFa zE80$a#`YXs6tf*e15f@KCv2H`H8A-M-6;xYoV$YQcQwjs>d7}`CwiZ-Gv043%3)I| z8LTTMr$$_Nxns~wXcVpD;`&;eOlV?M(2$C)X7z~&K3Zek7&^IXbR>B&W|-u-X0gG# z9eC?72?=Eo>}XRxU1!~og2i-l!T zKC;yyJEqk|hv{BFjJ-l^%pWlVDr-08c^c7i3{=_OrV)3gq4PGD%MddOZlt(?crtw{ zNSGCkM#^MWp^IaQaDtFwmM%p>WvMZ_)MJ12!;!661_+dkE2Qpo*u*m;i$`cL?f$1kyL zxPaC_5o*U8yBQOzy^%UVL8WG#AdBWx#9APjcrkbg)t@8qJL6)Jsw!|U83}0cC$llk z(k$wIbOmHk1Cz!CP_3Og?xVfkFW9m4v36%|h#=fPH-rfw%fSp#(s{0GI89q$0J#@` zIRb!Cuf!ckK$}euwOS4cy?hlsziFu?Yz=i$WOrvgY2th8?Qo-x(|Z{eF+nEOuY8tq zuVXp}hNTVE%q%d0YmihLRl=V1qP8)@rHMczE@}e4v?Hq+Mx9MeYdks+FGU6O@jX0$ zHClQ|Aj5Ovr!3`f?qCk$OeE~{0U&8|fud;2UFzyx+%SAN?BNodO0xnt%uaOUN|mtS zjw^*YMkdj=!DAtX6{T!ymL_^ox2HX$1tr(8u{tHQ_>AU5H;+lBrZlHl0macvH=O>?Xkt&1Tjm>LdVWgZ}85LRm_|2-=ZiR2K%V0|;v+ ze`z_Pk^JzNVvE^a*_ILO7LThA5~kV~)o^dE#sfo~ulvt+s}17`Q&*w;k){F|xPo3` zJhoKlcsn=aTDMT+78+^c3-oLWg??=s^?D!1ltv*|%?XwE#YBh@EA>vpR3Y;FZ_U}T zTjSwhDh1U8<*%7h?$$LPnjWJvvR+0!m?1<#W9M0Tq3{?uA@D)P4nQ@aR1Lqo&~80K zsUlY`S93?&#~TxOA#-WFr`G;ce|S5-f3F{A2WH+DVCd`M*`a=@uMu0<7kSgRks;SN zAWTjpa_V7QsfH2J$CUM7Yl5~_!&n8GQ#dW=PYzd{HA+}x>Hit@NXs`Z-J}3O)NKXK zBC+>JY3HJQfn$jNKra*iM_~}0ge9>JJmsc-FEh4SSr&#mmfR`>)dV+%SNpRWXwb1r z@ZnDZ^;g;d5-83r{rYFepA51~N2^&lg4fDjCa=!uw~=Y%)TzfX#BVW@u60Ia2{5Bj z!wGVPSxCdpK9OMBam)r|a>dtTFeUA7_{0JbRWSHtl3z0FTC?qx=rIJtATG+Vz9J9! zaJDndZe(?*`Jk#BSNr`Y*zQ>v2@i<^6~AjL8tYjAp-b4}4!o$~&gkl}RZL&Gy>cjM zt8q)_fbnU|0c9!Zequ$>;KLmC5R0LPP{de3qgDB_Fr4j4F>qqv?^Hn(61LZytCJpp zSc9T6e;DoJhcjFSLqI{OQH|@$aZA)0M)7Zw2dyTu&mAgD4&I)czfr~o79)yeU}7=C ze#7yXRjs3f?a1!|cm0V)WPI%jdKe@nTI7E0HWvaOZ)ag-CZUR!l5;N*|aF% zhYv>P-`fa@h$=!B;Az?>sWz`L5EWl-rNq)NEO;1n_1E<|(1;Oiu9U;%Q;1&YAAI7f zdLD=j*`OmVEf=E7bBqZ2lr%T;#)-I^pE+HP}YqF7C zEp1(QdZn#(HX@YW>QV>yb#h}~i0Y)JjdfnCxHm8CgH_zhLGxrIti(x36?|xgH(U9{UMdr{%fhbJh2d%6iok+G)B#~OB#=%ffE?wlLA1X331>{mb0<9@ni1y zlR3VmHUtUcEopTomAf{Tu3ATbeK{}gKG!SZ=)(4Tqd{;UM#Fp}e+4z>g?mu?)R$&M z)zU5vPsA>g=N4y20Ieu8J3(Dl98|x!JHI#?JKDzkKm&IZV&F+=O~857WFj0Q=QziF z?(lM1@Ku^r5XFE&uLHm(s;tO4H-Fknl602IDR#Ym%J9tD?`(w3{D{eRwMA#9kUorCBZmc zc9l8%D|&SgbTzhgof!}D4YY>c^&;4TV7Nx(#L3rDVyu#rut`ct3V4-FXaB|CfMIH6 zgXaiRh#@R`!VZd;p@JMZq_<++Cy7`!uvv9t*FRiJFg?amEnQafQd}cNp|WgkQZ%oW z?@cwOyAqJcZFo30)2F21^%>nwqZao2zmrsoEU8soEYq-jVD}t_FyzKj&c?o27NKBe zd%u=5B~FYfm8XFkB3LAzN%U#Xxg1jBBv*q5=Y@4zuhJa5Gf?t24%Yi*b@kU8coNRq z5AyH?Yw$35&lWjYWSz-CBDkDUO$;4veriG%;wgQp@!mT@WhteCY~U-R419h$_)KOB zT5=PDXoT=5Oez+#84}#}X|%JsPlbVfCk$er|8eVh;7EOdTv^}tvag2MU53PzdN1U+ z8x9%q^>6$+{*wO&&`Bosu?xBU`L?wg7G_HAXT92%N54mQy9GtE@uGL5=g~W{NthFjoohIQO28G;WqQDv{vLSm z6x;hZr~H*%l#^J(X$^sUuQ(}N^J(&DqQ1^C$e>$c)F9v3Z62nh$?l)EZkjxF=FNu% z;#Putit2sLaYyzA8atBlk=#bw22Jo4_me?{jq22gOGA$B`j_@SFo_@t=5A?%Zqp$+ zsEsayv3?x9=R2pT$3OivySeZ=9xgAvl`& zwe;dmua^GJaFJUq@$qcwG!~;Em%&9Qv$80u?vUQ_g?90ly8UHJ18+7ixP6ovp!C*= z|B|K(<3g^Yzm#?^J46X$oufM<6zUcTW!bd2u6C~@eri^56=5!u&)^-0Q)4cRHcKX$ zZBlNkrsy@t;`#FVynE9vbQO~v6ZvG9NSYV=Z*8L@>b!fEo@p}XL|%`x2vjeYH7fs| zR4}kv8R^tvn8qnH&`6>qzYoy2jBP12cgpWQi6do$cksefw5R|gZUAytqHxO6>3IU7 zf@mupD?*sxeGWw&|z+H8n$wz7#yu(&KwDAvVmgWM^czrSHXI41_>)5 zY-qD?&|$j0?R8Kd>g$bk5P*%2jxVI0huh+d^B%Q_sf{kQ&3)N@+J!DVakovcrw)!S ziE^svp`3_RWPR9&!e9FjFG%(&RO-iJs{o_Iu#^buz--`TL5VYUQe4KF+3@E$pZKew zF(bxGESdLp*9mtc@s`=%5dmS~Aj^Z(iZY9%3u2?5Q*fOq^S#a>B)MDq4cPGAaDCh( z2k^IqSFYtArWLu34TmMrvlUc&9+uUpu`s%-$nq7by$5f5u78u6r8%a zJfjWRGsMJT@$;J2;=Hk7cdPOfV0q{&EJVoQDu?R2DiOUGnR`}3Jk8mbO&cvsx84ND z&$GSbDQmSESUbif!DOjc2;74b<$3`sfd!GR9E)PVC1&CiteFwAHb>#pTHa7OS+G8c}3BlizTNsO`>TS z?}5T2t&wfq2la;8irXItSSKi8J-UnVy<(hBkI60BT_i5{w6ZGh#uwDH_IoX8m6HF8 z^U-PPM^}_3#&0<=QLRe9?c-$HWS$jgg1e|HHd#&H8WS-kAI?U~(6q#lnF`dEz0kt% zCtXYt_XI|{!enXJByet4(6+vBZhrOX-_JYMdrm$%cFeY@rd;W|BjJJr{6&jOUV_kR z#<|uPx?!#sgK3Ps%XYwE!2tCfvYqAqp2p8AB#O+j4U@av|rk`FI_$W%M zwm4`F*Q4^QI&3re3A@kPss<)#xd2S&5n<+QDm=#sd9Gvws;Iu)e+8kBm6CSg+ke0X+E?0?vf?l6+!mJv!4O-S~E8TAdU{ zwy7LfRZC-p^yu1V*(L0&V|0_g!zscON=PM4X3kXe&Nl<03ZG$x6mCytUcN4pe&hb* zsD(lNL-Z9pIE=IyaaU<=UVumKb~TY4PFWnb+fgv#6ZosjkGporP~#rbJix40LIFMR z!m0UBw4u0kkVO{4FoxDep1+vz9|J76%p?ID(ATJUXi_^k`<67eyUjZ6fneAoWKU9) zNoY-nH&n2xzZ!YvyaoD4Y zxrU?lnzPG{7h(u(j@JmTSd-rSYAu+gf1HW3eAL8^_fJwlE$X;#1iUK_jlFGhxBfV3 zTes6O8i_g%`t5mFO<$KX@b@*jJW%O%BN0-f=b-JD(KOkatDLhp`^N-m(9#l9Tjf^*K)66GFu;p*5@zT-cO0sP6?AAYB_?*3GTXNmTyt` z8d{-B-FLCYp@`SF2NqLB4=qq4DIgpMwZjBKo~HWX`WBV(M==LqSZvy|hT>r-MHm|@4@P!7l3{IpV2_57-BKCh!#4Q4eBZ5XXgB z8`#BrUbbQ6K@pm0SP9KzEhu&g*-NwXE`+S@1I~-I*okFYz|j+y67Dlbb@J>ksno7w zrW=K*1vR>oHZ7*~$-Ly^)r9_0vuy=15NB3_qeO97DtlS+Y*puV_gy8}YG@%E2_SO{ zaqq!$DkzhXXN(z9v24Wj>6|7IG}sobsSDoM<6HG8(2-{&ZK{2zgqTd>aJlV4dS(Cx zFC`wzAQ03W=*{RXaIPa-K=4x$0J>*0{$$UqxOlm8C-!MtkTSvk}Qnw z9!b4sqP$XKE+9eD@yE21V#(rJq1`3GK>ivH1B4FgOwCCfNP_J^HByjf&C!@arkncE zfCPpBp~$Qu*$GaqIBZ&^UAZE^Y{@$ZRZ&0g;KIh=z|P%a#rPD(sK|i(=?a`O3sM9n zCNV?827Gvke_4HntH1-YV4|y1vclRw>8$wiHYdH7n{Eg;{|*GnHRa}6ab z0GY{x7u)e(kTl$Q^Ioq?2G^L%DSz39!Dnz9?FB@cxM`&nnxM6s^-~a?%Us_X$f|m^ z5e4_lsJT0{E+clw5arv0!<|8`pD&em4c3GoErS|R=~-2XaZemRgx``4&h!GpbJvEL zsX&S^0^Qh!K($`Z#2xwz$N2+1k_NjI`OA}nm~f~uc0}ZXN$&Ofa{Ba$gQ(@Egjb~- z7)n%OI;$ifwp!h;9Vc9ngdx?NZ`5gw34o0d`G5*J>Z45>$-focV2pQyAQn?jZbRw+ zAu*8y!B3E5AxwIXEB7$Y!v^bLM+Uw7_&iK^CA~N2R8xZ$RCw(HTJzVm1aocRhOMOP z4ksK}D@#qU3W2#smu?Do{fQWm*rTwJ%L*dJ!t#j-#Ww~Dlc>b{aU@bBBXbPk7x=MG z^~ai!Zl>l9E3|wS;$dw2i}|$Bg^pL~e~=!jxn!nK+s$1#I|OsJwNdK2=7c=h zVNWd$C%WtXvt1@MEsSaNM0%RzpBOe`;z-F>fGFZ7On(7(;Vu6mG1iJ4%V5_P{ptxN z!5dRhwJf6S!U(A z40pLQ>B*r&!oP9L13qVxPVm0PGMugY)4%SLO-e@rAcqd6pXM_PXENUE?C=<8u zuZSVnJ;}ZM=4j{c$@)}~Tdj3b1Jeag*s{z`Sl3NdDQI+9@94>Xw;i zmpT*zj?WdF3@^~tK|Vp;E;H7B-zMd^{iYL*x};ONlq4DuA)FE6Z3frP1oIms9F!$4 z{gcWE>a44eT7uj*xQ2&*bCa<_K#jCAjjbISEGdSQGz55 zrLzP8nd2NUE7e=*h_$6Vme2_zu8A$e^3c0`c`%!pXKnVXUIJ&zY;NJX=V8Sjth zCv!LN1BMyh@8V4hHYVHe@AE0We*$J|cVlz=M1=pS?7q3T>6X4d+S6;dyI!BgEc|?* zQT>jvV*S2zL_R+qJl2T*`Mf-D#0oTdJLTgF2)LCR@eDH9w}LtK{AbB3+o1mKNE|eT z_RJv>Y9RR*5?QAbzRTr$1OKz3YHUcpwO~kz3{JHtdd9}y4YuyPANU_IypL0ehgU@J zYD5q#M5p{vhet7XJv>5b+q|71l%9DK9YSk9ZJYd$5a$=BIraW4}fsYjfXG@P^4&Ny~t;EJyhsyybSz2@p;|GF20!TBSfTy8gY z3KB{|v{zU%RsjcHQVemziKJLFB0Xwri*-rnpU|x7PrD*R=Mgg5_mb>jjMm{ zHs|stBL~&Ha3QQv^R&^Hz^w?9VI}Cdad~Y^CI$EYBqw<~y#D6sK3-qV7U2FBfNU({ z0Rs{V8}ppRNa!b-O$o$nM`bX~HQXSp_C zN?4_RR|Jh@4h?NO%PCz(jNC%zPIB9o{K}LB3U3QfM8=Q`@ySoWWSgbBz}MT`c2ph6j`V$-ocb}7kLzsF_acH zlqwV?Ye>?SzYQ^0k;PdAEeh|96TJX+F+LODyMdn5qdp(|#)P7iLF#1+xl3!-to){m z6e1U`f{{hac^|OPfQ(;8W6~<_&T~xG(oVC>xyOzjX(G13wN!UD_;ML@CFq)l{>(8V zA!UhxZ0$wZ0@$ZK%o>ij^!MUD%(cmak|V>qa%4?b_|cKeeV?O+QGTRb?7ij}OA;k> z^MpTD)%=WuRy zk~!5Rn!8ic&e5Pz0c{evE93Mu+v)JfZtWzp{=vE$Vr9qCQ+91~)v#Z^AVP?Ux-e=_ zM{S~Dw@I@FNQR@lv8~R%NaBm>QU~$3PynYB?u`C5jw;Ex`5u3Vu^ge?ogiJwX-?nS zVI<+|gd%Bf=x$`o-@S<&n1xH#z8Gw}duhIQ0WrfYS znX3Cm)nniU6+G$Kq3>_o3zjVGLngvO8GaK1h=rMneE=S_YQY_QH5G!DXCB*e0mbk& z(im@satuqB+gnoOy5i%!xIpX~wFrDv5bjvvB2_M!9Ve1)pVU}5u1^~JsA-d9s>;?(R1~IOs^&HA_rCmg zp!#TfzMI95i$YW?0*dqdAL3s<+{S*JkieiG{V=robQ%uN{@r1Dn@#FcPHTax(l&sg zJdQ1q@U{tpr*!Wps5nr!9qJrQgiX4Yxae4)4}y4rj#u1F0R+~dzS?9TOBRHSI;PXI z--j5VBEJ^20^*ldvtkt&yo^{^4{2{bFs|GM$gYf*2UR=?q9#t8ygwC++YYqfGz+7p z0SoId5uxF)CxgPs&7!&8g^Clj57qu&K6m9<7NU)3BO(?FOTC^C}-TfLb zX}L0fp}!$`!SD!MP|yKAwRzoiOv?wjwGHw!b?t}zy@mtAP@)1Ed+SJ#O9xtdeygik z-|*Byo7fYiVetF}#>Z0wKqGcmW&T7g!Dmw8^9Z9Mo1pegXpLeOPS--~uC~i&m@=x6{cGaoi(G5su}2 z648qhPQ*GsGAfayXkzpT9i($DO>-vQ2nT}1N_7IlJ#!JA+L%D^sgef3IQqLUP#h^x z3YM;N$$!F})Z9(%q7H;AxaYSbqd%C*l1{$jl4MdP?|=2M1_dA_B;59PL+8(5gz)n3b zw>!XqQYMBKj=r|kK zDp8gU#S}RO%4#@SM1o2|+rZf{+upWyuuG})51OQF1_xs-l^x;hD~sw&N>J&bC>gmJ z?rI+nvQD_wypPf94)Y#?sTyF{cXdhe5}i>B*4U9O zSv)kHZKbTSY#v$7RTZ=S#{ZlG8~q(;dAC>Ypnyv+#Mo5{0Ri?k&i9{&cIeH=zATLI z{@)BJaV|LrR)3DvvF`lI-W2o|S`2tI8TNb%R{hDi_rnO2JI!RM5k>>YvaTv!P2GzTo&O>j98kj z3^`d$(MVIQY}2@D@j$L2lhvTL8%pHC-LrX=C&$}s0|`!AWWar$sDc+rJ*#%D8gLH9 z{Hq4z)c-krQE|N2sC#<28)Q}%uogJ2@+ZR2L=1dk+Gbopqs?cYtKFt|pm}?3Xj;VK zQO2;9$)&Chs#q-29>>?T%i|L@PE)zZ418vMo3K+ne`xwUvfO$2jwbgUkO)9AS8L>TeTI?7V-%2+`xOscPG}5#bMe?B}JIz%V zS^qJ`cgESUhKTB8AO;Zw?{GP7KnYLhnds*22mXu7S4j;5f{(i~RI*AGR7DA=WW^9? z1%cMCC^3T+tC_@EnhDvt%B-!{avPikYoz7!^uW(tOn^r(UuIEM3?i)*4%$;%r^;L- zRZO5#IPuN`o8bD!9|mp29BH1~(tf48-P&J&W_b z;NGkVf%k7xiB7|5Zuoy)vR0jty7 zQ%;2ria;w|rX%6#JV5UrxPg9FAthkBbd95Ak3Lb(-KHlfZ(FBF7_M|4tEgMgw!;8% zD?Hr@jd*4-bbD_BqQPtyy})7O`r@IH`b)_1&&9?_O4 zE8W6($@Cz_o&>tbULUIriScaRoHoWSWewb(e^0}}iWZn;4DTG%KFh%hXvru<+K%qb z&7pMn0Tadv(*az=H<3L@)&v!<+-OfPu)~JVyECBq5dPx(bVj~W6P>wcY;BF7QZ8>6{|M=sj(w2P`GUi;?)7Nee zAyVB~D+EF8tF-pFS~SU94!j~YucLt|*oryy8XmjFVG7`EU^d_KJ3Zq?M{5{w8bDl> zPGz3Vu1$C_-(U<{IZH_GjVw57T{4X4t`!ZCE`Te82T_-o zBA_A~2uv0T7DRM&~$fQMn5|9hZL!E*fZ%V zKnFvxGf`H~W8LCBFoGR?wsn6B!PbI~_Fd3zbOzcWf3rIpwbB8c1BP>AerL+OgA<&= zzq?5tBs2eUu*|fg4lQKNU$2BMm{vuAhOi?N6!5G*&_g{&)6qWD*2 zHslfRzJavG1{DCENIr>*a^T`Tbw+=+0|U^PfOu$LPQgnZrTfXEcJi`~Z>Sh%;wnX@ zlPC{8!*Q4cc4Dk9s|4|;Lp2eYilEgFgg9wKcvhR#906%7why~@+RB89U_}uO#9|H+ zF~CE3gG_^+txWI@%4_19B1s;WojC*0n&Q47?rBIIF*%)D+!>rp!eJ;{axIH$VZ!}d zfRBJRaZGddm&Ge_z+1d22)N`F_u8)3#|Q0cp$^blBj>|^rOEcIc2;N(K#?e{<0^mm zhcS^YhAuJByAWnf%sFf_$SK#3+AU7*9&D!AaB<#@?bxh>##xM;Fc2JKG2*8I;@ir! zN?#x_%Hb9M#Wf;o(C&NWpe?|3y(=U*D8EX7F;hl#Gd0HYT_2u2orVP=oQ7SSENQcFl%aL-#PnKSKQe4%c7i+;Ow^jQU;HFZzwbpVd~{B zS6Q4C<_HG?>Vntt!ShJ4Mo-<8>BvHTQT0A*!KPI}Sw+V41-c)U>g9bEM$2~>8>2{W zrT8Ydy>!USGENHqYTKDA8b5jtC{4jtU=y5&bx~!TYa(vXuwr(4#nn^_1u_q1PAdumefb z>W8LJJErZG2~R$lH#;kiY$QWUZLie~RE9tEKNC9#4ZAB$^JQ;8i)qEJB)I%I_NKO& z-BXe+eXa;dlD@=PMfX!+8mcQE2Zs-e@GL!cPcx*;x;S_xDY7ss%~%@hMN4ig9S>2Y z-~AJO_)A~1ZUZJZb_P5Tx@ii`l`kN_Nc;{eF3^#=DAyouzj+vkhnVIR<=BG-J1vPM z*-(q6HvoMfH;DzzyYB+ia=D$b40wG|ZIF@0=T>oJB-6Lks5{I5I%hCGFdmjb)Ke!e z>-=%~0#_w$I*dKoNrR3lYB@Gun3}jHz5CC>c~@28HSO$S_?UeiE3lS6_iRHx_iXHA z#zAR#ps7!pn67=wF_5W#0gDYh&K-TxfZLAa%6c=GdNwg}+2c~e#|n}~#howezoJ#3SrLgv!fi|RC8EqcEd6u*VPg=1ss3+~rebh7l0 z5&zIxL=^fs)3a~Yx^wk&mU;j652Qbe1IY>L;hX!3Rf4y zubK}(96xup^50*rKlhz8Tt7=6i4R19S8Cs-BNsD-nG9S%|K|VuSB8|(D*t`Kr|0*+ zdi7bj6R;xS^Z94`We#aar^%b>Yp)ZFVLa1Kp2$ymdso74ICI1g+fML}XvHn|#L4^Y zsdMY*pVymXm&fBrtWw7g-QmO8f!}BFWtG=sEj{Q0@<{Q3N1@cRngntI{LwfWAi{&{x&LfU`S{P{Qd^r83jQ0x0}u@(7!D(3O* z7^?L4es%xjR=d;nK4H$$nfaF?M}Qj&jYtPu#Lf37bRq1fr#SC;BT?X0G5qHcX|v1o z>+0dI_3pDVdd2GN-||mw<`3nc6^Py1mhYY9OrnNwJR`fMA8eVf07@8yA?lpLBHxBy?xJrS-x$)-@bjH9m2lVKP&G)&C7mX zIzK-V7e6PuirwlzS6AN*&F3{wKhoOTPkxW@S1t{DwHGekH7idYx+@o+I{&Zp6I=5G NChaZc57-3({14DZwUYn< diff --git a/examples/ps/main.py b/examples/ps/main.py index ef5bb7b6..ba7b2b75 100644 --- a/examples/ps/main.py +++ b/examples/ps/main.py @@ -9,7 +9,7 @@ with Runner("ps-example") as rr: ps_svc = rr.service(ParameterServer(1e-2)) - workers_svc = [rr.service(DataWorker) for _ in range(4)] + workers_svc = [rr.service(DataWorker) for _ in range(2)] current_weights = ps_svc.get_weights().get() for i in range(20): diff --git a/examples/ps/pyproject.toml b/examples/ps/pyproject.toml index 48e417a3..53bd6eda 100644 --- a/examples/ps/pyproject.toml +++ b/examples/ps/pyproject.toml @@ -8,7 +8,8 @@ dependencies = [ "torch", "torchvision", "numpy", - "filelock" + "filelock", + "flamepy", ] [build-system] diff --git a/executor_manager/src/shims/host_shim.rs b/executor_manager/src/shims/host_shim.rs index f97ddaf7..edf492c2 100644 --- a/executor_manager/src/shims/host_shim.rs +++ b/executor_manager/src/shims/host_shim.rs @@ -27,6 +27,7 @@ use async_trait::async_trait; use hyper_util::rt::TokioIo; use nix::sys::signal::{killpg, Signal}; use nix::unistd::Pid; +use std::collections::HashMap; use stdng::{logs::TraceFn, trace_fn}; use tokio::net::UnixStream; use tokio::sync::Mutex; @@ -73,6 +74,73 @@ impl HostShim { }))) } + fn setup_working_directory(work_dir: &Path) -> Result, FlameError> { + trace_fn!("HostShim::setup_working_directory"); + + let tmp_dir = work_dir.join("tmp"); + let uv_cache_dir = work_dir.join(".uv"); + let pip_cache_dir = work_dir.join(".pip"); + + tracing::debug!( + "Working directory of application instance: {}", + work_dir.display() + ); + tracing::debug!( + "Temporary directory of application instance: {}", + tmp_dir.display() + ); + tracing::debug!( + "UV cache directory of application instance: {}", + uv_cache_dir.display() + ); + tracing::debug!( + "PIP cache directory of application instance: {}", + pip_cache_dir.display() + ); + + // Create the working, temporary, and cache directories if they don't exist + create_dir_all(&work_dir).map_err(|e| { + FlameError::Internal(format!( + "failed to create working directory {}: {e}", + work_dir.display() + )) + })?; + create_dir_all(&tmp_dir).map_err(|e| { + FlameError::Internal(format!( + "failed to create temporary directory {}: {e}", + tmp_dir.display() + )) + })?; + create_dir_all(&uv_cache_dir).map_err(|e| { + FlameError::Internal(format!( + "failed to create UV cache directory {}: {e}", + uv_cache_dir.display() + )) + })?; + create_dir_all(&pip_cache_dir).map_err(|e| { + FlameError::Internal(format!( + "failed to create PIP cache directory {}: {e}", + pip_cache_dir.display() + )) + })?; + + // Build environment variables for the application instance + let mut envs = HashMap::new(); + envs.insert("TMPDIR".to_string(), tmp_dir.to_string_lossy().to_string()); + envs.insert("TEMP".to_string(), tmp_dir.to_string_lossy().to_string()); + envs.insert("TMP".to_string(), tmp_dir.to_string_lossy().to_string()); + envs.insert( + "UV_CACHE_DIR".to_string(), + uv_cache_dir.to_string_lossy().to_string(), + ); + envs.insert( + "PIP_CACHE_DIR".to_string(), + pip_cache_dir.to_string_lossy().to_string(), + ); + + Ok(envs) + } + fn launch_instance( app: &ApplicationContext, executor: &Executor, @@ -109,33 +177,9 @@ impl HostShim { }; let work_dir = cur_dir.clone(); - let tmp_dir = cur_dir.join("tmp"); - - tracing::debug!( - "Working directory of application instance: {}", - work_dir.display() - ); - tracing::debug!( - "Temporary directory of application instance: {}", - tmp_dir.display() - ); - - // Create the working & temporary directories if they don't exist - create_dir_all(&work_dir).map_err(|e| { - FlameError::Internal(format!( - "failed to create working directory {}: {e}", - work_dir.display() - )) - })?; - create_dir_all(&tmp_dir).map_err(|e| { - FlameError::Internal(format!( - "failed to create temporary directory {}: {e}", - tmp_dir.display() - )) - })?; - - // Set temporary directory for the application instance - envs.insert("TMPDIR".to_string(), tmp_dir.to_string_lossy().to_string()); + // Setup working directory and get environment overrides + let wd_envs = Self::setup_working_directory(&work_dir)?; + envs.extend(wd_envs); let log_out = OpenOptions::new() .create(true) diff --git a/sdk/python/src/flamepy/__init__.py b/sdk/python/src/flamepy/__init__.py index b809d831..6bd3863b 100644 --- a/sdk/python/src/flamepy/__init__.py +++ b/sdk/python/src/flamepy/__init__.py @@ -11,22 +11,6 @@ limitations under the License. """ -import logging -import os - -_log_level_str = os.getenv("FLAME_LOG_LEVEL", "INFO").upper() -_log_level_map = { - "CRITICAL": logging.CRITICAL, - "ERROR": logging.ERROR, - "WARNING": logging.WARNING, - "INFO": logging.INFO, - "DEBUG": logging.DEBUG, -} - -_log_level = _log_level_map[_log_level_str] if _log_level_str in _log_level_map else logging.INFO - -logging.basicConfig(level=_log_level) - # Import submodules for rl and agent (only as submodules) from . import agent, rl @@ -46,6 +30,7 @@ Connection, Event, FlameContext, + FlameContextRunner, FlameError, FlameErrorCode, FlamePackage, @@ -109,6 +94,7 @@ "Task", "Application", "FlamePackage", + "FlameContextRunner", # Context and utility classes "TaskInformer", "FlameContext", diff --git a/sdk/python/src/flamepy/core/__init__.py b/sdk/python/src/flamepy/core/__init__.py index f1192c87..270c28b5 100644 --- a/sdk/python/src/flamepy/core/__init__.py +++ b/sdk/python/src/flamepy/core/__init__.py @@ -51,7 +51,6 @@ FlameService, SessionContext, TaskContext, - TaskOutput, run, ) @@ -74,6 +73,7 @@ CommonData, Event, FlameContext, + FlameContextRunner, FlameError, FlameErrorCode, FlamePackage, @@ -120,6 +120,7 @@ "Task", "Application", "FlamePackage", + "FlameContextRunner", # Context and utility classes "TaskInformer", "FlameContext", diff --git a/sdk/python/src/flamepy/core/service.py b/sdk/python/src/flamepy/core/service.py index 48931826..cd5a7f11 100644 --- a/sdk/python/src/flamepy/core/service.py +++ b/sdk/python/src/flamepy/core/service.py @@ -13,6 +13,7 @@ import logging import os +import typing from abc import abstractmethod from concurrent import futures from dataclasses import dataclass @@ -122,6 +123,7 @@ class FlameInstanceServicer(InstanceServicer): def __init__(self, service: FlameService): self._service = service + @typing.override def OnSessionEnter(self, request, context): """Handle OnSessionEnter RPC call.""" _trace_fn = TraceFn("OnSessionEnter") @@ -165,6 +167,7 @@ def OnSessionEnter(self, request, context): logger.error(f"Error in OnSessionEnter: {e}") return Result(return_code=-1, message=f"{str(e)}") + @typing.override def OnTaskInvoke(self, request, context): """Handle OnTaskInvoke RPC call.""" _trace_fn = TraceFn("OnTaskInvoke") @@ -193,6 +196,7 @@ def OnTaskInvoke(self, request, context): logger.error(f"Error in OnTaskInvoke: {e}") return TaskResultProto(return_code=-1, output=None, message=f"{str(e)}") + @typing.override def OnSessionLeave(self, request, context): """Handle OnSessionLeave RPC call.""" _trace_fn = TraceFn("OnSessionLeave") diff --git a/sdk/python/src/flamepy/core/types.py b/sdk/python/src/flamepy/core/types.py index c5dd328c..38ab0ed8 100644 --- a/sdk/python/src/flamepy/core/types.py +++ b/sdk/python/src/flamepy/core/types.py @@ -36,6 +36,7 @@ DEFAULT_FLAME_CONF = "flame.yaml" DEFAULT_FLAME_ENDPOINT = "http://127.0.0.1:8080" DEFAULT_FLAME_CACHE_ENDPOINT = "grpc://127.0.0.1:9090" +DEFAULT_FLAME_RUNNER_TEMPLATE = "flmrun" class SessionState(IntEnum): @@ -219,14 +220,30 @@ class FlamePackage: ) +@dataclass +class FlameContextRunner: + """Runner configuration for Flame applications. + + Attributes: + template: The name of the application template to use for runners. + If not specified, defaults to 'flmrun'. + """ + + template: Optional[str] = None + + class FlameContext: """Flame configuration.""" _endpoint = None _cache = None _package = None + _runner = None def __init__(self): + # Initialize runner with default values + self._runner = FlameContextRunner(template=DEFAULT_FLAME_RUNNER_TEMPLATE) + home = Path.home() config_file = home / ".flame" / DEFAULT_FLAME_CONF if config_file.exists(): @@ -250,6 +267,12 @@ def __init__(self): default_excludes = [".venv", "__pycache__", ".gitignore", "*.pyc"] all_excludes = list(set(default_excludes + excludes)) self._package = FlamePackage(storage=storage, excludes=all_excludes) + + # Parse runner configuration if present + runner_config = cluster.get("runner") + if runner_config is not None: + template = runner_config.get("template") + self._runner = FlameContextRunner(template=DEFAULT_FLAME_RUNNER_TEMPLATE if template is None else template) break else: raise FlameError(FlameErrorCode.INVALID_CONFIG, f"cluster <{cc}> not found") @@ -299,3 +322,8 @@ def cache_endpoint(self) -> str: elif self._cache is not None: return self._cache return DEFAULT_FLAME_CACHE_ENDPOINT + + @property + def runner(self) -> FlameContextRunner: + """Get the runner configuration.""" + return self._runner diff --git a/sdk/python/src/flamepy/rl/runner.py b/sdk/python/src/flamepy/rl/runner.py index 4e8bef82..b7af13b5 100644 --- a/sdk/python/src/flamepy/rl/runner.py +++ b/sdk/python/src/flamepy/rl/runner.py @@ -338,33 +338,41 @@ def __enter__(self) -> "Runner": storage_url = self._upload_package() logger.debug(f"Uploaded package to: {storage_url}") - # Step 3: Retrieve the flmrun application template + # Step 3: Retrieve the application template + # Use configured template if available, otherwise default to flmrun + template_name = self._context.runner.template + try: - flmrun_app = get_application("flmrun") - logger.debug("Retrieved flmrun application template") + template_app = get_application(template_name) + logger.debug(f"Retrieved application template: {template_name}") except Exception as e: # Clean up the package file if self._package_path and os.path.exists(self._package_path): os.remove(self._package_path) - raise FlameError(FlameErrorCode.INTERNAL, f"Failed to get flmrun application template: {str(e)}") + raise FlameError(FlameErrorCode.INTERNAL, f"Failed to get application template '{template_name}': {str(e)}") # Step 4: Register the new application try: - # Use /opt/{name} as working directory for the application - working_directory = f"/opt/{self._name}" + # Determine working directory based on template + working_directory = None + if template_app.working_directory is not None and template_app.working_directory != "": + # If template has a working_directory, append runner name + working_directory = f"{template_app.working_directory}/{self._name}" + + logger.debug(f"Working directory: {working_directory}") app_attrs = ApplicationAttributes( - shim=flmrun_app.shim, - image=flmrun_app.image, - command=flmrun_app.command, + shim=template_app.shim, + image=template_app.image, + command=template_app.command, description=f"Runner application: {self._name}", - labels=flmrun_app.labels, - arguments=flmrun_app.arguments, - environments=flmrun_app.environments, + labels=template_app.labels, + arguments=template_app.arguments, + environments=template_app.environments, working_directory=working_directory, - max_instances=flmrun_app.max_instances, - delay_release=flmrun_app.delay_release, - schema=flmrun_app.schema, + max_instances=template_app.max_instances, + delay_release=template_app.delay_release, + schema=template_app.schema, url=storage_url, ) diff --git a/sdk/python/src/flamepy/rl/runpy.py b/sdk/python/src/flamepy/rl/runpy.py index e5e0721d..303fa3af 100644 --- a/sdk/python/src/flamepy/rl/runpy.py +++ b/sdk/python/src/flamepy/rl/runpy.py @@ -28,7 +28,7 @@ from flamepy.core import ObjectRef, get_object, put_object, update_object from flamepy.core.service import FlameService, SessionContext, TaskContext -from flamepy.core.types import TaskOutput +from flamepy.core.types import TaskOutput, short_name from flamepy.rl.types import RunnerContext, RunnerRequest logger = logging.getLogger(__name__) @@ -199,22 +199,6 @@ def _install_package_from_url(self, url: str) -> None: install_path = extracted_dir logger.info(f"Will install from extracted directory: {install_path}") - # Debug: List contents of extracted directory - try: - contents = os.listdir(install_path) - logger.debug(f"Extracted directory contents: {contents}") - - # Check for pyproject.toml or setup.py - if "pyproject.toml" in contents: - pyproject_path = os.path.join(install_path, "pyproject.toml") - with open(pyproject_path, "r") as f: - pyproject_content = f.read() - logger.debug(f"pyproject.toml content:\n{pyproject_content}") - if "setup.py" in contents: - logger.debug("Found setup.py in extracted directory") - except Exception as e: - logger.warning(f"Failed to list extracted directory contents: {e}") - # Use sys.executable -m pip to install into the current virtual environment # pip install will upgrade the package if it's already installed logger.info(f"Installing package: {install_path}") @@ -222,13 +206,45 @@ def _install_package_from_url(self, url: str) -> None: logger.debug(f"Current working directory: {os.getcwd()}") install_args = [sys.executable, "-m", "pip", "install", "--upgrade", install_path] logger.debug(f"Install command: {' '.join(install_args)}") + env = os.environ.copy() + logger.debug(f"Environment from parent process: {env}") + + # Create a dedicated log file for the installation process + working_dir = os.getcwd() + if self._ssn_ctx and getattr(self._ssn_ctx, "session_id", None): + session_id = self._ssn_ctx.session_id + else: + # Generate a short random identifier when session context is unavailable + session_id = short_name("unknown") + log_file_path = os.path.join(working_dir, f"package_installation_{session_id}.log") + logger.info(f"Installation progress will be logged to: {log_file_path}") try: - result = subprocess.run(install_args, capture_output=True, text=True, check=True) - logger.info("Package installation succeeded") - logger.debug(f"Package installation stdout:\n{result.stdout}") - if result.stderr: - logger.debug(f"Package installation stderr:\n{result.stderr}") + # Open the log file and redirect subprocess output to it + with open(log_file_path, "w") as log_file: + # Write header to log file + log_file.write("Package Installation Log\n") + log_file.write(f"{'=' * 80}\n") + log_file.write(f"Session ID: {session_id}\n") + log_file.write(f"Install command: {' '.join(install_args)}\n") + log_file.write(f"Working directory: {os.getcwd()}\n") + log_file.write(f"Python executable: {sys.executable}\n") + log_file.write(f"{'=' * 80}\n\n") + log_file.flush() + + # Run the installation with output redirected to the log file + result = subprocess.run( + install_args, + stdout=log_file, + stderr=subprocess.STDOUT, # Redirect stderr to stdout so both go to the same log + text=True, + check=True, + env=env, + ) + + if result.returncode != 0: + raise RuntimeError(f"Package installation failed: {result.returncode}") + logger.info(f"Successfully installed package from: {install_path}") # Reload site packages to make the newly installed package available @@ -241,14 +257,18 @@ def _install_package_from_url(self, url: str) -> None: logger.error(f"Failed to install package: {e}") logger.error(f"Return code: {e.returncode}") logger.error(f"Install command was: {' '.join(install_args)}") - logger.error(f"Package installation stdout:\n{e.stdout}") - logger.error(f"Package installation stderr:\n{e.stderr}") - raise RuntimeError(f"Package installation failed: {e}") + logger.error(f"Installation log file: {log_file_path}") + + raise RuntimeError(f"Package installation failed: {e}. Check log at {log_file_path}") finally: - # Clean up extracted directory if it was created - # Note: We keep it for now as it might be needed during the session - # Future enhancement could add cleanup in on_session_leave - pass + # Read and log the installation output from the log file + try: + if logger.isEnabledFor(logging.DEBUG): + with open(log_file_path, "r") as log_file: + installation_output = log_file.read() + logger.debug(f"Package installation output:\n{installation_output}") + except Exception as read_error: + logger.error(f"Could not read installation log: {read_error}") def on_session_enter(self, context: SessionContext) -> bool: """ From 74caa16e48004c7cb80ba35bbb0f92f1b36e2df2 Mon Sep 17 00:00:00 2001 From: Klaus Ma Date: Sun, 25 Jan 2026 12:15:28 +0000 Subject: [PATCH 2/2] address review comments. Signed-off-by: Klaus Ma --- .dockerignore | 5 ++-- .gitignore | 4 ++- examples/ps/README.md | 2 +- executor_manager/src/shims/host_shim.rs | 38 +++++++++---------------- sdk/python/src/flamepy/rl/runpy.py | 5 +--- 5 files changed, 22 insertions(+), 32 deletions(-) diff --git a/.dockerignore b/.dockerignore index cfab4750..33c749b9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -20,8 +20,9 @@ ci/ **/*.egg-info/ # Ignore build related stuff -build/ -dist/ +**/build/ +**/dist/ +work/ .mypy_cache/ .ruff_cache/ diff --git a/.gitignore b/.gitignore index f9af9076..a2082527 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,6 @@ flame.db* __pycache__/ *.egg-info/ build/ -uv.lock \ No newline at end of file +uv.lock + +work/ \ No newline at end of file diff --git a/examples/ps/README.md b/examples/ps/README.md index eb438f34..17f3fccf 100644 --- a/examples/ps/README.md +++ b/examples/ps/README.md @@ -81,7 +81,7 @@ Iter 10: accuracy is 32.9 Final accuracy is 32.9. ``` -The accuracy should improve from around 10% (random guessing) to ~85% after 20 training iterations. +In this simplified run, the accuracy typically improves from around 10% (random guessing) to roughly 30–35% after 20 training iterations (as shown above). Higher accuracy may require more iterations, a larger model, or full‑dataset evaluation. ## Key Concepts diff --git a/executor_manager/src/shims/host_shim.rs b/executor_manager/src/shims/host_shim.rs index edf492c2..f5aa9ef1 100644 --- a/executor_manager/src/shims/host_shim.rs +++ b/executor_manager/src/shims/host_shim.rs @@ -74,6 +74,16 @@ impl HostShim { }))) } + fn create_dir(path: &Path, name: &str) -> Result<(), FlameError> { + create_dir_all(path).map_err(|e| { + FlameError::Internal(format!( + "failed to create {} directory {}: {e}", + name, + path.display() + )) + }) + } + fn setup_working_directory(work_dir: &Path) -> Result, FlameError> { trace_fn!("HostShim::setup_working_directory"); @@ -99,30 +109,10 @@ impl HostShim { ); // Create the working, temporary, and cache directories if they don't exist - create_dir_all(&work_dir).map_err(|e| { - FlameError::Internal(format!( - "failed to create working directory {}: {e}", - work_dir.display() - )) - })?; - create_dir_all(&tmp_dir).map_err(|e| { - FlameError::Internal(format!( - "failed to create temporary directory {}: {e}", - tmp_dir.display() - )) - })?; - create_dir_all(&uv_cache_dir).map_err(|e| { - FlameError::Internal(format!( - "failed to create UV cache directory {}: {e}", - uv_cache_dir.display() - )) - })?; - create_dir_all(&pip_cache_dir).map_err(|e| { - FlameError::Internal(format!( - "failed to create PIP cache directory {}: {e}", - pip_cache_dir.display() - )) - })?; + Self::create_dir(&work_dir, "working")?; + Self::create_dir(&tmp_dir, "temporary")?; + Self::create_dir(&uv_cache_dir, "UV cache")?; + Self::create_dir(&pip_cache_dir, "PIP cache")?; // Build environment variables for the application instance let mut envs = HashMap::new(); diff --git a/sdk/python/src/flamepy/rl/runpy.py b/sdk/python/src/flamepy/rl/runpy.py index 303fa3af..0d23ff04 100644 --- a/sdk/python/src/flamepy/rl/runpy.py +++ b/sdk/python/src/flamepy/rl/runpy.py @@ -233,7 +233,7 @@ def _install_package_from_url(self, url: str) -> None: log_file.flush() # Run the installation with output redirected to the log file - result = subprocess.run( + subprocess.run( install_args, stdout=log_file, stderr=subprocess.STDOUT, # Redirect stderr to stdout so both go to the same log @@ -242,9 +242,6 @@ def _install_package_from_url(self, url: str) -> None: env=env, ) - if result.returncode != 0: - raise RuntimeError(f"Package installation failed: {result.returncode}") - logger.info(f"Successfully installed package from: {install_path}") # Reload site packages to make the newly installed package available