From 3a0003e613c34568512b18bf87918df155196ebe Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 24 Aug 2024 14:59:52 +0200 Subject: [PATCH 1/4] add std to performance drifts in recursive selectors --- .../selection/base_recursive_selector.py | 1 + .../selection/recursive_feature_addition.py | 2 + .../recursive_feature_elimination.py | 2 + .../test_recursive_feature_addition.py | 70 ++++++++++++++++++- .../test_recursive_feature_elimination.py | 68 ++++++++++++++++++ 5 files changed, 142 insertions(+), 1 deletion(-) diff --git a/feature_engine/selection/base_recursive_selector.py b/feature_engine/selection/base_recursive_selector.py index d6ec3f092..879605da4 100644 --- a/feature_engine/selection/base_recursive_selector.py +++ b/feature_engine/selection/base_recursive_selector.py @@ -180,6 +180,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # Aggregate the feature importance returned in each fold self.feature_importances_ = feature_importances_cv.mean(axis=1) + self.feature_importances_std_ = feature_importances_cv.std(axis=1) return X, y diff --git a/feature_engine/selection/recursive_feature_addition.py b/feature_engine/selection/recursive_feature_addition.py index 6ceab91f7..564f1ee3b 100644 --- a/feature_engine/selection/recursive_feature_addition.py +++ b/feature_engine/selection/recursive_feature_addition.py @@ -178,6 +178,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # It is initialized with the performance drift of # the most important feature self.performance_drifts_ = {first_most_important_feature: 0} + self.performance_drifts_std_ = {first_most_important_feature: 0} # loop over the ordered list of features by feature importance starting # from the second element in the list. @@ -201,6 +202,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # Save feature and performance drift self.performance_drifts_[feature] = performance_drift + self.performance_drifts_std_[feature] = model_tmp["test_score"].std() # If new performance model is if performance_drift > self.threshold: diff --git a/feature_engine/selection/recursive_feature_elimination.py b/feature_engine/selection/recursive_feature_elimination.py index 36f873c07..1f4a482f1 100644 --- a/feature_engine/selection/recursive_feature_elimination.py +++ b/feature_engine/selection/recursive_feature_elimination.py @@ -164,6 +164,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # dict to collect features and their performance_drift after shuffling self.performance_drifts_ = {} + self.performance_drifts_std_ = {} # evaluate every feature, starting from the least important # remember that feature_importances_ is ordered already @@ -193,6 +194,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # Save feature and performance drift self.performance_drifts_[feature] = performance_drift + self.performance_drifts_std_[feature] = model_tmp["test_score"].std() if performance_drift > self.threshold: diff --git a/tests/test_selection/test_recursive_feature_addition.py b/tests/test_selection/test_recursive_feature_addition.py index 3d5a1ce2c..7162b860c 100644 --- a/tests/test_selection/test_recursive_feature_addition.py +++ b/tests/test_selection/test_recursive_feature_addition.py @@ -1,7 +1,7 @@ import pandas as pd import pytest from sklearn.ensemble import RandomForestClassifier -from sklearn.linear_model import Lasso, LogisticRegression +from sklearn.linear_model import Lasso, LogisticRegression, LinearRegression from sklearn.tree import DecisionTreeRegressor from feature_engine.selection import RecursiveFeatureAddition @@ -186,3 +186,71 @@ def test_regression( # test transform output pd.testing.assert_frame_equal(sel.transform(X), Xtransformed) + + +def test_performance_drift_std(load_diabetes_dataset): + X, y = load_diabetes_dataset + linear_model = LinearRegression() + sel = RecursiveFeatureAddition(estimator=linear_model, scoring="r2", cv=3) + sel.fit(X, y) + + drifts = { + 4: 0, + 8: 0.2837, + 2: 0.1378, + 5: 0.0023, + 3: 0.0188, + 1: 0.0028, + 7: 0.0027, + 6: 0.0027, + 9: 0.0003, + 0: -0.0074, + } + + drfts_std = { + 4: 0, + 8: 0.0293, + 2: 0.0175, + 5: 0.0205, + 3: 0.0173, + 1: 0.0087, + 7: 0.0242, + 6: 0.0234, + 9: 0.0169, + 0: 0.0204, + } + + rounded_perfs = { + key: round(sel.performance_drifts_[key], 4) for key in sel.performance_drifts_ + } + assert rounded_perfs == drifts + + rounded_perfs = { + key: round(sel.performance_drifts_std_[key], 4) + for key in sel.performance_drifts_std_ + } + assert rounded_perfs == drfts_std + + +def test_feature_importance(load_diabetes_dataset): + X, y = load_diabetes_dataset + linear_model = LinearRegression() + sel = RecursiveFeatureAddition(estimator=linear_model, scoring="r2", cv=3) + sel.fit(X, y) + + imps = [ + 750.02, + 741.47, + 522.33, + 436.67, + 322.09, + 238.62, + 182.17, + 113.97, + 64.77, + 41.42, + ] + imps_std = [18.22, 68.35, 86.03, 57.11, 329.38, 299.76, 72.81, 47.93, 117.83, 42.75] + + assert round(sel.feature_importances_, 2).to_list() == imps + assert round(sel.feature_importances_std_, 2).to_list() == imps_std diff --git a/tests/test_selection/test_recursive_feature_elimination.py b/tests/test_selection/test_recursive_feature_elimination.py index abd88b1a5..3d8d124df 100644 --- a/tests/test_selection/test_recursive_feature_elimination.py +++ b/tests/test_selection/test_recursive_feature_elimination.py @@ -208,3 +208,71 @@ def test_stops_when_only_one_feature_remains(): ) output = transformer.fit_transform(df[["x", "z"]], df["y"]) pd.testing.assert_frame_equal(output, df["x"].to_frame()) + + +def test_performance_drift_std(load_diabetes_dataset): + X, y = load_diabetes_dataset + linear_model = LinearRegression() + sel = RecursiveFeatureElimination(estimator=linear_model, scoring="r2", cv=3) + sel.fit(X, y) + + drifts = { + 0: -0.0033, + 9: -0.0003, + 6: -0.0007, + 7: 0.0001, + 1: 0.012, + 3: 0.0286, + 5: 0.0126, + 2: 0.0663, + 8: 0.1094, + 4: 0.0243, + } + + drfts_std = { + 0: 0.0136, + 9: 0.0168, + 6: 0.0169, + 7: 0.018, + 1: 0.0252, + 3: 0.0084, + 5: 0.0087, + 2: 0.0425, + 8: 0.0468, + 4: 0.0162, + } + + rounded_perfs = { + key: round(sel.performance_drifts_[key], 4) for key in sel.performance_drifts_ + } + assert rounded_perfs == drifts + + rounded_perfs = { + key: round(sel.performance_drifts_std_[key], 4) + for key in sel.performance_drifts_std_ + } + assert rounded_perfs == drfts_std + + +def test_feature_importance(load_diabetes_dataset): + X, y = load_diabetes_dataset + linear_model = LinearRegression() + sel = RecursiveFeatureElimination(estimator=linear_model, scoring="r2", cv=3) + sel.fit(X, y) + + imps = [ + 41.42, + 64.77, + 113.97, + 182.17, + 238.62, + 322.09, + 436.67, + 522.33, + 741.47, + 750.02, + ] + imps_std = [18.22, 68.35, 86.03, 57.11, 329.38, 299.76, 72.81, 47.93, 117.83, 42.75] + + assert round(sel.feature_importances_, 2).to_list() == imps + assert round(sel.feature_importances_std_, 2).to_list() == imps_std From a7e76e89c7681978f95ab19734a35ac7eb3a71ca Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 24 Aug 2024 15:19:32 +0200 Subject: [PATCH 2/4] add new parameters to docstrings --- feature_engine/_docstrings/fit_attributes.py | 9 +++++++++ feature_engine/selection/base_recursive_selector.py | 4 ++-- feature_engine/selection/recursive_feature_addition.py | 8 ++++++++ .../selection/recursive_feature_elimination.py | 8 ++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/feature_engine/_docstrings/fit_attributes.py b/feature_engine/_docstrings/fit_attributes.py index 6c833dc83..9263800f4 100644 --- a/feature_engine/_docstrings/fit_attributes.py +++ b/feature_engine/_docstrings/fit_attributes.py @@ -38,6 +38,15 @@ Pandas Series with the feature importance (comes from step 2) """.rstrip() +_feature_importances_std_docstring = """feature_importances_: + Pandas Series with the standard deviation of the feature importance. + """.rstrip() + _performance_drifts_docstring = """performance_drifts_: Dictionary with the performance drift per examined feature (comes from step 5). """.rstrip() + +_performance_drifts_std_docstring = """performance_drifts_: + Dictionary with the performance drift's standard deviation of the + examined feature (comes from step 5). + """.rstrip() diff --git a/feature_engine/selection/base_recursive_selector.py b/feature_engine/selection/base_recursive_selector.py index 879605da4..eb2a9b61a 100644 --- a/feature_engine/selection/base_recursive_selector.py +++ b/feature_engine/selection/base_recursive_selector.py @@ -78,8 +78,8 @@ class BaseRecursiveSelector(BaseSelector): feature_importances_: Pandas Series with the feature importance (comes from step 2) - performance_drifts_: - Dictionary with the performance drift per examined feature (comes from step 5). + feature_importances_std_: + Pandas Series with the standard deviation of the feature importance. features_to_drop_: List with the features to remove from the dataset. diff --git a/feature_engine/selection/recursive_feature_addition.py b/feature_engine/selection/recursive_feature_addition.py index 564f1ee3b..bf7b83566 100644 --- a/feature_engine/selection/recursive_feature_addition.py +++ b/feature_engine/selection/recursive_feature_addition.py @@ -3,9 +3,11 @@ from feature_engine._docstrings.fit_attributes import ( _feature_importances_docstring, + _feature_importances_std_docstring, _feature_names_in_docstring, _n_features_in_docstring, _performance_drifts_docstring, + _performance_drifts_std_docstring, ) from feature_engine._docstrings.init_parameters.selection import ( _confirm_variables_docstring, @@ -37,7 +39,9 @@ confirm_variables=_confirm_variables_docstring, initial_model_performance_=_initial_model_performance_docstring, feature_importances_=_feature_importances_docstring, + feature_importances_std_=_feature_importances_std_docstring, performance_drifts_=_performance_drifts_docstring, + performance_drifts_std_=_performance_drifts_std_docstring, features_to_drop_=_features_to_drop_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -91,8 +95,12 @@ class RecursiveFeatureAddition(BaseRecursiveSelector): {feature_importances_} + {feature_importances_std_} + {performance_drifts_} + {performance_drifts_std_} + {features_to_drop_} {variables_} diff --git a/feature_engine/selection/recursive_feature_elimination.py b/feature_engine/selection/recursive_feature_elimination.py index 1f4a482f1..978e63aba 100644 --- a/feature_engine/selection/recursive_feature_elimination.py +++ b/feature_engine/selection/recursive_feature_elimination.py @@ -3,9 +3,11 @@ from feature_engine._docstrings.fit_attributes import ( _feature_importances_docstring, + _feature_importances_std_docstring, _feature_names_in_docstring, _n_features_in_docstring, _performance_drifts_docstring, + _performance_drifts_std_docstring, ) from feature_engine._docstrings.init_parameters.selection import ( _confirm_variables_docstring, @@ -37,7 +39,9 @@ confirm_variables=_confirm_variables_docstring, initial_model_performance_=_initial_model_performance_docstring, feature_importances_=_feature_importances_docstring, + feature_importances_std_=_feature_importances_std_docstring, performance_drifts_=_performance_drifts_docstring, + performance_drifts_std_=_performance_drifts_std_docstring, features_to_drop_=_features_to_drop_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -92,8 +96,12 @@ class RecursiveFeatureElimination(BaseRecursiveSelector): {feature_importances_} + {feature_importances_std_} + {performance_drifts_} + {performance_drifts_std_} + {features_to_drop_} {variables_} From aece4cce38aa0c8b52e233acfcc6ac7b4f7550ec Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 24 Aug 2024 16:04:38 +0200 Subject: [PATCH 3/4] expand user guide rfa --- docs/images/rfa_linreg_imp.png | Bin 0 -> 21562 bytes docs/images/rfa_perf_drifts.png | Bin 0 -> 22528 bytes .../selection/RecursiveFeatureAddition.rst | 277 ++++++++++++++---- 3 files changed, 227 insertions(+), 50 deletions(-) create mode 100644 docs/images/rfa_linreg_imp.png create mode 100644 docs/images/rfa_perf_drifts.png diff --git a/docs/images/rfa_linreg_imp.png b/docs/images/rfa_linreg_imp.png new file mode 100644 index 0000000000000000000000000000000000000000..62b2da7d2a9bad4b28ce3dfa330c9ec9115fe25d GIT binary patch literal 21562 zcmd742{hMhyFUIwBt>K%GBuEjG#Sc}u|b7IDI^VsOeL8kM8*`&$P_Xc$vlPTB2$zx z855a8_+5{^-}k)dto1wRtl#>-YyJ1yJDa{f!}Hw3bzk>&Ki9MmXfiYLF;Nu7yl1zX zE=AF*Q54N5Mh5((%4+dWe35fdKjLu6_Kd?N)AOgP{iY6f=WHF$S)3GhJbnIxg{{pN zNf}A$4Z`LQ4t5vhrKGI?`2k7W^Jk@6m>B(VktKG!k6xgt<)-96nk1z}3yM0FxJPZ5 zp3{RLUoRa$)Hts+Y*5bVDj^);&(9%tI4k6__)vbJZl%ub^S~6#6k(g>lp*=Y4b3t7 z*7X6d*+cmTkx6>=&9TC}&bzX- zEGjB`LpmXADfyqx+Zphu=naMchc6Q04Qa;~`>J@kP7d)FaJU5jafzOu_z`%%`!(;F z+{FZK@lB^+hs#U^1_r9;+I(#v99)JE`!41F?rJ^N;k{f@dAz@V)LkL3C^5gdLC-0C zy#b5+%)~ZF-l;2IN!MS_RorN?F85h7_2hk5b*S0v@O4hpV;03bOCBV|$DhErvTnaK zv-ZWfZMLivAN>4uF}v7-7gw42KjvSwGP@m~s2M6~icdYa>$sA&PedYHE7*v#xG#TE zqOhr28Ri5Tg?yVD?b)=gJ31!z zTU9W>wnSi0F^A1HU#b8n2DjFwVUcGE&M*H~DBS&}&d>U&fi`KuKl;5_E^ZWPj^nwQh z`jwLN#iOJ%cv^~FW%y)lmRVU@H8nTK8WlJ`cNu@IAHDCkWQ=WFvA9X`&J7zjw15A8 zb#8X1t7Lxe=Csz`&GcPW!E}_9v-2Lmo}sS=e%Q@jG1@1-=3kU+nD3XKUw7z{a*TF+ zbBzo-RSbZ*nOtFs*2&#&sOylCxlbe(mIBQ zLXr<9sxZdX8>H!*y}ctnJUr~VH$f#r;_#+RLtg}DIhg3LE*A<3=9h^z%(28)9mzUL zzxURLC6vFv|8?V*!N&c!CwsaZCnkQh)Ee(}b#)yZXk5N--8wnrr$38lSYUPD5fPl^ zP2%^k4}L8$NC}l}IcS`2TomyA-Tex@5$v{Xtnbt4m%z9;E>GG*cU&5JXx;KGGbcyg z&Mq^1gKO~~4Go3|4<3BXFewoj*t2)<6<=T92IG6#*;{@5{Nhel2H?5y;H77y-LK+@ zY1NMs5=`robi|K7S$)phx+cp^^Uv&Lo2}c*C%OAY7TuCGk|-*XlG<_p8{<`$l}y~+ z+~MmEX}AnNIsGR7$BVCDS5{nQeq;Yq-u1wTgnhR^W?LLOa`b2n?$h+-^irz*W!S1D zlM=-Z8#n4C?ByJ6E1Az37gTiP`B^k6NIf(u@z5>GnDa0xaAbaWZyS00y|*QUU%q@P z@F8KqDkY^*@!;6!?`>mKzh-`x#K8i@k3CD(8%@%Q8~!tOA@zHhowLHWCP z%v@Yt;)fnByK?2q^*eXO$6r~q)O;a}Y-+b+-7TdvoMT|}M0dNM9qvI|D{{q0@v z@NkX|CdC55eA~+5y9r9dGZQ~5$el7wl%D71b@bP#9LYPshM34rv4bix+7IDW&z(ot zdi?HKyx~ZuvZ3ay8*4^BD9!g8U78&Fs*|j{=IsNyLrlzD&b%%C_UdL$oaS=3U$4Y| z_e5XLbQ(Fi)uMJ0Jm7Llg##W^Bg438xMY5IxzOc5(%;`l2C}S_KT#RL`TNi42Wf|1 zZK^c&=+ln~Gy(;eo<4a(BHUKx^P4ps7;DUZAtFF%Qu;!$*y-5W0Mb#I&1$*Eu6JeyB3%3m4^ zk~GSvD9gGe8s*%{p3@30W1hEfbDw#4&lfvm3eT~#w=a7jcOhPpjg9Tj=F?1k+t2yK zUCY0G5lB1`SzhM7xaRTU^;EvCQr@-TVD+bG-d(?Y_u3NnwWZC?eCrL;eG(E*hZIkz zif!4#?&alWUYEp%_g%VveTu+`vuDm+eIL2QY@i|S_w+!9sf~@~#+G9D9ksQ!tQ;JR z+A9U~VLF|mN^_+TcX_w}_!0c(wv<`f6?(D}&urTy6((*TR4gNr$+*ZR#=5*6h2bhR9lHxy_UNvNG}kD=Vv~ zi0ajLgzh87ZgTG*$a$MphgkgJbp7>hNpr4k=);G?4nJF0;^vy#+N|4czW9IqsJ%>Z zCl7q0{I-e5H5jnS+O-VNo;}NJ7{FsZtb$-mLoxGjuQN2kYt|&};~wj+ zp*!~MT-ooDE=0$`?bgk`v(3fpjh;ChDVc1yZ4H8LF5}(2=>7ZmvhLGcxulGP5W4yA z%h)y;W=OVlrL=ZGwIw@<4Mp@1(uvdPXf1N>txIM{h|9U+@XX@VqcZFdUP9f_kniog zclJnN*%qG|E)F!P!+65?u4za&3PkMdnryqQiMOCASVPH1d=tJO>-I>EsJW;vgLreoc87#G(}4nOt3Gj#mqDbNe9|i>_Hize+R2=jqce z4H+h(*uIl*Zf)%SlJ``9{`1qbs>ymM2b;2XYiX&SI<@+Wx3_(NU9wDZ!_)!om`ukX zhBU&B_gPhTx=v`x%F6nC(Ji)Aklemqzp16gDkm{M{+rXBkjL*`6e}`tM04?4y&i^Z2O5u)@6?Tol6hCM+c5w##!< zV%1g=`x__4W^#VCm6YhW!RD8qJ$trqF6~&Zk8UzY>RM~@ygbl=7o%&JoILrf+n zCWh8i_F{i{9hZ=h)ZmXFIjv&u4L4^!wC-#Ysh-9Y`6MPLIw;M!w`Q67>6bjSZVC9+ z{bt{!%(*6E-1;|CP0n=Mkt`L8wK`-ay9%RyM>#!>GVEV3#RS3Ums^`Lz49{t?OG`t zT??%1lEPlTeH-4~)z#&jmzQT=|2U$-HT%@76*S6jqwi#Nk0F52QzN+QKI_Y~|kLACuM%sYek|&E3xw+PJ=bSKu*`kdUwh>oL;m`tw>)(6Zz<;EY974RT)ckB>2` zw4OjRg$+(y`GS-5l5a3BTNduo__TVkyL$~%$hw@*EiFAw#wLz)f1GKQ&$Sj#<#bJq zjd6&Gh}6ywezNI>JZRS|6jju-Bu24aR?EZ@vE~qB#jmYk=_X`bO{%3YJVP<@waWS>k zwpHNtn_KT{JdimAK1jpo`g#Ucjr;ogYGeG`JZ6~Ju3gI`cRs$Mt15_>PGphe&(^g1 zR;&kHvv((hdt0=DNejZ&0;Y4k4Xn44FPOp9)Ku;}`>CJSwP$Buy|XQuy~@DMw|m~2 zR%&8xvCHlb>xM0-US4lZ=XV;JyCtqet28@yvu@(sEB3*b+~q>|Hy5lgsWWz`1q!)3>~EH+LhDf>}Do^YPz_{tEj5xw8p_t%iq73 ztnQ%a=jXS`;HLDFb?I4HScWGiZauSZA8*N1Ja_Kg@JO}LzR7p*-sxl*E2Q<*kLPUj zk+B|~b}KIYGxJD4W#nhcd@N!(tI9XoRF>Xn3dQOAMpiSUACyLr#!cHxy$spkczb(y z^!DBv$cGDLS6zTnf62eNl*D1|g;?6YH2paBKrUIAu_T8hEoEgiBu62Qo_8M_R8U50 zd^J9N%GhJ(hfH0H!NCE$nBZV$ERqipPzR!i;Vd>NG3U07YiM_YO~utx)ib_YxcK=< zJ@byRh*4-k41E zH?^s?P4?7FpS^rU(fQXRM@Pr!7kXI2_jX&U-c@WlXfxDSk}>Wldkpxv)k5LXqeppy zd`=O`$?NhRe*|C+1uZNr07jegFRtIbd2=>x4Wf+Q?DRzomd2DPCO@%rh{CZdzDyY_ zKeV`9y#KQGjIp~1LRz?#iDLEk!;iG?3iuuP@#DwE*7xHPEz?LU_IC38r)_6G-=2*!NIfF0$?Uwc_`S7JY2d@~+b>?I z$;!)%Zr;2C9tnu7j7p>Cne{Q?ZO!SOW50ft_twNoU!HS2_vJYoK)wb7)(C8+$NBjN zSAjhD?S+4wr+(e7&2Y614)#j&e|cbvfBQMl`}dE9bW6?q5uk+tE`UT!s<9JhW^}n!D9g0A z#zyYZo(~_!<&2p*rqtW4I}g(rMGpKbq@QXoP8LK$$K5|ZIlXW7>eVOy{2oa%%#lFN zGXiin_M=6n`saPc7PDtprU&0x8BxRCw@tn~%lQtDj{_oLC8`kCGcWx#cAM(HeP_#A zmS;9!CEl?E2ndq0+r(pv9(G%aQb5|OVZ(!+&bmJz-r+X$&@Wj+r^6OD8=VS}O-oDr zYj6@yE+A`10mEV2ZZLX~dP`U2`n+ zfwHLG^AXN2E+^;bW}PMm?=~36mds7>Gn`xNT=ZMjF%eqO*eBb!jejw6bE4uPR>q`8{TYt znkdTi^x2?wC&IRQL+ahSDBw@5%N{Y)LJL@+LPq{M5{1i=NeHxlQC&?C4GM>=9R5@( zHn6z3nA9*KLP{ZkLqhBIQ%gxLD4=OmA~Wq~%*FLURZndZ0|R0l%a*h6)ig9TLb^Yn zy?Q$Fr@<#j8_jKEfAHojz?bPUlI#lnbS|xx3T1H8X?Z1}4YcTZN1+_SY@h zo3H|=_ijYUcEZKstH-Ze{0=sJMdHEe2A8*3?40SM0T*LUI5r|sQPK7MtYyeD_Z?qt2+#-ry@6bzu zD%erF4mFe*TGh}Y^6pHhIE?C&80VjS;J-UV5>-nXhf|d=b*YMPxs10_R zo12F?i{)iyXZrvIcue<+>7^g%1%;AiTqOV4AR~QrCN3p_b3NRmxD+7$<8ym6hrU{R zi3p~%?;l{{cAy$d7@fFz`-9?)*0f6id`VCKN9(6ghpeJr6{Pp)gJ;^AKOz2qN0N+Kjaa{^%B>G!b6;Naje5_!*n8>f-g78#jpkR7CwAV>%` zXDJpd)8?x@wqRFcksYi(#UMiikE)$!xQ&oG;329i{FdAIeiUi?{Mi&40?~*Ug}*nf zE(QyfnUz%rlFR=4Ti&<#WtNu)zVTZoq$q$mrU|PGLr6dkR{G+_BH+?@Pp_grZTA!M zP{u7bo&UBQ+=3~h39P0Hu>N>;UQm?&O%5m?$qVS0T@d z_-*iKswY}E8nx0fhoLV=QL-?Cd?IAU2yE*3L9wXq*7wAX^0T!iA`BA`qJZ(Ip<^JQ zF}1QXL~{XHDeW-$<2&lgh*=CnpE^AcR2n%qLG)q^GBg?%cVvy00I#4||+O@Cu-d zazp~X$9j86A~8&S_C06lTy>}rH~fQeiiR2Y(|;Z`&ak%0K>0!Nek_U{l?2@-nwpvf zg=g8eDW)mrpHp(qbLhJPMkhHlGm{m2!iBPc-Krt=RSuUCsZPGF%{`Ru>tuf~%g;|U z{kHncTTxMxSin_ynYBe+pm(pdVA!NCnMA9Z+;nwD13t^)-h1}2@r zt;7O^lIMP&pC5qg++XZql=-*??wzEcW*Ac4|8%zJJN%1HQBkqHu1;@&<4-47HrUcBGfJ=zUWCn_wg+|<;2gQ)G71Y)xRXcKlZY5bZ2_BQUCbiDs!}LhdHKt8 z`&gJ@c|(If**E}$0HFIlAM7WGjA}|QU)G6Hp$9e>0ez?je&hx0G^b&_-(nI~S};G2NSK7i zxGiaPUSa2+sYJ#fZ@}tHDC!4_HGuyQ=Si@Uy{$z% z9z;gk4}FmX!NgFXVxWmY7HkwXxQP`z!@Dc*e0O_ChZm9qz#0vrRPU$61 zR)Xh&DpHx+lWIBgWwqLqqTFR59H0E2GHp;O8yR3kpg`M8lS; z$vchp>Ug=YTTO1GPpIRPhqt_TFaB&Y%*)PBhfu%DfLuL)H@H2^yyk~)%ON|y?nWQgJCu! zK}VpsPFPz9mzFBWKYFy+ozOVr{f9{v3Eun#0J*gJM_~(RUELK(G4!BV9w@kS0j{n= zX>lcDy}|dH-{cnltiFq=EbEr-eZ?AaY&JGFoeMz?l#&G84E+`%A6Av)1&;eER%F7k z>A}{bW8j<#ct%?bv=VzrNQmghjr;qzc>Hl9Jb0wlJ}oT{O6}yy&K8!RIX7y&8$b&di*4uY3qS}%A?dL@SMV!=)#v8-g=?+Hie0A|F$}HGwZ9WWIXXLOsozA&)8>rIy`ok$-@rNN7 zKmSs&`f1&$(Cp5iukF9ta-py=!~lH!QlydEcNG;~Ul$V(^3{)FR;$`-->C-5dT;KX zJ3MB$$fJ~D{P$jDI2$B#eIcf}vs;PRM9Mn(+Z06fCOAJ&WI_Bc7O%+O8N!^VF4 z=d*-?#QpN+i(PvuP4dQd>z3d(u9TE0QKfBdLLfpbLB%B~E?zF^1F8avFF_H`tFEDO zweBq9OpnitMiq0CXCpUYt)bZz@_+>y0;6RBLN1K@SWp0{-|&7eFatEWzxz4-K(Uq_>2(kp^phyvT(UTm(Bo zmqCIG&Aa2)ii*anBvS8P-h2YH9slW_NwKew52J|j0~aOdU#B|y`-z31e((j;eYN4j zAK>;_KpAwFG=LATVMDLsTZ{2Lr;%7t&S9vBV*#p8P7Ho_sekop=o+Glp!=N1dLPzu zbBk1e5_F1+6aa%pPb-F@R|j6eliT$jG81z#3*t>)P0!CuF_8=LPfz>b6Lesq|E_rU zZ@l!&6JXL&SL5G0%bct?$TzjTbZ`o`SAzfcFUBz}YG6+9fQ&@-iq7LT0y>+sEm&1{ zyZf-B*i}u|O&~w_^rXNP?Hh1jv0??QfWR^&9N@JRrJgjq4;+x}?Lhj|5jmVhSA%9p zrrXqcQd%@-m{j9iR$ks~m4V#*+)-m>nb(M>9?7C54waL$jh2os?A|?z@g*!QRY;}; zC=GteV~00YqGJ&xy_%#dACAuMuN!@}&|-q+25hrx*li6M>YI?Trg&y(2crlaB1GD( zihyOx6Lbk6&qUox_=FiMn*Y;J@DO)fx*GPPzf}gKXm92 zu=gH!V=T%BL>nF#SrT)0bM`B!kxB)XEg_w|zT{mvQsSXNYD=t0&wz{Afd~FrsXi`t z_L~_4XyS-=Nz}ReJ*z zNJ&5Z?%ZU_yvT+PTK%J+&qnV;g^f~{klK9PEPaxa#E%>~vfG`sv(U^Tv?=zsw>f7$ zn1EA3A|$Pe=p|*S>Kz%DSIb`~*(EF*+Zoez+zaR8NptlE#Jpu#gkT z`BOS)!}M&*B_pRTY7Zqu7l~=^valJs*V;Vv&L;7a5tcfB|4#)qm?9x@h(q4Fe}CT( zn0qzWhS^Ynyw|ReJ6Mg1-FVS>d0#Ws*?9%zaig&M!egv1T7FkfREg0D_U@Lmq`Vo_TZa{@3_5 zEby<0R=>AMkXe@X5?WC^c@5H!@4{y-8`FacyR7MsH&vTD@KZTF?cC=lq* zis*E})|nLixo=zSJaCfiR?x1W%46UF(`H~f&(C)=0b>#A1Apo!V&zw2F9|n;Q3}}( z7>`=(A;@8dbpZ?8%=_()5jbm&P{ClZQ#_r0XD><22zVHhHtzNw_b;C(q3(}2i&ZFOpMwV?|@Uh)Ll^tNSf0DfM z(U<1Sw*O^eNwpTsb$&GGU|}6x%F&K4I<$+tH2wZWynsM}jf?9lobJM_{pPZo}T@1%VVxLF0{R`?;2!H zak>e6v4-Be0!iGGV1$vOC?*a*$-mXy6Fm-|Y`iV@I(|}Q3&Nj|kIxBUy^rQwr*zoK zWxZ;5u%?-m2tqgWd$?k`PQHVM&(*7);6vVqHZSbqi(~sAw-!2Y>Fn&>bm6-ekU2A& zU&`QDAVsMAlN5$eiJKISU%LAlY9we-nhGvSL0_;4oEuh%=gD^R0JI|wUg8ilA-t;~ zi)tqxSV_on;OeWXB$p&9_qyNuj0iD|z;A|}xoO)rb+lFq5e^`}TB)DB;pP{cA;bgzXC&4-a4!A1-)uhy|v)jBjB2s)sMR0jNE z46;3hGYlScQ)j)ocjFe}ZDTBI1W_+E|686}d$ZCK)E%oHxXFFm;a@%ksui^XEv2Te z9%I#*4gyC7ERP*}P5<&CK|w*1g8{R^KYk=?i&B`~36yNZw1n7&K7t2g0Tg>a85vS@ ze?|{l2m4I3iXB$;FDeXm{> z#a}}RMJPwb-|cji6idR?^fX`gb5Jks*bF~VsOX*q<3UYf2X+n);nOG>Pl7=qiY$=o z^w?<-<)YiR9qwlr5C{T`cM?WMgaRRg3IV&k8F;hxB?SK~V2q`Wg-NUWv8baK8EG%0 z%Rcvc#oq2ft;5A`&JcuEJgj_ocgtd|!;u2Vw7TzT{LXIhDMdvh4TB)cJjxU8PA+g? z=*RCsi9z~ZSksQTcef0Wj?z(t9wlrua0K*bHXrYi*Y2%xU0#Do(gDQ=OsN9JgC%Kt zPW)`M{n+U-J8nRB7d{FN%q|2X5{fOH|NGkUdO0h#Xwf3&WW7#YIWJvi<3{E^d-fR0 z5Q8g*S-{ou-k0D+Kcazdh}E-V=TC*RNlzAoDOR7d-c|6S;?8NJuDcyREHlAMJJE zAjBCA3 znT4jB+9HS>5>BJK2J5BZFOI!_ot2VO*$w@d>6tUPY}HTm zZ?oheGA5FR;iW`mqo02K8i)y63RMhx+S2e+5_FA?jY(+%ZlxyqkQgO3A)aIN)lk0f z7CH^UOsmAj{mTo!TQU~4FHTezO}4r=J+tOt3(ShV zRSs;26;Ok)u&(Fn2Ssngv?>$}!()Aiwp!LP>gebga$iF@Tt!5Wz+$kaSh(DgZ1b?P zeiRh3vHOuQe|_6C)?dGL@@JbLtBTcKphtcETV(-so@j%Spy(`VAkJ|I;@3w9BJZ6Q z5)%`1Ll=M{jTlM5@*=t#wBkC@1VAA|n4-KWdksV`z3|?Z{-BF|Hy(TT5MskAo#l8t z9j{L_@@fFy)yQX~aa`C((YTLr9+IEVyyXG9FK=v=ss4Osn`PamBlaZB(gSkTviOb8&D=koYoB5FPCJx7vzX@1cBzBmt6hflmxtX68 zkU`=F7Y}!)D0obDb9-~aVAS+j;YJ`Es=<7bun!B+ySL?Ad%Gv3#(epTc(_=%#7n}4 zg-+5WX-gIVriGqDw;(!N2(3dHF>d6~bY+unJwKHJQa8Oos4%~bt`4^8%=OrXmAL_Y2d+H{zY#Kl|LMYi|A3HvEB*B( zIDs0@XDfO9;d%7vrvsM~UO!9+#)c})_wN{F>59^XG}I9OYyVjkT|h?J2^RM7$r{vxnGXrl) z(op{2M!b%77Rp0Xa=d-}c6cD8r1+UFx=YNuJPW2!vyPRtU$e5ZkOQ257+eDXh`N^d z@n29T=8$l$`BtndS!01-hvuT2H`(WACruWtegqnIw~c?n=sphr+E1Ut!owF)q>u$q z-!{xhEHzd&QT5E++#Heb^*5v~6H)D)k7 z!&R4V)bh{T?Zaj~t?KUGj1WUJ;z6N(=>#c2AYv9u2!Pb2Qt@{i+vSRfZK}K14%Q|| zubrJY5qs-;o+JEwek8Mttz$v%<3RtUq)!NNa3z1}Yfm7CWj=eR0oiD}NeM)ajt|j` znbKKB-nO{bbvYZg)#eKi!0HxPAm(%kP=D2*(7|55*4@Ye|9@j`MVYGjZT$|@?f>BsY~0HXpTY*w<| zF!4m}mrF=n^5{1M*wSW!qDfYDb#+rbEsgi}jBg=zYS}$O- zq{@gxHA;CR6O`lA0J~}s?viqYR5-YdAHof)bzd00DKtDp1x?&3&4xs2K2!vYDAf1J zm@h~^Jo&=kjail2gg}V}k*szhRsdCF+61g17Pb06TMcN|`=+F%)QN4{#0o$Nxu|9O zz`p-Pt3}e)pob3z@TQe45zXdU)}NoPZM-zz6{^$;Fned4m43)+gr`k1GPl$H!r#@` zbK*PQ=-U$jC38yhN)v;^rxhrcgidr_|7^YUd((aSg8TH<)m*{tZxuV`2LjZ?; z@Wm&cBRI^>Z6_qXbRhPivh4(&1Kh%8zlX%m1C`}tn7A~jX8(*3DM9m)75(hFw_o}D;e-{7#$r|1%euTilT%? zKvr1cJD%8SxSa+Vl7>P+g{pHiJS{;HUg-^K4xC}xKR{vhX?>Kp$Meq9bFSBadvOY_ zZ0*N->%2J`C}_AL%*fZj4|exKWMskwI`$_Z*+!Xm1y_Mb(g1Z5F+QY4253I_LTdqH zm{6xsoSQ;&j%EWx+HJf?z}m=3)*R~=E`&!7OH0d;(;uRhG$Fu-s2mlBL)M8h+HBi4 zL0~X-Fh68a2k~m}oBdi^nHPEvK(kExHDp9!daTbI(2>h2jf7#@zCu4}UiJ`)pM*hr z5TWk?%qdSI@W7mcI^3ifIxDyJTU+pVIYY;e968Zfn{Wg#9ThqBzoy2 zTo68g@nR+DE@{*rME0S=NJ*vd;o#x%$GUu%EsFH-HA?!6S9Y@Z(npQA$^dnAgt5Wn z_pP6Pfmixc?9Q*kNDwnRa#M$e7Q*7EnUoz=X-y3qG$T-VTdjC7VJgSoP^cR=fdts?l zNSnprX8+v`&lk9`|A(eC_nFnAE=&K;a0-W~KQAOWF8PeE)vXTsUVj&a_Qx+~4A|Q7K*`)&mG)=j-Ldg1r??^kOs2R-X zOA_(u_jj@wnG#fR@d2~6`wbb6!3E|lMun!kSeay=?41d>9_A8P^v1HJFxzhBn_65a5_ zZFerizd|%*GIXI=$jYXu6#4CDBxSG}7%nn+L&hv`HDb~adrG<^b&fEFole6f8ZXr9 z$3eNHd#xLd`m_vUN<>fkEvzJq)-jC>676>fYX54fl0US^jvcdjABvVN_$yOe+tB}O z6sCxQKo&xO(Xuz)&G{1NeK8|x8btI;KafmX8IDN;}w^3 zZBRyXs6mxVI{e@=5KvRHzd$7h9fJ?rJ;)_QiA>tsD5W1uUnS5kRzHPs{o3N$>1Q<; zar{zHQ}LIX$jZzth4?TFsCeg$lZ#7dZkxxJQxc)9X3LJ(~_8g4xn1~XfPssB< z5LzHp5ZQW>jaN0M^{wo4VnZ^Z3ZE)Z}xW^)z|0!1e@47G}DLecFL_CZZv4we% zd-ewNJV}1uC9(zE@ZV1bN1X;kpl(Rsra$pt-sb(l^M$w3sVP^$`usx|r0$P4x=IZC zTAqe!+a2X98UB6U;Hc!krkfvr@iCGg{}i9NSIPBY$d%eJDv8x!?|&Ui>%Xydh;&Cp zKbsNtP%LfYKQ<9{f3OkD!lSYMcMq6B_UXyPt+}2(zay(8Fz@MgFBED_R43xE;SYp#c1#VvsZrXw^ElNW zEYs@Lg+BtyHm+U!J^1J@E?57tZFmB-S`70uYRs=+zfPfX4I0Dk=vnlRAzfr$kb@Pt z9o5_=&eA2$Vn)e^l*h{k(TEHF^&N}Rl7DK;vNAT%>Rel~=ETX9SHMH2P2dlLDtjS` z0qL0CjL!A1r)t+CBaOmZpFidFv*#c$q-m-vP(^Ws{`H-oBp~v|ezCwk^&ldXj^#=c zM44l;m5+$#5xq20Nt2a{?&zQhtta$Nt5&aGR+|*o@uGgf$tV2OwRnEjzd8gs_Cq(3d=K(iALMRs?+`BOVgH z`zq*mMlk-sTd?}5^Pk_+goXHnY8n%PjfR-oAycTU4o^5@#TKYXv<`_1oQzF^;N>sa z5NTBC#0Fll6nQZI8A!x9Uf-7;a2*?y!` zeU-tK3*c;L^yN7~VxrIl=Qdi%c?KDh3;7u100)j38U~PG#h~l%H)dvLfiJd4SOaMj zMd~7Y`fC}*(|g^<>kSsfNf5K5Z}g(6yKl9Ef)TpJ`e;oDVp+(Ymtu4(>wNcKv)hku z!p(bJ4oO51LC(;2jHjA;*bAy}Pl28S7f)-U_ z6gl($!8)L?N;F+7arsKHnWWf+?)_OMhq5JP|Hzk@gqk`=q5}<@J?#)d; zN9OT;mavm))U*jyAtxashK@WYOyZLX;vldX11d}bjL5xoi^AV96;0LHBZ8;khr)w zB4g8Ufh5-l-+)wh2y0P2Wo^9?RD49-ESwUm@McENJ}F{22%T8h5opf|l4S%K%>vOt z42KNIl4Atiz#JO>K;_&C=1Al5UM1*qJ7ER2?jV2CiW(qn;zq}?N_@W*sintrtyZD;~1r;)b!qN1XP^NSZR-s$}71y4fJ&ELPB2qcBtv^Jm95!Hd=q?pzn zPms$37}_caunyL%Z|Q5wT7@0C6*o&JCxFd@wZrCcqy!;mJAqfp*0BgF24VafIEx;e zxy{kpmIGuOCInXk&+o!!1%CVa1l&`Dn>j?!pbEWUV_yzp!lRUAI~_y!DTG)!Q<(^d@+wP6SLimuC7KPA2uR zJHlcaOKD$WdU zA&O1(_VAc@5G({lS3P(9!KbUMi>Z>^1HV9a?L_{7fzThn>tP&u;Si*`U`&+rm8_e$ z!Xg$yb0O7R+qxH8SR#%O!s{S8$|?R{Iy#dD8VruT2s3^!FHS)mk$JL`31eW~cBgH7 z>Ydj+>xgqahjK{Kz|85dNyjQlRjJE=Mivv21l`&WEXVN3h_vG#+iZx7Wq0i8alT8k zgenA3NF8{nITV(eXWsEaa7Tx_feh?ICP-RNX6P%bi#DBxj`#(JjqH#IARi_JW2Br# zcs*U0&*LA|i$fz=RZ4Ll1!Xrs=YFo`88eC*GJ_G8^2x}!i|DlwUhz#N#$(<`D}`b+ z_zJ>v^^dm(-adUM!zv#aqmcuWklASTFm>r zQ3JMm0Xzz^V!~Eu2RM2s4glG7{$l=mmKHsG9&c8l7RU(SjyqyT)PD5)e&``BvzxI- zFF$`iny>#B*H*(Z8)|4zLWy$|6OfO|8rkz!E6x8|g-pF3s&LU~=RS*)7GK&i9c?TT zo_{HLCTY|1MSbJZn@q2u)4&E6ZM&nh{K&)v7b;G674zu*7jO=OsQsBk&pI>r1F16V z?5u)6l9~Fw3A*Ujq|M+ff@MwOL7@?(LfVYrn~#IV*Q`;$*PaW8mqZb$yh+6mv;tom z3V0(6)uR_uh$$u`Db-JFYb_+=le$! z@%-%G(DKK!=MDkYDUa|}BbIG~G(Cu4h8#jeM?Fw<=L3VGiWEi;@W9(?-P^hZeTj`9 zo1L1A*bb*2VZ)wk;(Q=D->Q{#A<*$fMn?X{(JP1XVVFaYr_#waNr_nL!x{3 ze*6jlRC?*;p0HDRSTchn9#&)CugsXMB zi7xECwb)v6R1g-tPvyyNDH9D)xsF3$u0g;>POt&^ssvVG%EmEXvGb$(fizt8OQ{

A}^ZOtl+H#PoHVuMKU^jX<*>=M1TFh0g2A4 zQf0O9r8+)R47+_*;qP|eUa}E+DjM2Vp!vx={=sk>8TiKb1RIHl)sW7}a>1P+r_<%y zD?i6ql|xc*`~(q|G$_Eixt;Lyhuf6U zL9?iuH~7uDgJ`TN&W;^|43Gwsy;aCXH8HA-gZQ>F0ftpch6ddE>k{tMeTh(Is-Vk| znwo0Y{hE`c3Y22xXfUKAku9YSc1mZ{kpFc)aq95o|8y)>V7imfxCv=F~7xBdY(JpeicpL>3rCs9T z4zX1ytc9pL06cB$n!ygA`q3RoBf-IxYFxyJ%V~RQLlo`K19)dnv_E3U!F^o0vX}~8mhda zf`&{71#)e?f}B0nS`>&ml}?CxDv&cjVv=^l@#m~C-P*~upPQO`4_A<4#sIf?M8BJ@zEFY80vkTftWM4LRc5h4rRTOVh8+(uaNz!i4yO5}V z1dT}MsGZJNh|(QQ1f>m`9L}0!j&`58%i$0GQbIL3TaOIpARuI&`@9*2COMA_ZA!45 zwucR%G|zYF6D6nQR2Mpr#&5^zD8#36^KA&G!|3J`2S(n$0{$*BvF8{XTEDnKkZ}^w z_G$Ww5rR2_Jspx^1tjh96RD{iFbzh|{{i*XRB~B~v<65+3aw4ZgzF6F(g9-Qqw6iV zk#!`5E?!*M^~{;IU|(no?+CXd#gLOjKnQ3MUKYsVQo6VQvOu(onAYHadZ|ZrQZ~xT z@q!sKbTxm11M=Dl!O>Q%aCI5)6`b!=`s&pZ0tHFkKQ=bj$RLUn2n|YJiLY9U3#8U4%i4;dCbFiuKBB^w)6bfFP8*#`(-FXHXEB2>vSv+p&NwDH${}JR zr`^FxqGrh)bd9D^M_j0)3c@5g*9+oTJau2rhl6VLWW+Ea2;B?P6$EG_y4Qz#5`>K) z)e|;U6NiM6Id|lqIFvu6P|#035;HD(&YiXJyPYmN`%V(7wmvqzx%x`y~@hSPw^WIL2Nce*$xArU$WYx)hPvNoatA{Z@& z=)M?+M}iVLxCIgI;=K!Fuf(EH03<=-xe=aE#*T?ULxD$8Xa?XH%P{~$ZizaUoRPD? z*;`%B*IT8~<>Ld8bGy&{mSTck{C_)i)H+YMwsg3Bv6dZ<@ua9d>Ic-)RZd*_KX$JYGNNHC4J&0N zdyC)kbah?#{kgyQXMMka{CeDXUFH3LjpsPe<2;VzeEMk|RApse$xKlc>s~b_EsCO5 zpeUL;CI( z-@L}m#^%B$DN#|&zds;ib@7boM`k7;Tx8(|wPTkkip_-lpC(>D&YYrjE%z$z*1mkd z@3Vt`K=+*dK+~>-x<}qx>Uw=b%K}`kR%|bO%`I_TlDkA~PbN1r|MhiFg8R5nZ1xoQ z+-$O_*j_9%P1xzwUG;7LZ%3y~QiiRy?+0I!YH*O{3mk6MA0NvG3O zK}ks|Ml~i!#NF4|H$Fz3;^5#2`eH%};8$J=g`X-p`q1EKz9Wn2*zv<|9)>meS*`S< z`LEh@lP{|Ei%~&uUFLp`U$}J1)5C*irKDLz@5Rd4@fJ6}{(`xD^S_G3w##y zXjvF#o#(M=dcwMA&z_^lj+y6{>F!;-WdB2%V}ml=w=c)1_U_$lksRkXH+#X_y2i{> zSu>+-_SbaDr%$;`7LHsZA|hQa1#`#03qNk@`C%0{)?UhTokx24!Gi~Hs>gOUCf_^Q z6HyyTr{yq~bO2GEVdyf44e9GN}jxp+DXXTw8R=lpiMs-Je z8hRfzK6vne_3+`tF^?Vn2FlMe@$vE9xpOB%D^{Ip(V~|Rr0s9rcEak>EO2$XFJ;S|te0^mAz{<{{dXuo zr7I0dx`M*O`>b=eV09E$4NqVhUa+_K*}vZN+BFt@$o>BP)v>lOva=)hg^Ij?c81N_ z#l^(b^a`Kr#BT8$r|KRZxFd0jc{LA@W}2z8P5&33?(Xi2SFehEIRrIg)j99nyC=AF zXLezZE>^>K%jO965TTo|PR1Y462aUZX|;Wo(pC{9XjU7u5O3c0?c3FivrWtI?AWq} zRZ#VY7oOpy#EL|nq^qvKrrK(vRs9_+qfQbV1|9V zFMRaq(Fp5~vI#7QWpze8bhmEZ8fgp@HOjw^bsUW;5Zk`}YD$Wz-H-R05f86%iAG_r z#KgsSXIM0dSvIFGoFWrV>#3Xe*49O;d@kC?&&fKe ztzF{D$mZHHrSsX%oBhsq%as@u1}rG&fg;B2%Y&woz1kTqDSwl2>9t zYceg5>le7@6%OsYz1id5y?eECGnv-Phyi%-otFpJh#2O^PfUL6FTePFwWg+Kylx6F zzGiafj8Bm#V@HeIoQ~eRl!96Rw-2OCii&6??k@WI^CzN`f}!C1G^y85Z7K&%E8}!t69NAueV2!?&6_xhd*nRIaE$?X= z=swZkonn|Lc6q>f(Mm}sjJ($VYuB%P`S~qW+_T5}*OcvMy>zClSFa+#O(0N+I}Dv% zE$_NIEiLVWtu3bbj_HR}RktGz^BlXHGlYe-=$v1QM-5VKAK^{IW^wThMicSq{~L#ynELVtDQX3MEf<->Ud2-5__cWeiY)NO-`Jk zd#-=uiNT9@?1YJ-DpAe2gFN@;-FSz;zbnTEO~YXBn7w}FiM}5uCHmO2)7;{w3(kMZ z^}~0fu*fc4yeK5*rlhps`t|E#jw5DYzkXdd^-5Mqi1}>OlhRDfkE!QANMI+H)YUER z&l~5xx@g4}L<;V3=b?pEVO3R9Mf1~V&uVgP&whGg&xUoTGTPH%{o}p(bv}7sHeMMG zCmH2)=Qih0w6l=+=_n8M2?^nRdw-W_cJ@x^*~#zp-?B0?N}ryp@_#(pUV4p0e?$b1 zl`EOIZrysWIep>h9NQAnf|>cdQaA74zs@E%L5UjXYP`;Jp1hcGL+V>+XCy}SG9FLN zY3$s~(o%jInbr72VSU13{vfsLxI=ueAIh#e^iXDZta`|@s@buRRuvT$3gL1tU$R+s z##>TT#rEIZ&iX*gRx$42g9}q5=IK@~yw9IMSMRM;Dme4@{zBvhL=Zt)S)ONS-l_C| z$+N~%iOun0U3C?+WBRosNKIW`G3)$iV@t~bELqOy=Z_yh{`h#-;<~s)s;5t{7Z4Pr zp=^8VX{bfKGTc}nOp6zn?PziKMC{#cn9Gfu6yXk>sw^i?oG>|m{>IIl>=^crLJvA7 z7M49KDh&B^vrbf@eT$3F$cW9SJSSc2#N=$ck1S4B+M_P57`m?7SeD$}Trs3Mgg0zm z9@RkJEl0LEsMv|e%VgrPw@OpkSbbaYH0EIU-_>$?4ON>c_CqSt!%iLJ#%6Xw|^w)JyEZ zWtMN>zQrf6T{|2$-df~I>mF}ZkYDlUjZXS_Ps2X7wM%N!PN>FC-rH^|bpOGF5VC4m z>Q}L|ule-p)2-&Avpbz9QlqvYKjGQME$TNaAII*Y)?}XltY@5ul*+{aGRL;pOIwH& z@j0$`q^rw}Mq#zAV|wA!$%~gRDTfH@D0wolDjhs{Bg$oRsFT*+>~l^=)RwtX>r0n} ztKZF-GqUk&r+fSOuuvpi_-FdC@i9{ZGBUXpVTPO>9Q=r)FR^7pnt^}3Jv=|>^-u#Bv1DHh1B z#s&lz&1e-?<&Q{a3n;Q$l#g3jB&MdPYbT~{#73ue=azHMuC}GoGE9q<0nFYb!&q zqC7M{?vtFn<^KKq?A+W;mo8m8_c4?8#*G{Ab8LmbUff0EGFIOv-m$SU1XhN{yfX3W zB6s&+ZEZCU$$W9PNrX~RQ}flD#t$h<+Wx1ug*^hMf`US1Z%T&FyPlQT`QSF* z-x7#VE#i~YH+S)7{n4OQ;OXfZGWKSspOCXzR=saCDu+?w@IK$C(qF~_WR`S*3r7%w{PFR zS=Fg_=#cLHc4^0vjeu4`H*dQC_;D_FvaYT!M6tB@BMo)-?Ah2s`W@##6~=|T>YYq3 ztZEuXLiA4)N~z+zu778ysdF^kFEWxJv4q*++kun^kAUo1zrTGTe5A!N)J-ZA5O8He z-_&R?yP|Pz%w9kTt)S6eQc~@W$;StVhlRy{H5^W{5)4sinb~jA_;?|uenJ(W_Ax2B z+J=u@u(K=s_U7K;%;cq!zK`R{A2m+RZLtVX%gETmzEiTMzsc>;P*%{K(;r6fdJCpG zg|sinP46O)07M&)EE`#8^m!b3$=7e`hIWK>WFL*N{!uo$Fpc+4(TV+ZvQA?!{W*oj zF0?9mUA-D*`7v|iYw)3J;DS=8ZJYr#zW((wdzY5#7^ehRbpn6A$4?@4zvsHjTls(E4{jjAoPx_gpFfosU^Aws;!kXd`L+pT?y#>Q(@ zZmjkLWI0jnwQypvoUg67ce=4U^5^VvEy+=*C=s9$A)UmHJO`X~T&vag?c0NWX<}mH z0c=|Q`7=*hS=mmPDao>cRdKbwN9v~=lRV>%bl&$zU<}r;UymdphEU;)%0oazWbvvU z=arB!QtkV{m>TR#IK=H9fArJm&-`BQNl8g%a~F0=OYad0?Z9jL(_E)+{_)2ZVDFvM z(r0a)yeIpzT5bjhd-6!zZFI44soNBvWo>U??#;sWMd}?&KmuEZgt(9`%G73b^M`zNNS3x+d4fBsBEaf#?_2;W(zb7{7Qd(WPd ztj6y@eq0Am;xsF`*y&$sThV#(Q?A3|Zu{>&4M|jlO$>qYU7wzBZA>u={F3L?(Nz;I z#JXJ9cFx5%++ozWzQl)(MV}|u&$Zxyt87z>Q8^=@bA*}OU_1REfBf-fXT|IRyl$G? zoU5eOmzjXbx$4M0mjP^%6#Wt6HZ>dN4Q(^hoYtDvOvanXjx`197bF|%v@oI?Bntre z%@hmZj`#_N{OO<4t7R^02o{{}ToD+gndUt7YJ-@VSmCWrhnbxnmnyanc9fSI+5-Sl z5rwNDS!&wMI;AAi4L5?c6BR7UIP6EqXDM zRwn%W4*zmpYGh=j{X~C-xVX6bbO|;+%K{18k**p#2L}gKD;vf7NZR=oncB2v#||!3 zvRhK`-@SV*GVbM2$z;-r=AogXR02eMbUIyHUj=WVfArn$!Og_1H`QlkXq8Iav5L&= z9+}Kc4Ek4T3mAu4ZNJm>TQ5hPtsWDJ8!c6+lg{!zGfyPzb$GHxF)iRrMB^lzf?PUPo#i{K0@o zq-1I;00ze3D|duFsn>zKOaUe4gTGFq7lw#ym&lpdcf{YPonXwd>5j@)RycBG`P*>$ zz%9o!*KFFfaQpV{LY5zki)l$2f_PVxZGD1({nX6NGE8vLg9icI&Njvy4F9oiUv-o+ z%VkGL)9Q%bV5#<~t21MHTp0gpa9`GGIVEQI<0ygAJW@7=SuL(f*{eT&_^<+z{cwUS zkTL=L7&=#1S46}eq^1NuTE;zI^Ze2oGN~kaV=%6*E*}!o88-qZzs@VW5~Vx=%Tl&I zUZ@t0$A5mNcU>OVHU5!%#R`+5udjg2b~dM*d-BRUlD8q%oV~qhBV|t!$U&w71kuc| z@fNd|eAz{;tk%eZ=f32wU|GId0T-(Q_&a+1IGNKH=V4X=52O%Xv-q*W!6HOHjwMU3 z;1AGdtI4b)RQQa4aU!LW?x*5bEZQT&tw~dI1jnzIbLK`Yy!!0fR;|N_(`>r6w3Cjo zZqZBM*myQpSA8aDtI+6>Xva*!+;lg$1Ww&c%Fowe?)y zOD}zUt(jv`)Y~i6Q`?TP?~CVL2u=s-hd)fxTwwdT_Yrqyr%nx6B~4=dql^k%i@$s^ zBqIQFt|oRLR~oibtnGbi`-AByJ3vsf;0Iu}_)$M644oP{efo4e`6i0$VgSbXA3pdE z3|t5b4b|zqq zTK8I4yN)u}w*G!m+$Htdv!zSd?`KEs6&{h8h_XQZS|cZC=sZ%RT3l4L8+&0Pu2qwy zn?jov7E!#{t9qzwtAK#x2C!={D?@~X?%#hLY81RCw94e{*=vX7CS$7W6qQPg#1)j3 zVnO%48|Vm>Z%4LhxHd5Sta^*K@{sWPI!kw8vw90yJ)(k^Q?IpA{6m0j>0Og z>f&O>@&?=?NJv6LqV?<7mX)00jmV5K)|NID9@hH#ee-ydDTz>sh2vSP&wROw2Nj#8U9Z}x9_esQy~ zFw3`Z-zazFUfjAn8`r3{@0t14*v%1vB8o8Mv$M1Q+Y_{zQ6J*@b8Q{?7qoYDkieU6 z(ZEdIymQC*>(`U9wt9Mc#qZu7inU#W$DY*P9lFYGq>BR(!uZUYxcc*W4xj+8=lZxi zSg@erU^@KTY>>08uQ`K`TDxI`aO2>V)Nov$$c7D!yLRpJ|BlBk?P0?xgs@7nDwOH~ zMv=@^@%purv$Kq%qGBtEaiFMzqrG4*Ujpq(9$!m7t|ui+e;WKDe#OMZY|My@i7EW@ zCBLE**b$Kx>6q^N%5BVG8C5|(P#bR_KJx%MO4ZRZr`;Mk!oGVsxy(h}6%ik?YPYj< zZqxyhP(e94UO)~V2U*ZI*34B64V+*B)O%l7R1~8A@z;-oLg4M~T|2)hL?49HvMgM< zTSJ2#@<0u$93^FCehhh0Q4!^ix~-_DM$_7BgM`Eq% ztXi&WPZiMV96NT5Tgql5>TMQ;>-r}KtK?@V1RckIB-}Xzp`fC&vKVxNAAokcSq&|| z41!){a2D6s?2aCurb1M5*{VMd$|3w|H{ZHDO*R@wH|>8d&*3Bgjjs2=jf*=(L?lvL zC+yI}>HULqQo-$OwrTxxYW(B0-H{g}Sd9nTIsU!)_upUrKT!?;fBNDkP8u`N43HN+ z($d6dW@at`jR|}Y@blaKCk4X_DgXr;*PQ~Wkf?*z1Az?9UaWFdwm7-yyLCZ{!>9h6 zD_8XX8SA=aD9caT9Q^$JOl)lB-F0zoeSHD$?i7@?0~MY1AN zp>_Wk+1N=bn=W_Ig&TQSZ9lis?ysjSX}^>rV-5l~bfDlA+dkW(@CiCf>71;rY+Gk1y}iBt zYRJYYw6>%^MAay`cCFH&jG&+=Ke@u{x^9j&t5&TdW!JiO$_cXT*3o1CTj|?@^}n<0 z()zKVpILDC9dCQTePh4}iO5vl4S8t!^5yDmSk6KPHY?r1iK3c`mUQ3crf>;T;Whp9 z$O5>5g1-K0M2eQ22N2WRar=0^j6{{5^%#w(2fpiiczb!RfeL%J@o`b?zEH7~73@fU z{5yAYQ`IR(@{lDA#pj^Ni6KZl*2`dqYOQwgpvuLI5)xat(on!l+*?mCy1^%ZBPM3; z%kuJ$=s>wEK0b`Jw6r$0s^sYmY~&c?fv&)$^A{Hv6G?)Bb>$ink%Qu5D7QY|RG$m2 z0~JZ=JgAxm=Ca6k5UeE@En0Np=ciq`IfJ>m`GpG?6tLj1(`fGQKN964h!?2cE8xYR zBNCPVs-nUjWF2zT=y{o&`??M$TM3$unSMspqUx5Eg4vNB)_@?jeqrUd^_@hQ&Zss=hbs-}0HmzmkCfgHu+`ZVbUFn-i zX>#t|@8_XQfkopgk|peGei3z+p_>3T?q~48UtYq ztbF#xc5KY;Mk}mfro{Y3Qq8nLiebKrKTEZ8Zpg!kkbF+S<3a_ae9m)Joxdv*syW zqJ|0rL2Ce%Ab#p?A*)60c>R<_Ju|0JE9ijs0Cs@T`2B~cKepbJG>@7v6vMZpo~rdF z6<%G;;}5{p2GSad-2RS5km+ALUYZSGP@^1v(b?=N2} zS=R5yr?gG$7b`#$YK|vqKK^C=GIuE1kweVTHWKYPpeYvtl>{OclWDApQbyIYH*&b_ zf_ zobm}*Lzjn#K8q_5=1@(I2@$#tpfEmuUAhYJeC$-oLOHg{#ILFi8*( zhkgA%<1E+8Z5Ce8H#1m>kpscO3#$>2$W>EQ1FuObsR$pqp)w(OgM6D1wNREoh5H5u zG7b!N9n#ibw*KIK-$1!3CPPERqc7~X{A?)*?E9FNWWYzpTHYd%W+kLtydlPpMfo^v z05B;Gc7_@etym~U_|N=OJTch8X{nX43 zOG{7p3J>Q4_p(ekEl=p5;HA*G2WGI6H^a*7GRj8)eonJ2AY?7AKlFOR{}K4g?3{e{ zUsDg0%w$!bG=pwRuxL4rsgT}NJgI~Ex-ZX3v~c*8s?Al25kHV=>}9UxebKED{N z*ZVXr4f36b)*V_y1|cV>=^Y}{XWp)UeSgAjgdmbNNA6$fbt(&aX&d}x) z?zj?(&Obh8vB7f0KtVia#Gt9`dM1E(U;_PsxzW`(zbzl(@IKWDydNZ>DG7d3Ek|2sqsl7KRL=GQ3#VPKvkWX znu>xXOR2pa8*rzZ`Cwc!cd$-HWetgU$TD7WiW_lE53?2TokM2-h}1Vfgf%Jo8! zc4RMhjVeTNu^wz!4p{zoPyybid-(9-PV4ps zgyKfMMHON*a`5s#8Kgz9@|8T$M=e{q(w`8jP~XHPB(&1AKw5$>Xfm@VcWREXqomws zW#ncjI%xZJ619uKjY$3@4MB<;Ao6q{1a}W;RVw}%E`i$hSdtJo@Q6Ox5Ga^Iwdnd;p+!hg zBx0c$hOj^^04ukj^F?4Vi})E6CH&GUVfEzRjR}A=>V=j(;jvkoBRy> z$1JPz>npYrt{EH?d><^+&z1oVRFOW)syJ@LjJ;+2+7%PGJ^p?2T^ z%r3eX@X<_3b1CCxVh-S{kWJdIiRKrkohZN6>`lM{Fl^n?zbCN=&Z>F8jN2R!VH1=D zYE%r!Na#h|QB%|X1=c}h zJK#p?~z=rz{V%11hT>oXs^p=o?MEmJ%~K2a-G`)^&){YP`6-?C<)tk zmh4v4lY3Te0|~%keyds49o9p4sL-6KaN#6^+NgpoX$;v6!P9lG5P1Se3qG34%1TA} z94HEsE$lysH?4#;ooGG1iadsx+t$x1q3b-SD0s;b5 zuFb=btEMfC8^hrT#G;Pz`+LaMm^`2Yv|^L}p82qR+ukF8L5I>DHVy3DAP|)>4Uohj z>s$H9|G`rU@neaQ3Li_1F#Dz_PC$LKi1o?2z4;g|2tJUrCGXy`flS43&*r(u^I!TQlptfs(3f zNqzk?7*mC8W)W06uwGRWUrtIK3+heQh@}0|*T>0b_&^2ImZIgvZWG8Wn`%A@7eLKW;Vm{rGW# zNdL)8Db(Cp1lJJLNQNc;Oufu=e!$fcrltPgB1>1SAe{Cxr+avE?M##cL~}u+iGX9T zIsW?tI9W9Z-GCM0Unqi71NpYtoraFt;4}mx;0@1Y~6m1Unv7E(xuEpx2^O`}(b3%hg1@rSbXmi(5OpwNRPy=wgIQ zW8bz|Wt34!CGFt59OBwZJg_;?x4W8H@lfDqXfTsGc%|XANF!k7wt1lnxe^G8{@@Dd z;Ws;03g5t2)K;Ma5Y~O-6%xXE@PU-~;kbjF4xn0?cc^RkAeSy#x->(RkzC79Q<};X znxlQ%?#jm9NIln8cX{PF2hhb9QFIu&I5vm17YW1#*3yfSwIguN@f&^79a@9x%KALq9N%!etZ>h6gLtn}Gz*s9KdiL4|PFsZ9s(@1y|%a;_zx2MHFT zmG$6589&%7kN%Y#)K%R<;UdA&V0ud7Wo z7`)+7y58@i+h##UX(>JSt`c|#elG<@McT*7$t(}DkPKFCnJ=2_P&C7-E1N&nBMdS~ z5ussMqbkZqQPD%u^z->COq~WO=RExTtH6*DF9#H%;>N}!A2Tp>9OmA7p+$I!6L1(p z-2-)Af8+p`k&<3)*X`+(-i?xM z8F)H46Bj!>I~xPeNPe!W+IOOK{vMvqG#kal*r|axJ6bk2BgT8N=g#vVl?wpeAfj#W zSyQ_(AxQjw`L8Fv5Hs{*N@C?_r$s<| z2^7YTP|GQN#4K8cEY<)lR@T)!*#RR6h=y5f5C%@-tA^;T@o6o=o$-=~r70Q(xKoL- z?Btm3V8|o&0W2H?$Zj2Y{$k|(zwjX7)eqG7KzpPHJQf=^`p8}RedB*M$RdCL}-^yAP)pbP&4x0(S>BVFvic=2h}mR-A6oG80q z6tg#2Qg1a`McbG4R%yWKb^~=^Nl6JEFE1}C2El7yg``e{prj?Eop$oo5)@1%N5UVl z6T3QS*bp^7*?ADN)&!~f&kX6!Gvs6c<+2r>ER3d(XkdhHZ6gdWZB{aMX%g_7(3 zW{T(L{DXdj0IwwmXCQKAy@567YSy`erwv{I(Do#E% zwG75GtK@yaLxUS%2@GU|$i-<1|38A88A^zR`4Is*0jye8wFky;d>|++jQP3#6UkpN zf`FHR`Um?f-1y4KQS6+Y-T;3K7#Zm(FzAfX+YyFnFYl_D1mB`LGl!yn0?P}IXwtS3 zGCx@;g{ENP<^WD1B?q`PR1c+JE@S%)4z%s@^7iJ(ia`wys`$Ps`!HA&{*c-JH4PAO zYf?@2lnx`Y8iS1{ZVpriF||@;ao6Xw-9!i`)F9H8dD8j^9WL(a*T#ufY+2poLMZ8F zarf{^-%0^Q0;o-?#znMU0}QofC-*C138~&#T?E{e<}`ku*xk_?pk4c|BFr1~G##nY^;a%32aLM2cR>YavyGdg#d z0JcHMLSP(p=@9P`L-0RDIywM*OJR$pC{zd~Na}w@ z20&6mZ&!7LhmR_vj$-r6+f6_Jd3xRcQg{6U%RPc8N z@b9_$cMPAFn5eGS0Jo5C`EfajI5Tt#ph<~*43_k>{|NPHymKD^Y)_g>E9EUGV^DDvD6Y1>-OD7IAMi+NHJE3zxXtiOx?k3;~8W@7^nyp(8AOEwy)lSQ2<)t7h&!=rtH)mbbt{C$!%s(HFDVUbB@km=%JDZ%?Pdt63vhQq$c8R1(mQOqlO~h z>w4gAI6`#(wQmyth;8`q^vE4=3qX(%eHd<5;83DC+X+fNLAyG~puiH7AJ1J$r3@NXVgZHHYDECJ^R{Hv-8D z_Da$h2o)=_dXj7NQFp{LW7zVD!yRu&dWDEH(p>KDeg=39OG_Xjtj*bjQ9I>+kWK^4 zAL1N6EDOr_w1vD23gkilG8?St=ihDaN+IiQ6eAibX~iRE?>$!+6+!>M_E-?yFkA=! zkm?AmhG>J*a#O~~4<8VGpI4uP3fUs%hQ88NyFNYWXeYqLOu!ISjSVkQkRb;>9}J?u zPjGOZ^cV?h>RUtl7Z}{eb{CrM@9tiR8WKesQ6SCGTH!H?Y9|nRxpWxy%vGX&Ufdt2 z>JD$Iv9Yl`c;whI{09D&E9W}YR=r_5cU1BC@da?Hig8sZCD3rUQ5H9fpQ$yc{(b(6 ztU}SKjkkZFWn~mJOhJ$xTRXF= z0ZX7Tl2`!R^ffHBV3W(fz6u^hyO#;(bKXY+E{%|us#;o0kRbF3skXCr=ilM-V)_;||`6bvAL0SwWJ|o;7Ag_PO3t;zP8N zC1#h&P4oKDA7y_sy8pB8V-C#(U|@PbC0e*NkmwCU_b7F){(rJWY-&5OX)hnKF=;-3 zxklBh_+Ot#b64S>U%*fQwp;ojy#bekQs+HLsv+Nkzsnt-DR>*RQ!U`S&KFV>TQn&v zO$Dfu=4o|t!>m)`DbJ z^77t*XRYMp$5mKM;NP~`%p&1Of*ax%Gp0z}9V(wllyeZ+u9!ajt#rXFT!ux2s_QD; z5i(9=k7II3ksEsyqb2qa(*xy>uB#uPo-YNlrD$em=JC%_hr*yS5%mOYoQjQ&q!&A> zuU`5sC99i;D6TAM8zHqWw3&5HyQ1t{m*gDobm)9O$1?tRoxgXhbdo`+oRFwIi_Wy> zCpoG!nr|krnoUZU_SU_7*HD-hwX_)hgADUh4BZP^D~ykVK)-Bw!}|3CyLKHj=LNn7 z1I79LEqai#z_Ib;Pwox4-X`LseX%pALw7p`pnlTO2R zN|X0iXS6@|vG&S;gSCSW(h;zUZHl&qsQQP0{6P!0Q86?vE6c58W^yj^*+E2Y78aIR z!jFU){bw$#EjD)_9k?g-hK)jvfB1EOPY*LfeXIlG8@gHj1e)fn8|B}=2sn~KF_VsI zkffwI%@%r$%m~-667hJoezYo32A&eP|EZ5AuoWch&QA4;lBRnK^%ALJNTcL7i+Xfm z2ouE~4pE!l#sj*a*CHZoDSfB;Pn;3HlYcrR>?0DPx{{`6G3$1v{ddG@aIr%CjP#h7 z%3!f7LjxlsdQ5CAX#)mn?wKZ@vYANR3jc>YVi&J(r5EA~Br!S)^n^nOOJ4BMQ~5N$ z=D4>y#`fRr8L6ECpQ1973hO7Q-y{`c!>ILl%`D?4qk7=fEqY`@HB;aRPa|| zS;dby8e)pMZ_PSl&P&IR$g=~~(&)ha<^MjC0sV4HO6rwWn;M2{*=iT_l7ixYc8bI0 z6*9P^xsG}6I-0=$wQ=*GV$*+TGW%@<?|zN#3XA7%b8={RBUBCPNdQV{wB7623CvCUs?yLR`Ssg3p#xR0 z9YTt>)jl=|6cvsB6LQAFix<*pytsmbwl`;U#(i{3z$HNR{dCgKjus|@kWEgN>Lwl8 zqW{9~K+~!CQ{Z-p4Gzt^6a|wG>5Hg&bYRtE!#ut6hCEOdtBJX$wUg@-1lhux8V*>p z8WMGgdwQSpWsL@aKO#Ut!HPl{zC6%MLm3yk({OLIxOM*8(wIwbbHB2nUgd~%1Q3%t zT1nXzeUS{95sSfy^UF>D*sA%$&JvzEp|=lZlRAW2UW4-i#L-~l2T2@)?huh7&9DbD z4q7l7EJW}__X!$wWnQ9dYgrvcgMGhdM-ow)_`<_NTKm&bf0O^iq%LY=X0{RJ%j4c5 z?c4Y638SXkf&%F!{12sP!d=$Ce)ER3|JxcgA9tA%;?E~Aj0aD`{ z=vZsX$54-Z<8iomUWgz~HKfPjeXhf^CsoeT&_xKP%fj!4&YP<^ir^_69uMEcErevZ z1V$UvZ?Er4J<+m7NyLd1+>XUx48aidzK4ll348>&)F%hec07pKSBR5GT6X>d!*OTp znZ+~zFfc|-G@OTJSkkQ!)CT?1n7oshsj4j#P)00@A4^VH@*UX8w z0!pJNFoIkguE>dR_pI3Y_yUP!3>-o%{4^9KVRDebg|8JHJi2<4Xd=; zgV3=fphVcQF9-IeA3 zvCK)=1;Xf&Fjv^Ru)L$L)%urWfnZ2<2Ipo373Rj><~G2b6H`^YpqvV!ASvP87(VbU z-g3~|5_*@ks0##vW1ucin%BU*X`T}j!a4hE+{ob8*gtRO(!$cM*$lO5f42kn z1v!ibIkp20TT+RMmvij;{Ndi@G@Bc3G9ryHJlfha00yL=izwxY>!j`?M@7g@eqFkV zM@qY?Gfxu&<}BEk$NJBgfI~IEJ7d5=p;tau+!~Pr+=TkOVaB%GN8m6Oe{?AP21-KG zOL8>FCZYCDI8u4!Zcm0BVT}WjhhGBTa=u%)WFc1g>30tUaHfNTPY4aCO^&TBx?DUF zUgqId^P{ zUyOFRT%u8d90=h@BU5yRu*jmpgG31)og*?}wsjUjd_j0>h0cgaciaw8oiwQe4ANRzS;643QEucGC-~=V=Sc}P zgwVHk4qSo5Pv}t#LOqsAosNUhU2HA1LXhBxy|fHXX=R|`e(Rb%IG3UbG`Wqzqe?Oq z*edNH#}rYXm!WfwXxr#2B>g}L8{82~2Hwfft$|4x^W}Ze4Nfi?!`RN8`M%`Ie%qA5 zy9qD=pvPRcbQo}a2Gl76l;PgcBb$*FkU5Ai7`+C%tpWLxbDNJmKI%az6JS1Jn<@C< z*m-yY@KQ^jJictmXRoykRU4kx;`z!x$jTo3*Voyu5R%5>NvJV>;E&bMzCeS~Y_Q!c z^dZx7l@QKL`D)1s>D&*P45;eXg;PBactp~>Lk>PcVdJGQqKaW5jfjg@NW|3pWH-ge z#gU#3ve5zIYCu~!{~B!~w%&uirj*cb(pKul4&xj>^(n`e1#G|`upc$~E>1kUabZzY zL`?zT*Oe{Rpo2~v;_UJ9@p-auQ|8dnv&%ZOA2p9j58Xsdx~>1f-J3VbmO_s~5&9DN zTtN-jwgU(BS&Vy)^AoHuX)!YkhZeCzYIBz5#068s5`fAd4Y%xO(@`seJH}f z+nt0S1V5bZvXI}4oc!Vq_cadA2}2#X92>vE+BR_$n+p;k9DHhEa+RA-O6|RNGSU$4e018vlMABz}LeK7LqI z*RT(L;X2Xj)?9uoG`tk@iWfo{6DzAu?kqwW$y@uwB={U{k<+u*8Lnd79U1}Az0^Gylx=(PAvq6i zp;D**JRek|-Iz9l>WDmfgICrclnd9LBh8GvzYB}XU&9Eaw^+q|zzzj3%;(vlAg+Mk3~ZM<4}gN+9~=D#7L|MWC}VhjyXxX#m${6t zq9-F8S=tyk%FAB^Tgm0<;T{-x=g2IJB%3M6|6zne-+M5(dH#6!>y8K439;O5+y`2|dJNX>DCy8;P5e zG=rl1aSedLqY1ZT8vmF$FS1@kvFxf{yKC3MnZ4PIT|iu1bHfh%uR6x-HbY0jd%L5N zIHop~u@&?Lfn~&YRo;;BNM?jI-;!_+X~))}QhHO8{58VS%^Brh_CQCKHoAzd&w7It zZ$-TBVW2_Rh#&S}Y_+RsFDHa3p=v8`^5=*I zscm>D=i-m)>cDQM48N$2!-}VyvoA{or}KSTdTQpIleQEb-mQ>@6hNV;nwBz?W2F$s zipar7IGsp8$7XNSx+UGOQIUJXDj@8lyJ}#_|Byl3seza*5?#^KOU^-{+yP>ZU;1*K za@A+=R&jWK?yg!eIU@zv=Ej26b?)y>;@fWP(6B|+b3@)TG(!<980kaa4$=uSNJj~q z{7c|A^hrJH9g1HhBpSM9sd+8604A5i7h|KnB1GqS=D8v?HWNrdln>Zv=qb|d4}sMCi%oER?z6PCA`~X@ za?qn%K!f2DoV8Vqe-T4&#$>CUnZHHs(8Xa{Az2t!Z(t#96gmF33HCi_;CZRX*VXKS zp_j#7wFKvjR!+iM!+%aN9}?##vD$r@EOO8kb2<)LTa7@}xLaA97mRT<_K&|)CZzxk zt@5?(abL+~Xo@5qQH79z(aBmyPy#tV@cHxZ^zA4Y^9Fm~j6={M2^3da?l>%q&RBaa zbZuv@FF+#%Rl!ZLgh*U)huG?^D#v7R4BXKoC?It9p^O79I+-4{IC{~5k-#C2<^N4} z->pA4`wL|?vkdg0*|)tn0s>H@G`!&2tJKNdPY13}k#OzABexy%z@u|G8`F*R5Hy-j zu?H~>FtKxlFaxHMPCPcW{Gq!#1&NO1_sMzibZvAocm?B3E2wvPO-KW6S9K)njRh2@ zn6x~RQ(9K*grSkeJ#^fuIp5`l7Ax?-Z$JP&wRYV)*vgCnmWZZ`i>6_NQ-uIRBsr6# z9+55y5;Wz3C6shf0G2bO6u@a6f^hLViLU10WC6vYnJ|ilLVB}F!zb~c!M)lFFY*Fn zRiq<2(sB`fCM-pwbi%n<3eZX#UQfyyhrNfut0u^br^HX~P8Yfoq>gBua=o|wI>sw%v0o6srV^;@1Gq-;C|kb@E)59d<46rayQd{ zIZPYoI}$L5NFOBbqFg0MF%pg$XK}S~yJKq@&aICiDN?T~X_Qjfzkd;|g}1(K6%(T< zNsD?W;2V_9J4k;F!X;s}5hNU{gcX$5`C(6}C!;~P4doO470rB91LW{cOv_DmBg48h z^Sbr14k+$#qVwMP7?%6IRyKq9SMm0(>j_!Z;iz&85zWA^^b34{mYS*n8M+373OQzy zoFxGjTO&a$cE^)&Uv+YVYKR_W7V`S=QQklqyhkXc9X;!$4+UA141tW>>}n+GVoa7V z;+lYzl%6>c%7NdVwrFJrvB3;Ij+~-FIB+Zl91Y-uGaeIc{HD=(OEh<)wBY^<5TyyT zN&q@+E=AaQgxe=&GZYdKa314X1Ll*QzzC%L(qOW>UJV#W8+F)k(t z50StDToQ$hi2dvha+e~8&pWbC+eue0j_<&Xn*f6O3u-HAOR2r=-B3h46P3GM*Aiy- qe|IMBe_&DnRldOwyC2Lk_C@ruZG3w#0jKd&dzB9=JyA5i^8W$#oDpXL literal 0 HcmV?d00001 diff --git a/docs/user_guide/selection/RecursiveFeatureAddition.rst b/docs/user_guide/selection/RecursiveFeatureAddition.rst index c9c81b1e6..2ee4970b5 100644 --- a/docs/user_guide/selection/RecursiveFeatureAddition.rst +++ b/docs/user_guide/selection/RecursiveFeatureAddition.rst @@ -5,56 +5,83 @@ RecursiveFeatureAddition ======================== -:class:`RecursiveFeatureAddition` implements recursive feature addition. Recursive -feature addition (RFA) is a forward feature selection process. +:class:`RecursiveFeatureAddition` implements recursive feature addition (RFA), which is +a forward feature selection process. -This technique begins by building a model on the entire set of variables and computing -an importance score for each variable. Features are ranked by the model’s `coef_` or -`feature_importances_` attributes. +This method starts by training a machine learning model using the entire set of variables +and then derives the feature importance from this model. The feature importance is given by +the coefficients of the linear models (`coef_` attribute) or the feature importance derived +from decision tree-based models (`feature_importances_` attribute). -In the next step, it trains a model only using the feature with the highest importance and -stores the model performance. +In the next step, :class:`RecursiveFeatureAddition` trains a model only using the feature +with the highest importance and stores this model's performance. -Then, it adds the second most important, trains a new model and determines a new performance -metric. If the performance increases beyond the threshold, compared to the previous model, -then that feature is important and will be kept. Otherwise, that feature is removed. +Then, :class:`RecursiveFeatureAddition` adds the second most important feature, trains a +new machine learning model, and determines its performance. If the performance increases +beyond a threshold (compared to the previous model with just 1 feature), then the second +feature is deemed important and will be kept. Otherwise, it is removed. -It proceeds to evaluate the next most important feature, and so on, until all features -are evaluated. +:class:`RecursiveFeatureAddition` proceeds to evaluate the next most important feature +by adding it to the feature set, training a new machine learning model, obtaining its performance, +determining the performance change, and so on, until all features are evaluated. -Note that feature importance is used just to rank features and thus determine the order -in which the features will be added. But whether to retain a feature is determined based -on the increase in the performance of the model after the feature addition. +Note that the feature importance derived from the initial machine learning model is used +just to rank features and thus determine the order in which the features will be added. +But whether to retain a feature is determined based on the increase in the performance of +the model after the feature addition. -**Parameters** +Parameters +---------- -Feature-engine's RFA has 2 parameters that need to be determined somewhat arbitrarily by +:class:`RecursiveFeatureAddition` has 2 parameters that need to be determined somewhat arbitrarily by the user: the first one is the machine learning model which performance will be evaluated. The second is the threshold in the performance increase that needs to occur, to keep a feature. -RFA is not machine learning model agnostic, this means that the feature selection depends on +RFA is not machine learning model agnostic. This means that the feature selection depends on the model, and different models may have different subsets of optimal features. Thus, it is recommended that you use the machine learning model that you finally intend to build. Regarding the threshold, this parameter needs a bit of hand tuning. Higher thresholds will -of course return fewer features. +return fewer features. -**Example** + +Python example +-------------- Let's see how to use this transformer with the diabetes dataset that comes in Scikit-learn. First, we load the data: .. code:: python + import matplotlib.pyplot as plt import pandas as pd from sklearn.datasets import load_diabetes from sklearn.linear_model import LinearRegression from feature_engine.selection import RecursiveFeatureAddition # load dataset - diabetes_X, diabetes_y = load_diabetes(return_X_y=True) - X = pd.DataFrame(diabetes_X) - y = pd.Series(diabetes_y) + X, y = load_diabetes(return_X_y=True, as_frame=True) + + print(X.head()) + +In the following output we see the diabetes dataset: + +.. code:: python + + age sex bmi bp s1 s2 s3 \ + 0 0.038076 0.050680 0.061696 0.021872 -0.044223 -0.034821 -0.043401 + 1 -0.001882 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 0.074412 + 2 0.085299 0.050680 0.044451 -0.005670 -0.045599 -0.034194 -0.032356 + 3 -0.089063 -0.044642 -0.011595 -0.036656 0.012191 0.024991 -0.036038 + 4 0.005383 -0.044642 -0.036385 0.021872 0.003935 0.015596 0.008142 + + s4 s5 s6 + 0 -0.002592 0.019907 -0.017646 + 1 -0.039493 -0.068332 -0.092204 + 2 -0.002592 0.002861 -0.025930 + 3 0.034309 0.022688 -0.009362 + 4 -0.002592 -0.031988 -0.046641 + Now, we set up :class:`RecursiveFeatureAddition` to select features based on the r2 returned by a Linear Regression model, using 3 fold cross-validation. In this case, @@ -62,20 +89,32 @@ we leave the parameter `threshold` to the default value which is 0.01. .. code:: python - # initialize linear regresion estimator + # initialize linear regression estimator linear_model = LinearRegression() # initialize feature selector tr = RecursiveFeatureAddition(estimator=linear_model, scoring="r2", cv=3) -With `fit()` the model finds the most useful features, that is, features that when added -cause an increase in model performance bigger than 0.01. With `transform()`, the transformer +With `fit()` the model finds the most useful features, that is, features that when added, +caused an increase in model performance bigger than 0.01. With `transform()`, the transformer removes the features from the dataset. .. code:: python - # fit transformer Xt = tr.fit_transform(X, y) + print(Xt.head()) + +Only 4 features were deemend importance by recursive feature addition with linear regression: + +.. code:: python + + bmi bp s1 s5 + 0 0.061696 0.021872 -0.044223 0.019907 + 1 -0.051474 -0.026328 -0.008449 -0.068332 + 2 0.044451 -0.005670 -0.045599 0.002861 + 3 -0.011595 -0.036656 0.012191 0.022688 + 4 -0.036385 0.021872 0.003935 -0.031988 + :class:`RecursiveFeatureAddition` stores the performance of the model trained using all the features in its attribute: @@ -85,59 +124,197 @@ the features in its attribute: # get the initial linear model performance, using all features tr.initial_model_performance_ +In the following output we see the performance of the linear regression trained on the +entire dataset: + .. code:: python 0.488702767247119 -:class:`RecursiveFeatureAddition` also stores the change in the performance caused by -adding each feature. +Evaluating feature importance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The coefficients of the linear regression are used to determine the initial feature importance +score, which is used to sort the features before applying the recursive addition process. We +can check out the feature importance as follows: + +.. code:: python + + tr.feature_importances_ + +In the following output we see the feature importance derived from the linear model: + +.. code:: python + + s1 750.023872 + s5 741.471337 + bmi 522.330165 + s2 436.671584 + bp 322.091802 + sex 238.619526 + s4 182.174834 + s3 113.965992 + s6 64.768417 + age 41.418041 + dtype: float64 + +The feature importance is obtained using cross-validation, so :class:`RecursiveFeatureAddition` +also stores the standard deviation of the feature importance: + +.. code:: python + + tr.feature_importances_std_ + +In the following output we see the standard deviation of the feature importance: + +.. code:: python + + age 18.217152 + sex 68.354719 + bmi 86.030698 + bp 57.110383 + s1 329.375819 + s2 299.756998 + s3 72.805496 + s4 47.925822 + s5 117.829949 + s6 42.754774 + dtype: float64 + +The selection procedure is based on whether adding a feature increases the performance of +a model compared to the same model without that feature. We can check out the performance +changes as follows: .. code:: python # Get the performance drift of each feature tr.performance_drifts_ +In the following output we see the changes in performance returned by adding each feature: + +.. code:: python + + {'s1': 0, + 's5': 0.28371458794131676, + 'bmi': 0.1377714799388745, + 's2': 0.0023327265047610735, + 'bp': 0.018759914615172735, + 'sex': 0.0027996354657459643, + 's4': 0.002695149440021638, + 's3': 0.002683934134630306, + 's6': 0.000304067408860742, + 'age': -0.007387230783454768} + +We can also check out the standard deviation of the performance drift: + +.. code:: python + + # Get the performance drift of each feature + tr.performance_drifts_std_ + +In the following output we see the standard deviation of the changes in performance +returned by adding each feature: + +.. code:: python + + {'s1': 0, + 's5': 0.029336910701570382, + 'bmi': 0.01752426732750277, + 's2': 0.020525965661877265, + 'bp': 0.017326401244547558, + 'sex': 0.00867675077259389, + 's4': 0.024234566449074676, + 's3': 0.023391851139598106, + 's6': 0.016865740401721313, + 'age': 0.02042081611218045} + +We can now plot the performance change with the standard deviation to identify importance +features: + .. code:: python - {4: 0, - 8: 0.28371458794131676, - 2: 0.1377714799388745, - 5: 0.0023327265047610735, - 3: 0.018759914615172735, - 1: 0.0027996354657459643, - 7: 0.002695149440021638, - 6: 0.002683934134630306, - 9: 0.000304067408860742, - 0: -0.007387230783454768} + r = pd.concat([ + pd.Series(tr.performance_drifts_), + pd.Series(tr.performance_drifts_std_) + ], axis=1 + ) + r.columns = ['mean', 'std'] + r['mean'].plot.bar(yerr=[r['std'], r['std']], subplots=True) -:class:`RecursiveFeatureAddition` also stores the features that will be dropped based -n the given threshold. + plt.title("Performance drift elicited by adding features") + plt.ylabel('Mean performance drift') + plt.xlabel('Features') + plt.show() + +In the following image we see the change in performance resulting from adding each feature +to a model: + +.. figure:: ../../images/rfa_perf_drifts.png + +For comparison, we can plot the feature importance derived from the linear regression +together with the standard deviation: + +.. code:: python + + r = pd.concat([ + tr.feature_importances_, + tr.feature_importances_std_, + ], axis=1 + ) + r.columns = ['mean', 'std'] + + r['mean'].plot.bar(yerr=[r['std'], r['std']], subplots=True) + + plt.title("Feature importance derived from the linear regression") + plt.ylabel('Coefficients value') + plt.xlabel('Features') + plt.show() + +In the following image we see the feature importance determined by the coefficients of +the linear regression: + +.. figure:: ../../images/rfa_linreg_imp.png + +We see that both plots coincide in that `s1` and `s5` are the most important features. +However, note that from the feature importance plot we'd think that `s2` and `bp` +are important (their coefficient value is relatively big), however, adding them to a +model that already contains `s1`, `s5` and `bmi`, doesn't result in an increase in model performance. +This suggests that there might be correlation between `s2` or `bp` and some of the most +important features (`s1`, `s5` and `bmi`). + +Checking out the eliminated features +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:class:`RecursiveFeatureAddition` stores the features that will be dropped based +on the given threshold: .. code:: python # the features to drop tr.features_to_drop_ +These features were not deemed important by the RFA process: + .. code:: python - [0, 1, 5, 6, 7, 9] + ['age', 'sex', 's2', 's3', 's4', 's6'] -If we now print the transformed data, we see that the features above were removed. +:class:`RecursiveFeatureAddition` also has the `get_support()` method that works exactly +like that of Scikit-learn's feature selection classes: .. code:: python - print(Xt.head()) + tr.get_support() + +The output contains True for the features that are selected and False for those that will +be dropped: .. code:: python - 2 3 4 8 - 0 0.061696 0.021872 -0.044223 0.019907 - 1 -0.051474 -0.026328 -0.008449 -0.068332 - 2 0.044451 -0.005670 -0.045599 0.002861 - 3 -0.011595 -0.036656 0.012191 0.022688 - 4 -0.036385 0.021872 0.003935 -0.031988 + [False, False, True, True, True, False, False, False, True, False] +And that's it! You now now how to select features by recursively adding them to a dataset. Additional resources -------------------- From 9df113cf2b676ec0ab960ea40be6b32a140895df Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 24 Aug 2024 16:31:31 +0200 Subject: [PATCH 4/4] update rfe user guide --- docs/images/rfe_perf_drift.png | Bin 0 -> 24945 bytes .../selection/RecursiveFeatureAddition.rst | 2 +- .../selection/RecursiveFeatureElimination.rst | 277 ++++++++++++++---- feature_engine/_docstrings/fit_attributes.py | 6 +- 4 files changed, 231 insertions(+), 54 deletions(-) create mode 100644 docs/images/rfe_perf_drift.png diff --git a/docs/images/rfe_perf_drift.png b/docs/images/rfe_perf_drift.png new file mode 100644 index 0000000000000000000000000000000000000000..570ca242f3c0adcba1bad46b011cb7a51401710b GIT binary patch literal 24945 zcmd_Tc|6u@yEgtsWh!GLV>IAaG+2?b43UuG)<}^wk|`N7RpyF>Bt=N3N-0U1GDf8+ zQ8Fi@WX#O(xUBU&d+q%```vs0@&4XF-agM74C}tX_cfg7aUREUo=>omq0S;UK{kq_ z7H!kjGNC9&Iz=(OXIX&nw3P*Xz&|#f(%yZ_)c)`(S1TtQ%E0QBgRT83+hYfpowjju zK4yP%gRFw=y4A~$o;u~=tRyFA_s?IDwRbuq*T}{ah!du;(n;EOOr=B^tm(O9bl+otO@#(fO&r80m<}G|tKQ!=^^JQ*V^Q2nk zN_XeJTuB~rKZ+ZF_^#9I8MyE--<4OT-Y97zLU$v`bSHl`s>%P)pYegJ2*OSnVp$# z*(XR)K5)q=q%XsO?K{nV^Dxv!ESK(}^V@G+yvG;d&KR>;uV0LCwqDh_al90m5TRyWhsV2#6JMWtZ^U3U7 zwtP90X^P>S2)C$169OBK9e#8Dk88@Vvd>(414er5tv{q1mpR=qh}|fzqLR1!w}IH! zr54Pa#lpV7PbsOZYNU$GWe^As>iTb!{gGeCcp z%oVRIr}sWSde&!ldROO&qQ|h*Vz&89WX*%p%`zCNIMp)&cu7N>z`|}@vcHO#lfxf=}<;ci+ z*47;o+wb^sy?p)J@8Lto@E^aduf)W##^^6dqq8kvzC6lh+qP|#(U0f}wu6?I6cZEE zrJN1&@~Z=mc6*%~e`{=Fa;3WZed6WGX5F%|=qcSJd$W(vr-CkAFugiGHT1TBco;9c z4r>yKkJ#Q;5=d+Gn)sfQYMR1v(Av5tQJekRwQH8`rNKB5WSeT=zmJOMufMmCxphP_ zXz7N-xAdZDmtS?_IJ_tg<_(XEO3t$9=jT7w{f>c|L)bSun*ZXZO9L&lGrKl%9PRyh z@7hK?K^i@ka&~q;)%$Ugw6yfgs;Xivjn$DO%o{drn73fTw@;&=vrp8do5?hM`m{wh zI%(&YZt^H*Q=sPTKmStZcqht}srF zR-yNF|Bp`_Y;NAYyV@@xAn|xzI(bMZr{brM9fPI2F5{QTlds*lacfDyqJauApRZ{V zlT)_Mc}pW!8EE*k@mU@`c<#f84=UEF=H}*Hf`~;Ava|iQB*>|^I&p%X!Y8>tIC94M zi>ddOh=_GgUv(8XZ6c40HydjU6r+2D51PIkyNw%EQc=PD?c28p4<2kjzhKdWWA9gW zPAF6ul~q&}DtsdceqYW>sa^MlFJ8RZ+S%#f^xU;8xT~iw<7izvAC}Mf;X&=!H#QCB z+qpI7pBZia^^4Qg)KpPP>EOihcPf_IKZIkmV+RnQyyu7@RZ9nngqP}sB?|Cx$a~<7&Mz#9-AM98SzwmHwPcJXWuIigA zjdB*x7UN{^+!WZNAS$6!jh*BZRe_dg5Sog>oh*Gt9f)}$ME;}%y_E)!NKAXJ{gDK zKla?+o6SbOySPd@h?clvPOLP$jUv{uY|oBl@o!nWtX3eIaJ6unE(G&@LMg#<4KrUlEEEu&6+ zyh{#M+xPFmSWzabwzl?Np1ZSop|{8ZpXon(>NCTRyp6LO?Q0tA%`CLEwr29zI2$UW zj9bYopyIan_{Y1!ucGCwU!0r&)TMXH>Ao-H)%8b5dg{oBxGsO}T=;UG_U8OE-3|>Ax1x7opt##85xk01VXzMIOf3#b(`c^$yL)&kvPNV@ z&YnFhpysLA-roM;P$iG~QwIjg9dY#EhwtNOhYufS;ot~}iCM}argl-<^v<$B{@`=& zs-90xb*ay8(bHp_ndp`4OfD;KK6<(3OwzMoOgN?5fgB>(kx*nhP1)#<>YK`n?t_OC z4vu%TBp!Tn`sZGT&B98jbXB}2=qjFLMu={EaF^Jvt*r;@5BLO+jk#G_S}w!QO3wP( z`6^l)hm~IC<>@((ppx*e-3$y2#UDPx9+4&G%f5j9l^6DOW(%<8hWPwuRBRp}^}*+%#=5o46mB&_-J>lQjAi0yM36 zxr7AXdGKqcO5?Yx>gpw@PoGZ2HI|T(*}kXXU>O_3=CQFcsaH<%@i^sO$By(pTdor- z^-4oib3QdN{N2>%)BBVi^C@mY!M&FH_Se^02du>+=bUUZk>+G&>0!IvNBe2KV3t^B?#Qbjg9-JCVv&uCSQ5U`n!2~=Z`-tkU4zdz=5`J-&U>Pu)*wi z9b${|y}f_fd_pdpPdR>jz0_t8vhRb(kLke=mtiHbfE#VERXKjnv6nlvK4x%Jf&0)M zY8fuu{VcDFP!azN7cS^24<%KP$6`(J8%J9T@~dW68Qzehf_!~_HL@EkDkNKm?qR=Z zUEhlR+3_~xgEm;EJ$v>PfBw8z{^+~Sd7k4Yd+dSW+^gHpVLgIy#xo+Pv4B|Fr;bT_ z?sPgmZg46tM_e<2q=YEvB)}r~<_&%?&)hGV7S0=YHG4OAz<^#}4!LgY=~$^gVL3H5 zWr?gODJePp;}b8AD+v@xr3t^ugXp*fHJBW>q?2qKl}8=dPK0xfvNUQ`6H!O}mh->E$(UXU=G{oXsqp*_#}flcON2 z<{5+|F8!)`>aFt1mGj@msh$rD z1IOAwFfcGHrNaO@i{e_kH2C@-Cla?_5~RI7(bxGgC%#?*VTA*zY3y69J>BE_#ON>1 z?b}~@=CHD`j9GTPkyyUG2GKWi&Ay$C47^pJMW9;o)-Cf( zmkF=j{=6b!rkeaSt^iZ&RSj|KJ{&$iK3@vFgn+df`zNMsKcsRCZnRDP(){t`5`>YD z-Pg8k*+N^#Jb%8pva%2^FeBwTSSi{6`*(TyB_Xr+FNHqp9;1EJ&8O3yXGE1wvEW9{ z9h{j#p`#Xye{w8bNZ+L3JbTz_)({syujSqlG#1UdWO2*|T>q@ba>iE4AIX4SlVQ zK;YKa*I&4P{d#&&%I({!E2Qnkn5n9(YckfWbzrZ%aeU7{WXz1O@2$_2shG%H4m%o(u>*fdDDh}=SwR%ZZk9tz#Uz^dNs+!^fZ(CDEkzBW>m_< z#4iXrQ;MO&o6g~4NYH3(`<~ct$=G?XAEB1Nr@A4G zm6t~228Lu5pJ#4iabfEGyHB5%viURS*thKDkecsXTB@}|QnKXJr(E5z==c`z$wi2S z4f$tss(k8xJ^T7w%`Bb%0zkHTUsrAN_WqHPy0Kcbz9FZ{-*%nJ43yjW;7VlU*0#0; z_jkW;9rMJmZ`r$75V7vD+UY2o8AYLPtC6aamiqRkSwd5@WVF2N$B&CRYF*NWh~A$a zQ6+wP?z*GXzU$+?eblX5UQ?XN$M!9vxLR7jn)8vawq-B)m2qaKz`Z)}ll5<46BXA& zKAa#)g*au`K7wtk1`+7Q)zoHgDyZgd$NM^TzY~|Y`#gMeR!dmbV=<6z#qXbuq#Cms zk9FPHMe8=N`suRW&~T@;&+{YGUIXcqYRjVZ+IoB0k&{R*+2S*+Ix`eM+lp+YJ(#oe z`a{kzJwI8CXGKLt)cZRIuZCKZxft=na>E6U*(aDOcQr+)*?lK)yAww5-TyNCquF*J zP{+sn7W=xrfRXs3Wi6_lquXOEzBlJ9i>i9~VO_ta4WW#eP*u$@nm*(1{sJKmz=7s^ z^xnQ4*|)K0j9=cA(P{Dmh!*hF@%F0SwM}2b&0P|V}1)bfy1xWtT2U)_sT0P z*UHQLWoBmTwD4$-vaN)vt_|?(AeN994Z{KnT1O)8Pb9<=QGpp#x*v6Azu1UUluLXD=9uaXK)ymgBqdj#po#U5VhOXYcc?lcT`KkKD z2lvPEHSiH!iiCb$PX-2yy?|bl#Yn{L8>t9>D z#eLtS+}uqB@>y4KEEH7{ur9wS{h>frMTLut%NKQ*)bmzkwhQ1f2$-bJ<}QiGNzz;u z(T0RlaammR^3|&WSnA^HYS|V}bC-)E%1cn?ltdr!4o0pNjEjqN8)%<@`d9O%pdc1E zH@92%Ed>Nb7mR-s65ixA54?!Bp&`e~FHg^6S7=?0Rbilf4p-j@1^~Eama((DCW)Cm zAU2t?!su8DDjg*7;;U=+^&>9cGS77a8(?_0<9d0h$OV+9Nt%HiRnjZL1Q7IL+xTSB zz`($RhYyW=8 z^5jVY`J-I4UH32I7K0LM#iuDo&Y>toBh=(V@9r1~T@+ZSvB}F9!FXnBXh(Lt0?K|= z5sZwCtv`QWLNRHLJ9bUc@nD?e22yoqXa7O+_Y=ED#tMe~ZPfmO*n5DuqsQt@ea=bRKl*fB{J;-HF9yQ7#UuOPcW zF9@4c^6niMBGA@DhgJ}R2oN-3Tf{0-2jIEZdX71u1hH*+Al|Z~32B$aGzW*Zg7QZx z)Ks1`<6Ws1&o>JGaUA8+DyfvbhCKI2TW8m%PO6cZ2}~?ZJ_tVXl`B^s*nV11zq^|o zgqO?*=Mm%a)=|D%9m&xrEwj_Ve16@VL_HV+ILHb*!0PyM+VKym;sG2Yw3E$wX?G)9 z>|W;%#%|ogBEG-CQ~HiU>?#CN|Ad4D@E|?e+xKn=V1PQhWP5H%Cfm>(=zT zCsTLklMw3Q>dKGkU0hbCJ=I^rc@WRwell2ADj>&RgkK{Fu;aq5Tg!}*{>39IwaP;H zv^Q_|B@i&n@bu{?gaFa{`8nI)T$Udfh!4l~;OEE3Bv!6m`72`VD!#?KYu7IE=vV&H z(b32PZfA2I7EVv)<@R~KU&SI`nr52fmy(ilx$@9)gM@zM^&qSk;o{h0^jF#1+G>HN zDGgJfNnM{`);LM)-Z_kud0~~obruviR#sNxz+6(^8~%Njc+e0b=eylh7wvx>OesrsHG)@b$#m88RBsI z^v;-#TOv+poGB9@A{QSZOPizB}u(h+R!DaQ7JI~Fxpu&io z+e-^!d;l>k=|uf~G41tHpk2Fo8o@pL@yqS+XTO)G>$s4QyGCc3v@|PFZ!zv@|J1lM zx#9KVUR;Rt<*CL={hiSVP|676cvS#C8K)SEc#d~QKjR3~<0f3V*jc=M0ygu(qemr) z+7}{S?W0c1%E}UIk9>Ll+2lxX;HO7NGE_IM4OdbO2nh-b(%mks`S0lTc7b%x%?U=W z!1c&Ma>H6qAlrwnSjEXnE2^m($UngK#;=FTZEDOh+=q1ylA4zQr26m-waOt6UdJ5-mza;nvz30&3!w0+H zB|LcYgfJTT)er>u#FK(V;gHVPOMPPd>0E28C79WEP%r)c{l0~T>WFuP_u^5NpnA9d z{o|8^gM*YqVfKmoa-8+ZSHZlJ^p9bJf3OiU2SMKPN5eW5?8sMF3lI*f%F4^P=E|bP z1cOwL=ON6qn}-Lvqlzv)ivc_=zS^z8c1Il8${-GgFef^6& zCjmb;;_?{hI@ypSj8jy_=>COEm&!oQnT764+PVlVSz1xYVH+FqKmIVY5tooitle(U zlMZH2TvC#-m9$R$O;=Y}YSDGFN)K}%@AV3zl_ub#FukYFA>udlh-G=(_1!^dY!aBXIuHNe8 zl%*$E_dHZ4`c=!O!xwO%^!W3?CtRsK1SLW1)Tzf=`l#-dlmt;R$YyoiPEKw^5r=T7 zLKPtw0*KKn^4*74iK@66-L6R93t#m0r2xzlK<=|}a_V?^C}9s7D8y_{ zu<=|=mMmx;+58)~U)slKxSJcfm<9Qies3=ywf6990g_k|X3IM&!a(q_K?oV{sbi5H5q>CsK?wUhNbW5$DH%?oA=?~K7P=O zRcCv!k-qc^pPMJc#MHOq;75k}ObzHvjdg?(2DBzghk^2$9^nT8;g6VA0=R)hSpq^1 z8bQXFli*}8>m{R#mC4G=0&yRP%;$A&D_QqoNp1}m3ufcb$#?D$#vW1#1cG-iiU>vA z+jr~;#MSYwFEQ6QFmMqzu1v1%%4xnh7dB{V*AQ{JxsL_2@{xmo_5blNhh-D?A__pK zu{?BWp1r;O4b?MGik=aIwEn&YiCz#r5oJ_dK=p$wVL~=7sjlX@spL}pCRV|$#}kql z1TNYug%(fK4+bC_B~+7lvDN_tlamR&a6{ck6o2|hM|Dh0ObSLetY2@0tNuDpRR~vO z{Ntw6KLmihLclO~Aia{x5|H82@bIZ955%3~xe#_^lXFO(7twX&tGbma15&8rhZ^8u zsbvxpj6@&-TgAo0b0vjDEApt+OIKA2fgnv~KS1e^A3wN+gn%3NO1(1B*JsnO!U68? zxPvy)BDSGqz3|-iS0c1H_um_=is#K^W|k34!HuY_OdDe)IrC<1tY6ii(MeZdpDw*a zUw`$LhX!EL8Xg_t3>8xkg(^_+=sq|@suhgb_$oS7K&M>u-fLJKUG?ra0yOBQKvuvy zGOqw$5|ff@paW|B+Qo7f7#6}0Qy`~UKt@LInVej=0SW3B#0h^jS@Qg%9`8XPzj*P2 zg@uKvnZT7nkV=qVWqwV7lM*8w=z|AqsZ8dq1J6$zj88%paTsXVz7`Xc040#xf8bl= z<72$2Jmy351b0YLtJkctvbDX4&rQJNQ53g#@7){diuXAPDf?(m;`}d7O)mYbVF)nZ zwtOx~Bm}SAe$}$?OD^)mW{|rNkJdIE|Np?6{|Yc&DQB%a&&;{Z z@1?1%UlIN-WQ7z?kc3yFqef>>PY6KfH)$V~%Y;q(w;!$a1H)En8h7>Tg5}GTBv|JV zQhcOw&OsDz6fl$}L=m=jFNNVXB<)%%P3Zw_cSVEq4G`0@M2u zNQeFXMIh2b)HLw+9+J_Q^N-h?9Xdku$=tMw%_PQSS9JT4oO;FkVZnP@>i}E{StlSQ zw0!MaE_QxdTp?}8imlj+gE(j<&_s)Rtz@B&DTM~<@nn}9&1Yuri1(RcponA#Z-D39 zCAh#^Vw~9?sX!qEs0$Op<#K6|s+zVojTE!t3v}@-^xCv2BZKSmtSGTHjf}Wf@UWsZ zy_{m$*W`nHuPwR)r`~wmot+wgxb<6-97@@{@ejY^0n}#mB{EBv~ zoW8ENscMb2FK8=`BcI;>om2v&FQH8f>%;JnL2QoEEv_wIfymMwSch{dDM#Xdt#@*D zmCI@+KOa6@u1ZJ1(=s;~4&W3EMd3vRrPN)Saf3%3_x)EX#iZ4DV)Q3K2rt5$Kd1*- zW8P(-*tKHSswFm$@cO!8$8H#*-s{{05R@{t=f-~<_YSM+fOaK>CTMy^EO%Iy?|7;R{12&q6oLqnrE_Z}PpTSJ9|A^skJ z{FcztpFfju1CCjUY6SvH-ScSG!Gi~(q9%=&-@JaEZTt4^w{PDrDlSgsT7$rA@fQZ6 zmaSQ{=Iv#E3n$i38K*6@Pp=Y{b(?noCVyP$_WI@7PcFna6&OAIzQI7Puuf1j`^@Oc zY|iF~);ig}$0eX}Jbd`@+FGk~e@jYbPu`-Ug;2!ZwsiP!+dzJ_H3GE3(hN`bJztgsFUZekW=?NHdiiW~1`+Rl#l<+fM51 z>iX)hIv*8f7N^9@$~xYsuzK}Ed}JBAJ~=Wz9|Sd%jie%95$h53_k$4KA@Bjzt_Qms zI5E<@oJLE3+EK?&(xaYUh?DzRf5LrDoW#U~w+9}xk`$<=H4iEJ&gvnY#^7Mq1)QQ6 zkp=jV`R0O^u(Y(i+|~CC#SSC$4gNAU5n&#X-~r;CoA4JZT=o-DeSZ4?^4hD|vDA@g?j3KnXyl z4bLhy;b`Ao|EU{fPLP<-6gScs+*hf4C+GRjXJ4|J7L^Fh514WeB_i|J+y@-9NTqOlg7kPV zF((sn5>ETq_Bt9}!v%_pKB6QLzlMnk4`qq7^3-GGc=!Sxa+Emb4O8L3SO7`}pl$&K z0UhWDR4a;Q&_inm|9l^%OT6h2cVOM&W#$z1Lu$C1bG(iXt07zaj{wA42jLy|iY}E- z=-J@U_Jq%y<0^LGEnv)pqas43lei~P1^q3Z)xqX+QA}MF=Nz%oNNO>Z{}sfs01b;z z=OP|bYvpyM!_gKW6wSwrDl1uGN=P-!SVCNGAj&LjZ*ME(UQVKY@SOqI@ybK%@0#cG zz`*hXPZSQ;88uA;>|$lyq_i{+-jyn+vw#A2fX#`|W;J;Sk8K^>y^&|iIMt^vr+8N1 z*fciyO73r&)dd8~;}i9=JKVupJ5G%a{3E#@yGq|{bzfzc(SbTd>I9@JQOXkC4R&cF4lfQpnP+EfjRwAF??Vbu3 zNI0B#{u&7;`_oKT-aAizQY$GaNeAg$4uSa+2;;O!c5y(g%Lqn?Uf{vGzcXR8A<`x3 z$E+u~v?^MTGg!++*m3TMi||9FI|*=t2zOGUZgU?fw0y`2_lz9^fK2G_xo;5!gT?fP zX`&6pyxYvwCKy-D$kQc;T`G?TlQ%_C1qB7PPT@#HY=M{zUK)XEv5xTEj%@Gn3we}4 zPInq@xHJw<9k>2g2I6d~YHDiA=J*raVEI+OCKkhyLa6Y9-c55nnc7CgBn24RKo@kt z_CnloVB3jZ3P_e1kp6jK969SCd_Tn2qGsQMdIQ2-yw~rgkp57rm&+L-#wCV@BENqB z{vBK$69qwi$&Prn`4ntL#Bif6B!Y(w7k#IgkiI`(36wP}MNpH^?BoFzk6{-45WInB zIP;&V1+uh#X%M&D;8#{^nY{dByU&kbAc=u{et?vn7)la;YU%r@f$4aYZunH^2nS%7 zirJY_bpn7@;pyxd8{GXL)B5k5BOtdx2P_g;y7a}HH;Zr{O!vqEKU;jOzL{~iqrLsg zk!cWL7n0{{Hy>Qg=EHCYB4L!2m*0_UcTZWf?+FRP;MR%Si9e*&M>6&7jaPDELl7zU zWngk>E1rL<=L0J#JAuR_U7&C!X5B^jgjE5z$m~-+)IqbJMM4|z?rzk>c1~#Gerd1} zjRp3*EeJ%5+5D>}aA^7WIM~NR>LnhvqN2^zX5^*ceN7ksIPsDA6Lw7dv-2lH`XO={ zWr?_eNV)9l>OJQm5>dYA&!4Z_NbIJsuC5`BRTQq=kctZlj=mbznsqt$2TIGzh!^kb z)vL7aj+LN(Y;ErjzKUT$K>n+?96t&7NRo*d1&)dRZv3@tK~QlI7Wp!`e|&?o_~XZq z08*k@!V4(G`2E)I^*8rL%7QZMlB&mJLpALhYsB~5fv7`JB1q|0s-hRdFQK>d)zb;?QCKp*OMrB zpcR{N+f?wtU@UBYfQJD-v(q_>?b4SoU0MW{@$$86sU~sPuI+AZSOwcdcJC~FHc!EC zK>-_U-kq19KL8~WMzOBv`(>j+*J=Dk{B`)Hqu3vV&P{Bu$RZyy4;-+Omc_#$(Ie~c zyb56*^=c9V!T8DdPaUnJCMbvg))$_=py!F|hr8ltzZ%@SADw1aEwF>c0)JKCxv*JqxH~iK4apeF`T` z8~dg%k$caxqr>kw78-&th19%Lcp%Rdc9?#6un-;RBW0yRABJ%(lIgA3KVSecD&oLe zAuU1BllTikB*Mo*n{3K+-{8m9jXEICx~kKOxO5SNwQ-_Q$Pq7I)H)M<{$a-m7ID8- zLUeo=olut*KT~%<<7IVe$^0gYGE921g|@-t z?UKy0wGmspO9@xsL=iz!yu@?!Yso@4P2?XGeS}j(hR@_Z!~YCE+goTRp>C2i3&N z#>O8AyR^JK{ceQriaDykJ1NOggO8to*FS2qOBF`>+gKPDN$w>i3(W;LlF=7}6)U#A z3=xKbiuW93C7Lg&?Ip(?@JwX}O__Gae^s%1KQnC^)T%S>#vAY<-+;=(BW*=R2|p(r zMI84;fmQ*Xl(h8{ZE)Ef@>-8%1;N}J%ujcHW_;FN>xxh!hu$%~Sv4n2s43wydLYQ- zI9y4|gRY8*zwK|32%kR$I`%i|mMj}~9Fm*4Mpe+po53bU4{(}4Xp_6-CuH@9ZTmjO=4P|CLc&Y7y*ht zd{|r&CZ+?Tq#uZa-_;h2KrcnrqZrnb)Vo$s1m!mDz!x7smZvr&mJVR&4=(yM#irJ2 zcDA>lgFiU|RtGMPWvf;#0QvhS=OpO65c$8b%00j1V0DNUjWFB*46VDRK~{p%mss={ zzaW!rj;RQvH%O1vTN6`r{8t` zd!fu~29&};;jrzxibfF@akpp!3%w$d4dEULS2BghU#pCctrZ-%3Q>8ExL`8U$U-Ht&Dp0o|OLjLsheu&OOdAW7m7gbu5{X3DelZPq>-OyxU-H~} zi+VHTU?W?T^>3hdcDzfBbd`jLhC)<03kMZA9GI2GfFKs@T%=ufhQ>Vp%{4` znSS|FmxMWxxDJ5JxT5rr06Stl!CE#*ewXxaA~TQ&f~4RZ6~#xYxW|tlOV|tHalcFA z8lGihCU*f=OgXeKgrkK5Oh_y40bJv{j-{!OZC)(_8M_=<3&I9GmCVTV0LGE_(}=wZ z$ms%WEh$;^r@jc$3HTNfy7lrm+I?lD3>*usiDzD+RHZR4|0I zLJ=7^fm^;A3K3Moq}3 zk`g$}=j-TH{0GiM$8mHopd19V7n#I_f366c<2cf`H*Gwc^t924 z=h)!*Jx9(ca6aI2ha&gw-J59pDeHQlq20fv;r|YmhCZkRh=RyB%J*>bocx&sjGtSj zhEgxGaxpbFJ`OZZHfCH@K}Lohl@X1d9K-!MhNLqE*8~z^qz#fN!FEU^=C@zufB%yW zLB`M5%O!tXZLg=R>t;Vqwj(9pTjSIzdGE0{W?|*i3+)=8n0f110a?X87kCvZeIC>t zpW7F3*t_@crSYS{yVSUyY>y@`2Pr1bW5yR zWnX=>^V_!th#4dmf=KYs%(PGVz3Fe1{`xfBM_L)K3W6DZp;cb|zeiCl%*=tFIWQ2v zOI4%b>|DBW<3`ddfwq*k*4E2M-c3TF?MEPB99JN|#3$+I858d2L(Dx7@BLWiI^Mw#CT)Kk3?m>$$Mr$kDa{pBOLJs^;N@(vT zK96wDk@1l8AnP!Y4jn|T^o}658qI7-F)+CoH#dtCmIfip%FgZ*(0u|f+P~Gy8rkr2 z-QE3=5i|fqf)eOYfb&}xim3@h`2E}Ojo^*tkJZkjEFlrWn8ZxILpe4DUgd8Uvy+2E z5nhO*U^Og;*vvpVIy&C?5rJ%@ribfrZ*O)nC=v#W^qj!FLtNlkxAQ}$WX*K8;bx;{ zg~4Bgfr2ki3i6X<&j;yKKN`4E3rQ3DiY;kH^L(NSma|6DY5tJFMfMS@oR}5mA*y&> z3KkFdmikZ{M?yA1!9d0xVA?5Plo+jOP0|(=CX1?u0)GM*tqomkFe{O^7ibQyljcw362#vkxZVwo)=)RQf|s$lmy^3iQj8RUWedQg5q8`F++0#6Nsg8#2H zo@_K^)RgNixnmp-5oofVf1{Ydjk7G@da($S4ak~)7^iH%JmrQT@I_9G_r8B7_#Lra zJUon8447wORJ;Yuh8QQX;qXF0m!1FoHAy<^^x5&ZK5I|^*o{QV{3oGy)hTW;bi5-Tt2k=j-xDU=FCQ19k=>oCzxh9H%m}%ZSuOBmlBAY+n%;3PwxNmbB@2hwQG4La8(!(jyU*u(44^H z^I$P#n|c=eH}Zxw_aQ{z!t>7|AtDt3O%c1YF#v@8@eeHULT|CPT@6D0R+b!Uaz#Z& zDSlJa_Xq9ea3H=j=WipNCby6fySlo%O$=^xe7qQxpTQnOkkx8dD{$bXtM&udP1zALw{+?X(nDrT~R2#m2{aV!Ayg&I10w6Np zOV?Pq1U-8tSFHHDPaWDAwm}D!g(aMFtHsd%M%o#P7>LXWU5F7J*tzkcuCC|t03iCY z8x119kxL@_KGrY&DcMH&Is>^FQ6cQw!~d^28!v-@$Jz8Fh9qxxeEW96;uc{$7@|Hk zG!WN&aBwiK6MuuR?4_#^p=)}&R+$op5KN|-N6KIaEXBArFyf1XX?XhxlucLcQ9R7d zm8y%aD&ArDOK$P#*_Rvr8_L*o2c57YB9{>Rx}F>0PMY;Fk#1Hq^3sGZPS6arw6mLT z-c9^UY4q#AZu;Z#h6ie5MQN=g$vOYh^pqDTlK~s*Tn8cik&KW;u7P?$<5(E%Djhge zyB{8$?^|CIq9#0C2b+d`14?7iobjj24zE%0147rHm`cV z1;cET$fKRflMWw$E7Hfc^)_rkm5{Y9U^S6uj#GE*-`uG@KlVT6WGG)iJi-W51camu z_oB~W3|cf$*+blV%n{&2ly`7(DZ|yIm+N9rYbp#fd1w?1Iw*~dj0$=~MOCk-{D4ob z6s1Cpy%#DR7V&2HJkY8gKuLlXq(%_q%wI~pn-emyF^<<+fK7hJ5t#b&FaP9~W{bh) z5);?@cX~-o7l{(L<-hV)CaSpUEa476{J;35jDugiLe9CJiJ$O^ufpDlN&ah1=wA;KPOou%&hk5U_= zO5M)~h4B1`SMZ27`ftV9w7dKomeBNww+}7q@IwYd1ALmkQ7ME>PW(aiSq=!)GJzgqGXG$$Wf4VC0%&p#e9HmViOtPT z&fhGN{P9^yG6(o#Lz^ASc3LlSTjJK1;nqDo@>Y}>=wbYj{rww-dARqx|E(o-1|>Y~ z)#KnUh%JOS%0qwqedVw0LZww~eSMIXbq>z> ztWbn%v?Aj&NBnYp^zQ<&!uwT(UHcDaEd+J+skcm^KCBwawfU89edI_=Rsncd3>dhP zRow^++*-TO@_+2xxt6$6>P!#Kwsclrk4Rb;wKOf;V(Q0l|B&sxJx>^7APQZnMk7D_ek{(|vB(0}er-O~9_Un;yjF91i@9;;oUe?yTQ?^ObBbIzuB6XJn5 z_6_%em>d8>15la@ES+w#Y=^IpRG3f_|MI88Q#jlvICpNZYq5x*645#q{dZG{#?PPf zxsQIdojXZUDaMU@ADSGm>;CCdB@Ph-xeBqUfQ?jgl1OsGBCXr|u6ug-nsL@u1TZCF zhSL({Quy~ZE%JEOY+Y3WJF9%Fc1Z4kU%#taTW8R zfx8#uI{X09UVtG(PbKm7pf4#kD-2FV4TqiD?GekSdLrg^>6JaQaj`BrhZUUmK*F1&%M`Cli;FX? zk6FDV^9np^XB&wMiB7K0oz&A0Mrn3Vo zqsw)Iay;}>ROtxY;~Zn&&bhd?%)tP1GMMKghsiS`pd(~nf%^(Y#~!iKlPizQOvag@ z!S_~?7a34eTWgYwV#_ON74Ug@X!qkl!jwP>pH=Q9WEyPBpk+*?eMWlt_UE}V%#m%* z=+;L=17e|>M;gx3kgfC2&wrqm_#mW4^e)$)={Sk-XLv)=xOD;@SP;2)k1L{80=&b) z9XQnCqLtHMcAj{e`;G}#YhJ1pPL@a;RHIyAq9Nmhd&7&9rn818cClGM0F)VhgMx#N zuUh0fX`^#E7^GObNEmF(&|n^Y9zls9ML{V?2_$Pi>;MN=+ZpxVk#<>i>S*JWHF!o` ztb`jIU%Yy?eUn`=jyO203q-d9?J$obgla>My%?5^84zTl&_-A%ADR^PzVZYoLOeu7 z(^t8Q@mM%>3{+cN8$hZyc;Rr|Eo>D@<6}5oFeYd0`~goLeE$J3IzmVdSCzEaGcfRv zh~R}xl3MS|b2Z>}udl=s7iqANi2yj@e)NexV-t^cQX&*uO@k2SvAi08 z+M0GhJ{s;cjoOB|r@=f(y;`z*_pQ6JQBm^&Z&vSnqMz#xK8mod-ZSGz$la@Qg6|rN zG82WGZk@;QQR0m{e|eweuA-YzsD`8Urv=B3F$PLh8-m(v&7 zQ4OJ2ZvXXF-$t;$DCF%uQoEbXvN5@mYmOu!?qBFToch+n;{F*qcmPNms zo3a$Ay3d}c1PN0Dx(7_*3n&68!Q^S9FC8tidqs3oZbD`1fMXE&dpWwI&%s%QYC6&e zoG_6|-~&7*5c7?g%o8-32^&t@%F*&hy1Fd_ldrJ2Rs8z3{+IkSyI91v==4Zf zyRL>Noyoz&T|mvvF77urGx-a))NG6dD1gnU5S+$b3m{YzrO?6YjkXQT5YLvQAeV!4 zwG=Foe^gWynes9=(VOY7zls;k$}^6kd?nYu8^0u?^hk5T;6pVu0+F6t?)5QqEzYDr zk30%ck4AmMu|wZ$lM?|q6QNTb#62c|k%>yA>k`0PtG+G&J=6;FR?w!nkg&*fu6BgG zsu6%z+>D;J8Y5)<1{qreJs-b_-dfwU+qWGC%YiO2cweC@nc>`q0DgynSu#PM&_oyy zf;r^7qTWyQJU8R6uEj_6>a7v>$O(ca2Q7P{_;~`NJZ6Q60@1_dcBhw%9T(3J%yR-X z4x*)jTPIzJwEAC7&-vl3T0M}z9L=Db=+Yo2VaN!i*XSCAMD$K)^&YlL8224>1gZ54OH;J_dv5+uZpxrG3Ck%1=bU@CM z0vueC>&HbHXR_J&m?14n0;vk#!fvR@5H*P92EJ)1YD6AHaYkswJH1BikMZ2D_epP+ zH#(e};!Kqcx^!iLGujbMt6H4Tq|S^M&IV)ozOR2gN&`wq97lJ8mA$*8#*qG5P<44LFwX;X`KFx&?ZTj9;pPusoYtA_&Cx8Of2{N80it%)~YeC zTyrurE2<(_fj#mNnDSDXSr#=aT0JdE+#j89we3fRO&M1pi4(em*5it6k6G3`wg z`5#$15cmSp8b55Lx9^vnNg<~Y_H06|67dXjv#^RA7qNuEuW%mtw6*9g6A2E)kU>V+ z!7WPdcU6v!jg_fy$v?x7WBtL*O%S>S8K1KbB`1ax5N^Lp!OE*0Q|_?AFJQ1J>g9~U zFC@Vv(cPGn8Jc8z0GT?H)vB+vigZ)K|0;mtP*p20BjokcUVGM|e+j$*!tV3AX;in2 zq~2xhk`vM1=?1Sk+H{IwYa05pc6qq2-f-u3@p&W-pih-FBcfS}?rw)qzy$tJWb2qW zaa&!wq>W7(vVi69l=pJp3Zw725GM!;fa|CMUoW97ky{hd74(ebXFdVV*w_we2qmz* z;r)#;oe=Xn7Q5nBH#iM?cd(^VY9?f3D56sPZjI(g>ysd)O}_GISlLo(KNwT1DFn4 z%wn{Siucs+luRaGT5TFgYkoF=jZd-U;b7fP#KXZhosInfgq&d)C)_ke5xNv-l$5XF zZF&`zM)E&}!k0qi5Hw7t9w;o>pD#+R=-!qUrKLqUe}6)(GX4Xe3BCRDS0^{#vsVpB z?)=3_y?*n?5uQk>Rc7@Gi}xOLQ)N4@89es-U0Kl@njjXx%5@wdsti|N>+s<__cc8; z2`XQGpuij>0x;ra&5xfywP8{7N21#O#5M*GE7oZ2M1T{FJwZU+Y>=Ef-w5Ik@{wlz z(4_aTku&Bd6Cl4KgJDkJKKPzZ>aEi5$=}$KOH|p1c8wb?0u!QY`#mDRZX|#LU>5WK z#E}9a9k{Q?KhaK4#tC3*Y36;+HJ6cQoxHEZ)yW109CJjLKRMY%#xW#xbbqKg9VrZU zX)%rkmqu#yvNN|7Hl`ji8km2yMMK}fPAWF&GHgHj7+#w+(5SB1Bj}JOO;RZ;@ zs#54i9~gqGwtT^kDnC2*oe^3uk{FrQq!eoCWvf%k&Cef*o7^cK{i<`^d!@m)Y|Jtw z>tjQ?V&mH|Y!2q3?`8TO$!(rhe7wBpAytz`3ippu!&|=R>Xq~wUui5l#yi}A0ROChkH|wa5I!Y^6Yl%-fS8XsR6gz5mc&9v|>6JX@zw>)nl!! z$ALi+Na*W!U~<`F$RXoxOtjmu{(MuW>fgUNwJ$jEkSr6Vx&Sq=31U8h8=iJSZH4LP z?{2Dkr-V*@R^PgJ-TL)B^_d6Wnj$&6zyZ3#k4r*U_P*AR%UBWVg^L#X(`kHgFvnz( zL2vx97uQ%Fx^AVc7m7_K^9h5HY1lDmV*rdbIaxHHdVcgEl9oMpGSzGi3E?7Iw{vS9 z)H*Q>f!DR~OO1=bUg<@dI@?Y%{Th644HR)60FfxKXh>`w@yRBwp%|k;oWZ2XhG|o} zCPxz53Hcasc%t?A*?X`;kFXq@(OQ2>_~rz_yET*sGI$cR9oVT<47I}92tk{3w_=tn zI|s+knHd0OqQ#?3Gih%3#xOecjF4XH5-hq4-<_>`CFAq*W{$^zv9d-af5xFLliW5U zm1bwDkz4^ok3TLspcqZ^7=~G{hcEX5xK$!H;c9Qz&|riwFr(vWu-Zih`;6+BFN?59 z4lwG$ZoE}5mc5(kEUi7+%~wcCnm~EQyoQ2 zw#4Tlqbq&ud7pT}T}6B3!t13^+zG_%j}EUbps#S77%=jSaGpr#6^OnWyXGRXZD4|FEz-<+1!Gr2g^iz>D9GGTi$Glu z!|v}|8T33wyK|!G$`mCj9|eL;VMX^FbqBB9A-`0qsIQIb+Y0Z&e2auyG-r?`TpM(c z5pQ!ozD0(yw<-u7h|{`a<;oYBIzW;$s#|!ENT%;mQjxeRRg+^D(R>|p3lRwM$%PTQ zK(>^k9tb7e3$P)XbwTaNae@7X7*POJtX#jy^D)F)|^nOSy)|U11cp z9#i->1O2%x%Ry67tf3t{$NDGT_K)cw|9P#ccN3uxb27f}0VYTqDIkN5;XV9$enp~* zKMXI%X&)dD+_iYVGx_z$k2`g0zWxHa19!p)7XdB$JWPAiO1@$)8%PZGn7>Ts=OrM( zK%OSki3#Y5%H)WFPoJE7h!#-mJx7~dfx4L}(xwZ0hVHg)#=kubAea^P<^~}SrtzSI zjgJr*6r4hV_`raitsNaq=u#l|1?mA{0U^lX{Uh*<0MaiyKyj!b2oNDQL8M(Yc9o5m zavLWQc2iAo&ZU9TD8!HIKJ@MBnNeBvt}Ud{t~8%QZd<+gF$-*xq%tCb?4pnY2i9E^ zmNPQx7XwM93gzA-GbNGk!nO9zDjza-?TDcpz|T$eXS6twAhr;O3lOUWgpsCJ^a9#8 zJ>w%L6_9rW{cR7O4R3De6WgRluiK@!}uc{4@bLVZDsBtX&>{ZuqC zrhUOE&iNaTRF|ST|2#1HmQ@iTyLbSTP}}*3ocI$#Ul??Rq=a*uvf3G{BYIN%TV`kY zF?W;nzu?G_bG$_&9Ec>>sA6Sedl~%3ddXP4bkCT3X=&29W@K0!rnhe2gC4{4_^c=l z$@CB$MGbgD$pkHwHF+K*dN%n_fT1D_F`|#K+a7=MIn& z8X6Obg6V2MJ&EwvpZ0?UKomIUFXg0=nTcdX8-cfrqJVpKU>y1Vt0fGtk^m19#*>r) zGc*q^&zyjGu1yeOYw<8Hv zRgh6vTep@gcRb9Q8|#b@9EehafA`zsVn$-bM6vqr-aZ8)XOOE13r=&ckHmxnnDH7N zYeY1Rf?*;=D;jl(xe-lr@QSjL5oFkxc3db-XIrN>JLOEG2k5;ylLi^Y2Mz^!g8_Xe zDDKIuGGc)sa>zjS$>=WEdjhawzZvwE1@G$%zHC*$OkCZRZJHX zTULwb&?2-nGVI&84<=1!imV=JS8}IF7$&1gvdhgE2`h4;XfFnC!-Uuckcdb&#{Htm z;B)dH5@G4xEr#}v4lVL+J;*9H7w8lf5h9dGXMYp literal 0 HcmV?d00001 diff --git a/docs/user_guide/selection/RecursiveFeatureAddition.rst b/docs/user_guide/selection/RecursiveFeatureAddition.rst index 2ee4970b5..a884d7a8c 100644 --- a/docs/user_guide/selection/RecursiveFeatureAddition.rst +++ b/docs/user_guide/selection/RecursiveFeatureAddition.rst @@ -104,7 +104,7 @@ removes the features from the dataset. Xt = tr.fit_transform(X, y) print(Xt.head()) -Only 4 features were deemend importance by recursive feature addition with linear regression: +Only 4 features were deemed important by recursive feature addition with linear regression: .. code:: python diff --git a/docs/user_guide/selection/RecursiveFeatureElimination.rst b/docs/user_guide/selection/RecursiveFeatureElimination.rst index ed2522458..6cad0a63a 100644 --- a/docs/user_guide/selection/RecursiveFeatureElimination.rst +++ b/docs/user_guide/selection/RecursiveFeatureElimination.rst @@ -6,47 +6,55 @@ RecursiveFeatureElimination ============================ :class:`RecursiveFeatureElimination` implements recursive feature elimination. Recursive -feature elimination (RFE) is a backward feature selection process. In Feature-engine's -implementation of RFE, a feature will be kept or removed based on the performance of a -machine learning model without that feature. This differs from Scikit-learn's implementation of +feature elimination (RFE) is a backward feature selection process. + +In Feature-engine's implementation of RFE, a feature will be kept or removed based on the +resulting change in model performance resulting of adding that feature to a +machine learning. This differs from Scikit-learn's implementation of `RFE `_ -where a feature will be kept or removed based on the feature importance. +where a feature will be kept or removed based on the feature importance derived from a +machine learning model via it's coefficients parameters or 'feature_importances_` attribute. -This technique begins by building a model on the entire set of variables, then calculates and -stores a model performance metric, and finally computes an importance score for each variable. -Features are ranked by the model’s `coef_` or `feature_importances_` attributes. +Feature-engine's implementation of RFE begins by training a model on the entire set of variables, +and storing its performance value. From this same model, :class:`RecursiveFeatureElimination` +derives the feature importance through the `coef_` or `feature_importances_` attributes, depending +if it is a linear model or a tree-based algorithm. These feature importance value is used +to sort the features by incraeasing performance, to determine the order in which the features +will be recursively removed. The least important features are removed first. -In the next step, the least important feature is removed, the model is re-built, and a new performance -metric is determined. If this performance metric is worse than the original one, then, -the feature is kept, (because eliminating the feature clearly caused a drop in model -performance) otherwise, it removed. +In the next step, :class:`RecursiveFeatureElimination` removes the least important feature +and trains a new machine learning model using the remaining variables. If the performance of +this model is worse than the performance from the previus model, then, the feature is kept +(because eliminating the feature caused a drop in model performance) otherwise, it removed. -The procedure removes now the second to least important feature, trains a new model, determines a -new performance metric, and so on, until it evaluates all the features, from the least -to the most important. +:class:`RecursiveFeatureElimination` removes now the second least important feature, trains a new model, +compares its performance to the previous model, determines if it should remove or retain the feature, +and moves on to the next variable until it evaluates all the features in the dataset. Note that, in Feature-engine's implementation of RFE, the feature importance is used -just to rank features and thus determine the order -in which the features will be eliminated. But whether to retain a feature is determined -based on the decrease in the performance of the model after the feature elimination. +just to rank features and thus determine the order in which the features will be eliminated. +But whether to retain a feature is determined based on the decrease in the performance of the +model after the feature elimination. By recursively eliminating features, RFE attempts to eliminate dependencies and collinearity that may exist in the model. -**Parameters** +Parameters +---------- -Feature-engine's RFE has 2 parameters that need to be determined somewhat arbitrarily by +:class:`RecursiveFeatureElimination` has 2 parameters that need to be determined somewhat arbitrarily by the user: the first one is the machine learning model which performance will be evaluated. The -second is the threshold in the performance drop that needs to occur, to remove a feature. +second is the threshold in the performance drop that needs to occur to remove a feature. RFE is not machine learning model agnostic, this means that the feature selection depends on the model, and different models may have different subsets of optimal features. Thus, it is recommended that you use the machine learning model that you finally intend to build. Regarding the threshold, this parameter needs a bit of hand tuning. Higher thresholds will -of course return fewer features. +return fewer features. -**Example** +Python example +-------------- Let's see how to use this transformer with the diabetes dataset that comes in Scikit-learn. First, we load the data: @@ -54,15 +62,34 @@ First, we load the data: .. code:: python + import matplotlib.pyplot as plt import pandas as pd from sklearn.datasets import load_diabetes from sklearn.linear_model import LinearRegression from feature_engine.selection import RecursiveFeatureElimination # load dataset - diabetes_X, diabetes_y = load_diabetes(return_X_y=True) - X = pd.DataFrame(diabetes_X) - y = pd.Series(diabetes_y) + X, y = load_diabetes(return_X_y=True, as_frame=True) + + print(X.head()) + +In the following output we see the diabetes dataset: + +.. code:: python + + age sex bmi bp s1 s2 s3 \ + 0 0.038076 0.050680 0.061696 0.021872 -0.044223 -0.034821 -0.043401 + 1 -0.001882 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 0.074412 + 2 0.085299 0.050680 0.044451 -0.005670 -0.045599 -0.034194 -0.032356 + 3 -0.089063 -0.044642 -0.011595 -0.036656 0.012191 0.024991 -0.036038 + 4 0.005383 -0.044642 -0.036385 0.021872 0.003935 0.015596 0.008142 + + s4 s5 s6 + 0 -0.002592 0.019907 -0.017646 + 1 -0.039493 -0.068332 -0.092204 + 2 -0.002592 0.002861 -0.025930 + 3 0.034309 0.022688 -0.009362 + 4 -0.002592 -0.031988 -0.046641 Now, we set up :class:`RecursiveFeatureElimination` to select features based on the r2 returned by a Linear Regression model, using 3 fold cross-validation. In this case, @@ -82,8 +109,19 @@ removes the features from the dataset. .. code:: python - # fit transformer Xt = tr.fit_transform(X, y) + print(Xt.head()) + +Six features were deemed important by recursive feature elimination with linear regression: + +.. code:: python + + sex bmi bp s1 s2 s5 + 0 0.050680 0.061696 0.021872 -0.044223 -0.034821 0.019907 + 1 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 -0.068332 + 2 0.050680 0.044451 -0.005670 -0.045599 -0.034194 0.002861 + 3 -0.044642 -0.011595 -0.036656 0.012191 0.024991 0.022688 + 4 -0.044642 -0.036385 0.021872 0.003935 0.015596 -0.031988 :class:`RecursiveFeatureElimination` stores the performance of the model trained using all @@ -93,58 +131,197 @@ the features in its attribute: # get the initial linear model performance, using all features tr.initial_model_performance_ - + + +In the following output we see the performance of the linear regression trained on the +entire dataset: + .. code:: python 0.488702767247119 -:class:`RecursiveFeatureElimination` also stores the change in the performance caused by -removing every feature. +Evaluating feature importance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The coefficients of the linear regression are used to determine the initial feature importance +score, which is used to sort the features before applying the recursive elimination process. We +can check out the feature importance as follows: + +.. code:: python + + tr.feature_importances_ + +In the following output we see the feature importance derived from the linear model: + +.. code:: python + + age 41.418041 + s6 64.768417 + s3 113.965992 + s4 182.174834 + sex 238.619526 + bp 322.091802 + s2 436.671584 + bmi 522.330165 + s5 741.471337 + s1 750.023872 + dtype: float64 + +The feature importance is obtained using cross-validation, so :class:`RecursiveFeatureElimination` +also stores the standard deviation of the feature importance: + +.. code:: python + + tr.feature_importances_std_ + +In the following output we see the standard deviation of the feature importance: + +.. code:: python + + age 18.217152 + sex 68.354719 + bmi 86.030698 + bp 57.110383 + s1 329.375819 + s2 299.756998 + s3 72.805496 + s4 47.925822 + s5 117.829949 + s6 42.754774 + dtype: float64 + +The selection procedure is based on whether removing a feature decreases the performance of +a model compared to the same model with that feature. We can check out the performance +changes as follows: .. code:: python # Get the performance drift of each feature tr.performance_drifts_ - + +In the following output we see the changes in performance returned by removing each feature: + .. code:: python - {0: -0.0032796652347705235, - 9: -0.00028200591588534163, - 6: -0.0006752869546966522, - 7: 0.00013883578730117252, - 1: 0.011956170569096924, - 3: 0.028634492035512438, - 5: 0.012639090879036363, - 2: 0.06630127204137715, - 8: 0.1093736570697495, - 4: 0.024318093565432353} + {'age': -0.0032800993162502845, + 's6': -0.00028194870232089997, + 's3': -0.0006751427734088544, + 's4': 0.00013890056776355575, + 'sex': 0.01195652626644067, + 'bp': 0.02863360798239445, + 's2': 0.012639242239088355, + 'bmi': 0.06630359039334816, + 's5': 0.10937354113435072, + 's1': 0.024318355833473526} + +We can also check out the standard deviation of the performance drift: + +.. code:: python + + # Get the performance drift of each feature + tr.performance_drifts_std_ + +In the following output we see the standard deviation of the changes in performance +returned by eliminating each feature: + +.. code:: python + + {'age': 0.013642261032787014, + 's6': 0.01678934235354838, + 's3': 0.01685859860738229, + 's4': 0.017977817100713972, + 'sex': 0.025202392033518706, + 'bp': 0.00841776123355417, + 's2': 0.008676750772593812, + 'bmi': 0.042463565656018436, + 's5': 0.046779680487815146, + 's1': 0.01621466049786452} + +We can now plot the performance change with the standard deviation to identify importance +features: + +.. code:: python + + r = pd.concat([ + pd.Series(tr.performance_drifts_), + pd.Series(tr.performance_drifts_std_) + ], axis=1 + ) + r.columns = ['mean', 'std'] + + r['mean'].plot.bar(yerr=[r['std'], r['std']], subplots=True) + + plt.title("Performance drift elicited by adding features") + plt.ylabel('Mean performance drift') + plt.xlabel('Features') + plt.show() + +In the following image we see the change in performance resulting from removing each feature +from a model: + +.. figure:: ../../images/rfe_perf_drift.png + +For comparison, we can plot the feature importance derived from the linear regression +together with the standard deviation: + +.. code:: python + + r = pd.concat([ + tr.feature_importances_, + tr.feature_importances_std_, + ], axis=1 + ) + r.columns = ['mean', 'std'] + + r['mean'].plot.bar(yerr=[r['std'], r['std']], subplots=True) + + plt.title("Feature importance derived from the linear regression") + plt.ylabel('Coefficients value') + plt.xlabel('Features') + plt.show() + +In the following image we see the feature importance determined by the coefficients of +the linear regression: + +.. figure:: ../../images/rfa_linreg_imp.png + +By comparing the performance in both plots, we can begin to understand which features +are important, and which ones could show some correlation to other variables in the data. +If a feature has a relatively big coefficient, but removing it does not change the model +performance, then, it might be correlated to another variable in the data. + +Checking out the eliminated features +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :class:`RecursiveFeatureElimination` also stores the features that will be dropped based -n the given threshold. +on the given threshold. .. code:: python # the features to remove tr.features_to_drop_ +These features were not deemed important by the RFE process: + .. code:: python - [0, 6, 7, 9] + ['age', 's3', 's4', 's6'] -If we now print the transformed data, we see that the features above were removed. +:class:`RecursiveFeatureElimination` also has the `get_support()` method that works exactly +like that of Scikit-learn's feature selection classes: .. code:: python - print(Xt.head()) + tr.get_support() + +The output contains True for the features that are selected and False for those that will +be dropped: .. code:: python - 1 2 3 4 5 8 - 0 0.050680 0.061696 0.021872 -0.044223 -0.034821 0.019907 - 1 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163 -0.068332 - 2 0.050680 0.044451 -0.005670 -0.045599 -0.034194 0.002861 - 3 -0.044642 -0.011595 -0.036656 0.012191 0.024991 0.022688 - 4 -0.044642 -0.036385 0.021872 0.003935 0.015596 -0.031988 + [False, True, True, True, True, True, False, False, True, False] + +And that's it! You now now how to select features by recursively removing them to a dataset. Additional resources diff --git a/feature_engine/_docstrings/fit_attributes.py b/feature_engine/_docstrings/fit_attributes.py index 9263800f4..3a91a61bd 100644 --- a/feature_engine/_docstrings/fit_attributes.py +++ b/feature_engine/_docstrings/fit_attributes.py @@ -38,7 +38,7 @@ Pandas Series with the feature importance (comes from step 2) """.rstrip() -_feature_importances_std_docstring = """feature_importances_: +_feature_importances_std_docstring = """feature_importances_std_: Pandas Series with the standard deviation of the feature importance. """.rstrip() @@ -46,7 +46,7 @@ Dictionary with the performance drift per examined feature (comes from step 5). """.rstrip() -_performance_drifts_std_docstring = """performance_drifts_: - Dictionary with the performance drift's standard deviation of the +_performance_drifts_std_docstring = """performance_drifts_std_: + Dictionary with the performance drift's standard deviation of the examined feature (comes from step 5). """.rstrip()