From 2e90da4278b531a69e99a535a313d6fb27c61bf0 Mon Sep 17 00:00:00 2001 From: Dedieu Lucas Date: Thu, 27 Feb 2025 17:27:01 +0000 Subject: [PATCH 1/5] feat: add checkpoint management for tuning --- edsnlp/tune.py | 168 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 26 deletions(-) diff --git a/edsnlp/tune.py b/edsnlp/tune.py index 48d305a24..34c85818f 100644 --- a/edsnlp/tune.py +++ b/edsnlp/tune.py @@ -7,6 +7,7 @@ import sys from typing import Dict, List, Optional, Tuple, Union +import joblib import optuna import optuna.visualization as vis from configobj import ConfigObj @@ -17,11 +18,16 @@ from optuna.pruners import MedianPruner from pydantic import BaseModel, confloat, conint from ruamel.yaml import YAML +from transformers.utils.logging import ERROR +from transformers.utils.logging import set_verbosity from edsnlp.training.trainer import GenericScorer, registry, train app = Cli(pretty_exceptions_show_locals=False) +# disable transformers warn logs +set_verbosity(ERROR) + logger = logging.getLogger(__name__) DEFAULT_GPU_HOUR = 1.0 @@ -284,7 +290,7 @@ def on_validation_callback(all_metrics): return score -def optimize(config_path, tuned_parameters, n_trials, metric, study=None): +def optimize(config_path, tuned_parameters, n_trials, metric, checkpoint_dir, study=None): def objective(trial): return objective_with_param(config_path, tuned_parameters, trial, metric) @@ -293,10 +299,26 @@ def objective(trial): direction="maximize", pruner=MedianPruner(n_startup_trials=5, n_warmup_steps=2), ) - study.optimize(objective, n_trials=n_trials) + study.optimize(objective, n_trials=n_trials, callbacks=[save_checkpoint(checkpoint_dir)]) return study +def save_checkpoint(checkpoint_dir): + def callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): + checkpoint_file = os.path.join(checkpoint_dir, "study.pkl") + logger.info(f"Saving checkpoint to {checkpoint_file}") + joblib.dump(study, checkpoint_file) + return callback + + +def load_checkpoint(checkpoint_dir) -> Optional[optuna.study.Study]: + checkpoint_file = os.path.join(checkpoint_dir, "study.pkl") + if os.path.exists(checkpoint_file): + logger.info(f"Loading study checkpoint from {checkpoint_file}") + return joblib.load(checkpoint_file) + return None + + def process_results( study, output_dir, @@ -376,17 +398,38 @@ def write_final_config(output_dir, config_path, tuned_parameters, best_params): config.write() +def parse_study_summary(output_dir): + file_path = os.path.join(output_dir, "results_summary.txt") + with open(file_path, "r") as f: + lines = f.readlines() + + sections = {"Params:": {}, "Importances:": {}} + current = None + + for line in lines: + line = line.strip() + if line in sections: + current = sections[line] + elif current is not None and line: + key, value = map(str.strip, line.split(":")) + current[key] = float(value) + + return sections["Params:"], sections["Importances:"] + + def tune_two_phase( config: Dict, config_path: str, hyperparameters: Dict[str, Dict], output_dir: str, + checkpoint_dir: str, n_trials: int, viz: bool, metric: Tuple[str], study: Optional[optuna.study.Study] = None, is_fixed_n_trials: bool = False, gpu_hours: float = 1.0, + skip_phase_1: bool = False, ) -> None: """ Perform two-phase hyperparameter tuning using Optuna. @@ -406,6 +449,8 @@ def tune_two_phase( output_dir : str Directory where tuning results, visualizations, and best parameters will be saved. + checkpoint_dir : str, + Path to save the checkpoint file. n_trials : int The total number of trials to execute across both tuning phases. This number will be split between the two phases, with approximately half @@ -423,53 +468,79 @@ def tune_two_phase( trials pruned in phase 1, we raise n_trials to compensate. Default is False. gpu_hours : float, optional Total GPU time available for tuning, in hours. Default is 1 hour. + skip_phase_1 : bool, optional + Whether or not to skip phase 1 (in case of resuming from checkpoint). + Default is False. """ - n_trials_2 = n_trials // 2 - n_trials_1 = n_trials - n_trials_2 output_dir_phase_1 = os.path.join(output_dir, "phase_1") output_dir_phase_2 = os.path.join(output_dir, "phase_2") - logger.info(f"Phase 1: Tuning all hyperparameters ({n_trials_1} trials).") - study = optimize(config, hyperparameters, n_trials_1, metric, study=study) - best_params_phase_1, importances = process_results( - study, output_dir_phase_1, viz, config, config_path, hyperparameters - ) + if str(config_path).endswith("yaml") or str(config_path).endswith("yml"): + config_path_phase_2 = os.path.join(output_dir_phase_1, "config.yml") + else: + config_path_phase_2 = os.path.join(output_dir_phase_1, "config.cfg") + + if not skip_phase_1: + n_trials_2 = n_trials // 2 + n_trials_1 = n_trials - n_trials_2 + + logger.info(f"Phase 1: Tuning all hyperparameters ({n_trials_1} trials).") + study = optimize( + config, + hyperparameters, + n_trials_1, + metric, + checkpoint_dir, + study, + ) + best_params_phase_1, importances = process_results( + study, output_dir_phase_1, viz, config, config_path, hyperparameters + ) + if not is_fixed_n_trials: + n_trials_2 = compute_remaining_n_trials_possible(study, gpu_hours) + + else: + n_trials_2 = n_trials + logger.info(f"Skipping already tuned phase 1") + hyperparameters_to_keep = list(study.trials[-1].params.keys()) + best_params_phase_1, importances = parse_study_summary(output_dir_phase_1) hyperparameters_to_keep = list(importances.keys())[ : math.ceil(len(importances) / 2) ] - + hyperparameters_phase_2 = { key: value for key, value in hyperparameters.items() if key in hyperparameters_to_keep or (value.get("alias") and value["alias"] in hyperparameters_to_keep) } + hyperparameters_frozen = { key: value for key, value in hyperparameters.items() if key not in hyperparameters_to_keep and (not value.get("alias") or value["alias"] not in hyperparameters_to_keep) } - + _, updated_config = update_config( config, hyperparameters_frozen, values=best_params_phase_1 ) - - if not is_fixed_n_trials: - n_trials_2 = compute_remaining_n_trials_possible(study, gpu_hours) - + logger.info( f"Phase 2: Tuning {hyperparameters_to_keep} hyperparameters " f"({n_trials_2} trials). Other hyperparameters frozen to best values." ) + study = optimize( - updated_config, hyperparameters_phase_2, n_trials_2, metric, study=study + updated_config, + hyperparameters_phase_2, + n_trials_2, + metric, + checkpoint_dir, + study, ) - if str(config_path).endswith("yaml") or str(config_path).endswith("yml"): - config_path_phase_2 = os.path.join(output_dir_phase_1, "config.yml") - else: - config_path_phase_2 = os.path.join(output_dir_phase_1, "config.cfg") + process_results( study, output_dir_phase_2, @@ -521,6 +592,7 @@ def tune( config_meta: Dict, hyperparameters: Dict[str, HyperparameterConfig], output_dir: str, + checkpoint_dir: str, gpu_hours: confloat(gt=0) = DEFAULT_GPU_HOUR, n_trials: conint(gt=0) = None, two_phase_tuning: bool = False, @@ -548,6 +620,8 @@ def tune( output_dir : str Directory where tuning results, visualizations, and best parameters will be saved. + checkpoint_dir : str, + Path to save the checkpoint file. gpu_hours : float, optional Total GPU time available for tuning, in hours. Default is 1 hour. n_trials : int, optional @@ -568,13 +642,36 @@ def tune( hyperparameters = {key: value.to_dict() for key, value in hyperparameters.items()} set_seed(seed) metric = split_path(metric) - study = None + study = load_checkpoint(checkpoint_dir) + elapsed_trials = 0 + skip_phase_1 = False is_fixed_n_trials = n_trials is not None + + if study: + elapsed_trials = len(study.trials) + logger.info(f"Elapsed trials: {elapsed_trials}") if not is_fixed_n_trials: - logger.info(f"Computing number of trials for {gpu_hours} hours of GPU.") - study = optimize(config, hyperparameters, n_trials=1, metric=metric) - n_trials = compute_n_trials(gpu_hours, compute_time_per_trial(study)) - 1 + if not study: + logger.info(f"Computing number of trials for {gpu_hours} hours of GPU.") + study = optimize( + config, + hyperparameters, + n_trials=1, + metric=metric, + checkpoint_dir=checkpoint_dir, + ) + n_trials = compute_n_trials(gpu_hours, compute_time_per_trial(study)) - 1 + else: + n_trials = compute_n_trials( + gpu_hours, + compute_time_per_trial(study, ema=True) + ) + + if elapsed_trials >= (n_trials / 2): + skip_phase_1 = True + + n_trials = max(0, n_trials - elapsed_trials) logger.info(f"Number of trials: {n_trials}") @@ -585,16 +682,25 @@ def tune( config_path, hyperparameters, output_dir, + checkpoint_dir, n_trials, viz, metric=metric, study=study, is_fixed_n_trials=is_fixed_n_trials, gpu_hours=gpu_hours, + skip_phase_1=skip_phase_1, ) else: logger.info("Starting single-phase tuning.") - study = optimize(config, hyperparameters, n_trials, metric, study=study) + study = optimize( + config, + hyperparameters, + n_trials, + metric, + checkpoint_dir, + study, + ) if not is_fixed_n_trials: n_trials = compute_remaining_n_trials_possible(study, gpu_hours) if n_trials > 0: @@ -602,9 +708,19 @@ def tune( f"As some trials were pruned, perform tuning for {n_trials} " "more trials to fully use GPU time budget." ) - study = optimize(config, hyperparameters, n_trials, metric, study=study) + study = optimize( + config, + hyperparameters, + n_trials, + metric, + checkpoint_dir, + study, + ) process_results(study, output_dir, viz, config, config_path, hyperparameters) + logger.info(f"Tuning completed. Results available in {output_dir}. Deleting checkpoint.") + os.remove(os.path.join(checkpoint_dir, "study.pkl")) + if __name__ == "__main__": app() From c286b0793d136000e5d2a5b073cb84f251000158 Mon Sep 17 00:00:00 2001 From: Dedieu Lucas Date: Thu, 27 Feb 2025 17:27:32 +0000 Subject: [PATCH 2/5] feat: add checkpoint management for tuning --- edsnlp/tune.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/edsnlp/tune.py b/edsnlp/tune.py index 34c85818f..694648d3e 100644 --- a/edsnlp/tune.py +++ b/edsnlp/tune.py @@ -18,8 +18,7 @@ from optuna.pruners import MedianPruner from pydantic import BaseModel, confloat, conint from ruamel.yaml import YAML -from transformers.utils.logging import ERROR -from transformers.utils.logging import set_verbosity +from transformers.utils.logging import ERROR, set_verbosity from edsnlp.training.trainer import GenericScorer, registry, train @@ -290,7 +289,9 @@ def on_validation_callback(all_metrics): return score -def optimize(config_path, tuned_parameters, n_trials, metric, checkpoint_dir, study=None): +def optimize( + config_path, tuned_parameters, n_trials, metric, checkpoint_dir, study=None +): def objective(trial): return objective_with_param(config_path, tuned_parameters, trial, metric) @@ -299,7 +300,9 @@ def objective(trial): direction="maximize", pruner=MedianPruner(n_startup_trials=5, n_warmup_steps=2), ) - study.optimize(objective, n_trials=n_trials, callbacks=[save_checkpoint(checkpoint_dir)]) + study.optimize( + objective, n_trials=n_trials, callbacks=[save_checkpoint(checkpoint_dir)] + ) return study @@ -308,6 +311,7 @@ def callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): checkpoint_file = os.path.join(checkpoint_dir, "study.pkl") logger.info(f"Saving checkpoint to {checkpoint_file}") joblib.dump(study, checkpoint_file) + return callback @@ -483,7 +487,7 @@ def tune_two_phase( if not skip_phase_1: n_trials_2 = n_trials // 2 n_trials_1 = n_trials - n_trials_2 - + logger.info(f"Phase 1: Tuning all hyperparameters ({n_trials_1} trials).") study = optimize( config, @@ -498,40 +502,40 @@ def tune_two_phase( ) if not is_fixed_n_trials: n_trials_2 = compute_remaining_n_trials_possible(study, gpu_hours) - + else: n_trials_2 = n_trials - logger.info(f"Skipping already tuned phase 1") + logger.info("Skipping already tuned phase 1") hyperparameters_to_keep = list(study.trials[-1].params.keys()) best_params_phase_1, importances = parse_study_summary(output_dir_phase_1) hyperparameters_to_keep = list(importances.keys())[ : math.ceil(len(importances) / 2) ] - + hyperparameters_phase_2 = { key: value for key, value in hyperparameters.items() if key in hyperparameters_to_keep or (value.get("alias") and value["alias"] in hyperparameters_to_keep) } - + hyperparameters_frozen = { key: value for key, value in hyperparameters.items() if key not in hyperparameters_to_keep and (not value.get("alias") or value["alias"] not in hyperparameters_to_keep) } - + _, updated_config = update_config( config, hyperparameters_frozen, values=best_params_phase_1 ) - + logger.info( f"Phase 2: Tuning {hyperparameters_to_keep} hyperparameters " f"({n_trials_2} trials). Other hyperparameters frozen to best values." ) - + study = optimize( updated_config, hyperparameters_phase_2, @@ -540,7 +544,7 @@ def tune_two_phase( checkpoint_dir, study, ) - + process_results( study, output_dir_phase_2, @@ -646,7 +650,7 @@ def tune( elapsed_trials = 0 skip_phase_1 = False is_fixed_n_trials = n_trials is not None - + if study: elapsed_trials = len(study.trials) logger.info(f"Elapsed trials: {elapsed_trials}") @@ -664,13 +668,12 @@ def tune( n_trials = compute_n_trials(gpu_hours, compute_time_per_trial(study)) - 1 else: n_trials = compute_n_trials( - gpu_hours, - compute_time_per_trial(study, ema=True) + gpu_hours, compute_time_per_trial(study, ema=True) ) - + if elapsed_trials >= (n_trials / 2): skip_phase_1 = True - + n_trials = max(0, n_trials - elapsed_trials) logger.info(f"Number of trials: {n_trials}") @@ -718,7 +721,9 @@ def tune( ) process_results(study, output_dir, viz, config, config_path, hyperparameters) - logger.info(f"Tuning completed. Results available in {output_dir}. Deleting checkpoint.") + logger.info( + f"Tuning completed. Results available in {output_dir}. Deleting checkpoint." + ) os.remove(os.path.join(checkpoint_dir, "study.pkl")) From b01ae99cacf00958c8d43a8f4a77196f4fc1c528 Mon Sep 17 00:00:00 2001 From: Dedieu Lucas Date: Mon, 3 Mar 2025 11:00:37 +0000 Subject: [PATCH 3/5] add checkpoints unit tests --- edsnlp/tune.py | 4 +- .../single_phase_gpu_hour/study_.pkl | Bin 0 -> 6032 bytes .../single_phase_n_trials/study_.pkl | Bin 0 -> 6566 bytes .../two_phase_gpu_hour/config.yml | 127 ++++++++++++++++++ .../two_phase_gpu_hour/results_summary.txt | 13 ++ .../two_phase_gpu_hour/study_.pkl | Bin 0 -> 5769 bytes .../two_phase_n_trials/config.yml | 127 ++++++++++++++++++ .../two_phase_n_trials/results_summary.txt | 13 ++ .../two_phase_n_trials/study_.pkl | Bin 0 -> 6456 bytes tests/tuning/test_end_to_end.py | 85 ++++++++---- tests/tuning/test_tuning.py | 21 ++- 11 files changed, 361 insertions(+), 29 deletions(-) create mode 100644 tests/tuning/test_checkpoints/single_phase_gpu_hour/study_.pkl create mode 100644 tests/tuning/test_checkpoints/single_phase_n_trials/study_.pkl create mode 100644 tests/tuning/test_checkpoints/two_phase_gpu_hour/config.yml create mode 100644 tests/tuning/test_checkpoints/two_phase_gpu_hour/results_summary.txt create mode 100644 tests/tuning/test_checkpoints/two_phase_gpu_hour/study_.pkl create mode 100644 tests/tuning/test_checkpoints/two_phase_n_trials/config.yml create mode 100644 tests/tuning/test_checkpoints/two_phase_n_trials/results_summary.txt create mode 100644 tests/tuning/test_checkpoints/two_phase_n_trials/study_.pkl diff --git a/edsnlp/tune.py b/edsnlp/tune.py index 694648d3e..6eb48c8fa 100644 --- a/edsnlp/tune.py +++ b/edsnlp/tune.py @@ -724,7 +724,9 @@ def tune( logger.info( f"Tuning completed. Results available in {output_dir}. Deleting checkpoint." ) - os.remove(os.path.join(checkpoint_dir, "study.pkl")) + checkpoint_file = os.path.join(checkpoint_dir, "study.pkl") + if os.path.exists(checkpoint_file): + os.remove(checkpoint_file) if __name__ == "__main__": diff --git a/tests/tuning/test_checkpoints/single_phase_gpu_hour/study_.pkl b/tests/tuning/test_checkpoints/single_phase_gpu_hour/study_.pkl new file mode 100644 index 0000000000000000000000000000000000000000..41662e0720b90e293d44e25c226ec78d6d502991 GIT binary patch literal 6032 zcmb7I2|QKX_cx@ty<}|EGAL8&fmX z*wV_>)Od~s$~HDPwXn2dvCVAIP&OFGRDn|vnGik)C^6)i$});jjPkf(_+N@p1l$DS z3y=tIgistwTPX0N8vYV$j0Q$P)szXKLT64Tg-FDFG#ue`5Q%VT2?P~z5P>v;#T5e@ z0Y=gpa^zn;_=O=48}jjB6hi?v6ykv@LlHI<)Q<*dQ>DJ}7~rnINHJ;aaYM z=0^Run8Zc{3+a~vfv;O!WKq|!nsg4IyS^@H+qZ`FpL8;{D6^64-qOxBXBU20LFCYNEsfn z1fkReChm@=bUUXvHSegQ)u=c^Dni60;92gDb&wl=Qfds5;#@I;N+e<&m=eH@ISLLT zcqA_2MhuY631Gq0gk-)z!WBnwIeaKcwh|632{4<^P;e(Dk{@H3yJI!DsssN?nW~V0 zaQHZM5=$!K3k5i^B7ij=GZ{4d-v()J;X+j6@&i8&%z;j{A%HENISdMeT8V(2H&YR| z!-cECTzAI_R?pw)I2%*L7@vm$dv`}4mRCbpUf%#44>)X=vanegDa(-nPIS!9-7#-3 z`R&8z4ylg8nFAih91^80sq6q!JB-}X^B8ha47sc36Tq2{=`iH6N!U~j?gbZj$2&`? z55<1{(QW!Fz%>rI`2lzAFL!DxshC-T1#w`ZAMn7S3mMQN&}G|urwe*2G3cV}GUcQg zbcrVh9pHsQLwo;h=+*;?_YWjKkYdoPi=Ye&p$`FkAwqA6Y|uC)vH_c5KiI;L0RI2g z7T7$7`~dpJ1Xx0%R~SUU)ZKAFlONGTluo%IAPzA7z_K6F(?*-U2g~EY3O^7?q7S0b zPl^j;r0)hK`tl#p2b1U#5`DzcdKLlL{|!Bd0Nnopy}|(cPy+Bs z^ooP%v48cw=14h(gMc3g!u-I>AJMA~|E>wb<3NNT5RmAF6naAd2}5d3JxiQ`QlHppGmR9iy>f)HDuX%w@~&iSS}d z#ax%cDKrqvP>2H?{J=((y2MyqTrw2qz$ONCATGPzBG5HMF%QJSctG8GBz5Sl$1@;e z*#$YYAPy3+80z-LQSyowgUv%4$_Ow5OGI3gfnnNWA6I|b01YIqK(VTdLPFm}TG6$i=xF7@XrG{lDz;yNe;b_YbzKni`cyW=~!68ttN{=2|I zH96c+R2nWpR&(L?DZy!AYa#(sC16`D*e(n10$E6dF@!P@4t6kTOf@JPfe`oz8@?O( zWc4^9~H>HdGpm2uDp z1PyvL1ByC8fP+{XMoK+IfWufC8@@6`ktVVkCWwg7UK!4%Ap=@bN+E3MZyLy?c9Pd( zGaDNVYmgNOe)9uI2=F^ywt4vygvS+dVayhaK{l3!O(H-Jor-8Hxsf239VTRj^I0a; zdV}kDlZ($2i_ypcF)9*q;WBwpwW9<$MyH-(xRk{XM{ykF%M_&vI4H?Z1IJ~LA`BIA zL4lVDo}?2LMJVRLjee3T4?W`y3Mm>#5(&9Z4Tzw6G*HA;fGko=P@K3q0hIWGQaVGC znl=}l_V)Jn_zpknhao2cWi0Bzqv0@$BXFmPKsk)j=L@r|-fv@<&2o;;4(0#C*E*EC zWbe$7Pmd3nUtQYcnJkR`ZG*d@@W|!3B^@y}?MsT^b!P5g+tRebhH*DE+~@`^EB?gi z>O*%q8@2y*^_;cYyM4ZW#>T&{Om-M^xgj;U$-UXl?(4&cwQlB7H{Z+?bzj#? z^AZ$Y?YELfJ-XbR3edSc)>_t^Pm0pERxAaVPiith`i<>loalPRK2*KG&gZ?^Mg2V= z5;oXg{8Zdm7G!uh&EiclD?j>J(U?PXTs&yLQM0GGm^8Z;MV&4v*1cl?&cdhs%+wQ~ zH7uoxNpadmioXjB=8S7?HY?7%TUVnWS;cGk^M%som4~zW*(v8&JRhf*CrH8O&p7C$ zJ|j8wQ_xpELqV@xr%v|pK<=67M{$4ZO?#R2-ZB4c)sdTQ_xo{|qcYMx>+jHMT*sx` z9tvi-S8F^=Q}fD5nAW1&u3&X=XF`tVt^;$=Wp&Nlmvjl_?WG6ZT2?NaxwGoylr@`d zQmXy?O)~3JHkQ=pyS6sR7GU&x57r9*l-Io2ZNDa2s;PV{$o`Dvda~qO;w6)6Z?5ZK z#_hJ)Y-;=PmwP3TCfe*A!Tfk_MscvV%ljqVWp?}4thAqg)~VOz>Xj6y@@g#})1mjn zDf{ku_ma=1f81sNYHLpA^NWh(J@<5FUE9~VVak`K7o7Y0KabD89sYg8%c79&)A=`j z_I*)SDp`If#9J-2^pO7ta3j^%kMY6S*DAq-_F9nUF{Y}H75R}j1D)xuM0x_!EM2_~Kjb5OcdEqv9F4?0xeWl_mM1 zw4Ckw`M5qduD;N(Td^#m?-p;jIN-PSTHpPQr56_zHcXqm$|1q$<;4qC&oQaK_0-$p zwa4dIe@l{XX(%vv*_}>v-gJ`jV7al$#osg6`o^?WtpI6_b^4ahDn_1NA1HoGJH-s!G$)JWNRa?>XL zCymmMuI-*pXV&YkQcPs8j^9?@v}n6+)6`vg8vF0E)rM`uPZeKR{A^(DY^C%qU|7Kg zmFku1AFoe0JCdWlty*JpFLTtB5f|Kh$`hDR15UPU8lzLTto_h3{&hgt?yRvVdzagF z?&$D!cr>%*OXWxl--uupJGw`F%W5Ti%?!5*79H(!PtJ$fINdzlGp4&^!I8@Cee43O zq{*YErf1sjF~bF;6~AcL-JUv+rWx24YLs-ZT_;`b2Co6+1emp7n-i3Q$Hz;*1xJhW zYNt4SnUdhoF6QGOVp3NP`absGyBMUtxTu4^UPv5 z%`{k{@@=!NN69MTwa@zBPxn{sw=Caa%lq;|rN?n=ZtZ*@gEH+AyL6M3(r#^&@Bx26&$$$R70vW++(Rpgr~;9?TH?9RA*%*dwwSNHyYts%eiB~lT*C%JN7U(S+B4ROYS zYLDW_N===PHS0wlJjF?tACTrYM)PyUnVM!+$}dfm70>oZ%u+2|r)t1fs=K3W{%67x z-##{v*5I?6lOAMJ1Z ztW|C6`l2HX%r%O8?rN26jA@@htWEHo_@%wz$%lm-H)RKD6&pSJo??ivSQfo!k3O5L ztGeTURh!z$iI+<|bLWmK?^ZnCQf%3MG*~?_v+GNvmhCR%FpuHutbZ#x`1$>@u~M%% zzh?&P&05ztzRgP0C!OkPJ&#!oA^<-gXVWU~XypzFEslG@LAw@dtR=28mD@(HlM7R!{Z&8LsEk z@m<0R4sRDeUc+yX9$9UD!nM?NM59QaAKJG_VTGg8_diuSZ@zldd)M0XE$uv={km0E z^3Y_t$9#3`s{4YH3PC2KwNr&N#x^GXw8ptKdeKOaE?Td>heO4=3C8v!k#^GMs=3aA z30gr`XH1ug4Y#`ZJhad6XP%y{eK%0P-oN^^2j(_Cw8-+CJfgF%!HXUr$VT7{1!M?bF^C-FsGLBR=VMz4Ki&wrj)mgBL!&rRxOyG_KbRFTg$h9(6}* zxDdt~85TOoW7}x^H=*^4PNiLZx9Gy{`oRgQ=VHqDyJKytq2|8Ldn+cMSheEV?(Bef ze|(Q#(A1Wrb46~9_Ji+*nN^I)!1C|6w!FjVMBmNLE-2Wh$Lp!9Zm|dmdVgv8rrUe% zay`9vE8Oc1j*&d}Xz!aquRx*tiP@^GX3^1lS4XNBV)EY zR$VI6(^?x}$jm`+JXkDP`dM>kchoDh5uGz4>q;H(wtlTsb=CO8y}J2q#S>M5_u949 zsb9}K8*H%|O+Phxtpojr$NCL%K~M9PRjVRg=d^TcCC0XXZBQ4@eEm?~!y+tmiRS6H z)5CUu_B99`MLkbNLL8i7z`7t=zQj-*rUFv<;ygQ0ca!zb4OLxWRtEnDQZSf?@K>R* zdntgX+jK z$bnkpAJlsnyX1{2dCQUwzJMJrd7 zR51z_!jVv^fDNvBGquUpeoh7vb~p~M522CeC>6IY6&x$jclBOWClTJ&Yg z*s=0&QV`xI0VT^dKp6unWI9PnNr_NEgbV=(sM4oEfly?P-&s34+OwP~woL1AJ13?! z#nFLc&E!z*tU0#MY^QLB6UE7y1=Q$TunQtL!eaw5gefLwg$1ljBfC^<2YV? z3{y*pX?A)5@8?+ zS0I7$H04BK8VDxHtwb^SG6=&g6qr-pD(75VqHW4%(Dm>E*ic`}g8_g>BvLSuH6S1j zHh_f(X~joPblR`1RORG8j_EM0!Iuuk!dPa_?Wn8 zYE-v|fdPg>5yIx7aFUoZDNiIsfiniClexA7VgKPF$(t`?NIk#PCxIDo5@%w-g-ln2 z#-LYXFw38=3fp{9G;pQ5jdpD7G4`+~9&>pST;N7^3t;-)esk!HLM;N!PLeUXc3j++ zI|k;ExwEKlhvZIjT?bv~!k+aG#KoPREMv;#Tas8|_=@(RDM2&%s`A8u7nwVbrpz_r zPT<0|;7xUVD!0erPW*1#l!tBs)I{JD2za7)bs#^cyg+@n4`?}xqA)`}E*7LFg0w)e?Yk%p*Ve*(KzbthGZ19p zQP@sI!EH9}UHtBOKktAYcocTxQONu`3g7R5T?0|rje)FRi$XRA_WX+|sLPlf`Y`zB zCC;87-s@m52KM0&?1rz|Y#G6azB^cQFtDGlgTKGOKUZ=waDaGi6P)QFSB>b#=1Dj# z`B{+{3i1aI3W6BfD9Fp8j+}{k4s;=bL-?#54g^OS#6ia7pwdC|0}Ak07%D&E1dNzL zCIkv0Cm_x}Tsru^KT3m$<%i_pfG9Y|%_mM^lz|`95^#LbL%9I*GO3tjqj*dT4)6++ zKOlh<@aQPQz)3RQ7{&sDVT7CofiCg{3`pE?0pw5cj7355|F-&b7ZMUc81Wjg0dEy1 zNZ=GXiR$(mjs&lyY70cYd{PbjA~kVg;! zLcv)YiLMV#BM<@)VL>*@!#NBJ%4ok4bPb7M2m}mp?$<^plu6(`eKc+pQWhcPw``7> z1HXh)gm9E6i~tvsh&6*qV7vUErumGxSjcpQY>@zW7&5qdP!dQ86pZd7oQ6vnxXdm0 zN11Z!Hqb#arxpLSE7u??3=Pca6=>=z1}eGbT-<6E2Ci|-S&(UpV{PO;Y!ESz70Kt2 z5QSHSQwR&rHwjb|E6Fd|Zu)cwCvZIx+z13U82F1UU%Wgi62TF2Aa4^%z)kLTt_cQe z$pov3TamEp{IfCfUwmc~M@hYAzK9SJnj zRiKKD0-BPO{s4~xK{J`AN_2ApE&l%gbHBlh_@XIEK`WEk@C-g=atK^0V(=L9w5COT z&W7#St}pi;$1#G}7fhP2w6%eE!Pex+Xv4E}4m{bX<=vF$C6-c6pMA?*z1b;b`ty9> zhdYm#RqTwhC{K=2IelPkP2Xp7){?znPA+=)VNr=)Mv0$6LO@m9%YK(MJEg)&3^P}j z{<>(bz?sgQ-W2zHMq3*`OPxNxfZv?H{?02$WdB0m1Sf=|GIx)gb9<^*$p*8PhPj8w zzk0sIN!cJt?FSTA2Df3BrIS&`RzG0y2(k;9BV2VOeW-y&V@9G15;kEQ#qynLII zYDD>%RO27UHD=C^u%qjebxt1NpY~w)X!dKfZ>v&szmA(w0JgqP4xl z?TBM#;WJCdcZ>=y**_chws!j)`)T= z4ErcAA`K@;z8-eiBkbzaEv+^B)di)ICV@Ws&VOjM_>tPr3b*J^)m&iKsJ<}!?cE`~ zkC7qbGSQ?XrzeVyjDJZH@0o|WoT;@mf7HC?Cs*Z)WV#j^9wIvc0Ar%+1DnzzN) z&=0@bTv;%)d4lz1|@Vmgs5mP3g5%B+J;W@|GZLvrnI=R%g8G*6zIy%$t1;A5s^)b$3~9-0*wPp2vpE zN0Ht~O)FL(Wn4G*jcDQ{Q!mxTU3I17i%&@V{+-+*)!O}b?du$#3MyuNF>tB7JCaYZxi}uV2 zk6rby^4x2ebzkxi^tX=OK9LtuFxzGOIS;+bcK06d>!xVfC%yPhFj8ooYg$?PdEJG1 z=htB~PayUOv`>fkx%aX3R-SeET4Ysu`FEFb1|I?pu@gh0wAm5IBOY_lQJ1FH=A3E_ zJ(@czFWhmBSydF3eR$2b$3554uz7y>?vEH#DEw-=e~E5KG*x4PYeh)>sLn$Lg*Wb( zCa_9EhnFt1v~f9i*s85z-Oit~Vjnx*Vn*~!B1CHgAi2rPp z1XC?8*-_Veghp+iTl$P*dNH5$rY-K{!8}X4m-?^;{tnkpgw%cXSW`B8o?xz7^81oG z{FWn^!ZJO1NBV5gr3vD+9ABQsvdynNeKwi67^fHORvT8hE_&AD9UrMZU$dx7Q2({> zMC5|@A^Y{(GD~>vQ4KZps={6KpTzHSTSBWSI=#5aeD#R@*V{^$xYfACX!fa?QaYj< zs&_?Qx)XCh*wa&E#jzLXTYQ%0S80yltuuo=KFD%!$oAWvUtI#`nj7gpxwcMs3bzi; zNe_5i_lC+uTk@VQA$Pm>ns!`dR0Oa}G6G)LPWwHjcgZ%{?@_+?Z?6?`bH`?`W{9oR zzJwpYtkpelCG%$Ki)xgfmPIvL(qXcSd!A?h)1xLc+B)jS=I06XXV3H3D43F9&=cf4wRRH2^6J<7 ziQ?`Yox1Ew{}Cyf6V64&-d}dMy3W{qtmT~(?z6IdO*-n<2=1Ke+SBVx+0o(Go#Bx_ zPO1^$IeShlwxdVG?cquLgKH&QORZ{kVw5s!Rf_BFe3j~Z!IxLn+if-n+q8eOU!iL) z^l2R{jx+c=)^hTc_mwkf&yrvLaimt;_H)Mvm9)Ju5Ygk zll@mu_wuOp?EG%s#8>DB+klIUxh7@1=hbP}EIh4WnAR0?(;V5Zt+zb4P2F{ELAlmJ z%g+-_yK0?sUeJpr?rW#i86A33Ml(>A)Oma6?$h_nnpL6G@E~K7)n@+=3b@%DUXXD> z-DPiu*_D{;_tGce!N(pmEEH{NsxqJ7^CCg%lBwg~(7xRrH?J&@L58=i-@f>!hUtz* zUDvC&R>fQX@l~3ZY8tb7VqNZpp7WOE4?K|P&RnucM z7SFF6;vstf-pb~Yui8yI|X%{X$-b%kDL?2HM^ zKQ~rA3|Ly6dUW;v&dv(W&Wslu)Y|A*o8v!Py`0=qXvCWHa;fFG`i{O`yPsrgGNMf0 z)opB^bT6U(S$s^C0cWAhj;Q>M=SE6rD(g4g-4NI{X7nCSBNQsr>0pxsyxgz92_W>P+D* zcW37iyGQiSpd{ZFR1Ok;LmtMh+*__k_k=>D`sOHqj&z====4T%oZnZi0xbPd@vCA|U)Bfgg-lArC z*(Lo8n~pp6R&}tv$2LvxcQ{v_+N4*Iv1pY>{3x&ecGqp02|MQrYLYibV~$D}d@Of* zprN`dck`EaYh2C5kXJIA{4aIM#?K_Tm}kL0U2bR?TXr~{I28g zIu{d#A4LiV(-2-11D0@MwX!o=kx58IGBM~%rjNsug9!W%VRP_GsR-q;5tPG_u(${+ zX0SM*+n*>_{vech0Z;tpWBj21bdcnY3?eK(3f>H&`5_(sS@$1_BMH2P zZs1jR!Yv8t9WW%nhiNTQ`4y7<@B*+f0Cyhd7VlsL-ecf{yaK2m4< zLUZuqo~5bFp$qZE4zESygK@v4t-lxaR@C>uvd1gYHN@XolN}`mq^(vx` z3KcD)wAk8s%iAU`l$N)8``zam{Hpi&`G4O3_4$nFo_p@O=X>tG=iGBPC@=*=v4oGB z<6;RbM)pfxMtoBN#Uu{MNr60#L8VYABm=@nd2FD-FoguZz>MgbIfSA1VfI#GW?`t! zEHfJ>%g!v6Z69hDI@@+OI*Vy-!?v>oBN)nX3L+E2VF5)tg)vG-5s1)mHW>L|MMwf} zj&S%0kIfT^VyKJw-sHzJF*!yBN>H^?QlLy@OeTbgMI4lia9D^~FuVkU@>vL9!V6`K z096W9XmkqkmjFi7<=}^00#K#P!w>lcFh;`A_^);ZC_)g_QcVifu@Q6%KBTLJ#bM(> zLkcwMl>bX%gf!tIewcs?w4koqQlLX)jD(Ajs^Va52pC5jl0eeTkuZ^94VzD0NdEID zAW^}1I9XQ;^k@tvLc>8k6QK1JVILQxVm8qKMGi@&k^+_y3{)_YE*B3B{J+3GCU$N zLaGPMJe(H|=-=j8@yVemRK$}A5fK5{+{5|s9-?JS$T38MvqcCh7K?CTAqAEURRRhg zgNxa`L9&$;%wlLkGKVi_i+F4n2MUs{gad0Su%XfAJ>W3e=p#IwUp*vRr-U&Ek`NXL zhno~C5px859N0>M9St)ZGW$=1)P-CDDt7;Yp9*HfO|+K+2O47p6b7{tf;m161^CSs zM1i>;&U$u_U+TM=k;51#90QIX&c31ExBCu%9b^j!PDzqb%p4miueYU~r4-NG)t67~B#% z1~;E8{_+9|$7Q2g}gA zA5%3*svyu0ZUTuugh2lrfgbr4{UdS6xCOVwB>GiD=us&M{dee@Qo#CupqH0~vKgB2 z+>tl$r&A5sQV>QcfWkw@l#nj}hiers1sG#Aae9BeRU9dZAP+4W79z0`*-X6 zIuwmS2poh7115*?;5gVoA5R7=Bpl`OP!RX;RwkLLU?W41kO>_&ABJ!iTgZk#d@({Q z%HfBDO-baLK}7I-^FKv%(KRv9&+}OV9-%OdUtW;nM-m3(Sv=f^1Sv?wHv8bAvcNSo zVQ^2A{wY_+K^+t{w4=#TREiXA!8T)r)U8sGifv}XI4g`Xm(4IogdAohmrX?mwIY>5 zm~g+TAdNgp-u_k&4%V~5ws?^42ewPW4w~%na>PhDo6m+(OCSOn*fvaG3U<;+pB~AM z0lSzHf>16e)SO&zXdNG7aaWNDjaeZ=g+ex5CKIZ*TMDvh8JMY zzWTD}h^&Hlk*Ap2^q8{VaIIAzd@tk=cy)B&x;j@XcRQ1+cAfFP?A@Hhd&+mJ)OmRo ztB(^Nc>O(JZHelR>^nLwfoDq0)Y;Emb{V%%jEc+dx+`Ikq?ayK-1gpAS%|`4m_+1;T z@Byh6f6w1ug}-ljo}oTFtK<9oX#ZkQ@xhxG_66%Ry9-T^r?PfEbl$e?<;R4IUEgL* z3v_r@`t)kBf(=EvXt$}$+r9W+*CVl?6EkgctW{VUAE$O-T+CFOx3@ZfRpFx2yOs&{ z3-7964hjRmH@n$=sJZw2wdYpf_-j;MO1|rNU8HtZ^nsMFSu>93o%&+ibeVbWzS5WK zjt$elsa{I#3m&)jOsS6fo7;ljb}o0#bT+z3zc$u>ZFafMOR_wv+8TdTQ%yM1_?YpZ z>`=dVNWuFlYY#HczB##~Z=$=aB0cCsgVAVDl~JwN4D8j5bQec6+C754D|9IxKK<^a z+_}is6EO;<=htpH>Ff4n+oKO>&owVU5d3<~o;=HtR4evM`_|xkubjy;t zbIkFlC-17=pU4d^)Y-elX9un&@t8U`PrSN%;dJ+24Qu@y^+_u2bhQ$Fn)yJ+wTI`Q z2e$9CC^rYUx?b`QZ}}XSXqjmL$RWaK%jInV2<_8Ms?r;Zt;UYR^7sjB4YfNP4v$v0?PlE0*rxktdDoe+UOeK$ z#ZOM3ow6USR#kpzm({C(+tH9^oN)G!Q%>76LvOyVm{@0cY((fsW__ol(lt4k+O`&# za7LQ8=JwUMcD1_A&PY|;x!5}0_~_2W;6vl}ZD(!Xeer#$(V`m552q)1#?ImTPP=PV zduOgISUT^l$$<7V2a{%V+a}$Ck#G7gR7?+lJx??J$V8fVxOr`x`LhCxG4STLBmJy+b3^{ zo6@OK@WgY~mhGwAM4NR^ZZ{NZk(O2J_-C%L#xXFw{OC7xL&}GLP1IMJmpqSx5g zZeH-(_Q`~tNhM|(o?|uF`L6(a>FXpsihKp9{?Pf>)&40+@trMu{?v6?*Yn`ur7w+7 z{axF4^3p13G5vLw^BO95a17*U3L1(>b|WJL9Bhq8xbJTF+Ntlgv!ks)UDT(w!c}+8 zK=SMw4Ihu7Fy5b=Rwh+BhTY!Bw5vY`Z@f$Ziz9(Q?WZ75aj-(qa z?ASAuqBNW;N-CUR;WI?nXH7KsZQAD(y`j5qQoC6LMXt-S?(&|d`5}3K8?{DFaHCUd zf|^z}Z(GF5i(Gl#y_>vQ2zyFbPiJxOhN*JVm8IejDaDQ~H^XM9ef()Br7L@bk4tosDr`cpLws?7`BRF_8OQ_kvhz&AHy%!r%bbpT3oUm8*&2O7N z&5z{;@t3g{^=^6--Rf<<_p!R-ScjTRf{oj+daW@D&}E$vsezGpeeqP4_~7!pd~z zh%u(Rw;gHviaK=(o*Q?(+BYyd1)CjQ@6F~SEz=rU-|DZ{ZgeUb)zw%1;e3FS-*$^> z$ECX#zESn{b3K>R)qSw0ct>@xV^dMai=-a?@zF(ow_a#@Zt3LZ-P>nZ_`9{sm}n2n z&*ed8E)yB8@n^?NW-TdZ%ZDxte+>dwSJQ}+u=uhqU9;L~3w=q`!+rjO+IAvgm z`n$65mEQwgc7KehzIUx_7GT{dW2ChwW1|0~Czxb9$V-zizAxkBwtmMsJF`5W=)ku8xg*{=6YF zV2P$xrPoEn{`j!T=AP&EE+-g$$Svuab|J#$w^?oW)C7ZBaZgKTdcNcYy|3Fc!)@fJ zgS%F*fBFEkZBAN${-(OCcKW9BbuAW;Uo@`gv|p;_YL^z*KK>peu+}&p*ZU-L6JoVc z*)**rNP4%Xt*z!#M{bDq>KkugEyi``VR>rI_{7Iow>~zhde=2_+PoZ_lBg(dMA~Uv z9?CS;obFwu^EC5J5;U01f@fo7`e6%VZC?wuQ@_M7zL zZ7DIuC%u*xjk%Ye{WqnKv z%4rO3LNO-F#GpE|405pARzbdZu~HvQ(MOhSaQIBFgvCaLA_|N_S5_xt!t#(9 zg~<+=gNne3WLe@xQbj0K2uH#sd?q;Q!_XmC`!yLvm|Ps38bB2YObNY;mGEm`;CQ@x4u7vGmq9_AKR&;nY-dO$;$5l|+W5q0w-MYFhO6&cZuomS>iV3`t`tEmDfKcW;V> zkRmr(+N7jKrHdl8FIrXp&->1B)%|_`pL^f&8P9v3^PJ~=&v}+}&aBtwh{Q6v5T!_E za&DOFn-qYzNkGFU8fYqj7MVrT(9j?l5Fsky0d1BYBnU3ek?sW1W^g$bAb+BgEc@!5lPSx9_atKA_PH7 zLHI&Mz!QihVI&{ne4-H`Bl<`{7pgWu0rbeMF}M(!gpYFBS}@Lj~zNvA{^qGWuI_ z94|fx1TaJaj4{Y9?SUcZ@_AA)Q~`$3HU2*=2w`0xVX%kK}=Lt!3iGP7OS`ruu2ai$!b23X8*H8`B0vc8azDs4Oj0Z;lB8Q-o z2mt{&0||_wYbwE5CK#uZvKkGQLm1VlM{W2 zT*{Lms7xl2f+-5%$Qp=4k%mcSJb{`#RRN~4j3JpXl<_129+wXVsb-Rb=?dUPrfbn) zH}TN>(WoyU;m==k)*whixO^#e5?e0gi-b}zLjj!0*yJ9w|I|sE8!AF&bNcX;z)a{w z7X@%7v-&|{P%AN*#b9Z}cBp6#aHCPprayi@%F~|c#`qx^pwg)R?D@Ce9{;9h3jyx& zayI6G;j%mwU^W?>MWY^9SpjqFRULyP>l}#(b5D@7<*F@7tT22gU=fa9s|P)cM!h@x@?!n5+@H`}YhBF(i9>vkKe_dwP;L<4gN1*5Q zpyw(8@86*hR)CQIfL=?D9#a55j$XS5{Yo0OS$z)l^}U_`Nj_Mm1ffhI=!@RP(25I$ zN+4naF^(P(=&99tZqqe8e)YWsM=!wA)x{(U)6C_y9>MD;;W8lv&;3s|oN(M+%bM<0XxTfOO| zxW6~zh?O|v*kAqqGzEWl6aJ13t*!9PK%}iBN^24kvnfRI$;Vo*^XgJ8s@TnBW*n zTrF&#RMs1gfDGItQq`R!K-c!hCg3={rHPver}mz}C+HBd>e}j^Pzo}!qr{ykMe&PM z0TXh?zgK{&g9J{(!#rC7PLWxope+z+Mi@mPP({7~g`pHqfDsM6 z^`s!@e@p#!2nq2ggn0GHfUjy1B#=vv@7*r(3IAJQkD6RwFe(p~A!~T>QjGoTtZc#0dID~KvGahC$R?OI3NT*!h!J}ACD@e;0)cA2%wM(Z8FS zU?zdHEHhjt3?hUu;^gwgJoqJ)A%voQVF)-EPplb40^7y^6dj6&hr!rG$Q222g<(wP z3n@&3ppLOhpc~FBKq*$tkg`?5RL=mlPyhURNYJoF5>zba!D+5T z)oK*r2AK$srE)eW6qQQBO_ib~As5BjNuX90s$!^^2X4(5!<|$|P(%_gT=OoDyUD0j|2Ca-~pMgO$>7d4;c)`yl(gqUvv!_XkZf? z9u0-D6#`d^7(9ZJQ+Ub11Jlg&N-PRSN}P5iB90=MoZ-=1Z^TfPX@-fu0N_D!&C&aoGH zd(Tu&4wHMW@X=yS8gueqyJgXDnI%uXPq}5IEp_!JxyXebYf>Q- zO?i6xaI4)W?f9LV`i8&1^GjWN;gYpcaAD;2+HO^XM@?}il(RyEyK=@ zHIWnun%-nKo@E!!`19aagYA!thBZamoa*oVpyb$QQx+1PFKjOO^z5$^tCTky?hQ*i zSruSb?lX@ot@<|W3TH&V$urC;aydg?pBQeGk-{~P8x$91mS;QUOS(RwX87tjesyJa z)%N@Lq@lyKtB__ZGNiUYZ>Jxw4s~{WH7X4Z?7B7HE`2DwVddPz*7vUcxO4CE*SX<$ z>jxyy->_n#)VA~MQclcWy588G=>nY-HB?z-z*>%u?Di^m{+Y5TyPjt~N#0|XS^Bce z#^Zcz0)NWp`fGI#`#h!_t0^@wf4Ajw`>6q@UdxZ! zUF*v5J?r*%L7WGfQ65rr#z_3kKt4drs$z#{^??g%;|ARg`gpxjz9RKhlKtS}=a#(O znA_Cuw#|iHm{A-vwM@_2q-8keZS3WM;s&HvBZ5yFpY*uGFF8JwGSXxBlGcfxZo223 zmK9|%OYM)>W6bA@_bJ;0d}EX|-OS#bV^{|IYW8d62YYh{Uy4F^v>=x@m$zk0pBq2P zxpy#o&&i>X7LZV&eRuq3(h(CiRD_NvRB<8%*ovEJ`R zLupCR`qxLb4bq+K>RWut!q~fX3iZd)vd0-mIOmRUv>TYXwJ_R~5)qYMJtVnEyz!}^skY(=!H%={R$9hwaBq@Y>Rn?l>} zKXYBMwrOA1*fh@N!4{NczfD22JxHThdl2a@XEy$WpJ^$PeX6aDIZG^z-=! zEnoVRd$wV<-`v07*zUXVamUkpcN05`5QRH>%pX=d*5!f#ok-0$CO!=E40T4ejo zVCb4x-@E-h(|7$LvO5qE=XbK=Zl;6qT4i(;XG`;xC|y7|OHMbLYkA#fr*kKmgtW_+ z+r{t~=e`_OKB&=wUDQ9YM5k_z@k8H3wSV}ghIduA#k$v4&RI3$x7?`(4K{BdDFpM4 z;tRLs>S7VrK~G;d24KG9Eh1UjZ~UlT)Otrly)t!w$$Sl@t>ClUygSVi&NCun^O=mMn$uRa_8)b0$D#J4TyNrQuwug@*py&%gWa?ZnwQxSJG4{ki*JF(U2hErW$3jG>bxu;0H(D_Tj z!{$QEocFmMi$9!K6t{1P`8@2LuDjk4?kKuv{;-7xUsFUYN#q-&<$Ghr*+P%XeQxv4 zZoBM$+kW$z_l?~%7@m`re7@=7l;g6H!pt2+1Ls-i* zLX;dY`>!t@jn?Xxq&q(enA*`K*vfvZf^~DRO%-=S}DRt6`PLQrJFSEjG5rrA;3fq5G{iZ`Mi7)V$!2 zoWn>%g=dT|fg4whCa?Pvu;%`2AFGcbb-$m@P~EQJ_SOkrqkOMF@m=`3L(`<;0Aj5P1+=eQ)aNGWy;v- zAWF=ABR#k7DQUv57T)8IO||LGRz?mlaIWsq!*PR0hUrH#eA@e5${?)ZF+B zzku@ENj~|+i7$3D4`@v4`lOEtUbF|I`%~B3#F<_G=rXp8#%;>l=qAJ_dr&J>hys7p`Kb>{@J%4Y*&=|XP=8E<|x3`skvT%w^ z@h`MKAH0XVsYN*}X2R#V%stepF*E12hMap5?r(eZqTQ__%>2Xp-kScdJh7$lqfWtK8p2nCyh|Q3-gSph~<5ssx1!Ns(Z=kON*YSi|tyeoY1uPN)>T z>_ziaI{GW`KN3e0cm>tK>)wP~67af5km?<#wL~peNScqAWQG2?@-Vk}11<1Y0p6*~ z!U)VaI51t~hw{LCn!67a2Oq4EuA_o3#1lKbI*gZArJ$3p)ngK9U|3C7(?8HPRgQu` fIl!MC)CuQDye3Gb43JGEMuJ1p5bz0>PAUHfX|q=J literal 0 HcmV?d00001 diff --git a/tests/tuning/test_end_to_end.py b/tests/tuning/test_end_to_end.py index 1febb4e82..b9c32c068 100644 --- a/tests/tuning/test_end_to_end.py +++ b/tests/tuning/test_end_to_end.py @@ -54,7 +54,8 @@ def assert_results(output_dir): @pytest.mark.parametrize("n_trials", [7, None]) @pytest.mark.parametrize("two_phase_tuning", [True, False]) -def test_tune(tmpdir, n_trials, two_phase_tuning): +@pytest.mark.parametrize("start_from_checkpoint", [True, False]) +def test_tune(tmpdir, n_trials, two_phase_tuning, start_from_checkpoint): config_meta = {"config_path": ["tests/tuning/config.yml"]} hyperparameters = { "optimizer.groups.'.*'.lr.start_value": { @@ -73,26 +74,62 @@ def test_tune(tmpdir, n_trials, two_phase_tuning): }, } output_dir = "./results" - gpu_hours = 0.015 - seed = 42 - metric = "ner.micro.f" - tune( - config_meta=config_meta, - hyperparameters=hyperparameters, - output_dir=output_dir, - gpu_hours=gpu_hours, - n_trials=n_trials, - two_phase_tuning=two_phase_tuning, - seed=seed, - metric=metric, - ) - if two_phase_tuning: - phase_1_dir = os.path.join(output_dir, "phase_1") - phase_2_dir = os.path.join(output_dir, "phase_2") - assert_results(phase_1_dir) - assert_results(phase_2_dir) - else: - assert_results(output_dir) - - shutil.rmtree(output_dir) - shutil.rmtree("./artifacts") + try: + if start_from_checkpoint: + if two_phase_tuning: + if n_trials is None: + checkpoint_dir = ( + "./tests/tuning/test_checkpoints/two_phase_gpu_hour" + ) + else: + checkpoint_dir = ( + "./tests/tuning/test_checkpoints/two_phase_n_trials" + ) + summary_src = os.path.join(checkpoint_dir, "results_summary.txt") + summary_dst = os.path.join(output_dir, "phase_1/results_summary.txt") + config_src = os.path.join(checkpoint_dir, "config.yml") + config_dst = os.path.join(output_dir, "phase_1/config.yml") + os.makedirs(os.path.join(output_dir, "phase_1")) + shutil.copy(summary_src, summary_dst) + shutil.copy(config_src, config_dst) + else: + if n_trials is None: + checkpoint_dir = ( + "./tests/tuning/test_checkpoints/single_phase_gpu_hour" + ) + else: + checkpoint_dir = ( + "./tests/tuning/test_checkpoints/single_phase_n_trials" + ) + study_src = os.path.join(checkpoint_dir, "study_.pkl") + study_dst = os.path.join(checkpoint_dir, "study.pkl") + shutil.copy(study_src, study_dst) + + else: + checkpoint_dir = "./tests/tuning/test_checkpoints" + + gpu_hours = 0.015 + seed = 42 + metric = "ner.micro.f" + tune( + config_meta=config_meta, + hyperparameters=hyperparameters, + output_dir=output_dir, + checkpoint_dir=checkpoint_dir, + gpu_hours=gpu_hours, + n_trials=n_trials, + two_phase_tuning=two_phase_tuning, + seed=seed, + metric=metric, + ) + if two_phase_tuning: + phase_1_dir = os.path.join(output_dir, "phase_1") + phase_2_dir = os.path.join(output_dir, "phase_2") + if not start_from_checkpoint: + assert_results(phase_1_dir) + assert_results(phase_2_dir) + else: + assert_results(output_dir) + finally: + shutil.rmtree(output_dir) + shutil.rmtree("./artifacts") diff --git a/tests/tuning/test_tuning.py b/tests/tuning/test_tuning.py index b5c1d603e..9967f6934 100644 --- a/tests/tuning/test_tuning.py +++ b/tests/tuning/test_tuning.py @@ -212,22 +212,33 @@ def test_compute_remaining_n_trials_possible(study): def test_optimize(mock_objective_with_param, mock_optimize_study, has_study, study): mock_objective_with_param.return_value = 0.9 metric = ("ner", "micro", "f") + checkpoint_dir = "./checkpoint" if has_study: - def pass_fn(obj, n_trials): + def pass_fn(obj, n_trials, callbacks): pass study.optimize = pass_fn study = optimize( - "config_path", tuned_parameters={}, n_trials=1, metric=metric, study=study + "config_path", + tuned_parameters={}, + n_trials=1, + metric=metric, + checkpoint_dir=checkpoint_dir, + study=study, ) assert isinstance(study, Mock) assert len(study.trials) == 3 else: study = optimize( - "config_path", tuned_parameters={}, n_trials=1, metric=metric, study=None + "config_path", + tuned_parameters={}, + n_trials=1, + metric=metric, + checkpoint_dir=checkpoint_dir, + study=None, ) assert isinstance(study, optuna.study.Study) assert len(study.trials) == 0 @@ -260,7 +271,8 @@ def test_tune( "param1": {"type": "float", "low": 0.0, "high": 1.0}, "param2": {"type": "float", "low": 0.0, "high": 1.0}, } - output_dir = "fake_output_dir" + output_dir = "output_dir" + checkpoint_dir = "checkpoint_dir" gpu_hours = 0.25 seed = 42 @@ -268,6 +280,7 @@ def test_tune( config_meta=config_meta, hyperparameters=hyperparameters, output_dir=output_dir, + checkpoint_dir=checkpoint_dir, gpu_hours=gpu_hours, n_trials=n_trials, two_phase_tuning=two_phase_tuning, From 4418cecb55eccee139b2a31dabc96f3284200292 Mon Sep 17 00:00:00 2001 From: Dedieu Lucas Date: Mon, 3 Mar 2025 11:17:29 +0000 Subject: [PATCH 4/5] update tutorial --- docs/tutorials/tuning.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/tutorials/tuning.md b/docs/tutorials/tuning.md index 0865a2e9b..d0189d839 100644 --- a/docs/tutorials/tuning.md +++ b/docs/tutorials/tuning.md @@ -76,6 +76,8 @@ To enable hyperparameter tuning, add the following `tuning` section to your `con tuning: # Output directory for tuning results. output_dir: 'results' + # Checkpoint directory + checkpoint_dir: 'checkpoint' # Number of gpu hours allowed for tuning. gpu_hours: 1.0 # Number of fixed trials to tune hyperparameters (override gpu_hours). @@ -92,6 +94,7 @@ tuning: Let's detail the new parameters: - `output_dir`: Directory where tuning results, visualizations, and best parameters will be saved. +- `checkpoint_dir`: Directory where the tuning checkpoint will be saved each trial. Allows resuming previous tuning in case of a crash. - `gpu_hours`: Estimated total GPU time available for tuning, in hours. Given this time, the script will automatically compute for how many training trials we can tune hyperparameters. By default, `gpu_hours` is set to 1. - `n_trials`: Number of training trials for tuning. If provided, it will override `gpu_hours` and tune the model for exactly `n_trial` trials. - `two_phase_tuning`: If True, performs a two-phase tuning. In the first phase, all hyperparameters are tuned, and in the second phase, the top half (based on importance) are fine-tuned while freezing others. By default, `two_phase_tuning` is False. @@ -252,6 +255,7 @@ package: # -> python -m edsnlp.tune --config configs/config.yml tuning: output_dir: 'results' + checkpoint_dir: 'checkpoint' gpu_hours: 40.0 two_phase_tuning: True metric: "ner.micro.f" From 6e964e9886db146dbe2472035067ff0ff3c8a8f8 Mon Sep 17 00:00:00 2001 From: Dedieu Lucas Date: Mon, 3 Mar 2025 12:44:16 +0000 Subject: [PATCH 5/5] add constant for checkpoint name and remove unused line in two phase --- edsnlp/tune.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/edsnlp/tune.py b/edsnlp/tune.py index 6eb48c8fa..20a21e33d 100644 --- a/edsnlp/tune.py +++ b/edsnlp/tune.py @@ -24,12 +24,13 @@ app = Cli(pretty_exceptions_show_locals=False) -# disable transformers warn logs +# disable transformers lib warn logs set_verbosity(ERROR) logger = logging.getLogger(__name__) DEFAULT_GPU_HOUR = 1.0 +CHECKPOINT = "study.pkl" class HyperparameterConfig(BaseModel): @@ -308,7 +309,7 @@ def objective(trial): def save_checkpoint(checkpoint_dir): def callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): - checkpoint_file = os.path.join(checkpoint_dir, "study.pkl") + checkpoint_file = os.path.join(checkpoint_dir, CHECKPOINT) logger.info(f"Saving checkpoint to {checkpoint_file}") joblib.dump(study, checkpoint_file) @@ -316,7 +317,7 @@ def callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): def load_checkpoint(checkpoint_dir) -> Optional[optuna.study.Study]: - checkpoint_file = os.path.join(checkpoint_dir, "study.pkl") + checkpoint_file = os.path.join(checkpoint_dir, CHECKPOINT) if os.path.exists(checkpoint_file): logger.info(f"Loading study checkpoint from {checkpoint_file}") return joblib.load(checkpoint_file) @@ -506,7 +507,6 @@ def tune_two_phase( else: n_trials_2 = n_trials logger.info("Skipping already tuned phase 1") - hyperparameters_to_keep = list(study.trials[-1].params.keys()) best_params_phase_1, importances = parse_study_summary(output_dir_phase_1) hyperparameters_to_keep = list(importances.keys())[ @@ -724,7 +724,7 @@ def tune( logger.info( f"Tuning completed. Results available in {output_dir}. Deleting checkpoint." ) - checkpoint_file = os.path.join(checkpoint_dir, "study.pkl") + checkpoint_file = os.path.join(checkpoint_dir, CHECKPOINT) if os.path.exists(checkpoint_file): os.remove(checkpoint_file)