From b5839fddbd78934a7b8e5cd431a6de3df0955ab2 Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Mon, 2 May 2022 16:40:44 -0400 Subject: [PATCH 1/2] initial reference implementation for Breast Density FL Challenge fixes pre-commit ci Signed-off-by: Wenqi Li fix typo remove data lists and example predictions add download script and instructions keep empty results folder --- .../breast_density_challenge/.dockerignore | 3 + .../breast_density_challenge/.gitignore | 12 + .../breast_density_challenge/Dockerfile | 36 + .../breast_density_challenge/README.md | 176 +++++ .../breast_density_challenge/build_docker.sh | 15 + .../config/config_fed_client.json | 51 ++ .../config/config_fed_server.json | 88 +++ .../code/finalize_server.sh | 8 + .../code/fl_project.yml | 60 ++ .../code/pt/learners/mammo_learner.py | 696 ++++++++++++++++++ .../download_datalists_and_predictions.py | 16 + .../code/pt/utils/download_model.py | 25 + .../code/pt/utils/preprocess_dicom.py | 67 ++ .../code/pt/utils/preprocess_dicomdir.py | 304 ++++++++ .../code/pt/utils/strip_testing_labels.py | 59 ++ .../breast_density_challenge/code/run_fl.py | 94 +++ .../breast_density_challenge/code/run_fl.sh | 29 + .../code/start_server.sh | 5 + .../code/start_site-1.sh | 4 + .../code/start_site-2.sh | 4 + .../code/start_site-3.sh | 4 + .../breast_density_challenge/data/README.md | 19 + .../example_data_val_global_acc_kappa.png | Bin 0 -> 51107 bytes .../result_server/.gitkeep | 0 .../breast_density_challenge/run_all_fl.sh | 8 + .../run_docker_debug.sh | 21 + .../run_docker_server.sh | 21 + .../run_docker_site-1.sh | 18 + .../run_docker_site-2.sh | 18 + .../run_docker_site-3.sh | 18 + 30 files changed, 1879 insertions(+) create mode 100644 federated_learning/breast_density_challenge/.dockerignore create mode 100644 federated_learning/breast_density_challenge/.gitignore create mode 100644 federated_learning/breast_density_challenge/Dockerfile create mode 100644 federated_learning/breast_density_challenge/README.md create mode 100755 federated_learning/breast_density_challenge/build_docker.sh create mode 100644 federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_client.json create mode 100644 federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_server.json create mode 100755 federated_learning/breast_density_challenge/code/finalize_server.sh create mode 100644 federated_learning/breast_density_challenge/code/fl_project.yml create mode 100644 federated_learning/breast_density_challenge/code/pt/learners/mammo_learner.py create mode 100644 federated_learning/breast_density_challenge/code/pt/utils/download_datalists_and_predictions.py create mode 100644 federated_learning/breast_density_challenge/code/pt/utils/download_model.py create mode 100644 federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicom.py create mode 100644 federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicomdir.py create mode 100644 federated_learning/breast_density_challenge/code/pt/utils/strip_testing_labels.py create mode 100644 federated_learning/breast_density_challenge/code/run_fl.py create mode 100755 federated_learning/breast_density_challenge/code/run_fl.sh create mode 100755 federated_learning/breast_density_challenge/code/start_server.sh create mode 100755 federated_learning/breast_density_challenge/code/start_site-1.sh create mode 100755 federated_learning/breast_density_challenge/code/start_site-2.sh create mode 100755 federated_learning/breast_density_challenge/code/start_site-3.sh create mode 100644 federated_learning/breast_density_challenge/data/README.md create mode 100644 federated_learning/breast_density_challenge/figs/example_data_val_global_acc_kappa.png create mode 100644 federated_learning/breast_density_challenge/result_server/.gitkeep create mode 100755 federated_learning/breast_density_challenge/run_all_fl.sh create mode 100755 federated_learning/breast_density_challenge/run_docker_debug.sh create mode 100755 federated_learning/breast_density_challenge/run_docker_server.sh create mode 100755 federated_learning/breast_density_challenge/run_docker_site-1.sh create mode 100755 federated_learning/breast_density_challenge/run_docker_site-2.sh create mode 100755 federated_learning/breast_density_challenge/run_docker_site-3.sh diff --git a/federated_learning/breast_density_challenge/.dockerignore b/federated_learning/breast_density_challenge/.dockerignore new file mode 100644 index 0000000000..6029ce1d95 --- /dev/null +++ b/federated_learning/breast_density_challenge/.dockerignore @@ -0,0 +1,3 @@ +# Ignore the following files/folders during docker build + +__pycache__/ diff --git a/federated_learning/breast_density_challenge/.gitignore b/federated_learning/breast_density_challenge/.gitignore new file mode 100644 index 0000000000..6f15968018 --- /dev/null +++ b/federated_learning/breast_density_challenge/.gitignore @@ -0,0 +1,12 @@ +# IDE +.idea/ + +# artifacts +poc/ +*.pyc +result_* +*.pth +logs + +# example data +*preprocessed* diff --git a/federated_learning/breast_density_challenge/Dockerfile b/federated_learning/breast_density_challenge/Dockerfile new file mode 100644 index 0000000000..e9d1ce585d --- /dev/null +++ b/federated_learning/breast_density_challenge/Dockerfile @@ -0,0 +1,36 @@ +# use python base image +FROM python:3.8.10 +ENV DEBIAN_FRONTEND noninteractive + +# specify the server FQDN as commandline argument +ARG server_fqdn +RUN echo "Setting up FL workspace wit FQDN: ${server_fqdn}" + +# add your code to container +COPY code /code + +# add code to path +ENV PYTHONPATH=${PYTHONPATH}:"/code" + +# install dependencies +# RUN python -m pip install --upgrade pip +RUN pip3 install tensorboard sklearn torchvision +RUN pip3 install monai==0.8.1 +RUN pip3 install nvflare==2.0.16 + +# mount nvflare from source +#RUN pip install tenseal +#WORKDIR /code +#RUN git clone https://github.com/NVIDIA/NVFlare.git +#ENV PYTHONPATH=${PYTHONPATH}:"/code/NVFlare" + +# download pretrained weights +ENV TORCH_HOME=/opt/torch +RUN python3 /code/pt/utils/download_model.py --model_url=https://download.pytorch.org/models/resnet18-f37072fd.pth + +# prepare FL workspace +WORKDIR /code +RUN sed -i "s|{SERVER_FQDN}|${server_fqdn}|g" fl_project.yml +RUN python3 -m nvflare.lighter.provision -p fl_project.yml +RUN cp -r workspace/fl_project/prod_00 fl_workspace +RUN mv fl_workspace/${server_fqdn} fl_workspace/server diff --git a/federated_learning/breast_density_challenge/README.md b/federated_learning/breast_density_challenge/README.md new file mode 100644 index 0000000000..06c668fea1 --- /dev/null +++ b/federated_learning/breast_density_challenge/README.md @@ -0,0 +1,176 @@ +## MammoFL_MICCAI2022 + +Reference implementation for +[ACR-NVIDIA-NCI Breast Density FL challenge](http://BreastDensityFL.acr.org). + +Held in conjunction with [MICCAI 2022](https://conferences.miccai.org/2022/en/). + + +------------------------------------------------ +## 1. Run Training using [NVFlare](https://github.com/NVIDIA/NVFlare) reference implementation + +We provide a minimal example of how to implement Federated Averaging using [NVFlare 2.0](https://github.com/NVIDIA/NVFlare) and [MONAI](https://monai.io/) to train + a breast density prediction model with ResNet18. + +### 1.1 Download example data +Follow the steps described in [./data/README.md](./data/README.md) to download an example breast density mammography dataset. +Note, the data used in the actual challenge will be different. We do however follow the same preprocessing steps and +use the same four BI-RADS breast density classes for prediction, See [./code/pt/utils/preprocess_dicomdir.py](./code/pt/utils/preprocess_dicomdir.py) for details. + +We provide a set of random data splits. Please download them using +``` +python3 ./code/pt/utils/download_datalists_and_predictions.py +``` +After download, they will be available as `./data/dataset_blinded_site-*.json` which follows the same format as what +will be used in the challenge. +Please do not modify the data list filenames in the configs as they will be the same during the challenge. + +Note, the location of the dataset and data lists will be given by the system. +Do not change the locations given in [config_fed_client.json](./code/configs/mammo_fedavg/config/config_fed_client.json): +``` + "DATASET_ROOT": "/data/preprocessed", + "DATALIST_PREFIX": "/data/dataset_blinded_", +``` + +### 1.2 Build container +The argument specifies the FQDN of the FL server. Use `localhost` when simulating FL on your machine. +``` +./build_docker.sh localhost +``` +Note, all code and pretrained models need to be included in the docker image. +The virtual machines running the containers will not have public internet access during training. +For an example, please see the `download_model.py` used to download ImageNet pretrained weights in this example. + +The Dockerfile will be submitted using the [MedICI platform](https://www.medici-challenges.org). +For detailed instructions, see the [challenge website](http://BreastDensityFL.acr.org). + +### 1.3 Run server and clients containers, and start training +Run all commands at once using. Note this will also create separate logs under `./logs` +``` +./run_all_fl.sh +``` +Note, the GPU index to use for each client is specified inside `run_all_fl.sh`. +See the individual `run_docker_site-*.sh` commands described below. +Note, the server script will automatically kill all running container used in this example +and final results will be placed under `./result_server`. + +(optional) Run each command in a separate terminals to get site-specific printouts in separate windows. + +The argument for each shell script specifies the GPU index to be used. +``` +./run_docker_server.sh +./run_docker_site-1.sh 0 +./run_docker_site-2.sh 1 +./run_docker_site-3.sh 0 +``` + +### 1.4 (Optional) Visualize training using TensorBoard +After training completed, the training curves can be visualized using +``` +tensorboard --logdir=./result_server +``` +A visualization of the global accuracy and [Kappa](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.cohen_kappa_score.html) validation scores for each site with the provided example data is shown below. +The current setup runs on a machine with two NVIDIA GPUs with 12GB memory each. +The runtime for this experiment is about 45 minutes. +You can adjust the argument to the `run_docker_site-*.sh` scripts to specify different +GPU indices if needed in your environment. + +![](./figs/example_data_val_global_acc_kappa.png) + +### 1.5 (Optional) Kill all containers +If you didn't use `run_all_fl.sh`, all containers can be killed by running +``` +docker kill server site-1 site-2 site-3 +``` + + +------------------------------------------------ +## 2. Modify the FL algorithm + +You can modify and extend the provided example code under [./code/pt](./code/pt). + +You could use other components available at [NVFlare](https://github.com/NVIDIA/NVFlare) +or enhance the training pipeline using your custom code or features of other libraries. + +See the [NVFlare examples](https://github.com/NVIDIA/NVFlare/tree/main/examples) for features that could be utilized in this challenge. + +### 2.1 Debugging the learning algorithm + +The example NVFlare `Learner` class is implemented at [./code/pt/learners/mammo_learner.py](./code/pt/learners/mammo_learner.py). +You can debug the file using the `MockClientEngine` as shown in the script by running +``` +python3 code/pt/learners/mammo_learner.py +``` +Furthermore, you can test it inside the container, by first running +``` +./run_docker_debug.sh +``` +Note, set `inside_container = True` to reflect the changed filepaths inside the container. + + +------------------------------------------------ +## 3. Bring your own FL framework +If you would like to use your own FL framework to participate in the challenge, +please modify the Dockerfile accordingly to include all the dependencies. + +Your container needs to provide the following scripts that implement the starting of server, clients, and finalizing of the server. +They will be executed by the system in the following order. + +### 3.1 start server +``` +/code/start_server.sh +``` + +### 3.2 start each client (in parallel) +``` +/code/start_site-1.sh +/code/start_site-2.sh +/code/start_site-3.sh +``` + +### 3.3 finalize the server +``` +/code/finalize_server.sh +``` +For an example on how the challenge system will execute these commands, see the provided `run_docker*.sh` scripts. + +### 3.4 Communication +The communication channels for FL will be restricted to the ports specified in [fl_project.yml](./code/fl_project.yml). +Your FL framework will also need those ports for implementing the communication. + +### 3.5 Results +Results will need to be written to `/result/predictions.json`. +Please follow the format produced by the reference implementation at [./result_server_example/predictions.json](./result_server_example/predictions.json) +(available after running `python3 ./code/pt/utils/download_datalists_and_predictions.py`) +The code is expected to return a json file containing at least list of image names and prediction probabilities for each breast density class +for the global model (should be named `SRV_best_FL_global_model.pt`). +``` +{ + "site-1": { + "SRV_best_FL_global_model.pt": { + ... + "test_probs": [{ + "image": "Calc-Test_P_00643_LEFT_MLO.npy", + "probs": [0.005602597258985043, 0.7612965703010559, 0.23040543496608734, 0.0026953918859362602] + }, { + ... + }, + "site-2": { + "SRV_best_FL_global_model.pt": { + ... + "test_probs": [{ + "image": "Calc-Test_P_00643_LEFT_MLO.npy", + "probs": [0.005602597258985043, 0.7612965703010559, 0.23040543496608734, 0.0026953918859362602] + }, { + ... + }, + "site-3": { + "SRV_best_FL_global_model.pt": { + ... + "test_probs": [{ + "image": "Calc-Test_P_00643_LEFT_MLO.npy", + "probs": [0.005602597258985043, 0.7612965703010559, 0.23040543496608734, 0.0026953918859362602] + }, { + ... + } +``` diff --git a/federated_learning/breast_density_challenge/build_docker.sh b/federated_learning/breast_density_challenge/build_docker.sh new file mode 100755 index 0000000000..467d587a6e --- /dev/null +++ b/federated_learning/breast_density_challenge/build_docker.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +#SERVER_FQDN="localhost" +SERVER_FQDN=$1 + +if test -z "${SERVER_FQDN}" +then + echo "Usage: ./build_docker.sh [SERVER_FQDN], e.g. ./build_docker.sh localhost" + exit 1 +fi + +NEW_IMAGE=monai-nvflare:latest + +DOCKER_BUILDKIT=0 # show command outputs +docker build --network=host -t ${NEW_IMAGE} --build-arg server_fqdn=${SERVER_FQDN} -f Dockerfile . diff --git a/federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_client.json b/federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_client.json new file mode 100644 index 0000000000..3a2729d717 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_client.json @@ -0,0 +1,51 @@ +{ + "format_version": 2, + + "DATASET_ROOT": "/data/preprocessed", + "DATALIST_PREFIX": "/data/dataset_blinded_", + + "executors": [ + { + "tasks": [ + "train", "submit_model", "validate" + ], + "executor": { + "id": "Executor", + "path": "nvflare.app_common.executors.learner_executor.LearnerExecutor", + "args": { + "learner_id": "learner" + } + } + } + ], + + "task_result_filters": [ + ], + "task_data_filters": [ + ], + + "components": [ + { + "id": "learner", + "path": "pt.learners.mammo_learner.MammoLearner", + "args": { + "dataset_root": "{DATASET_ROOT}", + "datalist_prefix": "{DATALIST_PREFIX}", + "aggregation_epochs": 1, + "lr": 2e-3, + "batch_size": 64, + "val_frac": 0.1 + } + }, + { + "id": "analytic_sender", + "name": "AnalyticsSender", + "args": {} + }, + { + "id": "event_to_fed", + "name": "ConvertToFedEvent", + "args": {"events_to_convert": ["analytix_log_stats"], "fed_event_prefix": "fed."} + } + ] +} diff --git a/federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_server.json b/federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_server.json new file mode 100644 index 0000000000..37f0e84abc --- /dev/null +++ b/federated_learning/breast_density_challenge/code/configs/mammo_fedavg/config/config_fed_server.json @@ -0,0 +1,88 @@ +{ + "format_version": 2, + + "min_clients": 3, + "num_rounds": 100, + + "server": { + "heart_beat_timeout": 600 + }, + "task_data_filters": [], + "task_result_filters": [], + "components": [ + { + "id": "persistor", + "name": "PTFileModelPersistor", + "args": { + "model": { + "path": "monai.networks.nets.TorchVisionFCModel", + "args": { + "model_name": "resnet18", + "n_classes": 4, + "use_conv": false, + "pretrained": true, + "pool": null + } + } + } + }, + { + "id": "shareable_generator", + "name": "FullModelShareableGenerator", + "args": {} + }, + { + "id": "aggregator", + "name": "InTimeAccumulateWeightedAggregator", + "args": {} + }, + { + "id": "model_selector", + "name": "IntimeModelSelectionHandler", + "args": {} + }, + { + "id": "model_locator", + "name": "PTFileModelLocator", + "args": { + "pt_persistor_id": "persistor" + } + }, + { + "id": "json_generator", + "name": "ValidationJsonGenerator", + "args": {} + }, + { + "id": "tb_analytics_receive", + "name": "TBAnalyticsReceiver", + "args": {"events": ["fed.analytix_log_stats"]} + } + ], + "workflows": [ + { + "id": "scatter_gather_ctl", + "name": "ScatterAndGather", + "args": { + "min_clients" : "{min_clients}", + "num_rounds" : "{num_rounds}", + "start_round": 0, + "wait_time_after_min_received": 10, + "aggregator_id": "aggregator", + "persistor_id": "persistor", + "shareable_generator_id": "shareable_generator", + "train_task_name": "train", + "train_timeout": 0 + } + }, + { + "id": "global_model_eval", + "name": "GlobalModelEval", + "args": { + "model_locator_id": "model_locator", + "validation_timeout": 6000, + "cleanup_models": true + } + } + ] +} diff --git a/federated_learning/breast_density_challenge/code/finalize_server.sh b/federated_learning/breast_density_challenge/code/finalize_server.sh new file mode 100755 index 0000000000..2570a65f30 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/finalize_server.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +SERVER="server" +echo "FINALIZING ${CLIENT_NAME}" +cp -r ./fl_workspace/${SERVER}/run_1 /result/. +cp ./fl_workspace/${SERVER}/*.txt /result/. +cp ./fl_workspace/*_log.txt /result/. +cp ./fl_workspace/${SERVER}/run_1/cross_site_val/cross_val_results.json /result/predictions.json # only file required for leaderboard computation +# TODO: might need some more standardization of the result folder diff --git a/federated_learning/breast_density_challenge/code/fl_project.yml b/federated_learning/breast_density_challenge/code/fl_project.yml new file mode 100644 index 0000000000..466bd3f9bf --- /dev/null +++ b/federated_learning/breast_density_challenge/code/fl_project.yml @@ -0,0 +1,60 @@ +api_version: 2 +name: fl_project +description: NVFlare sample project yaml file + +participants: + # change example.com to the FQDN of the server + - name: {SERVER_FQDN} + type: server + org: nvflare + fed_learn_port: 8002 + admin_port: 8003 + - name: site-1 + type: client + org: nvflare + - name: site-2 + type: client + org: nvflare + - name: site-3 + type: client + org: nvflare + - name: admin@nvflare.com + type: admin + org: nvflare + roles: + - super + +# The same methods in all builders are called in their order defined in builders section +builders: + - path: nvflare.lighter.impl.workspace.WorkspaceBuilder + args: + template_file: master_template.yml + - path: nvflare.lighter.impl.template.TemplateBuilder + - path: nvflare.lighter.impl.static_file.StaticFileBuilder + args: + # config_folder can be set to inform NVFlare where to get configuration + config_folder: config + # when docker_image is set to a docker image name, docker.sh will be generated on server/client/admin + # docker_image: + - path: nvflare.lighter.impl.auth_policy.AuthPolicyBuilder + args: + orgs: + nvflare: + - relaxed + roles: + super: super user of system + groups: + relaxed: + desc: org group with relaxed policies + rules: + allow_byoc: true + allow_custom_datalist: true + disabled: false + - path: nvflare.lighter.impl.cert.CertBuilder + - path: nvflare.lighter.impl.he.HEBuilder + args: + poly_modulus_degree: 8192 + coeff_mod_bit_sizes: [60, 40, 40] + scale_bits: 40 + scheme: CKKS + - path: nvflare.lighter.impl.signature.SignatureBuilder diff --git a/federated_learning/breast_density_challenge/code/pt/learners/mammo_learner.py b/federated_learning/breast_density_challenge/code/pt/learners/mammo_learner.py new file mode 100644 index 0000000000..eca65f5553 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/pt/learners/mammo_learner.py @@ -0,0 +1,696 @@ +# Copyright 2022 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import numpy as np +import torch +import torch.optim as optim +from monai.data import CacheDataset, DataLoader +from monai.networks.nets import TorchVisionFCModel +from monai.transforms import ( + Compose, + EnsureTyped, + LoadImaged, + RandFlipd, + RandGaussianNoised, + RandGaussianSmoothd, + RandRotated, + RandScaleIntensityd, + RandShiftIntensityd, + RandZoomd, + Transposed, +) +from nvflare.apis.dxo import DXO, DataKind, MetaKey, from_shareable +from nvflare.apis.fl_constant import FLContextKey, ReturnCode +from nvflare.apis.fl_context import FLContext, FLContextManager +from nvflare.apis.shareable import ReservedHeaderKey, Shareable, make_reply +from nvflare.apis.signal import Signal +from nvflare.app_common.abstract.learner_spec import Learner +from nvflare.app_common.app_constant import AppConstants, ModelName, ValidateType +from sklearn.metrics import cohen_kappa_score +from torch.utils.tensorboard import SummaryWriter + + +def load_datalist(filename, data_list_key="train", base_dir=""): + with open(filename, "r") as f: + data = json.load(f) + + data_list = data[data_list_key] + for d in data_list: + d["image"] = os.path.join(base_dir, d["image"]) + + return data_list + + +class MammoLearner(Learner): + def __init__( + self, + dataset_root: str = None, + datalist_prefix: str = None, + aggregation_epochs: int = 1, + train_task_name: str = AppConstants.TASK_TRAIN, + submit_model_task_name: str = AppConstants.TASK_SUBMIT_MODEL, + lr: float = 1e-2, + batch_size: int = 32, + val_freq: int = 1, + val_frac: float = 0.1, + analytic_sender_id: str = "analytic_sender", + ): + """Simple CIFAR-10 Trainer. + + Args: + dataset_root: directory with breast density mammography data. + datalist_prefix: json file with data list + aggregation_epochs: the number of training epochs for a round. Defaults to 1. + train_task_name: name of the task to train the model. + submit_model_task_name: name of the task to submit the best local model. + lr: local learning rate. Float number. Defaults to 1e-2. + val_freq: int. How often to validate during local training + val_frac: float. Fraction of training set to reserve for validation/model selection + analytic_sender_id: id of `AnalyticsSender` if configured as a client component. If configured, TensorBoard events will be fired. Defaults to "analytic_sender". + Returns: + a Shareable with the updated local model after running `execute()` + or the best local model depending on the specified task. + """ + super().__init__() + # trainer init happens at the very beginning, only the basic info regarding the trainer is set here + # the actual run has not started at this point + self.dataset_root = dataset_root + self.datalist_prefix = datalist_prefix + self.aggregation_epochs = aggregation_epochs + self.train_task_name = train_task_name + self.lr = lr + self.batch_size = batch_size + self.val_freq = val_freq + self.submit_model_task_name = submit_model_task_name + self.best_metric = 0.0 + self.val_frac = val_frac + self.analytic_sender_id = analytic_sender_id + + # Epoch counter + self.epoch_of_start_time = 0 + self.epoch_global = 0 + + if not isinstance(self.val_freq, int): + raise ValueError(f"Expected `val_freq` but got type {type(self.val_freq)}") + + # The following objects will be build in `initialize()` + self.app_root = None + self.client_id = None + self.local_model_file = None + self.best_local_model_file = None + self.writer = None + self.device = None + self.model = None + self.optimizer = None + self.criterion = None + self.transform_train = None + self.transform_valid = None + self.transform_test = None + self.train_dataset = None + self.train_loader = None + self.valid_dataset = None + self.valid_loader = None + self.test_dataset = None + self.test_loader = None + + def initialize(self, parts: dict, fl_ctx: FLContext): + # when the run starts, this is where the actual settings get initialized for trainer + + # Set the paths according to fl_ctx + self.app_root = fl_ctx.get_prop(FLContextKey.APP_ROOT) + fl_args = fl_ctx.get_prop(FLContextKey.ARGS) + self.client_id = fl_ctx.get_identity_name() + self.log_info( + fl_ctx, + f"Client {self.client_id} initialized at \n {self.app_root} \n with args: {fl_args}", + ) + + self.local_model_file = os.path.join(self.app_root, "local_model.pt") + self.best_local_model_file = os.path.join(self.app_root, "best_local_model.pt") + + # Select local TensorBoard writer or event-based writer for streaming + self.writer = parts.get( + self.analytic_sender_id + ) # user configured config_fed_client.json for streaming + if not self.writer: # use local TensorBoard writer only + self.writer = SummaryWriter(self.app_root) + + # set the training-related parameters + # can be replaced by a config-style block + self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + self.model = TorchVisionFCModel( + "resnet18", n_classes=4, use_conv=False, pretrained=False, pool=None + ) # pretrained is used only on server + self.model = self.model.to(self.device) + self.optimizer = optim.SGD(self.model.parameters(), lr=self.lr, momentum=0.9) + self.criterion = torch.nn.CrossEntropyLoss() + self.criterion = self.criterion.to(self.device) + + self.transform_train = Compose( + [ + LoadImaged(keys=["image"]), + Transposed(keys=["image"], indices=[2, 0, 1]), # make channels-first + RandRotated( + keys=["image"], range_x=np.pi / 12, prob=0.5, keep_size=True + ), + RandFlipd(keys=["image"], spatial_axis=0, prob=0.5), + RandFlipd(keys=["image"], spatial_axis=1, prob=0.5), + RandZoomd( + keys=["image"], min_zoom=0.9, max_zoom=1.1, prob=0.5, keep_size=True + ), + RandGaussianSmoothd( + keys=["image"], + sigma_x=(0.5, 1.15), + sigma_y=(0.5, 1.15), + sigma_z=(0.5, 1.15), + prob=0.15, + ), + RandScaleIntensityd(keys=["image"], factors=0.3, prob=0.5), + RandShiftIntensityd(keys=["image"], offsets=0.1, prob=0.5), + RandGaussianNoised(keys=["image"], std=0.01, prob=0.15), + EnsureTyped(keys=["image", "label"]), + ] + ) + self.transform_valid = Compose( + [ + LoadImaged(keys=["image"]), + Transposed(keys=["image"], indices=[2, 0, 1]), # make channels-first + EnsureTyped(keys=["image", "label"]), + ] + ) + self.transform_test = Compose( + [ + LoadImaged(keys=["image"]), + Transposed(keys=["image"], indices=[2, 0, 1]), # make channels-first + EnsureTyped(keys=["image"]), # Testing set won't have labels + ] + ) + + # Note, do not change this syntax. The data list filename is given by the system. + datalist_file = self.datalist_prefix + self.client_id + ".json" + if not os.path.isfile(datalist_file): + self.log_critical(fl_ctx, f"{datalist_file} does not exist!") + + # Set dataset + train_datalist = load_datalist( + datalist_file, + data_list_key="train", # do not change this key name + base_dir=self.dataset_root, + ) + + # Validation set can be created from training set. + if self.val_frac > 0: + np.random.seed(0) + val_indices = np.random.randint( + 0, len(train_datalist), size=int(self.val_frac * len(train_datalist)) + ) + val_datalist = [train_datalist[i] for i in val_indices] + train_indices = list(set(np.arange(len(train_datalist))) - set(val_indices)) + train_datalist = [ + train_datalist[i] for i in train_indices + ] # remove validation entries from training + assert (len(np.intersect1d(val_indices, train_indices))) == 0 + self.log_info( + fl_ctx, + f"Reserved {len(val_indices)} entries for validation during training.", + ) + elif self.val_frac >= 1.0: + raise ValueError( + f"`val_frac` was {self.val_frac}. Cannot use whole training set for validation, use 0 > `val_frac` < 1." + ) + else: + val_datalist = [] + + test_datalist = load_datalist( + datalist_file, + data_list_key="test", # do not change this key name + base_dir=self.dataset_root, + ) + + num_workers = 1 + cache_rate = 1.0 + self.train_dataset = CacheDataset( + data=train_datalist, + transform=self.transform_train, + cache_rate=cache_rate, + num_workers=num_workers, + ) + self.train_loader = DataLoader( + self.train_dataset, + batch_size=self.batch_size, + shuffle=True, + num_workers=num_workers, + ) + self.log_info(fl_ctx, f"Training set: {len(train_datalist)} entries") + + if len(val_datalist) > 0: + self.valid_dataset = CacheDataset( + data=val_datalist, + transform=self.transform_valid, + cache_rate=cache_rate, + num_workers=num_workers, + ) + self.valid_loader = DataLoader( + self.valid_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=num_workers, + ) + self.log_info(fl_ctx, f"Validation set: {len(train_datalist)} entries") + else: + self.valid_dataset = None + self.valid_loader = None + self.log_info(fl_ctx, "Use no validation set") + + # evaluation on testing is required + self.test_dataset = CacheDataset( + data=test_datalist, + transform=self.transform_test, + cache_rate=cache_rate, + num_workers=num_workers, + ) + self.test_loader = DataLoader( + self.test_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=num_workers, + ) + self.log_info(fl_ctx, f"Testing set: {len(train_datalist)} entries") + + self.log_info(fl_ctx, f"Finished initializing {self.client_id}") + + def finalize(self, fl_ctx: FLContext): + # collect threads, close files here + pass + + def local_train( + self, fl_ctx, train_loader, abort_signal: Signal, val_freq: int = 0 + ): + for epoch in range(self.aggregation_epochs): + if abort_signal.triggered: + return + self.model.train() + epoch_len = len(train_loader) + self.epoch_global = self.epoch_of_start_time + epoch + self.log_info( + fl_ctx, + f"Local epoch {self.client_id}: {epoch + 1}/{self.aggregation_epochs} (lr={self.lr})", + ) + avg_loss = 0.0 + for i, batch_data in enumerate(train_loader): + if abort_signal.triggered: + return + inputs, labels = ( + batch_data["image"].to(self.device), + batch_data["label"].to(self.device), + ) + # zero the parameter gradients + self.optimizer.zero_grad() + # forward + backward + optimize + outputs = self.model(inputs) + loss = self.criterion(outputs, labels) + + loss.backward() + self.optimizer.step() + current_step = epoch_len * self.epoch_global + i + avg_loss += loss.item() + self.writer.add_scalar( + "train_loss", avg_loss / len(train_loader), current_step + ) + if val_freq > 0 and epoch % val_freq == 0: + acc, kappa = self.local_valid( + self.valid_loader, + abort_signal, + tb_id="val_acc_local_model", + fl_ctx=fl_ctx, + ) + if kappa > self.best_metric: + self.best_metric = kappa + self.save_model(is_best=True) + + def save_model(self, is_best=False): + # save model + model_weights = self.model.state_dict() + save_dict = {"model_weights": model_weights, "epoch": self.epoch_global} + if is_best: + save_dict.update({"best_acc": self.best_metric}) + torch.save(save_dict, self.best_local_model_file) + else: + torch.save(save_dict, self.local_model_file) + + def train( + self, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal + ) -> Shareable: + # Check abort signal + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + + # get round information + current_round = shareable.get_header(AppConstants.CURRENT_ROUND) + total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS) + self.log_info( + fl_ctx, f"Current/Total Round: {current_round + 1}/{total_rounds}" + ) + self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}") + + # update local model weights with received weights + dxo = from_shareable(shareable) + global_weights = dxo.data + + # Before loading weights, tensors might need to be reshaped to support HE for secure aggregation. + local_var_dict = self.model.state_dict() + model_keys = global_weights.keys() + for var_name in local_var_dict: + if var_name in model_keys: + weights = global_weights[var_name] + try: + # reshape global weights to compute difference later on + global_weights[var_name] = np.reshape( + weights, local_var_dict[var_name].shape + ) + # update the local dict + local_var_dict[var_name] = torch.as_tensor(global_weights[var_name]) + except Exception as e: + raise ValueError( + "Convert weight from {} failed with error: {}".format( + var_name, str(e) + ) + ) + self.model.load_state_dict(local_var_dict) + + # local steps + epoch_len = len(self.train_loader) + self.log_info(fl_ctx, f"Local steps per epoch: {epoch_len}") + + # local train + self.local_train( + fl_ctx=fl_ctx, + train_loader=self.train_loader, + abort_signal=abort_signal, + val_freq=self.val_freq, + ) + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + self.epoch_of_start_time += self.aggregation_epochs + + # perform valid after local train + acc, kappa = self.local_valid( + self.valid_loader, abort_signal, tb_id="val_local_model", fl_ctx=fl_ctx + ) + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + self.log_info(fl_ctx, f"val_acc_local_model: {acc:.4f}") + + # save model + self.save_model(is_best=False) + if kappa > self.best_metric: + self.best_metric = kappa + self.save_model(is_best=True) + + # compute delta model, global model has the primary key set + local_weights = self.model.state_dict() + model_diff = {} + for name in global_weights: + if name not in local_weights: + continue + model_diff[name] = local_weights[name].cpu().numpy() - global_weights[name] + if np.any(np.isnan(model_diff[name])): + self.system_panic(f"{name} weights became NaN...", fl_ctx) + return make_reply(ReturnCode.EXECUTION_EXCEPTION) + + # build the shareable + dxo = DXO(data_kind=DataKind.WEIGHT_DIFF, data=model_diff) + dxo.set_meta_prop(MetaKey.NUM_STEPS_CURRENT_ROUND, epoch_len) + + self.log_info(fl_ctx, "Local epochs finished. Returning shareable") + return dxo.to_shareable() + + def get_model_for_validation(self, model_name: str, fl_ctx: FLContext) -> Shareable: + # Retrieve the best local model saved during training. + if model_name == ModelName.BEST_MODEL: + model_data = None + try: + # load model to cpu as server might or might not have a GPU + model_data = torch.load(self.best_local_model_file, map_location="cpu") + except Exception as e: + self.log_error(fl_ctx, f"Unable to load best model: {e}") + + # Create DXO and shareable from model data. + if model_data: + dxo = DXO(data_kind=DataKind.WEIGHTS, data=model_data["model_weights"]) + return dxo.to_shareable() + else: + # Set return code. + self.log_error( + fl_ctx, + f"best local model not found at {self.best_local_model_file}.", + ) + return make_reply(ReturnCode.EXECUTION_RESULT_ERROR) + else: + raise ValueError( + f"Unknown model_type: {model_name}" + ) # Raised errors are caught in LearnerExecutor class. + + def local_valid( + self, + valid_loader, + abort_signal: Signal, + tb_id=None, + return_probs_only=False, + fl_ctx=None, + ): + if not valid_loader: + return None + self.model.eval() + return_probs = [] + labels = [] + pred_labels = [] + with torch.no_grad(): + correct, total = 0, 0 + for i, batch_data in enumerate(valid_loader): + if abort_signal.triggered: + return None + inputs = batch_data["image"].to(self.device) + outputs = torch.softmax(self.model(inputs), dim=1) + probs = outputs.detach().cpu().numpy() + # make json serializable + for _img_file, _probs in zip( + batch_data["image_meta_dict"]["filename_or_obj"], probs + ): + return_probs.append( + { + "image": os.path.basename(_img_file), + "probs": [float(p) for p in _probs], + } + ) + if not return_probs_only: + _, _pred_label = torch.max(outputs.data, 1) + _labels = batch_data["label"].to(self.device) + total += inputs.data.size()[0] + correct += (_pred_label == _labels.data).sum().item() + labels.extend(_labels.detach().cpu().numpy()) + pred_labels.extend(_pred_label.detach().cpu().numpy()) + if return_probs_only: + return return_probs # create a list of image names and probs + else: + acc = correct / float(total) + assert len(labels) == total + assert len(pred_labels) == total + kappa = cohen_kappa_score(labels, pred_labels, weights="linear") + if tb_id: + self.writer.add_scalar(tb_id + "_acc", acc, self.epoch_global) + self.writer.add_scalar(tb_id + "_kappa", kappa, self.epoch_global) + return acc, kappa + + def validate( + self, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal + ) -> Shareable: + # Check abort signal + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + + # get validation information + self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}") + model_owner = shareable.get(ReservedHeaderKey.HEADERS).get( + AppConstants.MODEL_OWNER + ) + if model_owner: + self.log_info( + fl_ctx, + f"Evaluating model from {model_owner} on {fl_ctx.get_identity_name()}", + ) + else: + model_owner = "global_model" # evaluating global model during training + + # update local model weights with received weights + dxo = from_shareable(shareable) + global_weights = dxo.data + + # Before loading weights, tensors might need to be reshaped to support HE for secure aggregation. + local_var_dict = self.model.state_dict() + model_keys = global_weights.keys() + n_loaded = 0 + for var_name in local_var_dict: + if var_name in model_keys: + weights = torch.as_tensor(global_weights[var_name], device=self.device) + try: + # update the local dict + local_var_dict[var_name] = torch.as_tensor( + torch.reshape(weights, local_var_dict[var_name].shape) + ) + n_loaded += 1 + except Exception as e: + raise ValueError( + "Convert weight from {} failed with error: {}".format( + var_name, str(e) + ) + ) + self.model.load_state_dict(local_var_dict) + if n_loaded == 0: + raise ValueError( + f"No weights loaded for validation! Received weight dict is {global_weights}" + ) + + validate_type = shareable.get_header(AppConstants.VALIDATE_TYPE) + if validate_type == ValidateType.BEFORE_TRAIN_VALIDATE: + try: + # perform valid before local train + global_acc, global_kappa = self.local_valid( + self.valid_loader, + abort_signal, + tb_id="val_global_model", + fl_ctx=fl_ctx, + ) + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + self.log_info( + fl_ctx, f"val_acc_global_model ({model_owner}): {global_acc}" + ) + + return DXO( + data_kind=DataKind.METRICS, + data={MetaKey.INITIAL_METRICS: global_acc}, + meta={}, + ).to_shareable() + except Exception as e: + raise ValueError(f"BEFORE_TRAIN_VALIDATE failed: {e}") + elif validate_type == ValidateType.MODEL_VALIDATE: + try: + # perform valid + train_acc, train_kappa = self.local_valid( + self.train_loader, abort_signal + ) + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + self.log_info(fl_ctx, f"training acc ({model_owner}): {train_acc}") + + val_acc, val_kappa = self.local_valid(self.valid_loader, abort_signal) + + # testing performance + test_probs = self.local_valid( + self.test_loader, abort_signal, return_probs_only=True + ) + if abort_signal.triggered: + return make_reply(ReturnCode.TASK_ABORTED) + self.log_info(fl_ctx, f"validation acc ({model_owner}): {val_acc}") + + self.log_info(fl_ctx, "Evaluation finished. Returning shareable") + + val_results = { + "train_accuracy": train_acc, + "train_kappa": train_kappa, + "val_accuracy": val_acc, + "val_kappa": val_kappa, + "test_probs": test_probs, + } + + metric_dxo = DXO(data_kind=DataKind.METRICS, data=val_results) + return metric_dxo.to_shareable() + except Exception as e: + raise ValueError(f"MODEL_VALIDATE failed: {e}") + else: + return make_reply(ReturnCode.VALIDATE_TYPE_UNKNOWN) + + +# To test your Learner + +class MockClientEngine: + def __init__(self, run_num=0): + self.fl_ctx_mgr = FLContextManager( + engine=self, + identity_name="site-1", + run_num=run_num, + public_stickers={}, + private_stickers={}, + ) + + def new_context(self): + return self.fl_ctx_mgr.new_context() + + def fire_event(self, event_type: str, fl_ctx: FLContext): + pass + + +if __name__ == "__main__": + inside_container = True + if inside_container: + debug_dataset_root = "/data/preprocessed" + debug_datalist_prefix = "/data/dataset_blinded_phase1_" + else: + # assumes script is run in from repo root, e.g. using `python3 code/pt/learners/mammo_learner.py` + debug_dataset_root = "./data/preprocessed" + debug_datalist_prefix = "./data/dataset_blinded_phase1_" + + print("Testing MammoLearner...") + learner = MammoLearner( + dataset_root=debug_dataset_root, + datalist_prefix=debug_datalist_prefix, + aggregation_epochs=1, + lr=1e-2, + ) + engine = MockClientEngine() + fl_ctx = engine.fl_ctx_mgr.new_context() + fl_ctx.set_prop(FLContextKey.APP_ROOT, "/tmp/debug") + + print("test initialize...") + learner.initialize(parts={}, fl_ctx=fl_ctx) + + print("test train...") + learner.local_train( + fl_ctx=fl_ctx, + train_loader=learner.train_loader, + abort_signal=Signal(), + val_freq=1, + ) + + print("test valid...") + acc, kappa = learner.local_valid( + valid_loader=learner.valid_loader, + abort_signal=Signal(), + tb_id="val_debug", + fl_ctx=fl_ctx, + ) + print("debug acc", acc) + print("debug kappa", kappa) + + print("test valid...") + test_probs = learner.local_valid( + valid_loader=learner.test_loader, abort_signal=Signal(), return_probs_only=True + ) + print("test_probs", test_probs) + + print("finished testing.") + + # you can check the result for one epoch and validation on TensorBoard using + # `tensorboard --logdir=./debug` diff --git a/federated_learning/breast_density_challenge/code/pt/utils/download_datalists_and_predictions.py b/federated_learning/breast_density_challenge/code/pt/utils/download_datalists_and_predictions.py new file mode 100644 index 0000000000..41721b7a15 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/pt/utils/download_datalists_and_predictions.py @@ -0,0 +1,16 @@ +# Copyright 2022 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from monai.apps.utils import download_url, download_and_extract + + +download_and_extract(url="https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/dataset_lists.zip", output_dir="./data") +download_url(url="https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/predictions.json", filepath="./result_server_example/predictions.json") diff --git a/federated_learning/breast_density_challenge/code/pt/utils/download_model.py b/federated_learning/breast_density_challenge/code/pt/utils/download_model.py new file mode 100644 index 0000000000..4ea50e70cf --- /dev/null +++ b/federated_learning/breast_density_challenge/code/pt/utils/download_model.py @@ -0,0 +1,25 @@ +# Copyright 2022 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from torch.utils.model_zoo import load_url as load_state_dict_from_url + +parser = argparse.ArgumentParser() +parser.add_argument( + "--model_url", + type=str, + default="https://download.pytorch.org/models/resnet18-f37072fd.pth", +) +args = parser.parse_args() + +# will download +model = load_state_dict_from_url(args.model_url) diff --git a/federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicom.py b/federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicom.py new file mode 100644 index 0000000000..752835f58a --- /dev/null +++ b/federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicom.py @@ -0,0 +1,67 @@ +# Copyright 2022 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import cv2 +import numpy as np +import pydicom +import skimage.io + + +def dicom_preprocess(dicom_file, save_prefix): + try: + # Read needed dicom tags + ds = pydicom.dcmread(dicom_file) # , stop_before_pixels=True) + try: + code = ds.ViewCodeSequence[0].ViewModifierCodeSequence[0].CodeMeaning + except BaseException: + code = None + + # Filter image + dc_tags = f"BS={ds.BitsStored};PI={ds.PhotometricInterpretation};Modality={ds.Modality};PatientOrientation={ds.PatientOrientation};Code={code}" + if ds.PatientOrientation == "MLO" or ds.PatientOrientation == "CC": + curr_img = ds.pixel_array + curr_img = np.squeeze(curr_img).T.astype(np.float) + + # Can be modified as well to handle other bit and monochrome combinations + if (ds.BitsStored == 16) and "2" in ds.PhotometricInterpretation: + curr_img = curr_img / 65535.0 + else: + raise ValueError(dicom_file + " - unsupported dicom tags: " + dc_tags) + + # Resize and replicate into 3 channels + curr_img = cv2.resize(curr_img, (224, 224)) + curr_img = np.concatenate( + ( + curr_img[:, :, np.newaxis], + curr_img[:, :, np.newaxis], + curr_img[:, :, np.newaxis], + ), + axis=-1, + ) + # Save output file + assert curr_img.min() >= 0 and curr_img.max() <= 1.0 + + os.makedirs(os.path.dirname(save_prefix), exist_ok=True) + np.save(save_prefix + ".npy", curr_img.astype(np.float32)) + skimage.io.imsave( + save_prefix + ".png", (255 * curr_img / curr_img.max()).astype(np.uint8) + ) + else: + raise ValueError( + "Error: " + dicom_file + " - not a valid image file: " + dc_tags + ) + except BaseException as e: + print(f"[WARNING] Reading {dicom_file} failed with Exception: {e}") + return False, f"{dicom_file} failed" + + return True, dc_tags diff --git a/federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicomdir.py b/federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicomdir.py new file mode 100644 index 0000000000..66c76da592 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/pt/utils/preprocess_dicomdir.py @@ -0,0 +1,304 @@ +# Copyright 2022 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +import random + +import numpy as np +import pandas as pd +from preprocess_dicom import dicom_preprocess +from sklearn.model_selection import GroupKFold + +# density labels +# 1 - fatty +# 2 - scattered fibroglandular density +# 3 - heterogeneously dense +# 4 - extremely dense + + +def preprocess(dicom_root, out_path, ids, images, densities, process_image=True): + data_list = [] + dc_tags = [] + saved_filenames = [] + assert len(ids) == len(images) == len(densities) + for i, (id, image, density) in enumerate(zip(ids, images, densities)): + if (i + 1) % 200 == 0: + print(f"processing {i+1} of {len(ids)}...") + dir_name = image.split(os.path.sep)[0] + img_file = glob.glob( + os.path.join(dicom_root, dir_name, "**", "*.dcm"), recursive=True + ) + assert len(img_file) == 1, f"No unique dicom image found for {dir_name}!" + save_prefix = os.path.join(out_path, dir_name) + if process_image: + _success, _dc_tags = dicom_preprocess(img_file[0], save_prefix) + else: + if os.path.isfile(save_prefix + ".npy"): + _success = True + else: + _success = False + _dc_tags = [] + if _success and density >= 1: # label can be 0 sometimes, excluding those cases + dc_tags.append(_dc_tags) + data_list.append( + { + "patient_id": id, + "image": dir_name + ".npy", + "label": int(density - 1), + } + ) + saved_filenames.append(dir_name + ".npy") + return data_list, dc_tags, saved_filenames + + +def write_datalist(save_datalist_file, data_set): + os.makedirs(os.path.dirname(save_datalist_file), exist_ok=True) + with open(save_datalist_file, "w") as f: + json.dump(data_set, f, indent=4) + print(f"Data list saved at {save_datalist_file}") + + +def get_indices(all_ids, search_ids): + indices = [] + for _id in search_ids: + _indices = np.where(all_ids == _id) + indices.extend(_indices[0].tolist()) + return indices + + +def main(): + process_image = True # set False if dicoms have already been preprocessed + + out_path = "./data/preprocessed" # YOUR DEST FOLDER SHOULD BE WRITTEN HERE + out_dataset_prefix = "./data/dataset" + + # Input folders + label_root = "/media/hroth/Elements/NVIDIA/Data/CBIS-DDSM/" + dicom_root = "/media/hroth/Elements/NVIDIA/Data/CBIS-DDSM/DICOM/manifest-ZkhPvrLo5216730872708713142/CBIS-DDSM" + n_clients = 3 + + """ Run preprocessing """ + + """ 1. Load the label data """ + random.seed(0) + + label_files = [ + os.path.join(label_root, "mass_case_description_train_set.csv"), + os.path.join(label_root, "calc_case_description_train_set.csv"), + os.path.join(label_root, "mass_case_description_test_set.csv"), + os.path.join(label_root, "calc_case_description_test_set.csv"), + ] + + breast_densities = [] + patients_ids = [] + image_file_path = [] + + # read annotations + for label_file in label_files: + print(f"add {label_file}") + label_data = pd.read_csv(label_file) + unique_images, unique_indices = np.unique( + label_data["image file path"], return_index=True + ) + print( + f"including {len(unique_images)} unique images of {len(label_data['image file path'])} image entries" + ) + + try: + breast_densities.extend(label_data["breast_density"][unique_indices]) + except BaseException: + breast_densities.extend(label_data["breast density"][unique_indices]) + patients_ids.extend(label_data["patient_id"][unique_indices]) + image_file_path.extend(label_data["image file path"][unique_indices]) + + assert len(breast_densities) == len(patients_ids) == len(image_file_path), ( + f"Mismatch between label data, breast_densities: " + f"{len(breast_densities)}, patients_ids: {len(patients_ids)}, image_file_path: {len(image_file_path)}" + ) + print(f"Read {len(image_file_path)} data entries.") + + """ 2. Split the data """ + + # shuffle data + label_data = list(zip(breast_densities, patients_ids, image_file_path)) + random.shuffle(label_data) + breast_densities, patients_ids, image_file_path = zip(*label_data) + + # Split data + breast_densities = np.array(breast_densities) + patients_ids = np.array(patients_ids) + image_file_path = np.array(image_file_path) + + unique_patient_ids = np.unique(patients_ids) + n_patients = len(unique_patient_ids) + print(f"Found {n_patients} patients.") + + # generate splits using roughly the same ratios as for challenge data: + n_train_challenge = 60_000 + n_val_challenge = 6_500 + n_test_challenge = 40_000 + test_ratio = n_test_challenge / ( + n_train_challenge + n_val_challenge + n_test_challenge + ) + val_ratio = n_val_challenge / ( + n_val_challenge + n_test_challenge + ) # test cases will be removed at this point + + # use groups to avoid patient overlaps + # test split + n_splits = int(np.ceil(len(image_file_path) / (len(image_file_path) * test_ratio))) + print( + f"Splitting into {n_splits} folds for test split. (Only the first fold is used.)" + ) + group_kfold = GroupKFold(n_splits=n_splits) + for train_val_index, test_index in group_kfold.split( + image_file_path, breast_densities, groups=patients_ids + ): + break # just use first fold + test_images = image_file_path[test_index] + test_patients_ids = patients_ids[test_index] + test_densities = breast_densities[test_index] + + # train/val splits + train_val_images = image_file_path[train_val_index] + train_val_patients_ids = patients_ids[train_val_index] + train_val_densities = breast_densities[train_val_index] + + n_splits = int(np.ceil(len(image_file_path) / (len(image_file_path) * val_ratio))) + print( + f"Splitting into {n_splits} folds for train/val splits. (Only the first fold is used.)" + ) + group_kfold = GroupKFold(n_splits=n_splits) + for train_index, val_index in group_kfold.split( + train_val_images, train_val_densities, groups=train_val_patients_ids + ): + break # just use first fold + + train_images = train_val_images[train_index] + train_patients_ids = train_val_patients_ids[train_index] + train_densities = train_val_densities[train_index] + + val_images = train_val_images[val_index] + val_patients_ids = train_val_patients_ids[val_index] + val_densities = train_val_densities[val_index] + + # check that there is no patient overlap + assert ( + len(np.intersect1d(train_patients_ids, val_patients_ids)) == 0 + ), "Overlapping patients in train and validation!" + assert ( + len(np.intersect1d(train_patients_ids, test_patients_ids)) == 0 + ), "Overlapping patients in train and test!" + assert ( + len(np.intersect1d(val_patients_ids, test_patients_ids)) == 0 + ), "Overlapping patients in validation and test!" + + n_total = len(train_images) + len(val_images) + len(test_images) + print(20 * "-") + print(f"Train : {len(train_images)} ({100*len(train_images)/n_total:.2f}%)") + print(f"Val : {len(val_images)} ({100*len(val_images)/n_total:.2f}%)") + print(f"Test : {len(test_images)} ({100*len(test_images)/n_total:.2f}%)") + print(20 * "-") + print(f"Total : {n_total}") + assert n_total == len(image_file_path), ( + f"mismatch between total split images ({n_total})" + f" and length of all images {len(image_file_path)}!" + ) + + """ split train/validation dataset for n_clients """ + # Split and avoid patient overlap + unique_train_patients_ids = np.unique(train_patients_ids) + split_train_patients_ids = np.array_split(unique_train_patients_ids, n_clients) + + unique_val_patients_ids = np.unique(val_patients_ids) + split_val_patients_ids = np.array_split(unique_val_patients_ids, n_clients) + + unique_test_patients_ids = np.unique(test_patients_ids) + split_test_patients_ids = np.array_split(unique_test_patients_ids, n_clients) + + """ 3. Preprocess the images """ + dc_tags = [] + saved_filenames = [] + for c in range(n_clients): + site_name = f"site-{c+1}" + print(f"Preprocessing training set of client {site_name}") + _curr_patient_ids = split_train_patients_ids[c] + _curr_indices = get_indices(train_patients_ids, _curr_patient_ids) + train_list, _dc_tags, _saved_filenames = preprocess( + dicom_root, + out_path, + train_patients_ids[_curr_indices], + train_images[_curr_indices], + train_densities[_curr_indices], + process_image=process_image, + ) + print( + f"Converted {len(train_list)} of {len(train_patients_ids)} training images" + ) + dc_tags.extend(_dc_tags) + saved_filenames.extend(_saved_filenames) + + print("Preprocessing validation") + _curr_patient_ids = split_val_patients_ids[c] + _curr_indices = get_indices(val_patients_ids, _curr_patient_ids) + val_list, _dc_tags, _saved_filenames = preprocess( + dicom_root, + out_path, + val_patients_ids[_curr_indices], + val_images[_curr_indices], + val_densities[_curr_indices], + process_image=process_image, + ) + print(f"Converted {len(val_list)} of {len(val_patients_ids)} validation images") + dc_tags.extend(_dc_tags) + saved_filenames.extend(_saved_filenames) + + print("Preprocessing testing") + _curr_patient_ids = split_test_patients_ids[c] + _curr_indices = get_indices(test_patients_ids, _curr_patient_ids) + test_list, _dc_tags, _saved_filenames = preprocess( + dicom_root, + out_path, + test_patients_ids[_curr_indices], + test_images[_curr_indices], + test_densities[_curr_indices], + process_image=process_image, + ) + print(f"Converted {len(test_list)} of {len(test_patients_ids)} testing images") + dc_tags.extend(_dc_tags) + saved_filenames.extend(_saved_filenames) + + data_set = { + "train": train_list, # will stay the same for both phases + "test1": val_list, # like phase 1 leaderboard + "test2": test_list, # like phase 2 - final leaderboard + } + write_datalist(f"{out_dataset_prefix}_{site_name}.json", data_set) + + print(50 * "=") + print( + f"Successfully converted a total {len(saved_filenames)} of {len(image_file_path)} images." + ) + + # check that there were no duplicated files + assert len(saved_filenames) == len( + np.unique(saved_filenames) + ), f"Not all generated files ({len(saved_filenames)}) are unique ({len(np.unique(saved_filenames))})!" + + print(f"Data lists saved wit prefix {out_dataset_prefix}") + print(50 * "=") + print("Processed unique DICOM tags", np.unique(dc_tags)) + + +if __name__ == "__main__": + main() diff --git a/federated_learning/breast_density_challenge/code/pt/utils/strip_testing_labels.py b/federated_learning/breast_density_challenge/code/pt/utils/strip_testing_labels.py new file mode 100644 index 0000000000..e891ed8d9b --- /dev/null +++ b/federated_learning/breast_density_challenge/code/pt/utils/strip_testing_labels.py @@ -0,0 +1,59 @@ +# Copyright 2022 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + + +def strip_and_split(dataset_filename, strip_set): + with open(dataset_filename, "r") as f: + data = json.load(f) + + # remove labels + [x.pop("label") for x in data[strip_set]] + new_data = { + "train": data["train"], # keep the same train set in both cases + "test": data[strip_set], + } + print(f"removed {len(data[strip_set])} labels from `{strip_set}`") + return new_data + + +def main(): + datalist_rootdir = "../../../data" + for client_id in ["site-1", "site-2", "site-3"]: + print(f"processing {client_id}") + new_datalist1 = strip_and_split( + os.path.join(datalist_rootdir, f"./dataset_{client_id}.json"), + strip_set="test1", + ) + new_datalist2 = strip_and_split( + os.path.join(datalist_rootdir, f"./dataset_{client_id}.json"), + strip_set="test2", + ) + with open( + os.path.join( + datalist_rootdir, f"./dataset_blinded_{client_id}.json" + ), + "w", + ) as f: + json.dump(new_datalist1, f, indent=4) + with open( + os.path.join( + datalist_rootdir, f"./dataset_blinded_phase2_{client_id}.json" + ), + "w", + ) as f: + json.dump(new_datalist2, f, indent=4) + + +if __name__ == "__main__": + main() diff --git a/federated_learning/breast_density_challenge/code/run_fl.py b/federated_learning/breast_density_challenge/code/run_fl.py new file mode 100644 index 0000000000..36faa2fd39 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/run_fl.py @@ -0,0 +1,94 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import time + +from nvflare.fuel.hci.client.fl_admin_api_runner import FLAdminAPIRunner, api_command_wrapper, wait_until_clients_greater_than_cb +from nvflare.fuel.hci.client.fl_admin_api_spec import TargetType + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run_number", type=int, default=100, help="FL run number to start at.") + parser.add_argument("--admin_dir", type=str, default="./admin/", help="Path to admin directory.") + parser.add_argument("--username", type=str, default="admin@nvflare.com", help="Admin username") + parser.add_argument("--app", type=str, default="cifar10_fedavg", help="App to be deployed") + parser.add_argument("--port", type=int, default=8003, help="The admin server port") + parser.add_argument("--poc", action='store_true', help="Whether admin uses POC mode.") + parser.add_argument("--min_clients", type=int, default=8, help="Minimum number of clients.") + args = parser.parse_args() + + host = "" + port = args.port + + assert os.path.isdir(args.admin_dir), f"admin directory does not exist at {args.admin_dir}" + + # Set up certificate names and admin folders + upload_dir = os.path.join(args.admin_dir, "transfer") + if not os.path.isdir(upload_dir): + os.makedirs(upload_dir) + download_dir = os.path.join(args.admin_dir, "download") + if not os.path.isdir(download_dir): + os.makedirs(download_dir) + + run_number = args.run_number + + # Initialize the runner + runner = FLAdminAPIRunner( + host=host, + port=port, + username=args.username, + admin_dir=args.admin_dir, + poc=args.poc, + debug=False, + ) + + # Run + start = time.time() + # Wait for clients to be connected + print(f"WAITING FOR {args.min_clients} CLIENTS TO CONNECT...") + api_command_wrapper( + runner.api.wait_until_server_status( + callback=wait_until_clients_greater_than_cb, min_clients=args.min_clients + ) + ) + print("MAKING SURE CLIENTS ARE READY...") + time.sleep(30) # make sure clients are ready + + # Run Training + print("RUN TRAINING...") + runner.run(run_number, args.app, restart_all_first=False, shutdown_on_error=False, shutdown_at_end=False, + timeout=None, min_clients=args.min_clients) + print("Total training time", time.time() - start) + + # Move client logs to server + print("GET CLIENT LOGS") + for client_id in ["site-1", "site-2", "site-3"]: + result = runner.api.cat_target(target="site-1", file="log.txt") + if result["status"] == "SUCCESS": + if "message" in result["details"]: + log = result["details"]["message"] + client_log_file = os.path.join(args.admin_dir, "..", f"{client_id}_log.txt") + with open(client_log_file, "w") as f: + f.write(log) + print(f"Wrote {client_id}'s log to {client_log_file}") + + print("SHUTDOWN ALL...") + api_command_wrapper(runner.api.shutdown(TargetType.ALL)) + + +if __name__ == "__main__": + main() diff --git a/federated_learning/breast_density_challenge/code/run_fl.sh b/federated_learning/breast_density_challenge/code/run_fl.sh new file mode 100755 index 0000000000..31333d3b2e --- /dev/null +++ b/federated_learning/breast_density_challenge/code/run_fl.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# add current folder to PYTHONPATH +export PYTHONPATH="${PYTHONPATH}:${PWD}" +echo "PYTHONPATH is ${PYTHONPATH}" +export PYTHONUNBUFFERED=1 + +algorithms_dir="${PWD}/configs" +workspace="fl_workspace" +admin_username="admin@nvflare.com" +site_pre="site-" + +n_clients=$1 +config=$2 +run=$3 + +if test -z "${n_clients}" || test -z "${config}" || test -z "${run}" +then + echo "Usage: ./run_fl.sh [n_clients] [config] [run], e.g. ./run_fl.sh 3 mammo_fedavg 1 0.1" + exit 1 +fi + +# start training +echo "STARTING TRAINING" +python3 ./run_fl.py --port=8003 --admin_dir="./${workspace}/${admin_username}" \ + --username="${admin_username}" --run_number="${run}" --app="${algorithms_dir}/${config}" --min_clients="${n_clients}" + +# sleep for FL system to shut down, so a new run can be started automatically +sleep 30 +echo "TRAINING ENDED" diff --git a/federated_learning/breast_density_challenge/code/start_server.sh b/federated_learning/breast_density_challenge/code/start_server.sh new file mode 100755 index 0000000000..89b9792936 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/start_server.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +SERVER="server" +echo "STARTING ${CLIENT_NAME}" +./fl_workspace/${SERVER}/startup/start.sh; sleep 30s # TODO: Is there a better way than sleep? +./run_fl.sh 3 mammo_fedavg 1 diff --git a/federated_learning/breast_density_challenge/code/start_site-1.sh b/federated_learning/breast_density_challenge/code/start_site-1.sh new file mode 100755 index 0000000000..2f2e06a75b --- /dev/null +++ b/federated_learning/breast_density_challenge/code/start_site-1.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +CLIENT_NAME="site-1" +echo "STARTING ${CLIENT_NAME}" +./fl_workspace/${CLIENT_NAME}/startup/start.sh diff --git a/federated_learning/breast_density_challenge/code/start_site-2.sh b/federated_learning/breast_density_challenge/code/start_site-2.sh new file mode 100755 index 0000000000..86ed4fdeb8 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/start_site-2.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +CLIENT_NAME="site-2" +echo "STARTING ${CLIENT_NAME}" +./fl_workspace/${CLIENT_NAME}/startup/start.sh diff --git a/federated_learning/breast_density_challenge/code/start_site-3.sh b/federated_learning/breast_density_challenge/code/start_site-3.sh new file mode 100755 index 0000000000..9a5a3992c5 --- /dev/null +++ b/federated_learning/breast_density_challenge/code/start_site-3.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +CLIENT_NAME="site-3" +echo "STARTING ${CLIENT_NAME}" +./fl_workspace/${CLIENT_NAME}/startup/start.sh diff --git a/federated_learning/breast_density_challenge/data/README.md b/federated_learning/breast_density_challenge/data/README.md new file mode 100644 index 0000000000..f06a23217e --- /dev/null +++ b/federated_learning/breast_density_challenge/data/README.md @@ -0,0 +1,19 @@ +## Example breast density data + +Download example data from https://drive.google.com/file/d/1Fd9GLUIzbZrl4FrzI3Huzul__C8wwzyx/view?usp=sharing. +Extract here. + +## Data source +This example data is based on [CBIS-DDSM](https://wiki.cancerimagingarchive.net/display/Public/CBIS-DDSM) from [TCIA](https://wiki.cancerimagingarchive.net/) [1]. + +We preprocessed all files using `code/pt/utils/preprocess_dicomdir.py` and generated train/val splits for each client +and separate testing split. + +For more details on this example data, see [2,3]. + +## References +[1] Clark K, Vendt B, Smith K, Freymann J, Kirby J, Koppel P, Moore S, Phillips S, Maffitt D, Pringle M, Tarbox L, Prior F. The Cancer Imaging Archive (TCIA): Maintaining and Operating a Public Information Repository, Journal of Digital Imaging, Volume 26, Number 6, December, 2013, pp 1045-1057. DOI: https://doi.org/10.1007/s10278-013-9622-7 + +[2] Rebecca Sawyer Lee, Francisco Gimenez, Assaf Hoogi , Daniel Rubin (2016). Curated Breast Imaging Subset of DDSM [Dataset]. The Cancer Imaging Archive. DOI: https://doi.org/10.7937/K9/TCIA.2016.7O02S9CY + +[3] Rebecca Sawyer Lee, Francisco Gimenez, Assaf Hoogi, Kanae Kawai Miyake, Mia Gorovoy & Daniel L. Rubin. (2017) A curated mammography data set for use in computer-aided detection and diagnosis research. Scientific Data volume 4, Article number: 170177 DOI: https://doi.org/10.1038/sdata.2017.177 diff --git a/federated_learning/breast_density_challenge/figs/example_data_val_global_acc_kappa.png b/federated_learning/breast_density_challenge/figs/example_data_val_global_acc_kappa.png new file mode 100644 index 0000000000000000000000000000000000000000..1a28809cef26776d23da8054613b3520d3140c99 GIT binary patch literal 51107 zcmZ_0byQqU@GW|9C&=LL1b26LCqQr+T!MRWCukU4gA*VSAUFgkKnNBf1cJM}bI%Mi@ee2CXtT{b%rcZU%u3cTbiPO?h#6l-U2LJ#|SxHU@0N|hi0Plhd4_>h}xh4ny zAb83s>!G5eF0E;4!ivG zDD<-hpJ%)OKCN_jm5VMf-b6-5ic`m7;v(({qQ@7VqW`Fgs4R-$qQ$1t8lu_9*ocnc zyvWtoi_}KMO&$D#hB%_(+ddQc?>O+GPXF7r?#|Q3-@5L$F8X34+TSyLR=$QcA)xrllNUg-vhVkU{eq0=+$ zv%w0R;B0dW6Q%oaXC- zFQ*ir{I~zl+f%vaaCV<()CV1V7Si>Y&M&vRZ@7g#7z2LiqfC>cC!NqSn9E}0)=kgL z(-a@SnAl@OaxX90I)1iVbl&f!rH44pAgjT6f{HwkAIa}8=}yjx%!Awye%IC* z)OX+S*Ie9n`UVXrQGmZ^-0V0vNM!zZanYYoJlhArm3g^#duef)DQ|X~H}jm}Uv!3% z|NCk;sVH%t3Qouke8#mHs_UJ)WM8;j=F`g=&3cozX+HJDGuHafMz1{`d2MV;vDSBM zeGex+A%1?h-4DmcjW(mFr(8p~)irhPciXvkj{Bdeo_3LlOis_oI8PQDtS`1F$OK&n zh)KP5u zi58b(cMb8YSu(dycYFI?@N33h0ayL6Bu!>k_5S)Sxg-#>-=DlZpCF)M2R$8_n}f4y zaal48IvebH+~Pb5{P*yDSNK9=ZNieFA#u0K8N8=zYHFG+alf0x?{wVr^4=q8B8T7n zeme8|aPeZy{Mln~y3G8xkBDpN;pW7=!D=w#mBa#Q_784B8=yg$md(D}PS7Qb@wf=u zlJ*Xkaj3mT7iBs`(zzT1-Ns+jY6!*tM;!qGIPe3YAH_*B)rY@ZeiN zJ9oVx%cW~Hro2<|n^uWESG8`Y-gElI8#2XTd{4|y^5$v`B(E2&zg`R$%f^U^h+Loi z(|mrYF@LI#<}{fX-z!Z&?z%Da2)86A6L9uB{f>fW?)ZFgK?fb{~VW)4f26%eNy< zI5IUV1f3V8o>@`*_UCHEm(3vZ=X9hWTAG?LF)_cevBs}A&zl8;aa(Wn6*NSFuoab~ zD+MMp95rD=>dTMD&km#jxzIa{tNl5#$@Gm}0T)to@>%b$Z{K|H&bLCVk>ebThelxFgX}Bufr}tb&B=HIR{Qqz?(X}! zdNZFzXt&Sd1_g59$DMrN6@SpirfzO0tE;Opc4NTeWToprZ@)fTnv{5OF4&o`HxJ>{ zQPGCTYxCwyZT`ZSu^QJ@g{wQTN>rceJOqKH`*t04GF}LJ@b%Zi;F_VRojXSnSyZ@C zmnDe2zm0ap7-!F;HPu7b%|AI>?g#dA3aRPaS(t+S}>-!?N`?*WyU(5!Yt0#2LrDU&nO*te&(yItFOIl{q$*L z^?QWb6wb+TiF`spK)@d+Edv9^c0P=9ir#lZc}n!40Th3Z+VFh)>yZF8!XV-hd$$^T z(7t)g8FDu!`MCD#`SQv=?;MF+nKJOO!DXo>m5zdf0*u^O61S^hAbCvi&wm`-qWc5y z;^I<~@AC&s@b4GTou77v!PmzfCj|jN=M2oBmQUW_XolSD`ug60uQQOS5?LB+;yCE2hP`-f(y4(`0}{(sYx*@7K5`hrKB>~ zwY{pN;{kNrmz$868*mP3yGHRPf4%2TtE;;&r&Tq@&JcxPez4XD?N?QR!BW@V-8~G> zew=rboPr|%iI5j-mB7HH0kkNXH|oH9ff4O!XU8;oL1gAvTwVQb*~b`k6>!cSAaF3? z#of-#54$@i1HE_PWDd+<0zj1C>;ZGh-MF2E=aS2}GqI3GXy#%|E%OgtvbK#R(Yl`J zdsWUZuhBFH@MHd)sfvyot)LgFludsSTrp(H5ODo_`?c_i4F@$v($s589?YJazMxZr zkbV;K@_^Y-j~92e^eyDT_+R_W^TRjS<$yVTgR+^uZ#E`!+MOre53b#J+ZU6PlBr7n zrOS@>c-GjSjTmsr)CHF>LFBi8y1xQ*6S{B+h^xM!sllZ_0j=@-_wWDUu%W^I+IXBV zwD9?|B4KcSNAktrbRLX_L(${b2zxUSH%{2U{XI8-xiX)anE3YZ{lh3z;i0BE)d^SS zw<^kiU~b@O+f2)e0KJ}Ptg+zfrRya4WUr#|{cds0O7}2bL*hQ&znABmNwJG@U%!tb z=jnw(JEH%pBpMy(s?8tQiJl)X)-I5MP^b|d9UVW)&SYW8URh>=|8_PQ3L>PoAb;bl zY#Mz|`OX3u@a=Vz4Y+;Lo^JGolY)GFep?v-$h}OH6+W4j^|e_fY0}5jPF_4?1U#6f zxF!EzkMaNEYyW=??*FjD|HJ8o1Rk%BG}$5On<)KUFlbfvK83&M7nddPWnlrUobnQ8 zPiI76`&pvS*a=Lkm=hptU-EH}{V^ znh)ArTpl>*LFVhmy&~V{LB4a;ifHYeQ-09yiHZ21Hz^*8I2--$@b(>ROp4>-8fvej zI^#2)N7L;@lQ;)&oJBxy3>M{tl zdku*WMmq%^(1YJ!t6y?dKD@sXmE06+`RtHs{(kgNbIrE=!6R*^(tD;{!x>2e2_Hu zgh>yR(?RQ=7hO-!8IS{f;C?>85(Ej10A=C0!_$gy$ZZs-+4Afx=r)qi8wSU1JAs~m zwLKsEbIpcJ$LS)4k}g_o{V*PI^Wn`fzM=|PqT^)==8xs))$b{o_&}UJ{Tl2lQ2o2&CMZ=qy01eSC1L8)`hKUn`G8QoergD2K_Hgdf#zzu+ zQE5FtXh6MW*aq9rElg{!X&w*BhH-h~Q{4V@div;_Z@z*WmmIr9mZyk*Y9c;?=6Plzt zk#owCBEElG!zkEMpYbBM^QnzgN~K24DaUn*EnyyOG?~w!j~h^roh-@WB#xFw%@~8` zlsn`w>Y~!v7`>s!In}t9n{7F&%B_mM@8`lywObd!lXoX1h+sBSY!ggG&UVi!Z+)Pw)P8-LvjWx23wHyhHM8YCNIQp z2(6ITIgpMuZld;c5nIzDz@Yk*@-uv+sZ!KZb$!HS(cI~xyJ&w~qxA1puB9d%OD#udRUlh$%En|a88Cv$f0mTt{@O{6w+a4C zo-ALL;~&~rO0y+Z8*aqN{$Kgj(-p0W%e9y(#5VMzPFi=J;@&{o~_; zjn)dC*Cp0etj5AgvO|d0VCG3`%}S3-c(f&sX6ru(1t@CLken`a%^saZQr&^v7z1kv7qOoH zocFQ?m(mO$_sG)H5?!j&1cUBfcqN%Gk-qv2y>fMZJrmDs)V{+c1sz3Ja%}ue6d9#7 z`lJm7MMdtRO^!{Um5cb2-KXo@Bvht0Q5?_KUje~uz zVy#|_pRUH3o12?HYI7yc^QF`pB99ay4dQr!G{|4tf4b*YX!-SPl)F;n{Nk(CW63Tj zNghrgr6=K*A*`+r$W%%zJqTb-S1!}x6<_Z3zW|xdw&9DFm)E7=Jm*iLjgK*u0h?*6 z5{cZ;{nnb#7xQ&D&yWuaI#t{=O^9K`#68#h$2BiE1SchSVV#|eWzZyL{LOP~9W}9H z+5-3z1pJzUEI-|@U^AxCc*WAn@*PU{H=>;4gLZ`bmkYx)4`K>6-TV#%x+tBho%|*+ zXg?R2Bv~?k*ziQ&8p3Mp>xtyuzlS5OsE*ox1((5)4LL$OzQjJK?l*kes0}j&{2=}3 zBIbCzH#meBk-9J_42t$0%9E|eC`*bf&kaEbkx8dY4&5_#<@ekiZg9+okP*zui^Gi^ zA`d=vdA%z;UDG0-4xuY#-4N<y${Mc702cSb1Gqu2TgDo1X8mJ4C)pxsjG!+uYn-mnXe^4=kY67)he1_*@v- zmBFm-n3J0uSC^_%25uFEe=pi$TouxQ$jfW1&CK$#XQ<#(a$|xRD??TA5kyo1uu9VC zn`dWdpBw9_JE*QMpV`Z{;@E_X`<7uz>``y!W5TgBr$y1G%|&cO9Id5hZEbCAY%G$0 zyuV+~y(#|cXJ8=+>222-2J_^d2X&$&$#YI`9+!HS1M=iJMobD#BdB@n6Fn&xqm4XaF!nQH!XQXhvm;d?@>Zl^o$8=j>ED~J$Lsbk z)?3O|(FASaBi+X8ol4)@>Eqo++yW6g5~m#-VFE+C^1S!(&`{&z;q9#-7_g8*u0+`w zY;{gOwk6^;vT!`mKx4?X+Oc!^DCoez#u+%s&(ELw#YvdI*TMU3sYF_3Iz1XW8Sm%n zYGP|1PtfFw3JR!geunK{4il+2Z03&y-;HVP&~XB4?3-cIu;YDbg~+O4`e|UAl-MJ$ zJVOm9O(ZR<%m>H)Y=XIxt-F_4cu9WiG@%8hk{vax!4ROzu0CbZWX*hP&r z{+*rusK!VmjyN=6zd+dLK%2^i`ANijuH{HbRkv7+i@b;d%^6?k2?FFgGyHt*u@>iK zP!S!YT3&k~;vN(!LsQ6^#Hly-sD|c4sjg=?8o{{6X}o8PN6PpOSSmwCJ)3>4(E_YnF2-Z_<&T6-#a8JjjLMkGJf1B z!>0}o2*{Jae_eel*f%@hm(Ygj*XrsSGnX)@u_25#mWxP>sdvA{Q)*8tcc9OxjP7&H zOISh6RTAADUHj4n?J6wAPoUN)HrHLg7Vi=dGL%N@tGNU}droPQml_@=Ow6HTbmp1@t zM@tbme$pxnd}G&27l~*sqxcK*?m)V}lD-Lgx{eqoB2dY&I=CUt9MmmjdweR@xwtY; zx8uiwrgnrY)7yu7sMUl+b0T=$hG)+!2XUoj%HGqHH_mGN z;_iRy(b&&#G>up&i;*lFh(~rjHv5GKI&T<*Wo09noF7}PJP0wSYKa%vb#$iBpr?lV(!2()X2w<5eaA{Mvak4qpzw~@xZcf zAWroJdD3I0V+fojpIhqXdWh}^cpiAO%ceJ{+d@u8Eq`8%M!eT9`1nm}85b>9taWFj z$l+~T>Hamyse|AEcq{>Vf08%Nnx&a=0AgDSl$ew_r()Z0m7M~R!iMV)Ee->_LaXv$ zFu7{Pr1#~jV~b_Dwl4FJ@a2?9B#rZ?&ZG_8Gty&`XA4(xL@rps#()4mVqXQ(m-ni1BrP1r(~K7pI4vl|N*OTAp7IF<6K z6h`BS(v?5e@)exk)$nMkf7n78cwhA)Xo-L9R|E3X9Bb2yfqiGv#V>;pp~w8)SNPY3G?sydCkU*gm*q44^`iA z$0HoMoQdTmyQi(Nz|eqr7Y1VGw9@Qw#g$K~RAw7r&*Y&W2&ievrzo?b&SAGmrGOAN zG3hSU;+F1S4)tMjp&a@qwQFP0$;m-p7ogVUjfbjW=C1m2@x4u@Fa;aZ1#?ZK5cj9e1a}KjuEm*O^{Lv7i>_Dw<=61m zv`})N*tdzPDnQLzkeQyD(akTw`Jmv`eYW)H57>E_rn}Deveij-$1RO{eF)DAgznKs zZ{WzZ)J^o6%$-ZkG;O*3_BcdT8W0t7na=So-kGIW{>SU{$GA87Bm7RY$qZ>9hwRjV zUyUImd}y)sYmA0+x^DY41@DguqiVs_5@Yc4=``@??0GRz*$L zaDFFdbO{257-jOHmEvFiRbdM312rEATLc_+kYyXk*Xj&FppJ>;@yUWD%Q7Al1E_Yz5jS{;GmB?I zbaG3jg|5W`KLHGBkq+l~l%Ke~#+a%EK?0*QSVfojb45F)*h|O}Vop(7&Rb4?*kb}` z_iow06k-%*O={yTl`$S!Z(qx7UHMP%kA8Ub_zoZ+5|B3V@<_2_U68jEK19n^c9jv=UJ`ci6bsCGV}Xu7*nQC zqyk@aK9nU%HYa-}LP)>9fgLCULM@YVB8qn7k5(7V2*)F>d0si95!;H9$Dm#PDkdQW zl;{Tqn3|cEXdRVg3@kTsD37DVIQ*=c1THs>K#b2)Z>#(nSNHQv0qLJpt&(^BnA66~ zGTsjl*GIbg8PLV@az=f9uqU&cuW)S@*}s;9)1PIgp@|xjvXQBP&%4}^hUVyK!-Ak5 z;exTy{F?TWW0D8rgCtv)b-#F5lJ7~PIKi?<7{EHqFhN%GE`Pq~Hn z!l&xc)BnbwI{tIqXm+A*diOte`E6U@YDNzs;<*Vn&);kXCQ z`ib0MHb#>tmAlMmko-=;#<=R43RKdDlyKp{MA@+96)$?k_Hoo?>arECF${Z8d{0eD zh_pfmKI22U-fH$~+@sKtSp$GB4d z7#0FhzfJP#^O;Br70dtD0`x~9HH*wR?+3$KqYKYbu+mkSb0YM{TgWln+AkfFa8565 z@eNW^dT%O0?Y>xJL~#ROWVxB@eH|F)c?hpgd;-i^NueZ**Gd1fi(88}4Q1yxKB#@s zp7(6!s@8S>-mq!kMd`#cx;DN30Y@c8VkQY1nJzUG^u>Gh*?{@r%aMS!s(`nTPZ|;5 z@aVsVh1-HHjGS_?NptW-NkUTW5CFFFJRLlPf=u-1aui}|q&(U!OUe-%P)A(t8;dQb zsoaReq>7;5;p?)Tu;uYeD`i7SW4GAX`VmE3 zCyiax(oB!{%9!-0;z(LlI=h+ZcMC4k4?}*)D#G@TH%Y#P9@Z^$M&IqBYjXxY%fX|gRc02bWnU79K-UJwmy1%O7Qpu6zTA}nu~et-_`ss7_t=9 zAK_Pp1 zjSu!x`k@n^;%R}YAEd-4=Y>Qaa(sXpzKm}ZrQ}+WC}PsQGC{<$&$b^`Zo#&mi=$(X zpqrUUJ|!6$-NS1bn~~1;qI=mddZcm*Y~^3o*NL=;?V-Xj0^y``ryH%0>@!Ha^Rv*B zSjK>{1x78%h;(MTJp)xq@0vf>BGg7PPX&`$E#D(mP|+dBmMx#d2>@W)8T#r|d<2_O zolxRqOK#g`cWf4aK1&}YVF{TTfJ^HQ8_`^XhaOJYCH)^Kl2uYfeV#veKeQmPQM zzg4al8wP3vOnOFUO98B;y?=PK#K*Y+9u{IqSM1QuUwi~!xa!*2V-0**z@lDHz$z#1 zQptRQ@(iDgV-aE47UPW-F{wo+9{noPAdl>a+LV2SVcO(F;8;pQl8FR0O6f)@jF6#- z`PUevG0c)i5PhLch%$cTwk_sLHjsvw(K^P>TqQrKV5%X!5o>o5s|6_zSp|KqZeAs3 zC53Efk5zy>&X%mm#+WY0vAuG0L0JUU1ffz15FO;K#pg0rIL&>`-Pn};Jr?{j==_|J z^AQfuHEYn{k~YThimuIh{#ZO|12N*eer-a;2;b}N6i09~F+ERpJrOfX25eH|%cs(t zs9=f)-hRQ$qF!nui5@9+G@g5p*w~rNvCj6vCv`#<^9v?(&}=2k`#8=#kSm(yk6q(0=yK@a2VbFJ5rsw3OH#Tl;P({HY& zMg~SmN&VZI)x%s6hQpX;nbF%YcO9cRV~ry%Lv2|oRe;z5noKxV6m0-tPKtP2-HI6} z`%%c94Z*|V!xmgoSUFH6$J=nc92IjS{E-C1GSVeo;*`htk9?~r0yL`HEq4Ul8YLJ8J!v3gy^FfhyH%( zYqnQbTwXsF`zzju{wVbtMh2>(+FL%ULe1-?d4H(|1arf*^it5vS1Jh6Mp;_?^Twhk zi4?DWHOK_Bqq4@JN0TVN^V*nI8{;qxEk0!hpTIy-Z4ug3B`_PQs*bH4w|`>kR?$ee zd$&QPMgh>n`dfx*jjNpiesf$jAlDKu)HAjB3w@E+qH?x#w4P*TS-cedyG2N(3=U?5 z8FJU5iy1X6LQTGc^ZD-mX$qNt zioDofOu}U0PEYfMECWXLxg_AGPfK%{vs+n4lsNQl8}3^|9gO)^Jx;GKbh_Zy6!x>m z2<<%7qdJOH-%OV8LT=USR=tD_LH2@=XU`%t16)r#q+rr6@(){Sh;}_rp>qdTNq);D z$-kOC`soEvO83V9iWj!gj@XE9hhO(f=ANzmEt>jRLGjE_RB7_o4aS$Ud^_CZ(;+^7o_hozAg=7^o zD_!YmiUX$g+M~9sh+@+TH(i(yW<&3KeA`r}es&_Mj9DcN^?f(>E1kbR>XHVXh6HhB zhRufp5aM#X+rO%ixsZ8h%52*!kE5>how|` zpMm^lh^wEg|i=(Dusv-#la7#HDd$-i8WNR>bRv|Dgc_z{3xqqR7F(oLy^3(kBN| zzzj77)Wc!{R7(boo;%#*lh6Llizs!q`mg`>G0A^Ev*gT{kQOj zI0)kFi~>cP?_>vBNMEylq%Qu-dA#y?2IWx~<9}x_xCRFJ#CE0FGCnPbbC zzNdqZ+A+4Dj*+_$2e9GGkuYY3_thCl^t_?xk~0w1NJkq_9X*$159Qfvn%^ zWzfjx5m_chG|K$7f@qTvN2@taG>}ANw^1)J0(vH7eV5GyyE3FF!N^_&KNys*WS8)o z{3x&oLO0^V@!ehi$W#A-ZjL-Xq~q(T*DkXuOBYz~kU@ck`OIUM?cYpf8A`N}qz!|@ z@MF_oUx{{j0VxJ!YI3Bpfrt!E5v?f^)Ztwj|voEZ}ohM41uR#oC8D?%KcPsBluy14VtCct6K| z#vt(lTQ-YWKanX!_gu);_=trs_cvD27GE)O5ldwFUiGkFpo%lZYSf0}QKHQ6=|U5e zc`(hPC7Sl7)pjH5C;=?H&9u6nlkFV?xyE_%Tl&40DhcHMujUMJWn#+-ISm`TY0&fo z8VzPc3r5U8bEHWBek?kmZ^%W^%72|b#t|-q48IocAcr(H^--vCE=~@u-EpQgDkX|M z+7_{>S0HTX0+WnRa7%0nMKn^$gb1MTgF5SD%TKzBd7wrI<~)eBou&529{uV*Px@yW zjS<}^l!F78qlQm3+@LeU(;2=@tUx{l{CXNT4{mzz-lH0OIA#bwgqx5v#badJFH$HhHYmmvHF@P@`4+lPDBTh1-c#l{fqd1ZY z7OZkMS@qp6`XS`s(wt+}Je!HTM<KXIUS$UTwC`MQTI%N)9a2`-!jS#cmzNLA|0KY+mRxY5j@VIHA&jYH_LJjYdewKg zyvD8Yl)`%NJn05C4B-L6V}dVL4T=!?#WqhSZSvo>Z-==s-3>vZ&;H|@{;SSpz<^2D zK8eEHWzm~~TPX--NzOpcuHp5fH0u%zT2|LVV8l;JLX>XjpopC#;!7lCGufN(x zs)`R=GK4w^S|jU+%)lwYC~pL*Rrau>Cz;nb)_p_zr_X8|jlrON*Vd*6@*xxtIN6Yc zdxz7};lJ1YPS$t1lYv#EQ;2Y>G7}Ki;*b%Nt_}Ie*HR2=vQ9@>F&eQ{OJ&yJB6qWE zl_(1}@%>2D8nm4~+k3)iZAW9?WwqntHUFG1Ngzsu60v6Jq0ZdNonxpneN+nkQj?w^ z`Ne?9?PM<^g#(b284Zaq2hMn0#<#cI3Rkwe4)LTO*kXxuTY53WvZu%Ju}y_isCsr^ z^&J1uc*Dy_Oz5oS;MgO6zUlWzbOi@9IB0Lb+55wVY^rx$07%3F3U&NET3?tR%z}ma z{eB}zKF#i9k(hECH@l~YEZ-d>*_rNw8;@{qybGrTVyB3$LzM31JH5Oi5S;aNb{+ad zp2PtUi$Cer(iz=S9V7YDvS>7!LmY8PN%oGhJ52eEHo3ZOh(NH4Wkj(%;2r7GghxGl zOq&1kP?%GOyzmTx5#w@D+tnYGKsx42^kDs0kZ{tY&=gixiOK z&bIKXhY;p&=LQvN{rioRRHrK7?CKchf!6qmb~uwecxI-Dgo4cZt2*h1pr60cBJ4FQ z0Pw_W??i|_5%71t+qFQ%0V+FZXL+cA1*I;6#3N_>2jLRRqXSai42nRX=N}C@VyS`YTt*jtOzCY8nZokhdF2A^Yfg_iFlTP00ao2vK`CgJlWU%EW?Ra$mt-?Gh zTP>xLX;wh9o%xh|d(4MLD4`5-oBMLIfy`;VSp5yJ2tA==`>X_B zOSC4Z37(Q_hEl$HR<^fNG@`wyPh~+0^V5rmD6r3uoAJ&3uSf|HwN?*dcLudWYvyJv z?k6kxFAa7(m)t^QY{EX5Zeg4_GN8^+Gl0$d?eDf7*Twd$&pG&@=riQ9|D|T{{%Zb( z?(X+u$HqD0XT7t4D@AuP2g%vFI4pMNF$@DXQ7%4A08?K5jegA*#&CA!h~5Ozf&kJX zywqQL_=(-5Wup=QFl0RUQZ)o2ZXU}i#8LK-@W2;h&gP~xGse`}O6ren2!I)DrXt(d zPaGuQ)Y731I!c|FhU-zoh5&BI1u#V10w*jYUlX*-X|;M&+-_=x353EkI|8I?A0J&X zqb0w2^jCeXCB3(Mbx6j4`9{HTu92EF>4_%)XC?t)|6yR|U;FAsjX)C-Z-#mqo1_Gf z9wF56?k$jlVKmo57-hYLMJbcf8k^J@t1x2$@PYZo&;%UiN ziF~SGATvC`CWBifxwLZZZY6_b{sXf|WcsE8A<&GN3ES2j!r&si?N={~77z#V>e zwpT|ic|dLlciv?aQ8BG)5dqVUY)1yGK2HR!@4x(yR9#p6q7{Si#&XW0xyI_@h02PJ z8pgw(pFKs&-KDr7>gg%)6hruMXzZGzs;`zs@dv>%Lny?sk_|Rh{V=g4AIEQE@lLRCTz1k`Q?k%H4E0 zLN-Gw%`765X7~5EWMw2n4_`bP;4nKdD)jsuhr}V(`2%w0go!I3e)+&R;2-F{gPN^_cmsJ zsoQ=w!+y8``EZcPpTAvuCJNZja0~D_Yh`nBo3b%0BZL_6lRGJ2MR`T4Vcpl66;>_U z$>TV^_ezf_J3)N(F|&qGEjry_v)o`2>K;XnN3~gSkwq?Or7=Kn{=&ztsP}P=K9&zw zlMXlIBfSS3dy>hQFI7ED_J|oo96kEkc4S}d>H#J_BZ6zsY_hearCQ6%eZ+{OC2R(zvQgt z0etDMvy>e^$2Hr3y@_|H-BrvUSm`!bt^ZnGTx$gC-DN)E)I#TZm_2c%ZI(q9kCT*J zA`}bev_c&%w)Em`pVx*IUNzC<9_`kwEes8Z?Q&1`lTR_6;x_|bf#$XArRdwrVLfQw z3k@50-~u8vto;l5X~fxmdXDl_bok*Q_~z;Hp40q-BQ0u}h~dN-?$|PDt&I{?_9}L` zs;a6=!La7p>6ieVDztpPzaPjK1Y*clKL=rfI|~S9CNdwFWK)YOA{GP^rb^y9he)LV z9;^NS&Ra%q#M=t&#(l80A=3AejmDP;@+L<(;6wjmdA)4*9=ukc;DP^Aa=h^G{d3%0 z_{{GvGUoF1t$G*3Y4*rxU(XA{`2x9|rx*BQsS=+iO$U2_#}~@iIH)nQz%Qc_Np3Iu z(ky%)*@FYpp6)d-3*t(*HEy7HS_!lIVz@ItV+1+0cObeci@J=Dzcv#JyGRTP=gnRE z>FnC!GZ7aliaZu?Yit@tKKUzp1Cs!wV|ZBM>b$<=Iw13N)_emuxoBJP*&n$lOWJ)k z-F=zLbHOUUPj79-)kEG*#q(jJ2er%Y*rzkOPbOb5H`{AAt{zj-kLcx;8`#qvxjz=1(T(zqL0|nte7xf)J6ZmU zk6k}@R)$saTsxpRUOnkJ%amT<)m?F4|MVn_g6;MEukcxL==PjdqT6|K-8X?#jR<$^ zmx!2P-T*%+BGA*v@>GNJ&dD)y9xMPB=p`1Ud@>p0f7HtUkXCiaakq@Jryy|Pk&^)_ zq$@{(4WBKUtd@MCtmTQR`Xteooq>C=59%!U& zMuDX>VyzqO$yEp!iiMxM3z64{M14>+C#S&lw9-uDb!=ksX774?$}s${+HfMh~ZM zBLf9C;-|D($Oi61v6)o*=pQ@V)3V(}c4EIquas>uacvFmstr%I6>0Z<8h zr^b0fPp6B`r?758a?|pE%+@&Lw#NJ0UI95_BTxlgHIe~JX3tUQ6obShz+KYwayf`T}5a`H=w z_ZQnzEwFUWKWWB>ur1*D+ohHgiB1A)hezFjT2MvIMynd*C&M?{#naZ-IR`)}nwq(G z9F6F{;b#{XOLjd{6o4&J7d%n4;rto}D7V6YDw||>8i5N%ZEZ56+>8tu#2bnkLV4(D zZ^tCM%2bU`aMtY)52`HUuMtjTwN%_k|FkITBPq%C+Zad2+}U|0QQL7-t2ECzl_LAZ z@QWL*T8E!(z_TOoVmk}OxOh|g+jQ{L-X}^Js9$mxAj6JP{K2p!FC;GEnvJxic228b zrtymG?RNn%}S!C`R-R;@+u)2viD(KOiZYLc4a7>Z6n zu97CB190GR3(9$O@HP-9xOg|LTdb3LK1Y*(AWBVhrJ&sM z{EM6HJN-azVNi=w6YccY*jT&JIG1MNnOUPVa?oZ zbQ$iO!WqUlfHoNNspC)0mdIlmJDc_r18PMZe3@#$1^41+|OlUxUCO{pJBZueO#6jVR>ip-&Pjk}IayV-8q~8g7BaQ5( z9-C9&-NCaub!m{A8(ESjGCA{D*p~(qoVSm2Y`koyEn) z?OTMQ2!nZRLV(t*;uZ@bnpo5h(=vErVxqyjFfM@4DzkXks=?5PS%MNFl)AU%9pAv& z@MSzZQAUIE5}|?lKb%@kR9gSaP-YBFUCsn|ZD-*i3}J%J*Ugdssk4a6cA&ljBRWQJ zBA1075b?16qc27lQstC&5alRd$n1!|iHcBG%h#3&!;T8on4~;OS$e^O~Eb9tJiT>vYRsI*2nKB@fnNllTIBrwX&7W$NumAI%LAyoAS~Y3!yuGALX=I z8Pk;9j|c2bYTU!wya_>y^ev{O;N&7wDwN#$!vKPQKnHmIB&>*CzqE`yV4T1T9@w)G zB4R^|NX=*uJx&UwqkQkxDhw6+x z+dA`QkR*{pQ^Hwe9u%I>QNDf@uQ9ppjg&=H#JlW_`mzskoO?};WH((O8z!~>9sYvx zpSO3TL;pu&%1N%~(J6gnmQzggCmZGfZ6Sg3%o+p+OnKkP61p6bxsg|%K|XyEr`rWG zYl9G0wjGwZcQj@e_6kf(M@)bDVjjdc_c_z%5WQC3;Ao;(@yZC0tFjGG-hFx(Qu~Y% zZCUr!H^+|xo*_&6;en<1jHC?f#SMSaUhm?YJoLtBv$Cf$oKQU(VTYBX; zjK9doLZR-&7z6zM?|{`5biM*q+}C{^YC$K8Udt&5)Hy%7gTl7}3+h7u+g}I$9fkM# zI7V&|V+!l8Vh=K8Hgz& zoVrhvm?~IB%{6Y~4F=bT* zC|<3Sls{#~x1#dIBEJ*MI(C038MVcVE-4v?ir2`e5}!R&8Z2Mjt)t${OXMq{DyNyy zU}Sx^qiF^IWrMbl7XjdxI0o~d`{==ZdE6_LN=R7P~yJEP;*TwM1J z>+jcB_w9If5!`P`-ZK*iOC^eB*au$rCD*IY_6WeDq=;3FQvSg!a#oe|uF2=lG>Nv1 ziQBDl!)iie8ff46ip?xe+Xrd7ig}4;tAYX-WIVOOm6#SVkjM~2<*&SfqAAQ|Q^JkL zVvr6I1N#xFO$RbYwX)YehkqMt3=RJZy+noOzb!wQDUVL5g7p76>A8(-JnKJRW{XDp z7})NlJ9vl@w)?*rd+V^O-mYDAAt2pID+mZkOLs|1D!|NJ&Y<8T`KezUSNf{r0&I|9CmCwdS1bd7jZ@+&8x)Yexuex9IVk(1N2UQcRDY z-4nZvl_czU5BPhsc--#qZKA7XB|`M`xLun#s%VhdF#+F2K8H18vEU@7KQ_Th|a*<6+n;6s9ZN%jQ5rnVxE@ILn#86v@59Ech`6QtrIQC0oi*g< z6i$uAB*T3Xxy(B}-Jc^a-d^$}eny;nTu!1PNqpU8Ikjvmj2p~tq6_Em(fiXXf4)BQ zjhha6yjWDnQtuh<^QkRE`x9GJi<8Z-Fz0Dhuh8b_6_Y|tf&U#Q5$K*9mf!W)M+}hN zi9$tuF^yRcZ894c_cpz9qC2_E;O#=3O^ zBHLr0BPEZH@U`r!$KFdIz`ppBw}>8gmHGPEYxnOO>5bq>|7W$i57668)j!1}Atdo- ziL=Nh9BdWu8~&YH>MvO74s9(Wo?X*Ux^0@E)q)uLp*-@d3C)*XlGb_PF6xknPMNYi zkFU9(?jvp~ZiygLKKl8&N7VHDfq7O}E2n=>7K>J~I0T+)e?hIJFU7G)X!}|be|Bu- zVrhPufx|c&n0a+Nvt-aVBb0cm>s^jbd@rc;J56OJ*M_p}$JqBK?VN+_mTaTJU*VT+v-9@$@g|K@6Kor3TN;?Lym$oD&M z`yxj!pzw=Oqnum)lli!_U(Woc7dZ)Cw60UOH4HNNgIzvr?Vo@=`N;TQEp zd^0@ngh{QT*S%IKG9KesrpN4MIGVr3GBIm%gWEG;c1C;j$JQr;sGLkH(x0Q6Qd z8ex`h#^R*i(yBWkJz#T+%{EwctgYqeS``%ZTmEjidzT1mHE*~%eYU)JhxwFx{K~dgU)RiCzdPwYl8eSwpB#9b}a8vD%CFP z?F%BnmKg9%;E3+ZKJqj(`_36wTPtY34qfQO<8O5jO?qGLL=RB25ZNflUYQBU%%76xfqVlBO@x| z11^Namu5~@1k)ERglU9(+mA=Q66c{;9dik2vYIJAS2f+6++Ah~h4|6<@-!$YGl-AO zU(;!=Kb_X5SFvl$WU~iVYTHyX`(b@Oda5}4G2}6UwWN%O8Ud;qR+PVgVT=yf^z4rN zl|YwyGOIr}`r7^p^7hP(pwqTaSA|Xx|6!%)6hK+82y}YytFn1n( zCTOM_U%Zo@^nDS*=qTb>M1uI1-G!iWKx-G^=i@~}*jL=DecU{cveSfuaOk^&CuSXV{GyXP!1Q38DfhEKbn5GjX`qLEFK?u7IO>1@VYwMRu{|p$FEyH z=_cr>MP@hfFH+c9UEuRy{h4SHmAeeF_@id`>eoXX>V}%N>Q^4-W<-}wErbZio1htr z%NdEqDj^p0n_k_NRO2{B&k91M>ugoB#uFcH8&8kE+BoCa;L;Ly>YgY3ThmgNivGKe zLz@E~3GunBfbgdPd*^PkL%Pov31W?v1flj*N%IvBnl!;ks8OZSjcl;m*xzV-{agQ|b7RVZoG$k66Tvy=LL2BqZV&+Hs$t%WtF9VsHOet$?)0p}$4 z!oGJ~F_9U`SA4b48|mdq9c6JqnhUYtnh-){ zZ}t?U5x&r>($@ZlMsA*ZunKvnbiv*5{mOCE_Jn#DXV1^(dxwYN2Hvw9cH^Wn>MaWn zoMhQatOw*ev9Ap9ZoW;cMkV%%VWb#ws}OL=c4X=b$&B|>L8KQIv7Ja;szmEwlA)vU zC@T)J9GqQL#>TM5s4*=&Gnj9ipuT!4*>QSFsx)(ss;Rm3^yfzHrH_x#3J2cYc$;T!g^dWGQLjiT>jKaNvtShV8os&iz%?OJCt6boRu+wh}kgk9)Uu1v?~2c zPiMK(nf2qF>kCYJh3R*g*W$;pl6slZZ_KV{wQU@_;chbgxj53x8l1?A6v03!E3-w@ zw`I0e+LQ!cc}PmWc{wvM@N?Jg^l&3i$@{N%5oY)3gYPvyt6gR3?Mc%rH4YPZbMJ>_ zW-=-K^4C@EVkE>3+~41i@$$MK5`F#`Q)3u!9b;@|Sbz+hL5EdaQdDE$K@n+QnwZ68 z`K_mhf*ox+dy!)JUd=_9^$fV@7OI zPHG&9H?bm?Eao1FE-YutL^31`?)Riw))FJ*(@$a8^@bIXl0+=K&#kl^W}!RUPLOsO z5^xF!i)e@3kof-bqZ!p5kRb10-#sVWu8)PABIB6og%b^9UA-WW!qU>|^IrR)hpi!?!J$|N0i52LWYH5CUIe_h%=% zU%KIe^c&)vzCkPoygCmif@$(eMIZ2x-0;n3!|^^ZmqCGt3r&-%BZ6|x(zW5KZIXXL z{-tjx2ypq)h&r8b=~0HuuD>bXC}+j;GfrNo%f9f5>mFUrmR1+R7DN&V3#PB(1|wrp zF=@-k7$XlucqQdxjL}|Y#;AvClHz&|H2x!_qg6?=tSkyJ5)9T=o zmRqW-w-O-!t;N<)p`Fn<^OaeBW**Y-t4Fx32b7-e)LpA=1rloc6PR|1y& zh9v&=N~^*0|NFxA)u=`X;$N=TiC^4y71??X6=M5^?OVPFH-%hnY|m-}B#CQXFGEm^ zGQXj)ndN;gLcepz|5*)@s6(yne8LnjJNs+PLg%b&zvt!2J8cE@z3xM~v7$4_y=5mP zUz7NX{*%#w!+A0vnDK+0sr91}EQCIL-mO(jx=s##4tgCvf12(kE9<`$K8`{u0O~_7 z8X;@ylC;Ix8JS>$FqXjFJ_x1hNEOetIenA1>YkG{o-1}ZvftEjS-{>{TTpJqzOR8i3cjN0bfe1wg@som^2;Q^5G9!&i+~uq=Z&y%|Xttp7^l6e(E!|+%4$|J{C=d^yb+jYhV2@V2O!LV)eT*1cI>dM{Sy`$1_!d6QXAF`>4 zO1FkPsunG%Hsjs|n1Lm=r~C$ARau1{mphMXm)yMpouq4-@%RoJLgwAOo~Xt=x{%D_ z^|1M*EPKnu*ySyRZniHk{dH2(B+Y7Y}7-@uUc&hTh{InyJY%b2WRxu)*%=klz?q==@o45JTC z4u6^K3r7U+-w$ak_W8nj2*y-8kiuq5r&V5mUWe>w>NM!5bk~9mxh9Wmy@2WmLj1oj z{OfAx`(=TolAMiGsMM{O0ooh=r`+6t)tK5)W6u17-+=uTt>FKn`Z>d-( zwfy>Jf#`&kKEwMjU%nV~3K)Mw6sFT@T( zNTc@nD}`A5nw8Tx$f2dt*Lu{3gu`QfYNMF-{(a1MEl#VKqi!4+el%;5^g~m>S?=Gz z;roI3sfvo0Qw1pt1_*zxgy!fm#+Hsw>aSlUh4j~&_lAy^z_vuJTWX9sIeaF9fJ~~h zEQV4>{a4RP_fn*|yf{EbH36dCcMy=>@y*$~xI31K3<^VF>SE5-b#Gv|8rw*CznO6! zR&HoIo(OSGRAP|P3X5c8GmK>9UPufJ!}j?0G8v%W+(dMAbh^$h0Thd#ketg*J~~xH zPcP|xwb<~?O!D#z0>sb#lgF^DGqU+bMZJ(^E*di<%t)FMpo%afB#=>$&K?=j_-DyG zJ{ux@J7Q{V##4ueqScsS9ymHCCLeW|(jNwT78e%@=w#{lLv6+i31nW*e0{0#Wj!%5 zvAn#T-u?;@vnT}}&7y-hC(5#}HEOJefx(tb-7wrqA^X!O30m z{SOr!wG?A&a}_!_Ffk#Ihkkd#VPbXP+yc5LjR7Z#hvGJmr5abTWg-dYyJ&q@bcA8a z_|h(YLeTtox#lbsGh-;JshQLpyPRkP zQ4m8H=AwJ*s{20+9;ijy0dQ}>ezhmv#BTkbww^RmT~&22A&3S&Lyf6RdhPbTG9l&z>RUA z-g12fg~7w7y3?pey{AvF<~&)LS&rCs512%W&{0T58slMJ`}bD${78w9cmI8SDH1l1 zvf|=Y^9et{b3OYy0)&R9-oXJ@QIVKJG)K)r%gZGdK}!{_l1t~dkVq;Q;ns411FY*_ zQTjFJ?bB`Wggw=HyN2jAO8&`{?q2>`b!}}4@i}f!3vci3FLuI4mmM}Fp_3LBv+mnLySLP;A&CMKLmHJiwVw?Bw|*mCN>$ny3<79HaO7xsM9DU#Zn7V+ozXM6j( zbL9S)sf9f{72f@vO4Q7jy@^%b^G55Ag2O{I+&l3q&t3kO+Y0knuyYRt>TO^xllEL* z;Ot)>aEcn%ALoQqHf>zEZdf02=>JjD)a>2jwxDFZ+WTh8%Uk7-pVsfQn&Km3Ny{R3 zykk?+@Gz28^kVV!a2{VOmc!^|uKboizNRJ|St49H50&rJLmsQtm-!n)cg?21q^Pob z@g?Z#)i@r)O-ppEYVCv1|FlNLUG3_glPeEf%(z6-p!-MRP&T?RoW$WJ24&_qHLdjR zG)@!x?5TD0rCt^5?$2x5oJMty6-`sWKDh63{+q=fP&jMxZjH>$aBW1KX=vsMx?ddT zG;mC_C+WnI%jbG|^m9y6pDga+tE|kxX1{?8w;m&VG>$?fX}0uidw6o*&kDc@Aa0rhYZR>0;3@m68_L`4nO8$H~Ef*T2@DNGp?PcrA&MhdIe8wl{QI4DX zbLFXiV(i$Z(8La(9XYSH0D`H&nk!+gtB_UoNk#Rl^xmH1kM3zBtW&_Yd)_kv z%L51skBV$@zSz(6S`J90uAW78<`2?|T`W6UQxu*%9DXuY`TY>}CDmu9f?f6KW%_Bu zRc!hvVq{HgY~}P$UT2BVQdd{LK5ad^$E&k2fqd#IkFh;fB^R9y)(0Ume(+UVKql5R zre)fmO|Q`_r`N*s*W}6Tx~i>dSDm8R%uJ-pv?I3nU^G@%%yV+?y^;8nTDRk~*cE85 z5MlInHtFO|Z!ajcqw&CB)`@(pb4|p=2i;v=+fI}02B()r)pG;Yn7`-6eSUt+oL6e- zdr^(U#z851>JJ;x#Tc3Dw1fOS)xEaW#0VZ<-IioFy^;LJZ0gPwX+xVW-=_j=Z8Q>_ z$9fp?O7-t`J^fxr&&S~G>|f2#8`apgD<9AuvW9hPaqtU#-P}_&e7)7C=k0TGeiC}> zU{hOswM1QVx!s{&E%u>5U9t_{9uXD$E0y!*`54=@l0`*mP<|j1Zd=W1TTG zE(7<`5st=V=7jQl{$^&`M@4>(r`DzdgeOOJR~}o3aX0bUb!rbT#5B}&6VHAn`3Uc* z;?47~N=7oqSTH1f85cx;%}!m(f%~R#voJT8s!%;FCNR!4b^N=1=dl0+7FfdPKvai2rB( z#p3X$NCUqCrtn_c%ICG+y%m;@Na{EXvm9I{h`g(g8?UHNtO;bZhaNbK-{?m&Rd*3;{fRw{_T z89$qvQMo?-(5}b|Os@Vza}e04e0 zRHzl&W?gf(P2P~;v-^X^-fq58kuZlyRb=7EXYb*_a#Y`2)LzoQ`fdv~?FkQ+r8TP< z)7-{Z@f=vY_sb^M;Xy{YWAoMYFV zHgF>Uxm;mq#RQMSV{oNnN??+|e=*;oKXGDP>o&roBv@3aIbsW0G6L^xG5?q&Qn^%| z4@nQz&lx8oj~0FJ$s}n**^N_+`&2NRfP%JMW_WzNp&naupz+w9FxvQs-on-9=IED= ztI1DF?kq}yD&Wj5#-R&7Bc+4jMf|>;&|Jc^H!E><+d4!Wy=^hwN^6lc^5uWqqMlFT zQdq9_^2&u879c`U{iufjwy6^BNaHWBrjyFO^Bntbi9&oEJfX)PCKESA3#kR!S>JUq zOt^jt7(|l9ujKG(`%Jp@g8*ec(P+`*Z;YXJXy@yG?TFA5Gm7Hpa*$ae_Zm%~s}mZl zX5)0!hn0QVMD>rSnJwqgK>puCssA25{Nr`~%k}!-Bd+U@{bwBZ-@p9-MrZ%J+QCs@ zyrGTxUoo4b_TjVsRMG4I9z5b08X77k#z$GlIc_vZk@Ucd0Jq`)e*6D?m48Kp|Jy@5 zUlx<5`CJONauyX8$*t7^YPwM%4yB8HmrAn0?w9EoXI+@P4zsBJ*C^6$vcc)E|5seu zWY$0??6P?{Ze;IOSX^vDW&zNyh*H4+04O=VqA~A_Q$Tv0P0T&qdyak=Pvp-xUnFFR zJnX+N1f1-zg73hfpH=W1fV8!Bb$>exb$54@av0ifvi*2D15mQ_O+Hs2K73G!Vc_NA z8Q2F{-o`&ISi{4^2JGX2#C{EGJ~v-8YJG7H7W;Sd`O(IBNp|)!L{r|U>v@2$y1naG z4EAAZ?DAsAC-RP{(7k&(`T3UZVdU@S{ZoC8W?ciZ$)8nEcZ5?`dmg=@(@E=h1^i&h zsMXDG1IqYTMZeg_!y#;8_)|&Ae1qpv=X<(-fhFw3O&DM2H2*rPPJ)1IZ z0H~HlKLJc~?bz2BhvPEpg@uJwe6|ySYW@fvQZowz+hyekx3m7$3(#?gNaiJgyYp~L z?kt|~_c#6Y!=j+1yrL0%=^u3k+lKf9puKT(jk7O|DlBmBW)~N;GctmgEhx(Pj;0|4tHILFjyFMx+UpO} zg#hdk+$$2FR)Ns2`2(P99RL1YvYlg?{QB=?oW5i5?rvd4-|s?Fa&mG&y`B%VxyD6? z^!{L~1y{nkx47sPPy!%J*x0c2S_@F+t~Om=dLkw&o~!O0KD#_W2}pi`O$6!9j=jA- zxjXLu_7v_m>nG=Q7r@xOj+;3OtgLX#-Ex)8i~4n^4N4C;ox++CXZIRMgzSJMrlE z?;ltVWdq_dCBg+_X*eEhy|qS9BO`58rWE$3E+{d{9|G~v?H z{k-I82T2J0`&U_5Sh)MJ?})eop!0zMKmWL!>Z6x0A0S4?#%_E&n)eYm|M9`k&u?>c zGnk0+hQv+<3gYDCB)~p8HrAqgIy{mmMbz^zy_v%J;qTwSr>3TmkZ(Fr$@No?Z%$QV zAs#(?6pssl|8)fg1#{2W0o*(RkJC zoOXfY^JuNl)n!)FcD#g)^u^gxNl_86-#hX1KggnFJeK$f zFVIK3gNqM6=te*mciSoLK`MdYpp3swg^gmV8WR&!D6OQRAagb5lO)Qz76<5tkxx%Z z+4Uhs=LAwa9D{rld!o`Rh6eqO#Lp?463AErY~s#JzrI5=DvMU(F$klk=4<$9wcYgC zN#eA9(*;1`pIn~p@G75yoK*uPP?CLP!X==s!9vMq4aUaCzQZC-^|?H`fB!z*9^uBHz-Hlu=j%nQ3@N;sx8EA(W4FY2O!Q_1=i-~E2fd6E-N3O*6J6-E}dOmc2N6> zP~igTt~Vf%2N)_P@~i`=#Z<*}yl{NH+d|H*0XPj;G7(FKS{r(B1b`T!9#lOV+uv)T zZ+N3Yo-L5Hr*LOLN|)@n2gFeTe9$B|AI?<Jup5#$H2T6GeVE@8Y z5nzg%fPMl%tzGJ7LE)__*FTBQo_>((udT@@L1i~iJP>6aQz!_JKi~}eNEiI^dgoK7Z?@whzzae@BnhBSJr_;JBf}A@* z<%g}MC5=DaO&=H^tgNi~lI*?*5jfW`sddPq-@Qhbe_o{vq#r{DWy2e3UVB|2sX|l{ zkb(oh_Pw`ji$4bcEaV3HXFtG}ef-F4+Kw$WHTv)|`Vgge?NOd=mU>{TWMrOdn&H!@ z)z8{6M>foUx?lG4&nFALvTiwn+rhWk4C~24(NFH5zi;h>!;zAZP~)5i*wifct-ZKU*&7S_OkI zzqO|-md4@Tv%rb801R&gbf4cd5fPR64@kTgV6Ub_bBv`=@O3@7l?K6jS~q zzqhO%0IIb8u>^iL|Ibqm6v=_>wAcT&5c40o!GA9N&sYCl#rfCkL@nuXyr^Bz1(K7i zsK#;t>|b1b)-*rg%1a8E`9PfDN=78YdX4-KvhaJs53-|P#|wc5msVR-Rrl`Qn?NC` z{C8z2^EsIJR5}|1n!w`3?haPadE0ie7zfoL)`_R zKj(qt1GeaaY5TQ2wvPjx6X(lC$We!^59guXKw3HnZ-qfQk;itT6y(4Q(02kga6(Br z?(kY`fckOGIR#(G$aAX-NDQWf27oUdDfXo9K#!qlu?IubWz?=!~_i$ewcZd^8No5Rt%1b>3UWAPD z@;#6nDu8|m1*gyt{bA?VJSSkMK<912PrPP6z`s6a04slOU!g<(+J}a)U$#ONTpd6p zc79p_1UK~lta@JxD=QB+N}3q_;ToRkftrraBQHaKUy&ieJX#?{(5e z&cI^$M}e0EZ%#I@p`lTzTUGG9As=iWGxJpAYvGGyIHB}w?}|6Bd5D1c2+!jUQ%IIv zq&HYMs}Ak24tGVV(NP{LD#B6u%=823eE{ZA)Tb4P|C(E2JKXSE0UiiY;ga6E1rm|$ ztzT(52xLV{=#bkSR~1^PpIXJZ3u#d2*Vd6T@gI7@&Pvsw_rL zP1s=2S{xo(q(-j~L8Y#H?lj1$y8N>^1C>_8Vf#;Vm2^stk=SA(-8Xi;4a_9sSLaSr zoj@f~Z^$8f1+-4Dhq4tW#@;Wle-v-oKNnn0-Uq{n^bHthzzljJ(r90Gp<+u5QAYw0;t! zDppfmtb*?kKGofa0c5N0sr<5LvFrxAhSz`B3={J7kByAxDVMe(fMa22hvaYQT7qHh z@2I;Et!-?GRC053A&P)Ag(RV+H3sq}>V19;6k6h-fuiX%?uVZ{@AiQzGKQEAJ_Qar z7*B*RwfC_J2r>VRL+iDox1ZZ|sq-9GBo8nl%dNw*6z37Ti?B=_GY=eml>&bF5bwDD z6D_U3O;;E8K-WXWs3M}j@{BKOX+>$62LTvPCS(s;39mVXw4DEdZAgQL_J+Yw_Ckr~2X3&X$G6b=e9j=6!G7UmM z_PEf13s6yT5=AN|3z-1X5luO}v{Vf+r47|Ic^7U#K{qC*!fKe|6pCFyazOqUGC&73 z_~<-5Jj5g<+j9-m8;5gn)_D|1cm!CnDR?Xay{KLJHE2V(^))`8X9_#>f)k_OS505+b;6P$XVnYo~_@Y*2+CsM$)J3|J zY}ZsM*KQH`AsmMFU-x;3^HhDn*BOVbBRCgu%*UUVU3+{7s-P54K>vc@zy%f={y;hX z`t>WZ_g7mesXe$ofTs<_I652`@85Dkd>CNzt1qPvZgS0o1)gEeLI7oc$}QYLQZ`-U zVClxo(+%*Chc~VRcvZ0zGYEVE4f(TH5rw$7*OMntI7G9qId~vSfo;sowG#dCI{=3g zICD}!i?L~!Ueg>wS~F+$+9JS!@KY_VKG;z3A@fo_m_PmkPO6n_1~FLXG|%mtR!vsu zYq75Z(*x(*_7cp~jMsiIXtx2_@Bln&Z(wo?XkB>Au(y^KIc56=RILZZY1dxfW)jdT z@dC@zbq0dpea|ed{|JIRVJh6yJYZqlgyp~NwfC-oXM_01E`1FP3jp1+K37ru6jA$+>}@I*xiv{d{9)3rlj zr8BB`gOqG{Kd1{hkyoJB`ntN-FZxA6gH}~l^@|?qT4Eq3o z2j`I1NcH&r3EZZyfcy&tO03;Cz5!%)>%t$+rv_C(`leH1AxMS$0h``Eo_+S`@+6p1 zV17w(*aIW7;6C@~DhY^m5yO*R*BN6W|BBoFDIszr24PQTr?gGcNXHQ}S)X zFZul%6AwZVPV>JX+JATUKCm25?)oLNPs#Eq%rbWS8CyZL$8#Lm_ zk00S=U7L-_wXFs?j{%;lU44C*>xw5pOr;pqrproV>jhgUY*A5DT3Wh)9ZZNp!3)9~ zX$y;A*EtfqsY*Hz&wxR1XN7|DoYh#V^l-j2G3{i*!i?ncFa=mFVO0VMlYa zv$FxEG6sECWD+XAG4lWLvH(v}E2a?PEe&<`U8k|onHl}1c>L(pRBw5f4q?-ZN$pr= zE@*~8m6YcqClii+?ptTT9TXkwW7#7Q1qPxpD$nlGu$FG3$R1Af>4oQnIO zuZp`yR2kQ4l`D8ag#JXO=F1m8C{^qJ zS35@C>*-@fF0++j>;EtSl(T0aTD;9;vT5lUH zlOy^pN0+3(nHuqm+C4gw(AyfUnS)H4Az4iC)(kTYshkW9hASUh%ch(~78J-2;jHk}nCDGSyA9a>{5vR2#=Bi_?tsF+v z9~Rz#EN{UNPVC!q@UZ7ZhuM*VOAl7h)vdJu&u-G=CO$qs(Ub4{3`Er7Z4l7@AfOAl zAPmErBJ!9I;8D)XMd>aN@Ed4+q?x7Y>A!kH=ZY18d0-I~ktii9VudRCkNpd5CBS^Q zW@nukd(K8R=!o!@rw25MZZ2$77^1;^l)r5_Rs~&_xOI1qJj$$B66@<{61ek-q_!Jv z=AdFTG>o_T@_ZzwfBu1u2jCPlg@CKwugD=P(>s{tsU(CkEFs3Og?r%DISD9S!@A`zd`OnNyAxd?NPTXsnji>wP z8aWgD?oN_o*hSnKeleiqs-ekyXI%r!BZ7BG9YZ17gFKqkQ3ZXy@0&uW5+0#~!N=pk z3KJu>QBLngHBsVIV{ykuL<2{}QofS&eawS#&logaT)77Uua(x`(RT-|S-6WR_s$(R z{aOWVk#Zb_u5NvX!^D=##pFg?#uTZn8h4}MP=s>ckxN{};0Ydu4bOo*zP~)Z3A z4$cIn5tp?SkE!YP-xy1COSk-grRB<+kOi3Q7>v`sM}MUmW{=_LkT3IkSxSWGb;MGS z1$Qf_K|oq~&yoJDKYpUB0d&AZ|I92jOMx}k)6@hr=_7d=RD7F>A$9d5wRpsf)XV7& zA5T3ktzVEeO7r?{pTain-w{_(fc855kijd;jJBM$L;MJLJy?#Kj;doYYC)1HtQnQvm*gLCX z9M*2;kn&d>S`SH(-$-a`YP!1%vc=L*Z)K?X3uAf^7>O1XFciAEA7{M7N^{FEEln?C zJBUNObGDCsRIA^2+}Hr=x?{uZBpyAoREC=TaOG^BE8{29@bK{0e?ndvJk=as69*fN z8<@4~qQqu>S`Sn)m^gO1tlid$V>2^K=)sY~6tV(ae*-3Mos;?+S~d=4X$b;bO|pS~ zlFmO~iQ!ZOSSnWC156;Ti>R!sBG1_tRUsOPPQpx3 zW6ft@RJT2Qkj%P7ii7oz9gFLQ*7#w2yR7NJlo{7x5580_3rg0S<<>wX#D6;`T>cpN=p$4(6cJHoraF)?F28v%Ft#QW5a!BO&>l)?%!8v z>@WZH={^-MV*+@pd`U%+ZoymNSBTEHd=#B+r3)Jn-u3F}55_wwgmJ~>+S{oG2er87 zb?0(c#WTID%L}NQgqM)Lt>RIh5`@NZp%epIiPnYEN!z}@J{JKh9#2tLGkSkDEWvyC z+K$(0MxRg7%pcj7Kbzq$sP;igJqE6hsLOmmzhl9XMXrVEMzQ6t$rtx5zI0fHggE%vjbq&=lgz>Bc517q9yrn^hR`8vh66Qfa z9zhrcI6~1o53_GMD{Zu`=SAv#v2FYKROQ}3y_i1d!v^hXGutEW?G_5rY$V)wm+O@E za_9G^O+`)za!4O3zg`L+PeRx6@{6X2F%$N~malWL$LmUdGy{S3b|2bJYn;69HeY6| zKBWuPuw*5{5>=%x;4u~ToC+A8+Z8F5p(W6XweoKvTg!!hWH*7 zf~8!JkGR(7QIlW2?|tJF z&Gl_jvX`=f7iabvSPc(`UOV0SVDA3!ZgdXqKk1XUsSlQNYn-Snb&AU3 zIZYOi{rtQ;z8X5a(0t0Xo5&QcQ^ZK=jStFUdK8y+w0kO*;$?kBDeD;bkX{yLkn_xn zBRp*|-72x5z@kFXJO7>}k!e|cuZ1GMEh3QU6w?J%LJu?WaY*8Oe*y0f z|Ez+lYPB{i7;5ldLKwOofzw2q78FE#$?)i6Vs2Z1mQ&qqCX<$?=HQNp$ep-unM1!2 z1K_-v-{EJyq4Kf!aTsTqrRuw{uixfZ-|H@C3JGgcjzyJZV79-BP&W8(5(-|a3ZAV?2*OoN~P`eVQy z6iIo9pX(1*)jt4{ssx0aC}AOArnh+b;B(Vk*Xd=5p_qudsP=%$2Y+i-fE_9~EG*qi zyJU@qEvN_IK&T^ozS^|XawqlT2ENro>Uasc;xg?*V9$iw>d0`0d42(N_@!ERQ%+9X z+Gr+|%biYa6lfmRfMtqomwaA6TTDe{x*&+4vG^Px{mBa87brVk!q!~twDA(^yZ-rH z!(Z_b#4(zVxjL*QPtp_cGoSwg5~bhcC^z2hG|i>I6zAd#Z#g;HXzRzIn;UbtjFH`h z9L_8D+JmU|kFt@x#TmD!-4G1tRN-6XMm$Nq7;i}{`x)2X5fGT^@KJ;hkwK#cguVFu z-Jt<0^a_|LH$c_5Pds;V;lqg_8$w2C$Vsc)T3bKjJpOc_0Cq=jYIUD{{Evu{np#v` z-0IO8-TR=~7toxU(|0b(7)_4*)XyNsIEfS6G@?r33$bp1b0Y?h-{bJ~BPi}>4d%lU zRa9hQV&c3(GSdA-lb4R}ZTl1q%8aZQP3x`q{m^u;U{r3JTh?p?rl|D^QBBEkqOurWhL_EC*4DX7 z6$&$DMMZEG)IS(=!oZ+Bde@I1aFC!J_D15{i|Ra}m|k6_OnDXDBvJv_||5qNj> zinN5HOt=!Dr#>I^dkX?FiCXHz;mkt_`1WQxvF5(<_mB`lT`f$6jM7t0s0b+(+uIqI z2HlE^UEO_{dQUVoU)AW#y{u*YEi?as$|6V)$NcMn#%)W9RNPBf7j{BF(t~xmB;qZ{ zU3QF=w^5ugq&T!Rs|pH!1TCl=)k0*FkgGrD{cKBC#g^~d zy1|SOs^p&&FJ}v++Y8*=iN%}vD9tO5IOJUPVvk$*>)W1XFg$$pC1$^h@bhI<@w$qV)Wy1W5k-Etr5g7e!SFj!cPaqK}u(CUL12QIUUtGx;ROl9E zGEt~vQAC2-+-jzHa0BsE2`Scw|ACOKgg3F7`J1~!+FMGGgeo@!`Fv*F^frAB(bWfd z@H5$68RGTjs(Z5x4^VE;XTJ3@5Ta}UhB7;x)9nxd?>6i&BupA0OXc>9Gi>l3p*~^j zLAWYJZ7>Ew@fH=Tg&-}E?5>*L0vmczGD26X%?g!AhM2AuNA;b!_GxDWymwMSh_mke zxK3961H^Tp)+q9fb@-s_z|9@QtO9kbUt<>h-FXQ;C$|?oOm7}UrvABI+rIBre+%Cy zS>Q1x&YN6cQB=1lc!z}I+5;(uoX^6cp^dO2YlGUH~=X0iHGIhe{K8h@bLEpBoIc z5Hz#M-W)-LUxjc96?|_SwR#GFGxMu|#zfuv0b8y?;Mbaa9*IY4-#KqPDv=anATd+8 z5t_^_x%I0jdUSJNgIq2ny1&73^)HqkIGW^R*l2vgB z{ciha&+)c$8_-vWuQ3nm@d}PEI5=*}s<}#{(7ijx^4JsbGaTUgsU*7bZb9vUXEbE$ zFBMuDCpvq8!bst9MGtm*me!&A`XyECKiz5~9 z?;qg!JgTdvCQ+TdGou(zKvaNwaFaaXK~nE-O~@7S?mjSOt}Xb=rgd7x&S>cz(M?vt zU8k;wy4IQF5hd8SOQORcYl4FKW{aAKb#936ahHvML6$JLn(X=Fy)!whke4h68o||5 zpY%Ss>15*|zWrLu_fGwi{g)COjYbp=K?bd2oUh2t`OqAwzH(1T(%r?U`>FI+8~uyV zD+yNYIS;~OAL7Hscy9Kv2}!AnAiQ{Tt(&i-7X3K2_YRR$dXU)Ko@{_5${(DywLHu^eP zWms8f_{?4^RdgM7Ynt!uO-EJnJm{#JzrFaGz*mAoTD2En0k> zx5XIif)s;xBC2#)7++qmMo90gaD(%Ck z<4(5)H?>~pU)j4NA>vun+a%uCR;m%I{3UkzD1#VS7eSX%`6E5GTkV_sX4IRewySjO zNXlMqn1;f7T>7SOb>+C?!&@gNqUWV=Q0g1DYVT6RP@*A~|FJ^jf&+{=wx_Kl^ZXhY zBrwgOH;^R{mtE3hyq%UubY{uPg^%tWM|4UAIzwI(SWPvR8J@I^G)T~tLYd4zAwq+7 zE$m@r2V}bvTQdUGIA>?xy9DPHb;__o&#~TCJYc1DWwt;pBCT z&t*TkyrXU*AmmEP{+3^!{v|YpULm|pUV;#b2JurJue>j%Z}#_!p9K3ri}Uvrjj_H* z>Uv$g10v3>(3JLGMlxWVIfx)j zKmDEd?eDgCwa^xh@063`uSjsB<7<#B#;qf|tu06&c*~EUL~NYI-W<@NWk&L$Aw#X0 z>~TxlGGbk_VZjs0(YLAqAFE?;;Hoj@w*dAQnaU7c46^b>L=-4<3TX|Se0Q{3_{c>sTd`WC9ETJmj2~l7v zAtQEP%i3-CtWbuSxQ@FwdBXbPGo>e^QS?(fee%gqpM}uBIv!$XktUAC!|)@*-Fz4m zu$9fWIcmZ)f&qJk`3 z4sWC1kLRejD$-^#5tp{J}AeURU%t$wrxOr(LM#i{IWOzF`sstkt%u2+ZkMvzV^8 zd^hA=CiV~(zdGDhAiVnfNT}&s{P{4y%l1syOZ`>pwZ~FIx$?@$O9Z4%okOKGW;%bD zml3`-Pe{2)o>45}Ixl9GWnMRg@Gv*;pF_!bF-~OgKA}sU=$v~n%Iow_Lf_GqzS2v} zrk*bnHi`2)KYBVHvj=2iYK!ZZnTRT+YSGLvXzvc1?e1CRbtUfrL`_w1iOy{dx~g*4 zJZ@`dMH)I}M<*QQ=^cJyp3$D&@?49&+`69F)C&THyqDie(l=YA&)hHZr9*EZzRuv| zN_pJ$7j&3Hd~5kb5^h(LJ|^pn{Zn8~!$?fMv)lm@Rm4J)gu&%F6?kWgb-;q82)EgG zr}k4Rmq1}uBCZs+8Np-QsKm*SzvYnht-^R&I4Hl9xrw0V$}$t3-LX=DebvQS=iA zp9`@!3f6UPxYWhyr%mnadWRWNIiAyc3u^f>yGDq~C)V;)^;FcA+y1?i8R;K&T!T!> z|AJTsM);sfWNc)Fz7i4B^`Z2T0fBzI>}{gMKu`H@ik8|f#6Z`Zj_dKd!!&m+PAX|B zc7y#J(jWfH5mDoohM>pRqX3T(dAfSEHreB8AE_}7L}#fsrRn=pT1s)Y{DPvhiK0TQ zEv^D)d<^w|j2^Ad9=oSFQ{-zf2NfiPrxdC|Y%F~4MWH-W`FEjRpyhV8X(n1ME_u}i z(m8f`1~VZ#!2pkZw-COWCh<=J`Cb&o!q;;j=nf?oYj3i;(&+yvH7@fbL)JlwMV3J^ z2z0>b!RgV8LfN~-+G~%z7mko9H0E^b`#4aKSKy71xtW&~u@mx2(`(Mxcso4z5cE{LCov0SD2SUdt1Nq3omM+BE|@94NqM+`v9G#@rsF|03&(UYdGo zDb#u4%c!;J?sp>h)RIr16gG`h=AN?J00}!)cTF!z8ktRy%p+kTJM9;dlIXSVzbQ(B z5=GlyR6o??M)*$N0A#_x5y|KU)=$g_c=e28xUMm9h;B4Uw_*Wl02eV8;)RvN{N zgXp2MUvNW2Ed@kb)7d3xdHR((eIpE$!8G`Uxa~x}en78-qF}j`P$2y_bL*}IGqnXv zCvE2qZMJlqpZ2-^(gT%$Ibn=$EpZNEV`&Qm-e;5aYn~=Vf$v?0dz}RBDJJ6DAE-Bai%huo_1s4rTNGKVvFTF4A z`&bVT+Ri;S?e=7k#tEOO)P!F5y>uU|vo@WJbaS64%GZvU)1#X~WlEn@v?^tt5kKR% zmzviFlJ@+rud&A}=bPc^Y|cFjfCF3b+tq_OpEoDN*3=GV~qc zpxhAY$NNM^QX&S}ge>K|%L?dI9U7hILlA!9CX{&U1qCTalJdq3=>|ni3^q`_J!F)PU z-Wey+rg7$$YIOm2@&$bMas~CDnf?OSv*)5+)jFT(d zrTs9kLcX1zVMLsXkM)pM$?XYqw@eRT90f9h@Sw%BKc-Z3v1UQBbLT;(Cdr0FhpSGCSU7}c6PPR1o!DufnH{Zd2Mjy*?os1s!mj8R zHxMeQtzDW1!b+k#QRxJB!>+)evBqeB1?bbc#kOS#jw6PGuH?G4^J1+9 z8tWujZDm;ntrmL06P_d_0A;+qxBzeR*w6SNf6)cQ0YzW(n=S$~jH3S_$wv`H0S<6n~En z2I^j0NNE%c!Lg_bL?zt>B)8e z&ufz^Z}GEUi^&@ov;s;sU$2$$YAZ7oyS%OxKbRtozu}8rpBPCSNU;ertT}!A5aJ(Z zATG>uQbt2f3kArwA(%S(l)cF!3KZ6E*T!KH9{Mc3FS?CW`&aiD#G>4d$@~dNmI)^u zK@3)lM6b5J%xbwaHw=%xG9Vr!{)^j4vorrR*aIb}w_Iw5^uvRV^ zZvWNDaj&-dBd!A{2_NGf11+G2+;wt)i0c393zrB)<0k`6~Me6FGto9SCiL zo;A-R61onw7x%SW3&+aLRDaO+t`k)Dv^I5JiVzaLJ8Y9L4p#vx zhB6Y;|JX9}%vl9g4%N1}e6e6Wt4xH>IM3Di@W`h^KS&JQBKW>t4MB@KphXQt^?>dF z9;~*?8Ee!j?`xFBX@3C2E|gV#VQqi(k!^!EALXPK+SWV2_-Fmo4;mj-s<6~8ms6+n z3x1<0+&r{o9LjdAN#D=eD^USQ7MDncS8!HD&`jW%t3tuF|C~T!UawS9cPCNpEawn^6q|sLO-vf~d+zkddD^J3afJPjH zMFi*^QzF&(1ffTmvfq4CnMDnECDDaGFb~5Kj$K>fS;gyy&KEKw|3nNV2ITj#~W6$zpnS@xXCUtD2rPob%T8V!N*YGN7 zsGF4lCohNhU?j}1lhXG`n73A}4&MEjW1mzddGf9Wzr+kp$HT_xs9iz@K zRWI6@Bt7L%R`2Ei(7rdVXee&f>`69G^GJDvQ`$BU4g4O#Oeg)&DC?1hCbM;-9u2Wp zvmk{9C=CbS;HtC{w(Q+|1WJYtM?4+(^YU6-f$nLB{4(y-hKzj##C+hRi3WpK3CM(t8DA+NY3 zBl`fIQ1hF0o03tKVNC)cWS|D_Ncr{nw%If%EA6L+R^jw20^pzk^VXiZRt?E(H+b{c z0pn$a4pUjaR!4ybsL^Q%* zk+8CaVVE1F)scQ%Q8yI7z;7ajp|`&9wZrS;nYVlI57&!Vk`^7CKAsGnTqWK1iXPU` zl>EWRp66NReZnnM6mdFfxCF`U+-pQDG*$jA&Og9#RtdSUIv-uV$ZO0*EKbWXp_~0_ zqnXEw5hKT9y$z6Yxh&Bol0?q_%+#_HeXPD^d>_IKk1{E&9I2}TK;I1>3;Ct)lbR&T z9YYa{b?Z}-8BqpTHuF5*kG_x1F|#PB&!Ex~`%(DAjmp(Vj8bPbQ5w`DN!Yii)dwAN z0P+nj)0U^X&TM2x$r_&{tK!k}XvDKA!U%OSy+!hsx0COy?RB@pe!)cDY%-ud` z+0yliS;H%Bv4!*5)nzzVi}QBe?8|QasemR=S466l<4GQbFbl^h1G``H5?_(cI`R8w!hqN=QiRazbijR3KH`m`ZI9Plobv@MrxJoN8jrf zS|&;|@wgtq9ey#8933gC5S0}jDafT23*i0IBY?Rm)uiYqzQ{gGIIIHnQYN9Cc-PgM zP-?kJI><3;ecz*!NaJK)=NNF!e!;)~590iW&1X7{!fkG*9j zgi2k`-9hN_)E`?t_MZhF5hT;v5f1+7dt45&YibEt3iW8EVNC*P+U)UJM{j;33%Nk4 zS29Op%?ft5(4M}*`c^VL8C?krWTW81Gk%vCktLm0mA$E}9rjv%$uxunm(G8&nH+la z?K(Y;rDR$gb{&x+EQ;O}*`L@E>rp3!{XkZ2P#&f&d z=#}2+oFNn2ZCk+o)mQSKce$nrv^n5=Jr9z^8+e=#3>pn3I^vMiIp)TQ2^C)7*@`E_ zJzzsj06r(TUeYh1+cf9lArb#59~F%5lkDWtW-;pUl#_8uhL|B{jSTkcQ?jZ6{S^AY zmo~c!-y-o9ymxjH6%tc0A?ve@hw(1Xl~N*&D1Z@zeAzBGc!Sx$PX-uoo!xI1;2@r_ zNGq9ZW9RkDQ)$Kp2gk#~Dwyu;1iQ{3Yn`H)Z`{Ei`)JQKUA%sn{ie=jv|cbBW=NeA zC;x1?nLO$*fctm*VpcZ3L7Z`GH@!c(Inz}9tJ6%~603`&i)}gY#}g+6sq`h)-D{rt z20b-rZTi$Hy{KB{t)Iy&shXNDWWBZ4<|Vz@#%U&0EjJ%0_Z{4jzmveu3)xIwx;|oA z(iy)uhHT%oU>{*AT)*78{j9b~%h6ZYcC>CHfp4asRaHcn5)>#nGZpi*UL6@pEjFRP ztlv?Zp-#8?OARp*?^%pF4_IWeLcFQ%CPw-4%Qf0^h7oM^#k+m0IVDeR#$@cg9xMIh z$+Xt)223AgB4${)T8c4m_#CfxpMZ!ie_V5_yzcNc6Z?jmTMHH74P=J`mp?EJ$id!&myUAN%&yo&&FGVNPx`)CT{j*rGC=2?05-aUw8kkJ1 zACEAW*aneBiDP4@gTx|C5dCe;s3$638cNR3n8V#1&rP_7`}jcgAWhkIvDTa5r7W?@ z<7!r*g}G_v*vr3cZs1~keG_L#VOF6}){QDkCaz`~->iunZ2v zlgqXOenFR3ot1lLUe(Ysdzs6EQ7(a#y9<`zBj3ovr$Gc2*zOQh*u0;{F6kmD>JMH8XANLTJT z&PXh|%rw8qrS4;fNt=bxO zZFAdg`s64+l4gz(9RkzK$K|%}9>`=U#k(CF14W2N!G2u*C7IQVdp`%x?d`xx*Y8T| z+uPfS4?;YOGH9@!^c)-<7XiqSTy)ZTnjiozd#^a5?q~f+)%C&DG;N4_^*cbYHLInJ zRp%f^w6MB(hzDJcP>d+fU_*d1hjAs8zsga_lyzodr0W|VPqH#>L~Ww{#; zV<@rII2ue$TUZTCph9mRWX{v1EaK_OmgDqTXA1(`VhN!Z~ssQA{3`#4#SK}N2f*t;C)OR;vw*r#nV zTh*KI?M6pSS0yAM2IyUw+>s@}4y>#@fuj6|Luanfh_Ho)c#_zQ2(P=JDtskTqN!tW zfU^nnI9_cpzF#SpQ`PvOo9Aur# zRyPKZ&PB;msVA3#HC^R41yD2CS`0x{`_mt*5p5bjA(0)riDPHXSRdsuoeZCChL%t0 z=aouO$ve2~E?K~+@9z`f*sIIuA07pQY>Bmi=T9m>3urz!xwJF5CGf13;Jj(-@pYe+6 z0P-=wpgWSWI#x%taESQ_yHi{O0x&Vi&-j)Zh9j?nc9L@|10FkhT!EEfEh+Ql`CdT* z;PZ2Icz~udxV$rF-!rN30 zO<^bjBhi0IfHCtZ)A^ehe1c}EuFnF|RAptL!t?YN0o4LqPj_1Y)*D2#feJ&f z^}iVQ8d_~HHELFm{hDoQYW}bQCz16FxsZ1HDa5J^l{^k{P#gmAMdhrX*6~&&kZG2& zl8O63YP|DrT`~Ig+3>~z5YKh+b%2dhFF>On5qw{k2CQ%Ma$x<_#nKCY^#FhoE>D?~ zjmzD|k3P~RErBJ=!EjsOj`tZx1jF9Nbgb-+) zdEV!?9~iWdq191qIkDLT0$$pDSxirB+bdiyi@| z!}XZ~3SMaU;}&ZK?>#~4;^RdBlUNeMXnlJT4%aBB`P2O_y`*#tMasK#GbOKc*0z`B zfxI-yyx*hlH!+^CI-GnO_-RHAH}i){YN2bA8Zy(Me$b?68#PuzQE@ESuuK}d%Vbbi zzW)WEKJ*EMm6=l%6$0AxYOY5#i-pj%5iJ#eQSE{m)#FMN=19;Jc8yM(UBP*Wi;sY{ z;C=@qq0z*3ximbsP>Po3)pU^AZ1B_k?rRS$CpRn{azd_X_{I~~WcfH?aQpNTN7bnhDeHU-n&a{_SBQ?CkT_5C^W_)izKN~>)&hKn;PwUkn@E!?Y#L5WeDHT_-EAb->lI)EmkEj2 z@^b18gE8`Pfy8lzmTB8Z-(u0TE;Q1XP#>0l!cZpwI1BX!RG4Ct*9nci@hE--JiDpH zl%sojxJ@PyV#?_;JE#B^4Y$3ah&INtgqf|JELgUa%M!cf`{>frQa#<)Dr+9Wgu(0Q z4rL_xNa4xmBJqqCZi<-Ru9Wl(Q(D&DHK}~ESpx&mR{p(H|B2b|p zn*-(37yO zw|7<7<(PkDLPce#F{jry=c#drf?IGiM=F7K$x|=sO+959xyx&4KvjI8M>BEhrUV!i zgKINc{8<>h99Eg&*UF|Rr>-+&p!@q`J z`k@wL60@zOK!SU8ll&1l(rwnPyB*(7XPfR;ef7Xu(d;GF`>+YFg2n*vL=V~W=7y*2 zU53{8#OMsvgC}nrdR-U?tY>{;prQxH*i|1}7nd==KzdfnjC%|Yg%;eB<0%NxH9Em$;B^#LRK8 zQ3dQ~i57MYMkqBn`^VXrq}W}D@zcEz)(0hgtmzIG^hUYKOid#l zZ*Lr8Kbagi=E3WdXm1xBj8s) zHU?5BJ)4TTyurfYA!ZzG-@!+b@upa6K*Fg3C;0uhZ|{TR^FcLl92QnIxYi6RA$3e1 z;bmdrebap=B@M}OpJxX4Qm}!>-%}1{B^ry%>vj1N3R6qGb}Me<{D8E0(a)UzdAvic zn6sHn#oL>u?_3pY_R7v5m>euUgaP1{g8bXI&sa!NSo;~FSC`1(i~i|IhzC9Euumku zg}3zXIf{O_X@t{J_Z}RD>LV93hPLDAk4ntxj}(VkzY|kvzWWBdxw^!o7(deS3Ps+q zOy*?GWpmwjp&?C$$gJx}!2n-kG6L{YIC*%TcqBo7b%D+lE67kUC~$d3f(P=S6!#YN zcve}q$*DQ(t8$wc(*9OA4bReQ*6Lnh_8ESVpHdz7+zd17A1R@4jy6o+2YNk7c}Z4` zjT+}QU^wGLzl4NF({3+0?&@~cse9Ya_d*}fx2^e)oMH&4dY0L`9c2_(@Mb}cXk3}i z2u#Mg-8dbchX)ug1yFe~2t|VXML69I1){#PJu|&oZVLI>0rAD{36~wsRYb>m8WY!| zq@crexnb_veL?nn#1(3VUdooa6lZfZB485SU^ILtzkEuh z$&`mn5(^(?W|0AU1Jv{JklVMb)N_1q0Qwt=X(r7hT(?eU)H^lH=69yPb$8v2@j1fs zL+7-^9QvWIN+-6rhOlC@@{@^dG1&RoB;_KeypkjVO&dPBw2`pDo`FiL&g0{wxI}hq z>u2@PDspe6mvj3UJkTI2^Wf@gN-ZrM+FMF`-1_c)o7GOaEFy%;cp=7u;7CI7^~(>eDhs&x6yPpO~3nEm95+hI97bkzcpwZV(qm7Lg^LP-Foa*hdf`#|dnUGaV0 z;8=1sGN6yx7;(u{2!=MQ%gG%&VW&56*NpBIi{o#RU|Cg|Y7bJkdirX?1MDp{=o+t#&leqFPR*YJk zqT<26Gt!DjKeYFKo=SqpEHr)7L+WZcduT|Ggq^YYg!AxDD4SEf9SK4XDMC&G9v{hL zja*OTMDSnqt-w_-Z&VJ`>%YvyVV_ucUY$U*cWl)PB$dJ*Lxu@s9| zjvvN(prwQXJ_j={ci76gsQ9@b zXsaF;N?x1u(FGIoVC}J(rwLAUXv7hG-o$qPHR$?VMKx$jW#>!SY3a-#s!4wS(bPMdRS)M6uF-h^5qw7_|6^2gNCSPT+CY zvrr5T7_myJbe$V9OZ|54bA3q%_|%^VpJ*0=wUXl9Xa*WO&UvEt7Xh|^hb`E;?4kfJzypaCFDSC}<);^IQZZg-yY*txU$@Y*3OH*JaS z_#TOF^w#Z$_-*YH2v$=TJd2zA4*qHikm@H<+<|Cd3`B~%;STf_4jx*iL}Nq+ z)V6#I3NYnr zMx@wId`VDu4jinVC2uSxpCGQk3(oC#L>2!!cd7pfwn--a>`-;1uJ>Yd7x14x>Bqc(+l{Vd&20E+DNuks^h6dI!XZE#HoFQ-gv^01*~C%Os`hU_3H0t@v%Ii6)D| zOjbOuVsF4qAur0cq%!vX{OPe0b-T(Qwhx{fm?h^$1?X2z9K05{>}6f*p5J8>L_~tc z_9J)oK5LXH0J_qqmkC<2-cL^RcvduRI1{|?CtY@I|Jo&h0(ATD5Bhf<|B&u3B7m7E zy6@iZ{)F@9eR?U=dbq%Pe(n&^Vwq$)Rw{AfqCeB=3rrjT3< z5C$zifP>3#B<~}xUj-TsRpOsl%*>byZ9ONa8Z3Dzy9y)}cTrw>Pe_5BKiHI&xfHq_ zsgBC;0c%fAAW|w1EHi=oRXqk>4P|l#m;f8K7*(9DW?gVrhOcn(Vo=(<3ceTlN#@t9 z-nQ2(Bi=bHjB0ctP#w%s%qBSr+Wo?eA@ITa28@bBue{eJ@{zoY93Rp!f7UJNn zPE&paEbB49nGooEIo>;MIkK3R^Gk9R&+G4>yRWXG|Kh||Tf3ck81!3{ zZ}0Ez1@>;%MoEwf7)9fa7Px!UlvE;+yt{G&+DpVoAaVy)warjg&$DmAwHSK7c4>1% zLF*epU&T@LebepMk|>BrGB>xsi-qLwb3KiFlA?;b7yciA%I;|TZ4<&tb(xm?d7Lt} zXW*=y-JDTpkI*mJJ(;RD?qfYIWAh46m$)w|_|B3k4!9Lhc5E3!)-M}tvTR-l>m}h& z!)uMDF^y2bD!a`-w(733*^8x^N>=Tgcr7W3BqSW|;0#SXXmI2V+dS-xzcVP%El8va zf?5!o%R8McCPW0zGGkj!dt@U8ov%#*Z=^OBUG0BJM?8UYh-@i{2vfbWrxpU~K_~#F z56)Yegvzch6^aji708y%cqi9;9|39qc(vv?J=uWt$*lkv^Hc-dzR4v;upLth=tF%!}yr9MUq5iwyvsE+|F1<@kX1 zO+VXv5NlfFH9kEZ*A!1miTO4zmhA^VZE%+U9!v)zkRvV;Z?Jmu$yjH(=2;DjP@P-o z;AEu?#LxS5WON1s0OIfO%X)Wqzu&<7l8SL48y%dUV#5;gHkKhCNd#6>b21Pk`dp&X z*^e2cLiWxJQ2Uh-CYf6_%LG87M?3(UK|b&&@}&1}-eswt=(ab^V@Tl-es)la zmn(?vj0U_f7c0WysuswyY-ZaZg}@lzl-E`_VEUaP5VRhVzy^Y#ta|-^;&qAcG|a0P;KgkzkX6OKi9cq8;b0_1>MKq;D1rTP-y`%pf(T z-VIMj<$k@edz>J|^;=0v%X9ZH&1ps#B6#397wO_<%u`0;8|K?EC)StNfAl^be@?=_yB zIUZ@Mfn8kZJVE;CE>{Zpf^8b#Dv;spi0-e zPcZ+|9CR_&bLTBg&_Xc5XIADxJ$oCwYK_gW_n;s7Y*Jfy962l1>k->Pcm2v3v zSAdtd*%~Jb;h8K){2OlC`vT_^+7p#fM#1|nEsJ8_ToDbH93GCHNiq&OIBKbR@tAf5 zLOqxy(W4^~|BM=G!w9CCdl^TF%R z&;)>~11bQ~9m{g&JNVR|rL8^!8JBMumWJ@n0`iu^{WhdVx_gARzFoOo!*0EsFOXB4Fj)t=W-ce{qrSaK@o)g9lp}E;IxE&fNS0^X~mvs3aYI|kxh%F!l!K>G!hISQgJ={^VxyH7qSDx^X{=nUBTS%n-@0>vMm za~@b|idm!f*BG+Env$EBNA8r`ug_w+G`F&HvK5hL_z@VJpKc(!a2^+yn3Tsf09Tf% zW`r^Q|K8?IFmDV5>Tc1;fLIGmA$DZ1Y5{s&f1A4x49J@xq0~kf_TRJQxl9_xD{$?g zug*PmLFt}jZ8Cj{w@^?x#x#<8$5)MP8P#u-#Y`hZhtO7l*~^Xbeit8WHo`n8RL3xvwGZ6MgmT(xC$UHNk>IoDW&7Vj z)i!fDDa>x-^O#v#pNpXR-wNnqU42U(6IUE65ClW3jqKkOXDm2$ZJj5U#{2?$xBb`8;F}s7@pAiNy#KSbzCoG(gbjwl^Q4cS-f}YN z?g~`D(tHFPU(CY24Bzz(8=+e~9asKneGnLM(&p>y+gQ45BuZVnGPz$ZKvGG~guFS1 z25Lcpu zcfc(c4HiUZT}bX_^jz2)-~TFr-7ZTmH5tBd*${XCC8HqXyB9X)ObJ+8lMC9)S`rTP)}mTGd$*ncqJ{5&4y_+sI{2%r+SmbrdKXWvqycd>ihqHioAX1N*>-R1z%ar zFRwAg^vIo+)B(a^>uxZgiM|+a2R?A|Xnjbo9SN#l7OB_**zn-mXx9DfcOcfT75e>I zU&KGm&;ApTE$udzMmTpya6QeXuvdY#t9f=e-T-7-e$}D_?bB9PWigcojJT7`_#b5y z9IUO!j_U=so*V&fYwO5dlYl5VFtaA6AJJ#6D-b`aF#O^N4oMcpTOazBA&Ta3-MUiiAR$z+O78?;=Y?LTaEnSH&|A0zqaQ{teWgo+HIRxht?brW13lA6f0jKRX?&tIF_3SFKnCZb(d{L`?cfDs_ zUEM2)KD01fOyYEBjuKtDFLI+9oM3(iDQgEg{ttgYP!Bflrtx0g+{D1n&ndqSTY10% z_RICVC#xda3_>z=FJ1Q_Zt}uYywdSle3a)=tE^WbatnpkJ$ctd%JTS=R@=$nkP(em zcNKKu@0n+_tq-X?JHn4`tV{J-t*wrO@yA~Gmmq~Tjq+M6_{!0pt99tT;& z^ZpM3`}P*`!HrYfqn+2Pw}%CeFx8i(*Cws7Sww*A>8uT$lqBTj4HUY;PoVe^23wDT zR8*xB_2J9biHY{$Q+12eBRsw3lf@xnYCiNjMdTR<39Z%k2& z8Ib#(?s}mzSP5zaDmKXY(U3j``DM>~o{Hp}sK(bA4Blw1I<&>8skOKs*OfJ#H|)-Q zFw(!=vG{k#_vtwPHp6w6k)-{V#USwR@%G)5L7OnM__v&Yd~4||(>WsB&^^MP>Ivb=h` ziJv3Pov*_#M|c-oU+#KGJzSp6z#A1qxn6xuUul(;$ho;b1h;;Z z!PIiOuK}TZ6pfy`<9C{G@*5i+hZEVCR~|uFGP&jIN~;zLUGv#RK^_hJ%N@Ja0ZaPs zY^!}wWyA*^eIt=sAGph-!{6koQ&v%7V8e0Lm7H~mUrb#utu%dwr+d62{~-=@bNzVg!RGY>#(UeR z>fk(f78Y_}$V7;0Xbd0CGaVh0r#UZ3&B`v7Jipw&^SJ+-?j*}QuKWaI37bwDlgpl0 zB5Ey`!A9D+xX`rDC-jTS-lA2@;iapm_2ei>8rwwjt|x85SsdnWk008_p4Xnbm1VHvvI2C2O(2R2!=MN} zyKnh{lhzmPIzZT~!4D+hC@BIm(G%}}rSPHv^1d=GxqusW>u= z!FFRHd66V=1LO5&ktBkL&EsJTye}0$UvLm;JubxYw%#Z3*6$mEjh&-TN_C3|lC=G^ z_Ny+Qyn@p2?>cVQIB!uD72`Q4k;t^B5ClkaqF2BeU#q6;I@YDL;}l<&@Hm6+?G1ns z6K|;uj6_5Y-#n@~1euU84+$b6^)>H!=3<9MUmpjxVEDXV0W?9XB-$)bkRse>K3^cf z`j@^@N20LAbNBdG#GUnnbGsmeUfBgbb2gQ!jRO)8!CQNQkzSu+jL>rT_p6eo1Dsei zVBPR?bS_+2_IF3Q!>SMKu9A54J+yb?V?ZsqKT2+}+iK@pzK^_~k zvZf~3IGMD~fVN)OTf5R&0~HXLc}VBAZHMJDY_K0K1kyrlweaamR?FAAHg7pmUH3A* zkR=HOb#L@oR@9Pty;h#v9;az|TlVAASFVD7s$1{>!S+0;3<=3kPsP_bZM)6%FaxLO zk)foL;o-1Fr)ncc6}K!})@fO9rcsXL*O$}Po9oF@E~kqujF5aaV2s&f@#Y2`tysqy z{V~a_9!8=^ELICFCbAmnr~{`*OKzjS2_6r+0TxzW(3mp3Le|B-$=sJi5bza?(@CU^ zNxsZ`=otyN@_MavS~b-ok15bvwgGp8f(uKPZ5%pZX4=8qe;(6%Ww6`1T5`KyIBYe* z>EU*ICBh4*3bq~Z-lfrKbZ#?nKB|cO#M0dV*bEA~4sGT13rYKp(i|4RRusnDVsYW+Zf#YaHr&o;R?BNx`9|yM`Sf+E(IJ5ND_BIH zjOf~`0~_JIjnePS;AZZ3=|(*mHAxyRpLS+%dpF($lbS$UPadr9-CK$Ark1BK54BcD zZI2RKuNrWfa4AVnk!a1U+uI^NBvTypc!R^k!-@m&)KG7UDGC)-#LFujpPu#> zibQ6s)4dP!6R7AD6DE#oUl55T39%{8GleKFrE~|9#)yG=Ey}sPt9WB=ZM2_$U^Cgs zZIWAR?65b4_vHHG`G)GfWEB-1dcWR-cIR!dI#2!% z^Edf#0M^pMmD(X+Fg&TJ1RFtLGRONIJ%Cg%DTr-tsddQikVXP$_sprJ?mwu=t`2x=(WVa*j^vbUMfbj+V(<*cd-GB`t=Xve1^~++JcSfZ`X)ox!O2 zEJ7hXM9sub2lcyTn1|b0!w^pBY=3e~Zj8}E@W8EyZ@ky%6zMZ&IH3sM=HjY~s;Vnr zJd<*~PzB$=f%Gq*Z6#A*n+FP(oveps1c8hVnLkobiVnCWA_h05oIBhH|Ls`ufSU;l zV0xH2i&?r+2R*p2W7D)uJt&p3$&L=o{se|J^;H0^_vrr<@ciGc?|;Vx{J-%7TYX!v ZP^|;Div2~*ZUFctCL|?T`d!=S{{x=z)r0^5 literal 0 HcmV?d00001 diff --git a/federated_learning/breast_density_challenge/result_server/.gitkeep b/federated_learning/breast_density_challenge/result_server/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/federated_learning/breast_density_challenge/run_all_fl.sh b/federated_learning/breast_density_challenge/run_all_fl.sh new file mode 100755 index 0000000000..604d992b53 --- /dev/null +++ b/federated_learning/breast_density_challenge/run_all_fl.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +mkdir logs +./run_docker_server.sh 2>&1 | tee logs/server_log.txt & +sleep 30s +./run_docker_site-1.sh 0 2>&1 | tee logs/site-1_log.txt & +./run_docker_site-2.sh 1 2>&1 | tee logs/site-2_log.txt & +./run_docker_site-3.sh 0 2>&1 | tee logs/site-3_log.txt diff --git a/federated_learning/breast_density_challenge/run_docker_debug.sh b/federated_learning/breast_density_challenge/run_docker_debug.sh new file mode 100755 index 0000000000..bf4d542f8c --- /dev/null +++ b/federated_learning/breast_density_challenge/run_docker_debug.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +DOCKER_IMAGE=monai-nvflare:latest + +GPU=$1 +CLIENT_NAME="site-1" + +DATA_DIR="${PWD}/data" + +# interactive session +#COMMAND="/bin/bash" +# test learner +COMMAND="python3 pt/learners/mammo_learner.py" + +echo "Starting $DOCKER_IMAGE with GPU=${GPU}" +docker run -it \ +--gpus="device=${GPU}" --network=host --ipc=host --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ +--name="${CLIENT_NAME}_debug" \ +-e NVIDIA_VISIBLE_DEVICES=${GPU} \ +-v ${DATA_DIR}:/data:ro \ +-w /code \ +${DOCKER_IMAGE} /bin/bash -c "${COMMAND}" diff --git a/federated_learning/breast_density_challenge/run_docker_server.sh b/federated_learning/breast_density_challenge/run_docker_server.sh new file mode 100755 index 0000000000..9cedd95e17 --- /dev/null +++ b/federated_learning/breast_density_challenge/run_docker_server.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +DOCKER_IMAGE=monai-nvflare:latest + +OUT_DIR="${PWD}/result_server" +SERVER="server" + +GPU=$1 + +COMMAND="/code/start_server.sh; /code/finalize_server.sh" + +echo "Starting $DOCKER_IMAGE with GPU=${GPU}" +docker run \ +--gpus="device=${GPU}" --network=host --ipc=host --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ +--name=${SERVER} \ +-e NVIDIA_VISIBLE_DEVICES="${GPU}" \ +-v "${OUT_DIR}":/result \ +-w /code \ +${DOCKER_IMAGE} /bin/bash -c "${COMMAND}" + +# kill client containers +docker kill site-1 site-2 site-3 diff --git a/federated_learning/breast_density_challenge/run_docker_site-1.sh b/federated_learning/breast_density_challenge/run_docker_site-1.sh new file mode 100755 index 0000000000..b5afcad11c --- /dev/null +++ b/federated_learning/breast_density_challenge/run_docker_site-1.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +DOCKER_IMAGE=monai-nvflare:latest + +GPU=$1 +CLIENT_NAME="site-1" + +DATA_DIR="${PWD}/data" + +COMMAND="/code/start_${CLIENT_NAME}.sh; tail -f /dev/null" + +echo "Starting $DOCKER_IMAGE with GPU=${GPU}" +docker run \ +--gpus="device=${GPU}" --network=host --ipc=host --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ +--name="${CLIENT_NAME}" \ +-e NVIDIA_VISIBLE_DEVICES="${GPU}" \ +-v "${DATA_DIR}":/data:ro \ +-w /code \ +${DOCKER_IMAGE} /bin/bash -c "${COMMAND}" diff --git a/federated_learning/breast_density_challenge/run_docker_site-2.sh b/federated_learning/breast_density_challenge/run_docker_site-2.sh new file mode 100755 index 0000000000..7268baf088 --- /dev/null +++ b/federated_learning/breast_density_challenge/run_docker_site-2.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +DOCKER_IMAGE=monai-nvflare:latest + +GPU=$1 +CLIENT_NAME="site-2" + +DATA_DIR="${PWD}/data" + +COMMAND="/code/start_${CLIENT_NAME}.sh; tail -f /dev/null" + +echo "Starting $DOCKER_IMAGE with GPU=${GPU}" +docker run \ +--gpus="device=${GPU}" --network=host --ipc=host --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ +--name="${CLIENT_NAME}" \ +-e NVIDIA_VISIBLE_DEVICES="${GPU}" \ +-v "${DATA_DIR}":/data:ro \ +-w /code \ +${DOCKER_IMAGE} /bin/bash -c "${COMMAND}" diff --git a/federated_learning/breast_density_challenge/run_docker_site-3.sh b/federated_learning/breast_density_challenge/run_docker_site-3.sh new file mode 100755 index 0000000000..976724b28f --- /dev/null +++ b/federated_learning/breast_density_challenge/run_docker_site-3.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +DOCKER_IMAGE=monai-nvflare:latest + +GPU=$1 +CLIENT_NAME="site-3" + +DATA_DIR="${PWD}/data" + +COMMAND="/code/start_${CLIENT_NAME}.sh; tail -f /dev/null" + +echo "Starting $DOCKER_IMAGE with GPU=${GPU}" +docker run \ +--gpus="device=${GPU}" --network=host --ipc=host --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ +--name="${CLIENT_NAME}" \ +-e NVIDIA_VISIBLE_DEVICES="${GPU}" \ +-v "${DATA_DIR}":/data:ro \ +-w /code \ +${DOCKER_IMAGE} /bin/bash -c "${COMMAND}" From 265af90aa4bada1bf127e940acc536e786110c5e Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Wed, 4 May 2022 12:12:22 -0400 Subject: [PATCH 2/2] add FQDN definition --- federated_learning/breast_density_challenge/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federated_learning/breast_density_challenge/README.md b/federated_learning/breast_density_challenge/README.md index 06c668fea1..f7506bd439 100644 --- a/federated_learning/breast_density_challenge/README.md +++ b/federated_learning/breast_density_challenge/README.md @@ -33,7 +33,7 @@ Do not change the locations given in [config_fed_client.json](./code/configs/mam ``` ### 1.2 Build container -The argument specifies the FQDN of the FL server. Use `localhost` when simulating FL on your machine. +The argument specifies the FQDN (Fully Qualified Domain Name) of the FL server. Use `localhost` when simulating FL on your machine. ``` ./build_docker.sh localhost ```