From ea724cead93e59506c98cfcdcf88df88b96b256b Mon Sep 17 00:00:00 2001 From: "yi.wu" Date: Tue, 2 May 2017 23:13:39 +0800 Subject: [PATCH 01/13] update cluster train v2 doc --- doc/howto/usage/cluster/cluster_train_cn.md | 27 ++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index 274452fbf0c59..5fb087fcd4d5a 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -2,25 +2,30 @@ .. _cluster_train: ``` -# 运行分布式训练 +# 概述 +PaddlePaddle 使用如下图的方式完成分布式训练： -在本文中，我们将阐释如何在集群上运行分布式 Paddle 训练作业。我们将以[推荐系统](https://github.com/baidu/Paddle/tree/develop/demo/recommendation)为例创建分布式的单进程训练。 + -在本文中使用的[脚本](https://github.com/baidu/Paddle/tree/develop/paddle/scripts/cluster_train)通过 SSH 运行分布式作业。 它们还可以供那些运行更复杂的集群管理系统（如 MPI 和 [Kubernetes](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/k8s) ）的用户参考。 +- data shard（数据分片）: 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用 +- trainer（计算节点）: 每个trainer启动后读取切分好的一部分数据，并开始神经网络的“前馈”和“后馈”计算，并和parameter server通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients）然后下载优化更新后的神经网络参数（parameters）。 +- parameter server（参数服务器）:每个参数服务器只保存整个神经网络所有参数的一部分。参数服务器接收从计算节点上传的梯度，并完成参数优化更新，在将更新后的参数下发到每个计算节点。 -## 前提条件 +这样，通过trainer和parameter server的分布式协作，可以完成神经网络的SGD方法的训练。Paddle可以同时支持同步SGD(synchronize SGD)和异步SGD(asynchronize SGD)。 -1. 上述脚本使用 Python 库 [fabric](http://www.fabfile.org/) 来运行 SSH 命令。 我们使用 `pip` 来安装 fabric: +在使用同步SGD训练神经网络时，Paddle使用同步屏障(barrier)，使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大的提高了计算的并行性：parameter server之间不相互依赖，并行的接收梯度和更新参数，parameter server也不会等待trainer全部都提交梯度之后才开始下一步，trainer之间也不会相互依赖，并行的执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台parameter server上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 - ```bash - pip install fabric - ``` +## 使用分布式计算平台或工具 -2. 我们需要在集群的所有节点上安装 PaddlePaddle。 如果要启用GPU，需要在 `/usr/local/cuda` 中安装 CUDA; 否则 Paddle 将在运行时报错。 +PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务，比如[Kubernetes](http://kubernetes.io)，[OpenMPI](https://www.open-mpi.org)，也可以使用类似[Fabric](http://www.fabfile.org)的集群管理工具编写集群任务提交和管理脚本。 -3. 在 [`cluster_train/conf.py`] 中设置 `ROOT_DIR`， 该 ROOT_DIR 要在所有节点上存在。为了方便起见，我们通常在所有节点上创建一个 Unix 用户 `paddle`，并设置 `ROOT_DIR=/home/paddle`。这样，我们可以将 SSH 公钥写入 `/home/paddle/.ssh/authorized_keys`，以便用户 `paddle` 可以 SSH 到所有节点而不用密码。 +关于如何在Kubernetes上启动分布式训练任务，可以参考这篇说明：[Kubernetes分布式训练](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/usage/k8s/k8s_distributed_cn.md) -## 准备工作空间 +## 环境准备 + +1. 我们需要在集群的所有节点上安装 PaddlePaddle。 如果要启用GPU，还需要在节点上安装对应的GPU驱动以及CUDA。PaddlePaddle的安装可以参考[这里](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install)的多种安装方式。我们推荐使用[Docker安装方式](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)来快速安装PaddlePaddle。 + +## 准备训练数据 我们将放置依赖库、配置等文件的目录视为 *工作空间（workspace）*。 From eeeadd191f634842c3062702406230ac408da258 Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Tue, 9 May 2017 19:56:35 +0800 Subject: [PATCH 02/13] WIP cluster train doc --- demo/word2vec/api_train_v2_cluster.py | 116 ++++++++ demo/word2vec/prepare.py | 38 +++ doc/howto/usage/cluster/cluster_train_cn.md | 259 ++++++++++++------ .../scripts/cluster_train_v2/fabric/conf.py | 39 +++ .../fabric/docker_cluster/Dockerfile | 11 + .../fabric/docker_cluster/ssh_servers.yaml | 23 ++ paddle/scripts/cluster_train_v2/fabric/run.sh | 14 + 7 files changed, 415 insertions(+), 85 deletions(-) create mode 100644 demo/word2vec/api_train_v2_cluster.py create mode 100644 demo/word2vec/prepare.py create mode 100644 paddle/scripts/cluster_train_v2/fabric/conf.py create mode 100644 paddle/scripts/cluster_train_v2/fabric/docker_cluster/Dockerfile create mode 100644 paddle/scripts/cluster_train_v2/fabric/docker_cluster/ssh_servers.yaml create mode 100644 paddle/scripts/cluster_train_v2/fabric/run.sh diff --git a/demo/word2vec/api_train_v2_cluster.py b/demo/word2vec/api_train_v2_cluster.py new file mode 100644 index 0000000000000..b1f523e03fa8e --- /dev/null +++ b/demo/word2vec/api_train_v2_cluster.py @@ -0,0 +1,116 @@ +import math +import os +import paddle.v2 as paddle +import pickle + +embsize = 32 +hiddensize = 256 +N = 5 +cluster_train_file = "./train_data_dir/train/train.txt" +cluster_test_file = "./test_data_dir/test/test.txt" +node_id = os.getenv("OMPI_COMM_WORLD_RANK") +if not node_id: + raise EnvironmentError("must provied OMPI_COMM_WORLD_RANK") + +def wordemb(inlayer): + wordemb = paddle.layer.embedding( + input=inlayer, + size=embsize, + param_attr=paddle.attr.Param( + name="_proj", + initial_std=0.001, + learning_rate=1, + l2_rate=0, + sparse_update=True)) + return wordemb + +def cluster_reader_cluster(filename, node_id): + def cluster_reader(): + with open("-".join([filename, "%05d"%int(node_id)]), "r") as f: + for l in f: + csv_data = [int(cell) for cell in l.split(",")] + yield tuple(csv_data) + return cluster_reader + +def main(): + # get arguments from env + + # for local training + TRUTH = ["true", "True", "TRUE", "1", "yes", "Yes", "YES"] + cluster_train = os.getenv('PADDLE_CLUSTER_TRAIN', "False") in TRUTH + use_gpu = os.getenv('PADDLE_USE_GPU', "False") + + if not cluster_train: + paddle.init(use_gpu=use_gpu, trainer_count=int(os.getenv("PADDLE_TRAINER_COUNT", "1"))) + else: + paddle.init( + use_gpu=use_gpu, + trainer_count=int(os.getenv("PADDLE_TRAINER_COUNT", "1")), + port=int(os.getenv("PADDLE_PORT", "7164")), + ports_num=int(os.getenv("PADDLE_PORTS_NUM", "1")), + ports_num_for_sparse=int(os.getenv("PADDLE_PORTS_NUM_FOR_SPARSE", "1")), + num_gradient_servers=int(os.getenv("PADDLE_NUM_GRADIENT_SERVERS", "1")), + trainer_id=int(os.getenv("PADDLE_TRAINER_ID", "0")), + pservers=os.getenv("PADDLE_PSERVERS", "127.0.0.1")) + #word_dict = paddle.dataset.imikolov.build_dict() + fn = open("thirdparty/wuyi_train_thdpty/word_dict.pickle", "r") + word_dict = pickle.load(fn) + fn.close() + dict_size = len(word_dict) + firstword = paddle.layer.data( + name="firstw", type=paddle.data_type.integer_value(dict_size)) + secondword = paddle.layer.data( + name="secondw", type=paddle.data_type.integer_value(dict_size)) + thirdword = paddle.layer.data( + name="thirdw", type=paddle.data_type.integer_value(dict_size)) + fourthword = paddle.layer.data( + name="fourthw", type=paddle.data_type.integer_value(dict_size)) + nextword = paddle.layer.data( + name="fifthw", type=paddle.data_type.integer_value(dict_size)) + + Efirst = wordemb(firstword) + Esecond = wordemb(secondword) + Ethird = wordemb(thirdword) + Efourth = wordemb(fourthword) + + contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth]) + hidden1 = paddle.layer.fc(input=contextemb, + size=hiddensize, + act=paddle.activation.Sigmoid(), + layer_attr=paddle.attr.Extra(drop_rate=0.5), + bias_attr=paddle.attr.Param(learning_rate=2), + param_attr=paddle.attr.Param( + initial_std=1. / math.sqrt(embsize * 8), + learning_rate=1)) + predictword = paddle.layer.fc(input=hidden1, + size=dict_size, + bias_attr=paddle.attr.Param(learning_rate=2), + act=paddle.activation.Softmax()) + + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 100 == 0: + result = trainer.test( + paddle.batch( + cluster_reader_cluster(cluster_test_file, node_id), 32)) + print "Pass %d, Batch %d, Cost %f, %s, Testing metrics %s" % ( + event.pass_id, event.batch_id, event.cost, event.metrics, + result.metrics) + + cost = paddle.layer.classification_cost(input=predictword, label=nextword) + parameters = paddle.parameters.create(cost) + adagrad = paddle.optimizer.AdaGrad( + learning_rate=3e-3, + regularization=paddle.optimizer.L2Regularization(8e-4)) + trainer = paddle.trainer.SGD(cost, + parameters, + adagrad, + is_local=not cluster_train) + trainer.train( + paddle.batch(cluster_reader_cluster(cluster_train_file, node_id), 32), + num_passes=30, + event_handler=event_handler) + + +if __name__ == '__main__': + main() diff --git a/demo/word2vec/prepare.py b/demo/word2vec/prepare.py new file mode 100644 index 0000000000000..14109266cb78c --- /dev/null +++ b/demo/word2vec/prepare.py @@ -0,0 +1,38 @@ +import paddle.v2 as paddle +import tarfile +import os +import pickle + +SPLIT_COUNT = 2 +N = 5 + +def file_len(fd): + for i, l in enumerate(fd): + pass + return i + 1 + +def split_from_reader_by_line(filename, reader, split_count): + fn = open(filename, "w") + for batch_id, batch_data in enumerate(reader()): + batch_data_str = [str(d) for d in batch_data] + fn.write(",".join(batch_data_str)) + fn.write("\n") + fn.close() + + fn = open(filename, "r") + total_line_count = file_len(fn) + fn.close() + per_file_lines = total_line_count / split_count + 1 + cmd = "split -d -a 5 -l %d %s %s-" % (per_file_lines, filename, filename) + os.system(cmd) + +word_dict = paddle.dataset.imikolov.build_dict() +with open("word_dict.pickle", "w") as dict_f: + pickle.dump(word_dict, dict_f) + +split_from_reader_by_line("train.txt", + paddle.dataset.imikolov.train(word_dict, N), + SPLIT_COUNT) +split_from_reader_by_line("test.txt", + paddle.dataset.imikolov.test(word_dict, N), + SPLIT_COUNT) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index 5fb087fcd4d5a..a019dfd4c1625 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -1,9 +1,24 @@ ```eval_rst .. _cluster_train: ``` +* [概述](#概述) +* [环境准备](#环境准备) +* [启动参数说明](#启动参数说明) + * [启动parameter server](#启动parameter-server) + * [启动trainer](#启动trainer) + * [准备数据集](#准备数据集) + * [准备训练程序](#准备训练程序) +* [使用分布式计算平台或工具](#使用分布式计算平台或工具) + * [使用Fabric启动集群作业](#使用fabric启动集群作业) + * [启动集群作业](#启动集群作业) + * [终止集群作业](#终止集群作业) + * [检查集群训练结果](#检查集群训练结果) + * [检查模型输出](#检查模型输出) + * [在OpenMPI集群中提交训练作业(TODO)](#在openmpi集群中提交训练作业todo) + * [在Kubernetes集群中提交训练作业(TODO)](#在kubernetes集群中提交训练作业todo) # 概述 -PaddlePaddle 使用如下图的方式完成分布式训练： +PaddlePaddle 使用如下图的架构完成分布式训练： @@ -15,118 +30,189 @@ PaddlePaddle 使用如下图的方式完成分布式训练： 在使用同步SGD训练神经网络时，Paddle使用同步屏障(barrier)，使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大的提高了计算的并行性：parameter server之间不相互依赖，并行的接收梯度和更新参数，parameter server也不会等待trainer全部都提交梯度之后才开始下一步，trainer之间也不会相互依赖，并行的执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台parameter server上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 -## 使用分布式计算平台或工具 - -PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务，比如[Kubernetes](http://kubernetes.io)，[OpenMPI](https://www.open-mpi.org)，也可以使用类似[Fabric](http://www.fabfile.org)的集群管理工具编写集群任务提交和管理脚本。 - -关于如何在Kubernetes上启动分布式训练任务，可以参考这篇说明：[Kubernetes分布式训练](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/usage/k8s/k8s_distributed_cn.md) - -## 环境准备 +# 环境准备 +1. 准备您的计算集群。计算集群通常由一组（几台到几千台规模）的Linux服务器组成。服务器之间可以通过局域网（LAN）联通，每台服务器具有集群中唯一的IP地址（或者可被DNS解析的主机名）。集群中的每台计算机通常被成为一个“节点”。使用不同的集群管理平台时，会要求集群节点是否是同样硬件配置。 1. 我们需要在集群的所有节点上安装 PaddlePaddle。 如果要启用GPU，还需要在节点上安装对应的GPU驱动以及CUDA。PaddlePaddle的安装可以参考[这里](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install)的多种安装方式。我们推荐使用[Docker安装方式](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)来快速安装PaddlePaddle。 -## 准备训练数据 - -我们将放置依赖库、配置等文件的目录视为 *工作空间（workspace）*。 +安装完成之后，执行下面的命令可以查看已经安装的版本（docker安装方式可以进入docker容器执行：`docker run -it paddlepaddle/paddle:0.10.0rc3 /bin/bash`）： +```bash +$ paddle version +PaddlePaddle 0.10.0rc, compiled with + with_avx: ON + with_gpu: OFF + with_double: OFF + with_python: ON + with_rdma: OFF + with_timer: OFF +``` -这些 `train/test` 数据应该在启动集群作业之前准备好。 为了满足训练/测试数据放置在工作空间中不同目录的要求，PADDLE 根据在模型配置文件中使用的名为 `train.list/test.list` 的索引文件引用训练/测试数据，所以训练/测试数据也包含 train.list/test.list 两个列表文件。所有本地训练 demo 已经提供了脚本来帮助您创建这两个文件，并且集群作业中的所有节点将在正常情况下处理具有相同逻辑代码的文件。 +下面以`demo/word2vec`中的代码作为实例，介绍使用PaddlePaddle v2 API完成分布式训练。 -通常，你可以使用本地训练中的相同模型文件进行集群训练。请记住，在模型文件的 `setting`函数中设置的 `batch_size` 表示在集群作业**每个**节点中的 batch 大小，而不是使用同步 SGD 的总 batch 大小。 +# 启动参数说明 +## 启动parameter server +执行以下的命令启动一个parameter server并等待和trainer的数据交互 +```bash +$ paddle pserver --port=7164 --ports_num=1 --ports_num_for_sparse=1 --num_gradient_servers=1 +``` -以下步骤基于 demo 目录中的 [demo/recommendation](https://github.com/PaddlePaddle/Paddle/tree/develop/demo/recommendation)。 +如果希望可以在后台运行pserver，并保存输出到一个日志文件，可以运行： +```bash +$ stdbuf -oL /usr/bin/nohup paddle pserver --port=7164 --ports_num=1 --ports_num_for_sparse=1 --num_gradient_servers=1 &> pserver.log +``` -你只需完成 demo/recommendation 教程文档到 `Train` 的部分，之后你会得到训练/测试数据和模型配置文件。最后，只需使用 demo/recommendation 作为集群训练的工作空间。 +| 参数 | 是否必选 | 默认值 | 说明 | +| ------------- | ------------- | ------------- | +| port | 必选 | 7164 | pserver监听的起始端口，根据ports_num决定总端口个数，从起始端口监听多个端口用于通信 | +| ports_num | 必选 | 1 | 监听的端口个数 | +| ports_num_for_sparse | 必选 | 1 | 用于稀疏类型参数通信的端口个数 | +| num_gradient_servers | 必选 | 1 | 当前训练任务pserver总数 | + +## 启动trainer +执行以下命令启动使用python编写的trainer程序（文件名为任意文件名，如train.py） +```bash +$ python train.py +``` -最后，你的工作空间应如下所示： +trainer需要和pserver保持网络联通以完成训练。trainer启动需要传入端口、pserver地址等参数使trainer可以正确连接到pserver。这些参数可以通过*环境变量*或编写程序时`paddle.init()`中传入参数。如果同时使用`paddle.init()`参数和环境变量，将会优先使用`paddle.init()`中传入的参数。 + +PaddlePaddle在集群训练时会使用[环境变量](https://zh.wikipedia.org/wiki/环境变量)获取集群训练所需要的参数配置。这些环境变量都是以`PADDLE_INIT_`开头，以防止和其他的环境变量重名。 +使用环境变量： +```bash +export PADDLE_INIT_USE_GPU=False +export PADDLE_INIT_TRAINER_COUNT=1 +export PADDLE_INIT_PORT=7164 +export PADDLE_INIT_PORTS_NUM=1 +export PADDLE_INIT_PORTS_NUM_FOR_SPARSE=1 +export PADDLE_INIT_NUM_GRADIENT_SERVERS=1 +export PADDLE_INIT_TRAINER_ID=0 +export PADDLE_INIT_PSERVERS=127.0.0.1 +python train.py ``` -. -|-- common_utils.py -|-- data -| |-- config.json -| |-- config_generator.py -| |-- meta.bin -| |-- meta_config.json -| |-- meta_generator.py -| |-- ml-1m -| |-- ml_data.sh -| |-- ratings.dat.test -| |-- ratings.dat.train -| |-- split.py -| |-- test.list -| `-- train.list -|-- dataprovider.py -|-- evaluate.sh -|-- prediction.py -|-- preprocess.sh -|-- requirements.txt -|-- run.sh -`-- trainer_config.py + +使用参数： +```python +paddle.init( + use_gpu=False, + trainer_count=1, + port=7164, + ports_num=1, + ports_num_for_sparse=1, + num_gradient_servers=1, + trainer_id=0, + pservers="127.0.0.1") ``` -虽然这些文件并非都需要集群训练，但是也没有必要删除无用的文件。 -`trainer_config.py` -表示模型配置文件。 +| 参数 | 是否必选 | 默认值 | 说明 | +| ------------- | ------------- | ------------- | +| use_gpu | 可选 | False | 是否启用GPU训练 | +| trainer_count | 必选 | 1 | 当前训练任务trainer总个数 | +| port | 必选 | 7164 | 连接到pserver的端口 | +| ports_num | 必选 | 1 | 连接到pserver的端口个数 | +| ports_num_for_sparse | 必选 | 1 | 和pserver之间用于稀疏类型参数通信的端口个数 | +| num_gradient_servers | 必选 | 1 | 当前训练任务pserver总数 | +| trainer_id | 必选 | 0 | 每个trainer的唯一ID，从0开始的整数 | +| pservers | 必选 | 127.0.0.1 | 当前训练任务启动的pserver的IP列表，多个IP使用“,”隔开 | -`train.list` 和 `test.list` -文件索引。它存储当前节点所有训练/测试数据的所有相对或绝对文件路径。 -`dataprovider.py` -用于读取训练/测试样本。这与本地训练相同。 +## 准备数据集 -`data` -数据目录中的所有文件被 train.list/test.list 引用。 +参考样例数据准备脚本[prepare.py](../../../../demo/word2vec/prepare.py)，准备训练数据和验证数据集，我们从paddle自带的公开数据集中下载训练数据，并根据分布式训练并发数（计算节点个数），指定`SPLIT_COUNT`将数据切分成多份： +```python +import paddle.v2 as paddle +import tarfile +import os +import pickle + +SPLIT_COUNT = 2 +N = 5 + +def file_len(fd): + for i, l in enumerate(fd): + pass + return i + 1 + +def split_from_reader_by_line(filename, reader, split_count): + fn = open(filename, "w") + for batch_id, batch_data in enumerate(reader()): + batch_data_str = [str(d) for d in batch_data] + fn.write(",".join(batch_data_str)) + fn.write("\n") + fn.close() + + fn = open(filename, "r") + total_line_count = file_len(fn) + fn.close() + per_file_lines = total_line_count / split_count + 1 + cmd = "split -d -a 5 -l %d %s %s-" % (per_file_lines, filename, filename) + os.system(cmd) + +word_dict = paddle.dataset.imikolov.build_dict() +with open("word_dict.pickle", "w") as dict_f: + pickle.dump(word_dict, dict_f) + +split_from_reader_by_line("train.txt", + paddle.dataset.imikolov.train(word_dict, N), + SPLIT_COUNT) +split_from_reader_by_line("test.txt", + paddle.dataset.imikolov.test(word_dict, N), + SPLIT_COUNT) +``` -## 准备集群作业配置 +上面的这段程序，会把imikolov dataset的训练集和测试集分别分割成多个文件（例子中为2个）: +``` +train.txt +train.txt-00000 +train.txt-00001 +test.txt +test.txt-00000 +test.txt-00001 +``` -以下选项必须在 cluster_train/conf.py 中认真设置 +在进行分布式训练时，每个trainer进程需要能够读取属于自己的一份数据。在一些分布式系统中，系统会提供一个分布式存储服务，这样保存在分布式存储中的数据可以被集群中的每个节点读取到。如果不使用分布式存储，则需要手动拷贝属于每个trainer节点的训练数据到对应的节点上。 -`HOSTS` 所有节点运行集群作业的主机名或 IP 。你还可以将用户和 ssh 端口附加到主机名上，例如 root@192.168.100.17:9090。 +对于不同的训练任务，训练数据格式和训练程序的`reader()`会大不相同，所以开发者需要根据自己训练任务的世纪场景完成训练数据的分割和`reader()`的编写。 -`ROOT_DIR` 用于放置 JOB 工作空间目录的工作空间 ROOT 目录 +## 准备训练程序 -`PADDLE_NIC` 集群通信通道的 NIC(Network Interface Card, 网络接口卡) 接口名称，例如以太网的 eth0，infiniband 的 ib0。 +我们会对每个训练任务都会在每个节点上创建一个 *工作空间（workspace）*，其中包含了用户的训练程序，程序依赖，挂载或下载的训练数据分片。 -`PADDLE_PORT` 集群通信通道的端口号 +最后，你的工作空间应如下所示： +``` +. +|-- my_lib.py +|-- word_dict.pickle +|-- train.py +|-- train_data_dir/ +| |-- train.txt-00000 +| |-- train.txt-00001 +`-- test_data_dir/ + |-- test.txt-00000 + `-- test.txt-00001 +``` -`PADDLE_PORTS_NUM` 用于集群通信通道的端口数。 如果集群节点数量少（少于5〜6个节点），建议将其设置为较大，如2〜8，以获得更好的网络性能。 +- `mylib.py`：会被`train.py`调用的一些库函数 +- `word_dict.pickle`：在`train.py`中会使用到的字典数据文件 +- `train.py`：训练程序，代码参考[这里](../../../../demo/word2vec/api_train_v2_cluster.py) +- `train_data_dir`：此目录可以是从分布式存储挂载过来包含训练数据的目录，也可以是在任务启动前下载到本地的。 +- `test_data_dir`：包含测试数据集，同样可以是挂载或下载生成。 -`PADDLE_PORTS_NUM_FOR_SPARSE` 用于 sparse remote updater 集群通信信道的端口数。如果使用 sparse remote update，则可以像 `PADDLE_PORTS_NUM` 一样设置。 +# 使用分布式计算平台或工具 -`LD_LIBRARY_PATH` 为集群作业设置额外的 LD_LIBRARY_PATH。你可以使用它来设置 CUDA 库的路径。 +PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务，包括： +- [Kubernetes](http://kubernetes.io) +- [OpenMPI](https://www.open-mpi.org) +- [Fabric](http://www.fabfile.org) 集群管理工具。可以使用`Fabric`编写集群任务提交和管理脚本。 -默认配置如下： +在使用分布式计算平台进行训练时，任务被平台调度在集群中时会使用计算平台提供的API或环境变量获取启动的参数。 -```python -HOSTS = [ - "root@192.168.100.17", - "root@192.168.100.18", - ] - -''' -工作空间配置 -''' - -#工作空间根目录 -ROOT_DIR = "/home/paddle" - -''' -网络配置 -''' -#pserver NIC -PADDLE_NIC = "eth0" -#pserver 端口 -PADDLE_PORT = 7164 -#pserver 端口数 -PADDLE_PORTS_NUM = 2 -#pserver sparse ports num -PADDLE_PORTS_NUM_FOR_SPARSE = 2 - -#集群作业中所有进程的环境设置 -LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/lib64" -``` +关于如何在Kubernetes上启动分布式训练任务，可以参考这篇说明：[Kubernetes分布式训练](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/usage/k8s/k8s_distributed_cn.md) +## 使用Fabric启动集群作业 ### 启动集群作业 +PaddlePaddle可以运行在不同的集群平台之上，包括Kubernetes, OpenMPI或使用fabric管理的Linux集群。对于不同的集群平台，会分别介绍集群作业的启动和停止方法。这些例子都可以在[这个路径下](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2)找到。 + `paddle.py` 提供了自动化脚本来启动不同节点中的所有 PaddlePaddle 集群进程。默认情况下，所有命令行选项可以设置为```paddle.py``` 命令选项并且 `paddle.py` 将透明、自动地将这些选项应用到 PaddlePaddle 底层进程。 `paddle.py` 为方便作业启动提供了两个独特的命令选项。 @@ -162,3 +248,6 @@ sh run.sh ### 检查模型输出 运行完成后，模型文件将被写入节点 0 的 `output` 目录中。 工作空间中的 `nodefile` 表示当前集群作业的节点 ID。 + +## 在OpenMPI集群中提交训练作业(TODO) +## 在Kubernetes集群中提交训练作业(TODO) diff --git a/paddle/scripts/cluster_train_v2/fabric/conf.py b/paddle/scripts/cluster_train_v2/fabric/conf.py new file mode 100644 index 0000000000000..f85b20a52b12f --- /dev/null +++ b/paddle/scripts/cluster_train_v2/fabric/conf.py @@ -0,0 +1,39 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HOSTS = [ + "root@10.1.9.7", + "root@10.1.18.7", + "root@10.1.32.9", +] +''' +workspace configuration +''' +#root dir for workspace, can be set as any director with real user account +ROOT_DIR = "/root" +''' +network configuration +''' +#pserver nics +PADDLE_NIC = "eth0" +#pserver port +PADDLE_PORT = 7164 +#pserver ports num +PADDLE_PORTS_NUM = 1 +#pserver sparse ports num +PADDLE_PORTS_NUM_FOR_SPARSE = 1 +#trainer whether use gpu +PADDLE_USE_GPU="False" +#environments setting for all processes in cluster job +LD_LIBRARY_PATH = "/usr/local/cuda/lib64:/usr/lib64" diff --git a/paddle/scripts/cluster_train_v2/fabric/docker_cluster/Dockerfile b/paddle/scripts/cluster_train_v2/fabric/docker_cluster/Dockerfile new file mode 100644 index 0000000000000..6606c01265af1 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/fabric/docker_cluster/Dockerfile @@ -0,0 +1,11 @@ +FROM docker.paddlepaddlehub.com/paddle:0.10.0rc2 +RUN apt-get update && apt-get install -y openssh-server +RUN mkdir /var/run/sshd + +RUN echo 'root:root' |chpasswd + +RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config +RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config + +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/paddle/scripts/cluster_train_v2/fabric/docker_cluster/ssh_servers.yaml b/paddle/scripts/cluster_train_v2/fabric/docker_cluster/ssh_servers.yaml new file mode 100644 index 0000000000000..0784b2d1b8785 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/fabric/docker_cluster/ssh_servers.yaml @@ -0,0 +1,23 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ssh-servers +spec: + replicas: 3 + template: + metadata: + labels: + app: ssh-servers + spec: + containers: + - name: ssh-servers + image: docker.paddlepaddlehub.com/paddlessh + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi + ports: + - containerPort: 22 diff --git a/paddle/scripts/cluster_train_v2/fabric/run.sh b/paddle/scripts/cluster_train_v2/fabric/run.sh new file mode 100644 index 0000000000000..f6324bcb13680 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/fabric/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +python paddle.py \ + --job_dispatch_package="/root/wuyi/fabric_submit/workspace" \ + --dot_period=10 \ + --ports_num_for_sparse=1 \ + --log_period=50 \ + --num_passes=5 \ + --trainer_count=2 \ + --saving_period=1 \ + --local=0 \ + --config=./trainer_config.py \ + --save_dir=./output \ + --use_gpu=0 From 02889735837334349b92cf002a5b967a32eef33b Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Tue, 9 May 2017 20:17:27 +0800 Subject: [PATCH 03/13] update --- demo/word2vec/api_train_v2_cluster.py | 20 ++++++++++---------- doc/howto/usage/cluster/cluster_train_cn.md | 21 ++++++++++++++++----- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/demo/word2vec/api_train_v2_cluster.py b/demo/word2vec/api_train_v2_cluster.py index b1f523e03fa8e..6eaaf488c3723 100644 --- a/demo/word2vec/api_train_v2_cluster.py +++ b/demo/word2vec/api_train_v2_cluster.py @@ -37,21 +37,21 @@ def main(): # for local training TRUTH = ["true", "True", "TRUE", "1", "yes", "Yes", "YES"] - cluster_train = os.getenv('PADDLE_CLUSTER_TRAIN', "False") in TRUTH - use_gpu = os.getenv('PADDLE_USE_GPU', "False") + cluster_train = os.getenv('PADDLE_INIT_CLUSTER_TRAIN', "False") in TRUTH + use_gpu = os.getenv('PADDLE_INIT_USE_GPU', "False") if not cluster_train: - paddle.init(use_gpu=use_gpu, trainer_count=int(os.getenv("PADDLE_TRAINER_COUNT", "1"))) + paddle.init(use_gpu=use_gpu, trainer_count=int(os.getenv("PADDLE_INIT_TRAINER_COUNT", "1"))) else: paddle.init( use_gpu=use_gpu, - trainer_count=int(os.getenv("PADDLE_TRAINER_COUNT", "1")), - port=int(os.getenv("PADDLE_PORT", "7164")), - ports_num=int(os.getenv("PADDLE_PORTS_NUM", "1")), - ports_num_for_sparse=int(os.getenv("PADDLE_PORTS_NUM_FOR_SPARSE", "1")), - num_gradient_servers=int(os.getenv("PADDLE_NUM_GRADIENT_SERVERS", "1")), - trainer_id=int(os.getenv("PADDLE_TRAINER_ID", "0")), - pservers=os.getenv("PADDLE_PSERVERS", "127.0.0.1")) + trainer_count=int(os.getenv("PADDLE_INIT_TRAINER_COUNT", "1")), + port=int(os.getenv("PADDLE_INIT_PORT", "7164")), + ports_num=int(os.getenv("PADDLE_INIT_PORTS_NUM", "1")), + ports_num_for_sparse=int(os.getenv("PADDLE_INIT_PORTS_NUM_FOR_SPARSE", "1")), + num_gradient_servers=int(os.getenv("PADDLE_INIT_NUM_GRADIENT_SERVERS", "1")), + trainer_id=int(os.getenv("PADDLE_INIT_TRAINER_ID", "0")), + pservers=os.getenv("PADDLE_INIT_PSERVERS", "127.0.0.1")) #word_dict = paddle.dataset.imikolov.build_dict() fn = open("thirdparty/wuyi_train_thdpty/word_dict.pickle", "r") word_dict = pickle.load(fn) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index a019dfd4c1625..156bebe711c65 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -62,8 +62,8 @@ $ stdbuf -oL /usr/bin/nohup paddle pserver --port=7164 --ports_num=1 --ports_num ``` | 参数 | 是否必选 | 默认值 | 说明 | -| ------------- | ------------- | ------------- | -| port | 必选 | 7164 | pserver监听的起始端口，根据ports_num决定总端口个数，从起始端口监听多个端口用于通信 | +| ------------- | ------------- | ------------- | ------------- | +| port | 必选 | 7164 | pserver监听的起始端口，根据ports_num决定
总端口个数，从起始端口监听多个端口用于通信 | | ports_num | 必选 | 1 | 监听的端口个数 | | ports_num_for_sparse | 必选 | 1 | 用于稀疏类型参数通信的端口个数 | | num_gradient_servers | 必选 | 1 | 当前训练任务pserver总数 | @@ -103,8 +103,8 @@ paddle.init( pservers="127.0.0.1") ``` -| 参数 | 是否必选 | 默认值 | 说明 | -| ------------- | ------------- | ------------- | +| 参数 | 是否必选 | 默认 | 说明 | +| ------------- | ------------- | ------------- | ------------- | | use_gpu | 可选 | False | 是否启用GPU训练 | | trainer_count | 必选 | 1 | 当前训练任务trainer总个数 | | port | 必选 | 7164 | 连接到pserver的端口 | @@ -195,6 +195,15 @@ test.txt-00001 - `mylib.py`：会被`train.py`调用的一些库函数 - `word_dict.pickle`：在`train.py`中会使用到的字典数据文件 - `train.py`：训练程序，代码参考[这里](../../../../demo/word2vec/api_train_v2_cluster.py) + - ***注意：***对于本样例代码，在使用不同的分布式计算平台时，您可能需要修改`train.py`开头的部分（如下），以便获得训练数据的位置和获取环境变量配置： + ```python + cluster_train_file = "./train_data_dir/train/train.txt" + cluster_test_file = "./test_data_dir/test/test.txt" + node_id = os.getenv("OMPI_COMM_WORLD_RANK") + if not node_id: + raise EnvironmentError("must provied OMPI_COMM_WORLD_RANK") + + ``` - `train_data_dir`：此目录可以是从分布式存储挂载过来包含训练数据的目录，也可以是在任务启动前下载到本地的。 - `test_data_dir`：包含测试数据集，同样可以是挂载或下载生成。 @@ -210,7 +219,9 @@ PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务 关于如何在Kubernetes上启动分布式训练任务，可以参考这篇说明：[Kubernetes分布式训练](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/usage/k8s/k8s_distributed_cn.md) ## 使用Fabric启动集群作业 + ### 启动集群作业 + PaddlePaddle可以运行在不同的集群平台之上，包括Kubernetes, OpenMPI或使用fabric管理的Linux集群。对于不同的集群平台，会分别介绍集群作业的启动和停止方法。这些例子都可以在[这个路径下](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2)找到。 `paddle.py` 提供了自动化脚本来启动不同节点中的所有 PaddlePaddle 集群进程。默认情况下，所有命令行选项可以设置为```paddle.py``` 命令选项并且 `paddle.py` 将透明、自动地将这些选项应用到 PaddlePaddle 底层进程。 @@ -220,7 +231,7 @@ PaddlePaddle可以运行在不同的集群平台之上，包括Kubernetes, OpenM `job_dispatch_package` 设为本地 `workspace` 目录，它将被分发到 conf.py 中设置的所有节点。 它有助于帮助频繁修改和访问工作区文件的用户减少负担，否则频繁的多节点工作空间部署可能会很麻烦。 `job_workspace` 设为已部署的工作空间目录，`paddle.py` 将跳过分发阶段直接启动所有节点的集群作业。它可以帮助减少分发延迟。 -`cluster_train/run.sh` 提供了命令样例来运行 `demo/recommendation` 集群工作，只需用你定义的目录修改 `job_dispatch_package` 和 `job_workspace`，然后： +`cluster_train/run.sh` 提供了命令样例来运行 `demo/word2vec` 集群工作，只需用你定义的目录修改 `job_dispatch_package` 和 `job_workspace`，然后： ``` sh run.sh ``` From 7614c217d0433d58216532eeb64dd923994bf7cf Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Thu, 11 May 2017 17:17:49 +0800 Subject: [PATCH 04/13] cluster train doc --- demo/recommendation/api_train_v2.py | 8 +- demo/word2vec/api_train_v2_cluster.py | 18 +- demo/word2vec/prepare.py | 13 +- doc/howto/usage/cluster/cluster_train_cn.md | 154 +++++---- doc/howto/usage/cluster/cluster_train_en.md | 299 ++++++++++++------ .../scripts/cluster_train_v2/fabric/conf.py | 2 +- .../openmpi/docker_cluster/Dockerfile | 43 +++ .../openmpi/docker_cluster/head.yaml | 25 ++ .../openmpi/docker_cluster/mpi-nodes.yaml | 26 ++ .../openmpi/docker_cluster/ssh/config | 1 + .../openmpi/docker_cluster/ssh/id_rsa.mpi | 27 ++ .../openmpi/docker_cluster/ssh/id_rsa.mpi.pub | 1 + .../openmpi/start_mpi_train.sh | 28 ++ 13 files changed, 471 insertions(+), 174 deletions(-) create mode 100644 paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile create mode 100644 paddle/scripts/cluster_train_v2/openmpi/docker_cluster/head.yaml create mode 100644 paddle/scripts/cluster_train_v2/openmpi/docker_cluster/mpi-nodes.yaml create mode 100644 paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/config create mode 100644 paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi create mode 100644 paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi.pub create mode 100644 paddle/scripts/cluster_train_v2/openmpi/start_mpi_train.sh diff --git a/demo/recommendation/api_train_v2.py b/demo/recommendation/api_train_v2.py index f6a061799e3ac..afbfd316a592c 100644 --- a/demo/recommendation/api_train_v2.py +++ b/demo/recommendation/api_train_v2.py @@ -68,10 +68,10 @@ def main(): parameters = paddle.parameters.create(cost) - trainer = paddle.trainer.SGD(cost=cost, - parameters=parameters, - update_equation=paddle.optimizer.Adam( - learning_rate=1e-4)) + trainer = paddle.trainer.SGD( + cost=cost, + parameters=parameters, + update_equation=paddle.optimizer.Adam(learning_rate=1e-4)) feeding = { 'user_id': 0, 'gender_id': 1, diff --git a/demo/word2vec/api_train_v2_cluster.py b/demo/word2vec/api_train_v2_cluster.py index 6eaaf488c3723..57f7e6e6c9913 100644 --- a/demo/word2vec/api_train_v2_cluster.py +++ b/demo/word2vec/api_train_v2_cluster.py @@ -12,6 +12,7 @@ if not node_id: raise EnvironmentError("must provied OMPI_COMM_WORLD_RANK") + def wordemb(inlayer): wordemb = paddle.layer.embedding( input=inlayer, @@ -24,32 +25,39 @@ def wordemb(inlayer): sparse_update=True)) return wordemb + def cluster_reader_cluster(filename, node_id): def cluster_reader(): - with open("-".join([filename, "%05d"%int(node_id)]), "r") as f: + with open("-".join([filename, "%05d" % int(node_id)]), "r") as f: for l in f: csv_data = [int(cell) for cell in l.split(",")] yield tuple(csv_data) + return cluster_reader + def main(): # get arguments from env # for local training TRUTH = ["true", "True", "TRUE", "1", "yes", "Yes", "YES"] - cluster_train = os.getenv('PADDLE_INIT_CLUSTER_TRAIN', "False") in TRUTH + cluster_train = os.getenv('PADDLE_CLUSTER_TRAIN', "False") in TRUTH use_gpu = os.getenv('PADDLE_INIT_USE_GPU', "False") if not cluster_train: - paddle.init(use_gpu=use_gpu, trainer_count=int(os.getenv("PADDLE_INIT_TRAINER_COUNT", "1"))) + paddle.init( + use_gpu=use_gpu, + trainer_count=int(os.getenv("PADDLE_INIT_TRAINER_COUNT", "1"))) else: paddle.init( use_gpu=use_gpu, trainer_count=int(os.getenv("PADDLE_INIT_TRAINER_COUNT", "1")), port=int(os.getenv("PADDLE_INIT_PORT", "7164")), ports_num=int(os.getenv("PADDLE_INIT_PORTS_NUM", "1")), - ports_num_for_sparse=int(os.getenv("PADDLE_INIT_PORTS_NUM_FOR_SPARSE", "1")), - num_gradient_servers=int(os.getenv("PADDLE_INIT_NUM_GRADIENT_SERVERS", "1")), + ports_num_for_sparse=int( + os.getenv("PADDLE_INIT_PORTS_NUM_FOR_SPARSE", "1")), + num_gradient_servers=int( + os.getenv("PADDLE_INIT_NUM_GRADIENT_SERVERS", "1")), trainer_id=int(os.getenv("PADDLE_INIT_TRAINER_ID", "0")), pservers=os.getenv("PADDLE_INIT_PSERVERS", "127.0.0.1")) #word_dict = paddle.dataset.imikolov.build_dict() diff --git a/demo/word2vec/prepare.py b/demo/word2vec/prepare.py index 14109266cb78c..24f5c5b26d37e 100644 --- a/demo/word2vec/prepare.py +++ b/demo/word2vec/prepare.py @@ -3,14 +3,16 @@ import os import pickle -SPLIT_COUNT = 2 +SPLIT_COUNT = 3 N = 5 + def file_len(fd): for i, l in enumerate(fd): pass return i + 1 + def split_from_reader_by_line(filename, reader, split_count): fn = open(filename, "w") for batch_id, batch_data in enumerate(reader()): @@ -26,13 +28,14 @@ def split_from_reader_by_line(filename, reader, split_count): cmd = "split -d -a 5 -l %d %s %s-" % (per_file_lines, filename, filename) os.system(cmd) + word_dict = paddle.dataset.imikolov.build_dict() with open("word_dict.pickle", "w") as dict_f: pickle.dump(word_dict, dict_f) split_from_reader_by_line("train.txt", - paddle.dataset.imikolov.train(word_dict, N), - SPLIT_COUNT) + paddle.dataset.imikolov.train(word_dict, N), + SPLIT_COUNT) split_from_reader_by_line("test.txt", - paddle.dataset.imikolov.test(word_dict, N), - SPLIT_COUNT) + paddle.dataset.imikolov.test(word_dict, N), + SPLIT_COUNT) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index 156bebe711c65..b1ff118f68a2a 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -1,34 +1,34 @@ -```eval_rst -.. _cluster_train: -``` * [概述](#概述) * [环境准备](#环境准备) * [启动参数说明](#启动参数说明) - * [启动parameter server](#启动parameter-server) - * [启动trainer](#启动trainer) - * [准备数据集](#准备数据集) - * [准备训练程序](#准备训练程序) + * [启动parameter server](#启动parameter-server) + * [启动trainer](#启动trainer) + * [准备数据集](#准备数据集) + * [准备训练程序](#准备训练程序) * [使用分布式计算平台或工具](#使用分布式计算平台或工具) - * [使用Fabric启动集群作业](#使用fabric启动集群作业) - * [启动集群作业](#启动集群作业) - * [终止集群作业](#终止集群作业) - * [检查集群训练结果](#检查集群训练结果) - * [检查模型输出](#检查模型输出) - * [在OpenMPI集群中提交训练作业(TODO)](#在openmpi集群中提交训练作业todo) - * [在Kubernetes集群中提交训练作业(TODO)](#在kubernetes集群中提交训练作业todo) + * [使用Fabric启动集群作业](#使用fabric启动集群作业) + * [使用kubernetes启动一个测试集群（可选）](#使用kubernetes启动一个测试集群可选) + * [启动集群作业](#启动集群作业) + * [终止集群作业](#终止集群作业) + * [检查集群训练结果](#检查集群训练结果) + * [检查模型输出](#检查模型输出) + * [在OpenMPI集群中提交训练作业](#在openmpi集群中提交训练作业) + * [使用kubernetes启动一个测试openmpi集群（可选）](#使用kubernetes启动一个测试openmpi集群可选) + * [启动集群作业](#启动集群作业-1) + * [在Kubernetes集群中提交训练作业](#在kubernetes集群中提交训练作业) # 概述 -PaddlePaddle 使用如下图的架构完成分布式训练： +本文将介绍如何使用PaddlePaddle在不同的集群框架下完成分布式训练。分布式训练架构如下图所示： -- data shard（数据分片）: 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用 -- trainer（计算节点）: 每个trainer启动后读取切分好的一部分数据，并开始神经网络的“前馈”和“后馈”计算，并和parameter server通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients）然后下载优化更新后的神经网络参数（parameters）。 +- Data shard（数据分片）: 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用 +- Trainer（计算节点）: 每个trainer启动后读取切分好的一部分数据，并开始神经网络的“前馈”和“后馈”计算，并和parameter server通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients）然后下载优化更新后的神经网络参数（parameters）。 - parameter server（参数服务器）:每个参数服务器只保存整个神经网络所有参数的一部分。参数服务器接收从计算节点上传的梯度，并完成参数优化更新，在将更新后的参数下发到每个计算节点。 这样，通过trainer和parameter server的分布式协作，可以完成神经网络的SGD方法的训练。Paddle可以同时支持同步SGD(synchronize SGD)和异步SGD(asynchronize SGD)。 -在使用同步SGD训练神经网络时，Paddle使用同步屏障(barrier)，使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大的提高了计算的并行性：parameter server之间不相互依赖，并行的接收梯度和更新参数，parameter server也不会等待trainer全部都提交梯度之后才开始下一步，trainer之间也不会相互依赖，并行的执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台parameter server上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 +在使用同步SGD训练神经网络时，PaddlePaddle使用同步屏障(barrier)，使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大的提高了计算的并行性：parameter server之间不相互依赖，并行的接收梯度和更新参数，parameter server也不会等待trainer全部都提交梯度之后才开始下一步，trainer之间也不会相互依赖，并行的执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台parameter server上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 # 环境准备 @@ -74,10 +74,10 @@ $ stdbuf -oL /usr/bin/nohup paddle pserver --port=7164 --ports_num=1 --ports_num $ python train.py ``` -trainer需要和pserver保持网络联通以完成训练。trainer启动需要传入端口、pserver地址等参数使trainer可以正确连接到pserver。这些参数可以通过*环境变量*或编写程序时`paddle.init()`中传入参数。如果同时使用`paddle.init()`参数和环境变量，将会优先使用`paddle.init()`中传入的参数。 +trainer需要和pserver保持网络联通以完成训练。trainer启动需要传入端口、pserver地址等参数使trainer可以正确连接到pserver。这些参数可以通过[环境变量](https://zh.wikipedia.org/wiki/环境变量)或编写程序时`paddle.init()`中传入参数。如果同时使用`paddle.init()`参数和环境变量，将会优先使用`paddle.init()`中传入的参数。 -PaddlePaddle在集群训练时会使用[环境变量](https://zh.wikipedia.org/wiki/环境变量)获取集群训练所需要的参数配置。这些环境变量都是以`PADDLE_INIT_`开头，以防止和其他的环境变量重名。 使用环境变量： + ```bash export PADDLE_INIT_USE_GPU=False export PADDLE_INIT_TRAINER_COUNT=1 @@ -91,6 +91,7 @@ python train.py ``` 使用参数： + ```python paddle.init( use_gpu=False, @@ -117,57 +118,30 @@ paddle.init( ## 准备数据集 -参考样例数据准备脚本[prepare.py](../../../../demo/word2vec/prepare.py)，准备训练数据和验证数据集，我们从paddle自带的公开数据集中下载训练数据，并根据分布式训练并发数（计算节点个数），指定`SPLIT_COUNT`将数据切分成多份： +参考样例数据准备脚本[prepare.py](../../../../demo/word2vec/prepare.py)，准备训练数据和验证数据集，我们使用`paddle dataset`中下载公开的`imikolov`数据集，并根据分布式训练并发数（trainer节点个数），指定`SPLIT_COUNT`将数据切分成多份。 + +在线上系统中，通常会使用`MapReduce`任务的输出结果作为训练结果，这样训练文件的个数会比较多，而且个数并不确定。在trainer中可以使用下面取模的方法为每个trainer分配训练数据文件： ```python -import paddle.v2 as paddle -import tarfile import os -import pickle - -SPLIT_COUNT = 2 -N = 5 - -def file_len(fd): - for i, l in enumerate(fd): - pass - return i + 1 - -def split_from_reader_by_line(filename, reader, split_count): - fn = open(filename, "w") - for batch_id, batch_data in enumerate(reader()): - batch_data_str = [str(d) for d in batch_data] - fn.write(",".join(batch_data_str)) - fn.write("\n") - fn.close() - - fn = open(filename, "r") - total_line_count = file_len(fn) - fn.close() - per_file_lines = total_line_count / split_count + 1 - cmd = "split -d -a 5 -l %d %s %s-" % (per_file_lines, filename, filename) - os.system(cmd) - -word_dict = paddle.dataset.imikolov.build_dict() -with open("word_dict.pickle", "w") as dict_f: - pickle.dump(word_dict, dict_f) - -split_from_reader_by_line("train.txt", - paddle.dataset.imikolov.train(word_dict, N), - SPLIT_COUNT) -split_from_reader_by_line("test.txt", - paddle.dataset.imikolov.test(word_dict, N), - SPLIT_COUNT) +train_list = [] +flist = os.listdir("/train_data/") +for f in flist: + suffix = int(f.split("-")[1]) + if suffix % TRAINER_COUNT == TRAINER_ID: + train_list.append(f) ``` -上面的这段程序，会把imikolov dataset的训练集和测试集分别分割成多个文件（例子中为2个）: +示例程序`prepare.py`会把训练集和测试集分别分割成多个文件（例子中为3个，后缀为`-00000`、`-00001`和`-00002`）: ``` train.txt train.txt-00000 train.txt-00001 +train.txt-00002 test.txt test.txt-00000 test.txt-00001 +test.txt-00002 ``` 在进行分布式训练时，每个trainer进程需要能够读取属于自己的一份数据。在一些分布式系统中，系统会提供一个分布式存储服务，这样保存在分布式存储中的数据可以被集群中的每个节点读取到。如果不使用分布式存储，则需要手动拷贝属于每个trainer节点的训练数据到对应的节点上。 @@ -187,9 +161,11 @@ test.txt-00001 |-- train_data_dir/ | |-- train.txt-00000 | |-- train.txt-00001 +| |-- train.txt-00002 `-- test_data_dir/ |-- test.txt-00000 - `-- test.txt-00001 + |-- test.txt-00001 + `-- test.txt-00002 ``` - `mylib.py`：会被`train.py`调用的一些库函数 @@ -214,15 +190,16 @@ PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务 - [OpenMPI](https://www.open-mpi.org) - [Fabric](http://www.fabfile.org) 集群管理工具。可以使用`Fabric`编写集群任务提交和管理脚本。 -在使用分布式计算平台进行训练时，任务被平台调度在集群中时会使用计算平台提供的API或环境变量获取启动的参数。 +对于不同的集群平台，会分别介绍集群作业的启动和停止方法。这些例子都可以在[这个路径下](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2)找到。 -关于如何在Kubernetes上启动分布式训练任务，可以参考这篇说明：[Kubernetes分布式训练](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/usage/k8s/k8s_distributed_cn.md) +在使用分布式计算平台进行训练时，任务被平台调度在集群中时会使用计算平台提供的API或环境变量获取启动的参数。 ## 使用Fabric启动集群作业 -### 启动集群作业 +### 使用Kubernetes准备一个集群（可选） +可以在`paddle/scripts/cluster_train_v2/fabric/docker_cluster`目录下，执行`kubectl -f ssh_servers.yaml`启动一个测试集群，并使用`kubectl get po -o wide`获得这些节点的IP地址。 -PaddlePaddle可以运行在不同的集群平台之上，包括Kubernetes, OpenMPI或使用fabric管理的Linux集群。对于不同的集群平台，会分别介绍集群作业的启动和停止方法。这些例子都可以在[这个路径下](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2)找到。 +### 启动集群作业 `paddle.py` 提供了自动化脚本来启动不同节点中的所有 PaddlePaddle 集群进程。默认情况下，所有命令行选项可以设置为```paddle.py``` 命令选项并且 `paddle.py` 将透明、自动地将这些选项应用到 PaddlePaddle 底层进程。 @@ -251,7 +228,7 @@ sh run.sh 提供 pserver 运行日志，有助于诊断分布式错误。 `server.log` -提供 pserver 进程的 stderr 和 stdout。训练失败时可以检查错误日志。 +提供 parameter server 进程的 stderr 和 stdout。训练失败时可以检查错误日志。 `train.log` 提供训练过程的 stderr 和 stdout。训练失败时可以检查错误日志。 @@ -260,5 +237,48 @@ sh run.sh 运行完成后，模型文件将被写入节点 0 的 `output` 目录中。 工作空间中的 `nodefile` 表示当前集群作业的节点 ID。 -## 在OpenMPI集群中提交训练作业(TODO) -## 在Kubernetes集群中提交训练作业(TODO) +## 在OpenMPI集群中提交训练作业 + +### 使用kubernetes启动一个测试openmpi集群（可选） + +执行下面的命令以启动3个节点的openmpi集群和一个"head"节点： + +```bash +paddle/scripts/cluster_train_v2/openmpi/docker_cluster +kubectl create -f head.yaml +kubectl create -f mpi-nodes.yaml +``` + +然后可以从head节点ssh无密码登录到OpenMPI的每个节点上。 + +### 启动集群作业 + +您可以按照下面的步骤在OpenMPI集群中提交paddle训练任务： + +```bash +# 获得head和node节点的IP地址 +kubectl get po -o wide +# 将node节点的IP地址保存到machines文件中 +kubectl get po -o wide | grep nodes | awk '{print $6}' > machines +# 拷贝必要的文件到head节点 +scp -i ssh/id_rsa.mpi.pub machines prepare.py train.py start_mpi_train.sh tutorial@[headIP]:~ +# ssh 登录到head节点 +ssh -i ssh/id_rsa.mpi.pub tutorial@[headIP] +# --------------- 以下操作均在head节点中执行 --------------- +# 准备训练数据 +python prepare.py +# 拷贝训练程序和字典文件到每台MPI节点 +cat machines | xargs -i scp word_dict.pickle train.py start_mpi_train.sh machines {}:/home/tutorial +# 创建日志目录 +mpirun -hostfile machines -n 3 mkdir /home/tutorial/logs +# 拷贝训练数据到各自的节点 +scp train.txt-00000 test.txt-00000 [node1IP]:/home/tutorial +scp train.txt-00001 test.txt-00001 [node2IP]:/home/tutorial +scp train.txt-00002 test.txt-00002 [node3IP]:/home/tutorial +# 启动训练任务 +mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh +``` + +## 在Kubernetes集群中提交训练作业 + +此部分的使用方法可以参考[这里](../k8s/k8s_cn.md) diff --git a/doc/howto/usage/cluster/cluster_train_en.md b/doc/howto/usage/cluster/cluster_train_en.md index c60876721cbf5..9a599a78b4ad6 100644 --- a/doc/howto/usage/cluster/cluster_train_en.md +++ b/doc/howto/usage/cluster/cluster_train_en.md @@ -1,128 +1,197 @@ -# Run Distributed Training +# Introduction -In this article, we explain how to run distributed Paddle training jobs on clusters. We will create the distributed version of the single-process training example, [recommendation](https://github.com/baidu/Paddle/tree/develop/demo/recommendation). +In this article, we'll explain how to do distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the mail architecture of a distributed trainning job: -[Scripts](https://github.com/baidu/Paddle/tree/develop/paddle/scripts/cluster_train) used in this article launch distributed jobs via SSH. They also work as a reference for users running more sophisticated cluster management systems like MPI and [Kubernetes](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/k8s). + -## Prerequisite +- Data shard: training data will be split into multiple parts, trainers use some parts of the whole dataset to do the training job. +- Trainer: each trainer reads the data shard, and do Neural Network training algorithms like "forward" and "backward". Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training. +- Parameter server: every parameter server stores part of the whole Neural Network model data. They will do optimization calculations when gradients are uploaded from trainers, and then send updated parameters to trainers. -1. Aforementioned scripts use a Python library [fabric](http://www.fabfile.org/) to run SSH commands. We can use `pip` to install fabric: +Trainers and Parameter servers coordinate form the distributed SGD(stochastic gradient descent) training algorithm. Meanwhile, PaddlePaddle can support both synchronize SGD and asynchronous SGD. - ```bash - pip install fabric - ``` +When training with synchronize SGD, PaddlePaddle uses an internal "synchronize barrier" which makes gradients update and parameter download in strict order. On the other hand, asynchronous SGD won't wait for all trainers to finish upload at a single step, this will increase the parallelism of distributed training: parameter servers do not depend on each other, they'll do parameter optimization concurrently. Parameter servers will not wait for trainers, so trainers will also do their work concurrently. But asynchronous SGD will introduce more randomness and noises. -1. We need to install PaddlePaddle on all nodes in the cluster. To enable GPUs, we need to install CUDA in `/usr/local/cuda`; otherwise Paddle would report errors at runtime. +# Preparations +1. Ready your computer cluster. They are normally a bunch of Linux servers connected by LAN. Each server will be assigned a unique IP address. The computer in the cluster can be called "nodes". Different hardware setup may cause some issue when using different cluster management platforms. +2. Install PaddlePaddle on every node. If you are going to take advantage of GPU cards, you'll also need to install proper driver and CUDA libraries. To install PaddlePaddle try to read [this](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install) document. We strongly recommend using [Docker installation](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst). -1. Set the `ROOT_DIR` variable in [`cluster_train/conf.py`] on all nodes. For convenience, we often create a Unix user `paddle` on all nodes and set `ROOT_DIR=/home/paddle`. In this way, we can write public SSH keys into `/home/paddle/.ssh/authorized_keys` so that user `paddle` can SSH to all nodes without password. +After installation, you can check the version by typing the below command(run into a docker container if using docker: `docker run -it paddlepaddle/paddle:0.10.0rc3 /bin/bash`): -## Prepare Job Workspace +```bash +$ paddle version +PaddlePaddle 0.10.0rc, compiled with + with_avx: ON + with_gpu: OFF + with_double: OFF + with_python: ON + with_rdma: OFF + with_timer: OFF +``` -We refer to the directory where we put dependent libraries, config files, etc., as *workspace*. +We'll take `demo/word2vec` as an example to introduce distributed training using PaddlePaddle v2 API. -These `train/test` data should be prepared before launching cluster job. To satisfy the requirement that train/test data are placed in different directory from workspace, PADDLE refers train/test data according to index file named as `train.list/test.list` which are used in model config file. So the train/test data also contains train.list/test.list two list file. All local training demo already provides scripts to help you create these two files, and all nodes in cluster job will handle files with same logical code in normal condition. +# Command-line arguments -Generally, you can use same model file from local training for cluster training. What you should have in mind that, the `batch_size` set in `setting` function in model file means batch size in `each` node of cluster job instead of total batch size if synchronization SGD was used. +## Starting parameter server -Following steps are based on [demo/recommendation](https://github.com/PaddlePaddle/Paddle/tree/develop/demo/recommendation) demo in demo directory. +Type the below command to start a parameter server which will wait for trainers to connect: -You just go through demo/recommendation tutorial doc until `Train` section, and at last you will get train/test data and model configuration file. Finaly, just use demo/recommendation as workspace for cluster training. +```bash +$ paddle pserver --port=7164 --ports_num=1 --ports_num_for_sparse=1 --num_gradient_servers=1 +``` -At last your workspace should look like as follow: +If you wish to run parameter servers in background, and save a log file, you can type: +```bash +$ stdbuf -oL /usr/bin/nohup paddle pserver --port=7164 --ports_num=1 --ports_num_for_sparse=1 --num_gradient_servers=1 &> pserver.log ``` -. -|-- common_utils.py -|-- data -| |-- config.json -| |-- config_generator.py -| |-- meta.bin -| |-- meta_config.json -| |-- meta_generator.py -| |-- ml-1m -| |-- ml_data.sh -| |-- ratings.dat.test -| |-- ratings.dat.train -| |-- split.py -| |-- test.list -| `-- train.list -|-- dataprovider.py -|-- evaluate.sh -|-- prediction.py -|-- preprocess.sh -|-- requirements.txt -|-- run.sh -`-- trainer_config.py + +| param | required | default | description | +| ------------- | ------------- | ------------- | ------------- | +| port | required | 7164 | port which parameter server will listen on. If ports_num greater than 1, parameter server will listen on multiple ports for more network throughput | +| ports_num | required | 1 | total number of ports will listen on | +| ports_num_for_sparse | required | 1 | number of ports which serves sparse parameter update | +| num_gradient_servers | required | 1 | total number of gradient servers | + +## Starting trainer +Type the command below to start the trainer(name the file whatever you want, like "train.py") + +```bash +$ python train.py ``` -Not all of these files are needed for cluster training, but it's not necessary to remove useless files. -`trainer_config.py` -Indicates the model config file. +Trainers network need to be connected with parameter servers to finish the job. Trainers need to know port and IPs to locate parameter servers. You can pass arguments to trainers through [environment variables](https://en.wikipedia.org/wiki/Environment_variable) or pass to `paddle.init()` function. Arguments passed to the `paddle.init()` function will overwrite environment variables. -`train.list` and `test.list` -File index. It stores all relative or absolute file paths of all train/test data at current node. +Use environment viriables: -`dataprovider.py` -used to read train/test samples. It's same as local training. +```bash +export PADDLE_INIT_USE_GPU=False +export PADDLE_INIT_TRAINER_COUNT=1 +export PADDLE_INIT_PORT=7164 +export PADDLE_INIT_PORTS_NUM=1 +export PADDLE_INIT_PORTS_NUM_FOR_SPARSE=1 +export PADDLE_INIT_NUM_GRADIENT_SERVERS=1 +export PADDLE_INIT_TRAINER_ID=0 +export PADDLE_INIT_PSERVERS=127.0.0.1 +python train.py +``` -`data` -all files in data directory are refered by train.list/test.list which are refered by data provider. +Pass arguments: +```python +paddle.init( + use_gpu=False, + trainer_count=1, + port=7164, + ports_num=1, + ports_num_for_sparse=1, + num_gradient_servers=1, + trainer_id=0, + pservers="127.0.0.1") +``` -## Prepare Cluster Job Configuration +| param | required | default | description | +| ------------- | ------------- | ------------- | ------------- | +| use_gpu | optional | False | set to "True" to enable GPU training | +| trainer_count | required | 1 | total count of trainers in the training job | +| port | required | 7164 | port to connect to parameter server | +| ports_num | required | 1 | number of ports for communication | +| ports_num_for_sparse | required | 1 | number of ports for sparse type caculation | +| num_gradient_servers | required | 1 | total number of gradient server | +| trainer_id | required | 0 | ID for every trainer, start from 0 | +| pservers | required | 127.0.0.1 | list of IPs of parameter servers, separated by "," | -The options below must be carefully set in cluster_train/conf.py +## Prepare Training Dataset -`HOSTS` all nodes hostname or ip that will run cluster job. You can also append user and ssh port with hostname, such as root@192.168.100.17:9090. +Here's some example code [prepare.py](../../../../demo/word2vec/prepare.py), it will download public `imikolov` dataset and split it into multiple files according to job parallelism(trainers count). Modify `SPLIT_COUNT` to change the count of output files. -`ROOT_DIR` workspace ROOT directory for placing JOB workspace directory +In the real world, we often use `MapReduce` job's output as training data, so there will be lots of files. You can use `mod` to assign training file to trainers: -`PADDLE_NIC` the NIC(Network Interface Card) interface name for cluster communication channel, such as eth0 for ethternet, ib0 for infiniband. +```python +import os +train_list = [] +flist = os.listdir("/train_data/") +for f in flist: + suffix = int(f.split("-")[1]) + if suffix % TRAINER_COUNT == TRAINER_ID: + train_list.append(f) +``` -`PADDLE_PORT` port number for cluster commnunication channel +Example code `prepare.py` will split training data and testing data into 3 files with digital suffix like `-00000`, `-00001` and`-00002`: -`PADDLE_PORTS_NUM` the number of port used for cluster communication channle. if the number of cluster nodes is small(less than 5~6nodes), recommend you set it to larger, such as 2 ~ 8, for better network performance. +``` +train.txt +train.txt-00000 +train.txt-00001 +train.txt-00002 +test.txt +test.txt-00000 +test.txt-00001 +test.txt-00002 +``` -`PADDLE_PORTS_NUM_FOR_SPARSE` the number of port used for sparse updater cluster commnunication channel. if sparse remote update is used, set it like `PADDLE_PORTS_NUM` +When job started, every trainer needs to get it's own part of data. In some distributed systems they will provide a storage service, so the date under that path can be accessed by all the trainer nodes. Without storage service, you must copy the training data to each trainer node. -`LD_LIBRARY_PATH` set addtional LD_LIBRARY_PATH for cluster job. You can use it to set CUDA libraries path. +Different training jobs may have different data format and `reader()` function, developers may need to write different data prepare scripts and `reader()` functions for their job. -Default Configuration as follow: +## Prepare Training program -```python -HOSTS = [ - "root@192.168.100.17", - "root@192.168.100.18", - ] - -''' -workspace configuration -''' - -#root dir for workspace -ROOT_DIR = "/home/paddle" - -''' -network configuration -''' -#pserver nics -PADDLE_NIC = "eth0" -#pserver port -PADDLE_PORT = 7164 -#pserver ports num -PADDLE_PORTS_NUM = 2 -#pserver sparse ports num -PADDLE_PORTS_NUM_FOR_SPARSE = 2 - -#environments setting for all processes in cluster job -LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/lib64" +We'll create a *workspace* directory on each node, storing your training program, dependencies, mounted or downloaded dataset directory. + + +Your workspace may looks like: +``` +. +|-- my_lib.py +|-- word_dict.pickle +|-- train.py +|-- train_data_dir/ +| |-- train.txt-00000 +| |-- train.txt-00001 +| |-- train.txt-00002 +`-- test_data_dir/ + |-- test.txt-00000 + |-- test.txt-00001 + `-- test.txt-00002 ``` +- `mylib.py`: some library functions. This is optional. +- `word_dict.pickle`: dict file for training word embeding. +- `train.py`: training program. [Sample code here](../../../../demo/word2vec/api_train_v2_cluster.py). + - ***NOTE:*** You may need to modify the head part of `train.py` when using different cluster platform to retrive configuration environment variables: + ```python + cluster_train_file = "./train_data_dir/train/train.txt" + cluster_test_file = "./test_data_dir/test/test.txt" + node_id = os.getenv("OMPI_COMM_WORLD_RANK") + if not node_id: + raise EnvironmentError("must provied OMPI_COMM_WORLD_RANK") + ``` +- `train_data_dir`: containing training data. Mount from storage service or copy trainning data to here. +- `test_data_dir`: containing testing data. + +# Use cluster platforms or cluster management tools + +PaddlePaddle supports running jobs on several platforms including: +- [Kubernetes](http://kubernetes.io) +- [OpenMPI](https://www.open-mpi.org) +- [Fabric](http://www.fabfile.org) A cluster management tool. Write scripts to submit jobs or manage the cluster. + +We'll introduce cluster job management on these platforms. The examples can be found under [here](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2). + +These cluster platforms provide API or runtime environment variables for training jobs. + +## Cluster Training Using Fabric + +### Prepare a demo cluster using kubernetes(optional) + +Run `kubectl -f ssh_servers.yaml` under the directory: `paddle/scripts/cluster_train_v2/fabric/docker_cluster` will launch a demo cluster. Run `kubectl get po -o wide` to get IP addresses of these nodes. + ### Launching Cluster Job `paddle.py` provides automatical scripts to start all PaddlePaddle cluster processes in different nodes. By default, all command line options can set as `paddle.py` command options and `paddle.py` will transparently and automatically set these options to PaddlePaddle lower level processes. `paddle.py`provides two distinguished command option for easy job launching. -`job_dispatch_package` set it with local `workspace`directory, it will be dispatched to all nodes set in conf.py. It could be helpful for frequent hacking workspace files, otherwise frequent mulit-nodes workspace deployment could make your crazy. +`job_dispatch_package` set it with local `workspace`directory, it will be dispatched to all nodes set in conf.py. It could be helpful for frequent hacking workspace files, otherwise, frequent multi-nodes workspace deployment could make your crazy. `job_workspace` set it with already deployed workspace directory, `paddle.py` will skip dispatch stage to directly launch cluster job with all nodes. It could help to reduce heavy dispatch latency. @@ -134,23 +203,69 @@ sh run.sh The cluster Job will start in several seconds. ### Kill Cluster Job -`paddle.py` can capture `Ctrl + C` SIGINT signal to automatically kill all processes launched by it. So just stop `paddle.py` to kill cluster job. You should mannally kill job if program crashed. +`paddle.py` can capture `Ctrl + C` SIGINT signal to automatically kill all processes launched by it. So just stop `paddle.py` to kill cluster job. You should manually kill the job if the program crashed. ### Check Cluster Training Result Check log in $workspace/log for details, each node owns same log structure. `paddle_trainer.INFO` -It provides almost all interal output log for training, same as local training. Check runtime model convergence here. +It provides almost all internal output log for training, same as local training. Check runtime model convergence here. `paddle_pserver2.INFO` -It provides pserver running log, which could help to diagnose distributed error. +It provides parameter server running log, which could help to diagnose distributed error. `server.log` -It provides stderr and stdout of pserver process. Check error log if training crashs. +It provides stderr and stdout of parameter server process. Check error log if training crashes. `train.log` -It provides stderr and stdout of trainer process. Check error log if training crashs. +It provides stderr and stdout of trainer process. Check error log if training crashes. ### Check Model Output -After one pass finished, model files will be writed in `output` directory in node 0. +After one pass finished, model files will be written in `output` directory in node 0. `nodefile` in workspace indicates the node id of current cluster job. + +## Cluster Training Using OpenMPI + +### Prepare a demo OpenMPI cluster using kubernetes(optional) + +Run the following command to start a 3-node MPI cluster and one "head" node. + +```bash +cd paddle/scripts/cluster_train_v2/openmpi/docker_cluster +kubectl create -f head.yaml +kubectl create -f mpi-nodes.yaml +``` + +Then you can log in to every OpenMPI node using ssh without input any passwords. + +### Launching Cluster Job + +Follow the steps to launch a PaddlePaddle training job in OpenMPI cluster:\ + +```bash +# find out node IP addresses +kubectl get po -o wide +# generate a "machines" file containing node IP addresses +kubectl get po -o wide | grep nodes | awk '{print $6}' > machines +# copy necessary files onto "head" node +scp -i ssh/id_rsa.mpi.pub machines prepare.py train.py start_mpi_train.sh tutorial@[headIP]:~ +# login to head node using ssh +ssh -i ssh/id_rsa.mpi.pub tutorial@[headIP] +# --------------- in head node --------------- +# prepare training data +python prepare.py +# copy training data and dict file to MPI nodes +cat machines | xargs -i scp word_dict.pickle train.py start_mpi_train.sh machines {}:/home/tutorial +# creat a directory for storing log files +mpirun -hostfile machines -n 3 mkdir /home/tutorial/logs +# copy training data to every node +scp train.txt-00000 test.txt-00000 [node1IP]:/home/tutorial +scp train.txt-00001 test.txt-00001 [node2IP]:/home/tutorial +scp train.txt-00002 test.txt-00002 [node3IP]:/home/tutorial +# start the job +mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh +``` + +## Cluster Training Using Kubernetes + +The details can be found [here](../k8s/k8s_cn.md) diff --git a/paddle/scripts/cluster_train_v2/fabric/conf.py b/paddle/scripts/cluster_train_v2/fabric/conf.py index f85b20a52b12f..e96503d093a43 100644 --- a/paddle/scripts/cluster_train_v2/fabric/conf.py +++ b/paddle/scripts/cluster_train_v2/fabric/conf.py @@ -34,6 +34,6 @@ #pserver sparse ports num PADDLE_PORTS_NUM_FOR_SPARSE = 1 #trainer whether use gpu -PADDLE_USE_GPU="False" +PADDLE_USE_GPU = "False" #environments setting for all processes in cluster job LD_LIBRARY_PATH = "/usr/local/cuda/lib64:/usr/lib64" diff --git a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile new file mode 100644 index 0000000000000..1a2d19e823541 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/Dockerfile @@ -0,0 +1,43 @@ +# Build this image: docker build -t mpi . +# + +FROM paddledev/paddle:0.10.0rc3 + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y openssh-server zip unzip vim sudo \ +gcc gfortran openmpi-checkpoint binutils wget curl git openmpi-bin openmpi-common libopenmpi-dev && \ +pip install mpi4py numpy virtualenv scipy matplotlib lxml sqlalchemy suds ipython obspy && \ +mkdir /var/run/sshd && \ +echo 'root:tutorial' | chpasswd && \ +sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ +# SSH login fix. Otherwise user is kicked off after login +sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \ +echo "export VISIBLE=now" >> /etc/profile && \ +adduser --disabled-password --gecos "" tutorial && \ +echo "tutorial ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \ +mkdir /home/tutorial/.ssh/ + +ENV HOME /home/tutorial +ENV NOTVISIBLE "in users profile" + +# ------------------------------------------------------------ +# Set-Up SSH with our Github deploy key +# ------------------------------------------------------------ + +ADD ssh/config /home/tutorial/.ssh/config +ADD ssh/id_rsa.mpi /home/tutorial/.ssh/id_rsa +ADD ssh/id_rsa.mpi.pub /home/tutorial/.ssh/id_rsa.pub +ADD ssh/id_rsa.mpi.pub /home/tutorial/.ssh/authorized_keys + +#--------------------------------------------------------------- +#LD_LIBRARY_PATH +#--------------------------------------------------------------- + +RUN export LD_LIBRARY_PATH=/usr/lib/openmpi/lib/ + +WORKDIR /home/tutorial +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/head.yaml b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/head.yaml new file mode 100644 index 0000000000000..34835e5eb8d7c --- /dev/null +++ b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/head.yaml @@ -0,0 +1,25 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mpi-header + labels: + app: mpi-header +spec: + replicas: 1 + template: + metadata: + labels: + app: mpi-header + spec: + containers: + - image: typhoon1986/paddle-openmpi + name : mpi-header + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + ports: + - containerPort: 22 diff --git a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/mpi-nodes.yaml b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/mpi-nodes.yaml new file mode 100644 index 0000000000000..2fd5cb4d44a25 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/mpi-nodes.yaml @@ -0,0 +1,26 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mpi-nodes + labels: + app: mpi-nodes +spec: + replicas: 3 + template: + metadata: + labels: + app: mpi-nodes + spec: + containers: + - image: typhoon1986/paddle-openmpi + name : mpi-nodes + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + ports: + - containerPort: 22 + imagePullPolicy: Always diff --git a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/config b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/config new file mode 100644 index 0000000000000..a9ecad07c39e4 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/config @@ -0,0 +1 @@ +StrictHostKeyChecking no diff --git a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi new file mode 100644 index 0000000000000..23768343edf52 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA7PWLZmgdJ508dD15T6+xqGDvL9Ehzo9SgsnN6xJ+qpUvvOi4 +1axW0AqR4MnPTg/uuvk+x4tUpuufOW4w22UTGjsdvmIVWa9ujLtcRiN3YPY+SU+Y +O5FfqKg7r/hBn+/GMcSoffwSs7vVgmhBBnp/mJh2O1cOAFZEe98/47mbg3/kHBAk +36NOQktaU3l48B38EhBTnjWfcEGm1HcTRPFxXV5Wiko6ZhKFEuHcTVKng4ROtUqE +mgHyI0aB7TAxg4na0ejItsYWEPWGeDOw6ms/4MwylxNosWzHFPW9p4zgLCLNr+b6 +bDDfYKjXZflAuTQtQhLmJUwD9uuYLAijpSE2fQIDAQABAoIBADgcgRET8Gt0CV/B +OtvKz/f+VEVvcWD3gWNlJDTZIVOFllNWjIZUlA4ZoqenQkbK8Q4nfV1FOht4yjCQ +TlN1oMtiWk297i5Zo4UBzPzy4w774I39oh/g8dT/WXr2/5s+7SDV38xNh6Q2A34o +79T35wUcfUrZ93/O7dKjb/6d8hx2FMha0wVKqY4lmG1lQE3bbx3kakec0PdvU5kO +YHKlpqj3pMR7CpMa+4yL/iXFwWYmnK+uu+zw7JR7PwvH1CzrnvW438wjQ1QmYbSx +mHHOE89X67Lsl5hn81qYWBhpwAlBwi1qscsE0cV9GcFyKqWFqZsj5coM9u3CRfvy +lrWe1OUCgYEA+LBUFEd3Hxs4sFiYElJ8R9SAs1udaqPvAl01hTEijJLfYlMMVs/y +rgNN7j22zjDak2f8QdyMJZX7EZdRmdYcHO0csYOwbYvalzcnwk+U3mxmdD3r4xSo +DSvkJ70fogAqUlcVIg2re6fCmZVJQTvMQYTVEM8zQomJRt/Lb2esSfsCgYEA8+zv +44aToe8uqiDs4w8guRW7LCDkTw4z4IVo9JUibIaPjaAs5bZEBXSB43EEywXCR75H +fML0rU1PVvKh1rqcvZdVzm+XMWVr3asPk0sapaiHaTcmyZvJRDxxqbLFp0zRP1T6 +cCtXNFdHWU4KiuKrUi6cDyOKchpfkSZa4seiT+cCgYB+n4FgBfdQPlMB70oW4irn +g/q32CjxuGCk6oKqu5bkzo+xB6obtavSEFqouIGQwO056tNVUY+GP7Rjg5GH663K +yKw4cl3tmS0Gm43B8TVSfw03mKO3rrfWZQe5eCFYIg9qd26KNT2gK435FzsCXQkm +PxUhhu6JrW/ZR2/U3Iur6wKBgADrWLAb1ryagSuE+j+U1AO+kDkHWrTtkcZ72jxp +v3p3O11GSEUJXdJDcSXhTCpTuDq6/dv7hB6PFwh126RKicKxKlKf2wsFndV1Cpb8 +hnovW2tLGOtTmfuW2rrQAKyzvmolsNfxYd/BoHQ2thV16z1hDZeFA8WQUeHjKh6G +sBbrAoGATdtQlaUxx4izua6k02ihkxx/cRYwDl2N8UDvDBHokS7vJFMX8b8NpsGg +zMElnqSpu/pe/0UG7N2MtPF6uyMcX8AZzzcsRkiMkDvWJzYt8Jpf+Eyd/uryF+Yv +yrXaOEY83tm6x/fny5ZaZmk8lNth7bfWywuTMkZLX3fYpWtIeE4= +-----END RSA PRIVATE KEY----- diff --git a/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi.pub b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi.pub new file mode 100644 index 0000000000000..015f2b42e7192 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/openmpi/docker_cluster/ssh/id_rsa.mpi.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDs9YtmaB0nnTx0PXlPr7GoYO8v0SHOj1KCyc3rEn6qlS+86LjVrFbQCpHgyc9OD+66+T7Hi1Sm6585bjDbZRMaOx2+YhVZr26Mu1xGI3dg9j5JT5g7kV+oqDuv+EGf78YxxKh9/BKzu9WCaEEGen+YmHY7Vw4AVkR73z/juZuDf+QcECTfo05CS1pTeXjwHfwSEFOeNZ9wQabUdxNE8XFdXlaKSjpmEoUS4dxNUqeDhE61SoSaAfIjRoHtMDGDidrR6Mi2xhYQ9YZ4M7Dqaz/gzDKXE2ixbMcU9b2njOAsIs2v5vpsMN9gqNdl+UC5NC1CEuYlTAP265gsCKOlITZ9 oweidner@peahi diff --git a/paddle/scripts/cluster_train_v2/openmpi/start_mpi_train.sh b/paddle/scripts/cluster_train_v2/openmpi/start_mpi_train.sh new file mode 100644 index 0000000000000..c645495448f98 --- /dev/null +++ b/paddle/scripts/cluster_train_v2/openmpi/start_mpi_train.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# General trainning configurations + +NICS=eth0 +PADDLE_INIT_PORT=7164 +PADDLE_INIT_PORTS_NUM=1 +PADDLE_INIT_PORTS_NUM_FOR_SPARSE=1 +PADDLE_INIT_PSERVERS=$(cat machines | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/,/g') +PADDLE_INIT_USE_GPU=False + +PADDLE_INIT_NUM_GRADIENT_SERVERS=${OMPI_COMM_WORLD_SIZE} +PADDLE_INIT_TRAINER_ID=${OMPI_COMM_WORLD_RANK} +PADDLE_CLUSTER_TRAIN=True + +env + +# start pserver +stdbuf -oL nohup paddle pserver --port=$PADDLE_INIT_PORT --ports_num=$PADDLE_INIT_PORTS_NUM \ + --ports_num_for_sparse=$PADDLE_INIT_PORTS_NUM_FOR_SPARSE --nics=$NICS \ + --comment=paddle_cluster_pserver \ + --num_gradient_servers=$PADDLE_INIT_NUM_GRADIENT_SERVERS &> logs/pserver.log & + +# start trainer +# NOTE: train.py will use the above environment variables as configuration +python train.py &> logs/train.log + +# kill background pservers when train finishes +ps -ef | grep pserver | awk '{print $2}' | xargs kill From c03642832204c95911322574a0d9ee407681b0a8 Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Thu, 11 May 2017 17:21:00 +0800 Subject: [PATCH 05/13] add TOC for en doc --- doc/howto/usage/cluster/cluster_train_en.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/howto/usage/cluster/cluster_train_en.md b/doc/howto/usage/cluster/cluster_train_en.md index 9a599a78b4ad6..af520c83dba77 100644 --- a/doc/howto/usage/cluster/cluster_train_en.md +++ b/doc/howto/usage/cluster/cluster_train_en.md @@ -1,3 +1,21 @@ +* [Introduction](#introduction) +* [Preparations](#preparations) +* [Command-line arguments](#command-line-arguments) + * [Starting parameter server](#starting-parameter-server) + * [Starting trainer](#starting-trainer) + * [Prepare Training Dataset](#prepare-training-dataset) + * [Prepare Training program](#prepare-training-program) +* [Use cluster platforms or cluster management tools](#use-cluster-platforms-or-cluster-management-tools) + * [Cluster Training Using Fabric](#cluster-training-using-fabric) + * [Prepare a demo cluster using kubernetes(optional)](#prepare-a-demo-cluster-using-kubernetesoptional) + * [Launching Cluster Job](#launching-cluster-job) + * [Kill Cluster Job](#kill-cluster-job) + * [Check Cluster Training Result](#check-cluster-training-result) + * [Check Model Output](#check-model-output) + * [Cluster Training Using OpenMPI](#cluster-training-using-openmpi) + * [Prepare a demo OpenMPI cluster using kubernetes(optional)](#prepare-a-demo-openmpi-cluster-using-kubernetesoptional) + * [Launching Cluster Job](#launching-cluster-job-1) + * [Cluster Training Using Kubernetes](#cluster-training-using-kubernetes) # Introduction In this article, we'll explain how to do distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the mail architecture of a distributed trainning job: From c7ced327bfa6a8ad7a819badde4666c9db9d29ec Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Thu, 11 May 2017 19:27:54 +0800 Subject: [PATCH 06/13] fix sphix build issue --- doc/howto/usage/cluster/cluster_train_cn.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index b1ff118f68a2a..a6c4f870b3394 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -33,7 +33,7 @@ # 环境准备 1. 准备您的计算集群。计算集群通常由一组（几台到几千台规模）的Linux服务器组成。服务器之间可以通过局域网（LAN）联通，每台服务器具有集群中唯一的IP地址（或者可被DNS解析的主机名）。集群中的每台计算机通常被成为一个“节点”。使用不同的集群管理平台时，会要求集群节点是否是同样硬件配置。 -1. 我们需要在集群的所有节点上安装 PaddlePaddle。 如果要启用GPU，还需要在节点上安装对应的GPU驱动以及CUDA。PaddlePaddle的安装可以参考[这里](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install)的多种安装方式。我们推荐使用[Docker安装方式](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)来快速安装PaddlePaddle。 +1. 我们需要在集群的所有节点上安装 PaddlePaddle。 如果要启用GPU，还需要在节点上安装对应的GPU驱动以及CUDA。PaddlePaddle的安装可以参考[build_and_install](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install)的多种安装方式。我们推荐使用[Docker](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)安装方式来快速安装PaddlePaddle。 安装完成之后，执行下面的命令可以查看已经安装的版本（docker安装方式可以进入docker容器执行：`docker run -it paddlepaddle/paddle:0.10.0rc3 /bin/bash`）： ```bash @@ -74,7 +74,7 @@ $ stdbuf -oL /usr/bin/nohup paddle pserver --port=7164 --ports_num=1 --ports_num $ python train.py ``` -trainer需要和pserver保持网络联通以完成训练。trainer启动需要传入端口、pserver地址等参数使trainer可以正确连接到pserver。这些参数可以通过[环境变量](https://zh.wikipedia.org/wiki/环境变量)或编写程序时`paddle.init()`中传入参数。如果同时使用`paddle.init()`参数和环境变量，将会优先使用`paddle.init()`中传入的参数。 +trainer需要和pserver保持网络联通以完成训练。trainer启动需要传入端口、pserver地址等参数使trainer可以正确连接到pserver。这些参数可以通过环境变量(https://zh.wikipedia.org/wiki/环境变量)或编写程序时`paddle.init()`中传入参数。如果同时使用`paddle.init()`参数和环境变量，将会优先使用`paddle.init()`中传入的参数。 使用环境变量： @@ -170,7 +170,7 @@ test.txt-00002 - `mylib.py`：会被`train.py`调用的一些库函数 - `word_dict.pickle`：在`train.py`中会使用到的字典数据文件 -- `train.py`：训练程序，代码参考[这里](../../../../demo/word2vec/api_train_v2_cluster.py) +- `train.py`：训练程序，代码参考[api_train_v2_cluster.py](../../../../demo/word2vec/api_train_v2_cluster.py) - ***注意：***对于本样例代码，在使用不同的分布式计算平台时，您可能需要修改`train.py`开头的部分（如下），以便获得训练数据的位置和获取环境变量配置： ```python cluster_train_file = "./train_data_dir/train/train.txt" @@ -190,7 +190,7 @@ PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务 - [OpenMPI](https://www.open-mpi.org) - [Fabric](http://www.fabfile.org) 集群管理工具。可以使用`Fabric`编写集群任务提交和管理脚本。 -对于不同的集群平台，会分别介绍集群作业的启动和停止方法。这些例子都可以在[这个路径下](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2)找到。 +对于不同的集群平台，会分别介绍集群作业的启动和停止方法。这些例子都可以在[cluster_train_v2](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2)找到。 在使用分布式计算平台进行训练时，任务被平台调度在集群中时会使用计算平台提供的API或环境变量获取启动的参数。 @@ -281,4 +281,4 @@ mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh ## 在Kubernetes集群中提交训练作业 -此部分的使用方法可以参考[这里](../k8s/k8s_cn.md) +此部分的使用方法可以参考[k8s_cn.md](../k8s/k8s_cn.md) From 82d829096159905acfdc15a05d24ee1aae06b5d7 Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Thu, 11 May 2017 20:16:22 +0800 Subject: [PATCH 07/13] fix error links --- doc/howto/usage/cluster/cluster_train_cn.md | 6 +++--- doc/howto/usage/cluster/cluster_train_en.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index a6c4f870b3394..03246d8f37382 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -20,7 +20,7 @@ # 概述 本文将介绍如何使用PaddlePaddle在不同的集群框架下完成分布式训练。分布式训练架构如下图所示： - + - Data shard（数据分片）: 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用 - Trainer（计算节点）: 每个trainer启动后读取切分好的一部分数据，并开始神经网络的“前馈”和“后馈”计算，并和parameter server通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients）然后下载优化更新后的神经网络参数（parameters）。 @@ -118,7 +118,7 @@ paddle.init( ## 准备数据集 -参考样例数据准备脚本[prepare.py](../../../../demo/word2vec/prepare.py)，准备训练数据和验证数据集，我们使用`paddle dataset`中下载公开的`imikolov`数据集，并根据分布式训练并发数（trainer节点个数），指定`SPLIT_COUNT`将数据切分成多份。 +参考样例数据准备脚本"prepare.py"（位于`demo/word2vec/prepare.py`），准备训练数据和验证数据集，我们使用`paddle dataset`中下载公开的`imikolov`数据集，并根据分布式训练并发数（trainer节点个数），指定`SPLIT_COUNT`将数据切分成多份。 在线上系统中，通常会使用`MapReduce`任务的输出结果作为训练结果，这样训练文件的个数会比较多，而且个数并不确定。在trainer中可以使用下面取模的方法为每个trainer分配训练数据文件： @@ -170,7 +170,7 @@ test.txt-00002 - `mylib.py`：会被`train.py`调用的一些库函数 - `word_dict.pickle`：在`train.py`中会使用到的字典数据文件 -- `train.py`：训练程序，代码参考[api_train_v2_cluster.py](../../../../demo/word2vec/api_train_v2_cluster.py) +- `train.py`：训练程序，代码参考"api_train_v2_cluster.py"（位于`demo/word2vec/api_train_v2_cluster.py`） - ***注意：***对于本样例代码，在使用不同的分布式计算平台时，您可能需要修改`train.py`开头的部分（如下），以便获得训练数据的位置和获取环境变量配置： ```python cluster_train_file = "./train_data_dir/train/train.txt" diff --git a/doc/howto/usage/cluster/cluster_train_en.md b/doc/howto/usage/cluster/cluster_train_en.md index af520c83dba77..5cf2252650d58 100644 --- a/doc/howto/usage/cluster/cluster_train_en.md +++ b/doc/howto/usage/cluster/cluster_train_en.md @@ -20,7 +20,7 @@ In this article, we'll explain how to do distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the mail architecture of a distributed trainning job: - + - Data shard: training data will be split into multiple parts, trainers use some parts of the whole dataset to do the training job. - Trainer: each trainer reads the data shard, and do Neural Network training algorithms like "forward" and "backward". Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training. @@ -121,7 +121,7 @@ paddle.init( ## Prepare Training Dataset -Here's some example code [prepare.py](../../../../demo/word2vec/prepare.py), it will download public `imikolov` dataset and split it into multiple files according to job parallelism(trainers count). Modify `SPLIT_COUNT` to change the count of output files. +Here's some example code “prepare.py”(located in `demo/word2vec/prepare.py`), it will download public `imikolov` dataset and split it into multiple files according to job parallelism(trainers count). Modify `SPLIT_COUNT` to change the count of output files. In the real world, we often use `MapReduce` job's output as training data, so there will be lots of files. You can use `mod` to assign training file to trainers: @@ -175,7 +175,7 @@ Your workspace may looks like: - `mylib.py`: some library functions. This is optional. - `word_dict.pickle`: dict file for training word embeding. -- `train.py`: training program. [Sample code here](../../../../demo/word2vec/api_train_v2_cluster.py). +- `train.py`: training program. Sample code: "api_train_v2_cluster.py"(located in `demo/word2vec/api_train_v2_cluster.py`). - ***NOTE:*** You may need to modify the head part of `train.py` when using different cluster platform to retrive configuration environment variables: ```python cluster_train_file = "./train_data_dir/train/train.txt" From 810bd66041d6901717f28c0e9d5fb0b26525d777 Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Fri, 12 May 2017 09:21:13 +0800 Subject: [PATCH 08/13] fix link errors --- doc/howto/usage/cluster/src/trainer.png | Bin 0 -> 145107 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/howto/usage/cluster/src/trainer.png diff --git a/doc/howto/usage/cluster/src/trainer.png b/doc/howto/usage/cluster/src/trainer.png new file mode 100644 index 0000000000000000000000000000000000000000..6537d3d56589ca9f19a77a50a970e4b5275e6ce0 GIT binary patch literal 145107 zcmeFZbzD_V7d9-Vgea|aNJ~kBlyrA@cXK2KJSZS2f*{@9B@GgS0t!fXiFD^7j>J0$ z@B8t7;{Egc>wEnv9`>F+duGjAYp%K0S|?0ZSq2-E1oOs?8`yHPlIk~Z+>=naOmp2r5PU{lW~ZULZUS)-rqNYWrIK)Q zx1!=<<7VTa5y7OQq7rhqv=&sCl>X~+@S8A=Ed=5!$jK!BZt zlbw^36|`XW@NtHid9yluJow$o-~C8hd04pHxkBt*oT(7~nwh&ig$UEoAYSyZKflKb zv9td3CTEYorUfR*j=01Ah>e5&Uwwl|g%DQ-C0v|b-K{)4K>s2jH5BQgFAkkaLDuxr2uxW{A;=aQ^l5|GmY3_NC%(X9Z?`{q&>1p8n^xzuF72BPRY2 zLHw5T^;IBd5lkWWe=V5^X2l}s?2Q{@H{>LrXnG@U&HA-z4qtZdn|xz6dM_KigRa() zA}=Y!bmz-E0y(yZ993r~=0<)wbOz?7so;}?*{@|n)<6x5*NV)l6cMz?E+&A~U7fAYU>BPRDhN00jd zCimZR`Tz0C$%_HQ5uo2Y=wa_q5#Z_)l=8aP!z`A8YP;dFQUjTW;A-r@MfU0~Q1LZG zEd3xcdsKq3rrnlb>KgZIej6W7I=Fe7!WjIwIiBAg*7U*uFUv&Sj-p1A?zBQr8vb*a zmbq;TD=b`%0`W2`$>&cA#a$<2<$%2nm@ieqtPT>>eD-oKIn?S6`7$xfjq z5(qPezCr(Zb@>bH9$RmBAwS07 zz9Q-cuu1p5tRXs7A>{DSvros3(iC}dukBY>`N#3J3&Z_*GIE)JxgI>zBxaZYn>T6e z{&%+kZN~yrSimks0yF5wM;g=Wj=@Ten5DfRAE6V$`{3_6_#h#d5)X1C4qD;K~noalAeGd$s>@8T3SqM|D4*+E)3+|GfQ)e-7pEH~d=zOWEh9hW~GJ|F*RM zZ{_}5RsKI#OI*U!jpB0sTF0}`_$rds{@hr9d%)i=zz}HZPirzLLU*GApfwKTwJ!4k zm*-~=Vo9ld&4TCPP2_Xq-X&b zFde11H~Kw+zqaepYnip=AA9Y5+$?LNS4y&cIAfFU?w3ikHr zM|8`Y8$B$Ru;uK1#Cxn!#bXSBL$gUfWYoH%1D1Wy0E3G+3W?%p5bOwx5LLue2#O&!ZN*5&2rM zo3wn?EL6Wm=K8$B60c0FIBsqgdh4dOb(6GdzPXjPkxhBiP9xv^GmY#gC$nBA7hL1T zko6%1u-yUMssArerTH-vio>aTw`^zkkzyT}^V9vxMimjT#KzC9h&Wxo6N7yS>PP_m z>-BCCMUK1+^y0^-7lhR69x#g`Zhf^kV4GvgF#Tnk#mi;${o~1;U$J)SRFl^(5a~p- z8@=b&MCOh8y@if@EV@4^vYLkN+p79Sel5kQ%z5++ha1-WS>zICk#Jge_prB?Am1W~ zL@^zMr8f?|fTjB%CkorckJfivuUfKbD~ptzid!y@r_DM-gn>GcJkFblv+MMl_a86T zVGl-kDt?bs?pvCXDI^m?lAFG{V?3IXX;Z9IUYn?{{=UnRnU=YLS*vKGd7l=7d+QEa zLB{jOtvHhg_gr(2Z)%ErNe*QV>v_rID=7iz`(5!yo{7KaIJDLe*0&e59freS8vCy2 zdJd<{%5}`xYry7QH*d>5j5~@9JeRkQF+m~$SUQx^Lt#zCPYSijD2UAaw!(FgR^XaU;$_%Aw1ghKzg}XGkFGJq{RNWje z*ICmhD3bZ)Y%oV3UQx9WNaToxg5O2v@uqBRC|BVrJ9WkLzL8d@3l!0_r5L9>7yzLJ z0b7z>U>>-Muw^*F6($+T|I`);xZ2dwlaTJ#hw%L9e(K)HE);5lcq3mzsN<5Vt37K2H0p?z#S zemI;-Xdk?JswzPnqsXK+fW*l5P#D7}?dh1_1DUs)^EC?8$qwbs+V7B69(ifFH3R4I zhC*2IlG>fU<@9F+-&}!rc;V`+2XE^SzC9#VAIuQ{atdj@Md2+b;Z2~W)~2Z=k3u-h zm2_0^w&;7jb=E~0ca5qilXalWbgj~E@6Ruu2%glkLaXyf|S7_!%CrB!Ds z^d<9oagp-a=j>HpCOo^kIB{-~dsyc6U10U}@O5n!bs*8X5^RaFz9gQlVCUhQ%UfqV zZ%kW_D{+^WGwX{?jc3EH@hKyiwuxx7;*95~Gb6P{ZBs||BlDIKg6tKR^jK@d@__?c zV?_j*YubsIlh$82v2dUHapIIn* z9_0l5Q{g9#v;&e=_mxB&=7gtvpKkei`tXjI=w<5}S#QM#ut(f~^z#kZ_X;a+6IiqY z+u#l2m~13|<`R?j%d?{0kB4uO?MakY?hYWVdZIqcjE2Kgt$GE! zg~Y=zo88Mj)6Xp(G2*D|Kd5%pH;mR7KWOB8k)wl?W6|}N1l8uw;5d~LGWiF`NzQ)i zprQL>XoCdndyui#!ivc5MRgqq%_s-H42l)+d1{gpgmWqF3vb(S=Vk;7yshu61+1#lP@%B=~v4 zwcnXsiLNq2#;$BW%`Zohk(cQvJKLrBA5Q|T4^gl$DZ3o9kIG1v6I4P@G_8*YWm6`j1MnZW$^DwD5HbWo6Gd%G6PH$Rr}!NVd)=Jc@F|c(W|tGUvhK16yN-m)IzPKIA8%9?B%)2%xMdFSc5&GvxEs$v?r z&*jrMJvv|W@;tzC6RnpOu2AVZ{DRU<4?z-@XvPd zKDU{xZ91AhSnPlHGmOExVVS7zBhd%VrKvuq=<3UEA^2MMP9ZE`Y|(f&MKkyHl0=`8 z^^;bHzIoy+PAq3ip0x*IU|ql7ukQ>h)k|mpT3VSAs0z2q^*wiBVbrKAkJh!IZqW7_ z)8#0|G_?3M#@_2ardzQbE4xjCRcq}(MFbwi?9#W+v>VRzti&?NmwU-xh2xObTuS>r zWb#11zK4ZiWqV&t{q| zVW9PGrA1d~(hQEbtb1bOd7FG$Fj;IJ*d`8+hlcl42pOa2S*10gco6~W&&5QC&R;YL zLe?@KX;j{PV|yb=y2ZOw+6XczRT(gAm(wf9kZLK@T!@r}TR2&mwpsM!ZOd-sm!lqS zBo>uN7i2?Xw6o=;f{=o^?1Qb*6p}BPuvZu{<@ii9g$v%{n09UPzdHD9O4S!(qk5%7 z?2n{ugi41d82!U4AMtpVza7dZ6-r zuINO%WbEdOgZ76@>S-f|IF>M;J*6jtMvsF_30-QIXC+a2B#V$tuHfUXGoJReC3Drz zZy8Dla>9HczRGxh3cW?K#+1>A?m0H!{nGu&MIJ?AR)Qc~#~Yy)5crJECLigYvg{!8omThcT?Q*6tVI!LXR z$?JQ?hg>3cXZM%78hV4?4zZ5v&fiu_RU6nl`rcZ&R~d)2K2jJnn>#8Bt5cX~y_gTx zl1^WTYj3q`Py~PPg!h;>5v2|A$wh3A3y18PTbI8&S{6-<4rYq^dbgy#I>zEq^|@V< zkN4m@C-OSpfR4?q!>EEA*~JZ46^~D7_txSSyImpv$D5tmmzxy1QJ;hF85dyAD#mC# zIln3eX!}w&MSkNAkHZ*5fw^s|T&a$jr(ThL_LKJKZ>$+>(Kp{Vc-8VW9ln#HP$Nd` z5fYYnD%LzJJ9`yspU9z7%24zZFJQ`8Z$3*^ zRXJ0}$o_g!;j8VA2XmE$A`=^1Q)>Zd?n&Waq@U~RlU7ohaiaAtUQc#Z(yxZK#w8N@ z3U?Bj3HW>(lBCegX<4Yn)r0y@aa?0xCcD$xh|lYOB_icdue7Lj{P(MNagqm6lDL~Y zQccXg-hsmys%KBA@{kN5ZYrJ!QWUrgxjhVw;b!lDD^lfG^X9!$6GmT?8=nLPQvrXf zfqG;ngiqM6&trdykz`r==S=e@g;j~x%09ZYg^-=fp0eLIijy$icAhyIuM-|zWczDn za+1xGh_3dT{!0H2wxYLqXp(X{UV$kYiEHB~?tT{%M z+-4~pd`OazXGKK7IkrtUyE<^21v~x{90961Upyx2BrD4-X}q#bE63B#yvgpMTdB9L ziFU+6)OTwXyY5cYWRCUp9)s{1($~B!qV=1t^q~rOm(yNmyD&$$MTxpoFNKF;I z@p>1Y{wpoXMO~gfnT|jR2DhQ?F3~LWvkk=t64{&VIQ9a zLf5_WszVR@3McCG{$sqF5(@vO}FO(bE zhYmy(%Gvl*r2yOCh!HoFc#ba%(;6~5AJyLM{KcK4tU%M+n@P@3qV)_9en(wvkj1IP}3e(DTM**rI zJNn#)^*z3!GQu8ElRn-&4u)`}9las=qR0JXobtvfC7lxEaOgYHjZ&%G1R4XDD*X(v zk){3NM;Kd$+FS!3jz^(`J5G}ya+RiE{^Wt%Rwc`OJ0B+)k?fX8@xo9hL^neD718~ypC-s<~OtUSybk*Xe7$c&|~wboJI_J=|_-Bs~QsO|>9NjLcWuahH7J@}EX zPiXVFdZb59hJ5)>!_Ymii-3U`U11(WaD>IYVHak)$t<;13rYVg+~1RNaBgRsT0AvPES*onfX4P zpdR0mhfQo)Yldy2+B315;iV0hRP+c}#tG zTzA0FV6ZV~et{|7wmdFpWBZ+IzWRq=ac-UuRuREjjYrmIw!%KWRU?JJF4fxf`>h9Y zbdt}`R?{5_N+WuvpW&e|y#PUjXZ7EXaG{QP11{8j_w(}{c*8~S=aar^q$|^b`YUkX zUnPF8X?o!dm93E<7RSfzEPN2&h4xkg@~fcQJR^+^92yKM9;bF`w6zxM79kg2p0_0` zU7AA$?3@z5>n5B?o4&BH0z^}o??7gg&OX1Ar3RxsU&`WbPZ>jrMHB8h$*NvQJ{~r? zMZI(AQ#4N=EX6O}Xo~&)2<-9s@g{F92ZlqQ3~?acCr=GN?-Eov2qW~ME=;s zv;G@=SD&Q~hPcqV1IKjtSiJRrbcK)Uy518a%m^{+(%<8zJDlX%Qgb8r%XQ4+;ENfW zs>lMk^zQ2lNoVmMmbFS+C|@E?u^D*ALQdb}Nz6Bt`MY^liFy{)V}6`5@y1u~JmIaz z&AvNY$N4;Nb7&*+P#oc_g(a#e1w@=;-G1mEs{50+)R*S)#k2KjTL$4z^~kMb4EllY z%UlCRv@J6Pr!R&f+6!tb<%HDXHPP|;meAy2V!f!b5W!lFcn6=IgkkS~S1gS#bH$nWr#pzSPG z83JL>`MsQ2zQ~cug#&3uXq?LqEVRNUj|EG^+extC7RB)*c{M{pD|dbKh?01S&N~{P z#$w^Wd2OQw%^}SrwI(K=+&-hpEKE(3H_THcJ~4sH70c;ce@ZV;VRTrF>ApzebYS^R zGL~AzL-gV2P^jpa9%EbV@H#uXYO?%Q;l&-g2r*|Y5E9+qRM^Go7C{Q+V0*S7uPZ$B z!Rqx`$)Ju{>Fi?MBa8Z(6ve#k$s$R#qtRiHS1sy(?$L8By^)F4ye#V!bZ!~qok=`- zR?+7UZGeLqvlwzs257g|ev~=qV9mkS4HA5Vz@e&nV2|iT=IvLVb2W^J?4$XGx?9q% zRkek*G;BDl-5P=HKLNfFEGDiudZk?a!7BpIhyLO$b|>uE-k>wcE!f1G`X|X`O{X|L zd-E>_I@;1;s@Z1Wq=+Ut66i-n|{)QEI9ps0dr_SyLAiAQ^iA0`px~!(_rcJ!sjhs6oKdajSI&E5_cOE@~ym+ z?70mH(eDT5U4_#=qT?M~$Haf5;Lrh}j{ljyj#T6m&H8QNIO;qEW{2%QB+o7QDwZN^ z0dB`8IK=#|@0(q~IND5?+wjzXss|^k4c1IitjpjB+tl)wQZ}>(zsaeQ`KaPEzLp&vOpk-i$BRi*E)@Owz{{qQji3tg1BLszg8JOH)HU z;6^kQ`&}CLJ4v%^JH=5AV%H41uh1R@JFpZyx@_%mtL2!*eqBY(#AF?($Ef!qri%)dw`mgnmH^=*}jh~Kc0U|womKG{TfeGxZ zH}>m;Xk5j?P`?7QuS(_mUi9Lu#uD~U!^EK(TYeY5~Vg4`0uG|yKKKW-eD!n;1J+- zryao$4>0Xf4m)A2FVgxjRC_*-Rd&TY$Y)dlxlxvJ`b$zJqY9}+X%5%8`kCH3@-MUM ziL>Z@^9Wx;(-Dy2aI;-v%#&%C)`rl0KZGtXV-kpLx6<^)u|9lOlNXYcf3~&-mQ<35bo`)YQ$}_kWzNr?HZRXQu*!s`i0AF@_Jf- z8ia#w7F&U!Xm7cX6Kgd+Frb9^vvp#yTDJ6Ae=H$l$?NuyV|pwS4N?>`gL$6obcfh%4KnR3un_&#t{ZS{i#fsEnfO+ zkDLCjdy>F6HH)usJ57yfbbwgLqT4aStH3G_SWb=Ez87^V9S^wy{xjdBwS}|bD!XC& z3vW-AR4I})STgK^)A~>@37J_Qryc$4>GC!R(No@v3(J*|=j2CbO^a8WADZ63TCu-5 z=oq^h2N9e-oQ%0Vm-p(3CVJQlm1D1pA|xrPj2iHh;G5!U_XmueV3Ax5y^6@0rG9G! z!w4_t1J##c&kV;&*Ld(;?D=zsj10V_;0^ZtyD$*?8C-E_c@bDiu?@TZT0>c1^fG&r zx%hq2%Me?ixEF&esu2_Nau=7BT$jm9mCEnOLcR#vtsL?=9gZ2X_ZPNaUHb7cyyjNs zUwHZ8Eq}YyNBTMDk7d;2*d*NiJN1h*M$mPZ4M)>oUwg5k2=d{28d{k1q?Z%6nB%&t zk1x_q+;~NLe%Rv1gegbr8xy|`*9Dw^zu)6f>*oBU9u?2otW&nulw z@3t)dnb@;#=YtpOZC_!eEjC=W*5ZjxAv96$ba$VYdu<&=1dc6}l&;VY?($*?sV0KG zW#iZr=5Gund>LwH*_U5JXt;AFs5{f=A<&o>`>SW zVl%W89=k{OQzT%{qZ^T1b|#q1)EFrOFUZ&Zk*A;a)bgkxqK$sE%xK!s5D7u!u#<(=D(_gXi$?RH=%R?I^?h22vW#0!X*OkI13D z7zuvn3#%GyGWf$YoZ&K>#oB#+kL<2uYU9{VcG&aC`acUVd;erg%RZ7A=c*bN9s1<) zYPQ$1SiIe`Ryd6_P065_zPcrCg>h1ej@NPGhjZgP${l_&e7ae(T-a^sk$YKTexsa&5qJikjhCCuw-UJ% z*}VsmXli|xTRgi>ZEFp}w66snrVUNouTHwIyz<)Q8KWK{T|L(Zob8}tA5~JR{8_NV z$um!Y=zuJSW-Hz+&>NWvfe`h(n+_~5KjK4dDiz|{T$*8V-)7u_9z$uvnZ6G^-0Vgz5~3z5sqlh%3H4s7VM@bJE- z^f2W)UCpz8#2)0 z{~oLU(t5nR(1Ji+8_&35c9&tJfc&PK9;<#e&WBc?^k)ZQY4;1{F!hsL0f`QjBY#!5 zO0ULVtN*j*nO|0Q-V)g4>KrG3meVR>KkNl`d3O}q7slWWWZ&wa(R2#tu_IeXq;FpH zglKkIzYB3rZb}_Z;=G(3a7GFgd@Z=?^33$@<*4_8O@G?M^gJdODL(+mQNU?bbaS*qTthw0<~<%yby@pw;X!OHKugl zVcMfU)6cJP;bFz>w@9*J02fXo`_g;{p=}2~r!2pdolCLUSqFSO``DGQaD$qF3pjvz zQyPAF@aT)Zj=O%Gg!LG=?Gnx5Q%mWVMXK@Ip2I9CDq5b0pKeh)PE=&ha$^LuKRqK* z7KDvx5Grqsm#5P+rGQf*=1^fDTrXF4#v)Rf$l-HLOP$@BfJ$l-)Ekj;!2A%ia6Tei zCUR^`ij@Gd`o8i>eed~7qIM(Nq(dx|KDjj>VaomLno2<7K_48;=w1OaTNggzFvr0G zE;4Sac3xwY9g zuH$V;?L~bad{P3Hm)28kex3eRok6i~_;#uNJ8a~p$|S%rfHx+IIhyVmeaKPz%auvb zn))?2O-rYECtLHcxq*|;hh_ER^BYe!whQyq8u!|zebz4ObQkbg8G&6QI{B$vr&~YdMIYzHU_fhQND_%OsM=Ij) zQn|kK1Dlz1Jp?Oq(S~x@cGd)~s2ZoH1(UIa%k_s?1n#hJWo-_!7TA1wlHosea8#)o zzCgf<`2?iR-6}KBlg-K_vEvLfCYjKxb2c(aPL_2lK2lJeHyvUAk|s2t*(dijc^}|O z%{H~{(5)&Z9wQwj_6P~X3RCDbZ_HTq6(EG}$2l+URCZC&YiOWA?2fk2K*|e@GAlj4 zX#2f}vYKTnAkET{lk^J&a*UchpQ04zy*VfRbad-c&D-55YKithE#|n@))vgR^<~}+ z=?Ie5G{3TBi{V0h-J&_tS~EBLEp|a8@-J*WgA&=20K+-l8JGNWdGSjniTkVLM!<*m zRB9fZchalhQ@7D2VS+gtm7Z2=HYiOQ={q3dLUTax>iDALXD>C}AU|*g({RH-OEmv7 zaTdxGlUV(nEz~B_Gf=BBcd20-x4t(hFz%593#xfz;<~QI6R)rt4ciE`(*A-#J-GId zl_t9jyP;G8f4jl?XR z0v{MG%?7D8?c0ucz>0t6*3FLV)9Hswda1-_m-mru5jlKJZZWs>4Hr4+J)kc&sAB_Z zE2W{R|G;}0YiI<|1}}yJFE&lW(-`wIIKo?cnBj|hv5FF-)n%UtWhjkIUx(fyFFH-d zUPwG3X~~I~qYOA*fcuv22Ejv0cah*c%Y_;a9S+aN0L`CoMh`f{{VkZI{f~(hqjDyV zywgzLiFPK(QIHnEP1HLEf?;jM&6W-pkGy7|#zzdIQe+)(=I*}r?fuBa0Z8C}7ukL& zSPN2+4}qM7nXFxAI9a75YJ^&#<}&2jYTy@cr_po8#9O4WHNV_H$RTWIKa;g|;YTPp z@?QGF^@EyhVB&jjMYC_3&3xyqx4#aG;xy@8xn4GGYWa^Vd?GQz(2ShV?N<@WhAU%Dkn)!?R@iP8wDGb>GD7r{2obtPu~bRa`XG_);RcsA3HRu z?ntvy1Km4O2I!4mc&nUP&|>0K9fdc5w9fNSoU>~yxrCFucT?9^@rFBj^par7kUxFj z7ZwYk*pXgIs~vSh<(MvdgXn7OW;Pj!OsX`tu{nq|wGi4Sd~DJI0ZPVqNAZY}G4e7{+GnXMO91j2Cy@)Ah4EhQ}+~KMEV>A`b)14>EO? zsbyz#4@K{dj^OljV(H~%vePCNVV~Wm?Rp3)?gjJKi%jWSHz*ZZWpiSe8uKyaekZFU4Nb}ABM&9&6$(P6@VKR<^xqUF1+*4U2@MR{Wn4n=xu z6+F=;e5kG2(jvLx>Mx+J&%Qm>C+MF!?kX^wIIai5S9ym#eKU|IxKy1(+(;v)R{>ii z#i-UW*u>39Nw3^~m>tWoB@zmP|CSKtyjN(NSy?PKD zsiqM<@?4L^s+zoFx3qtL)s*`( zHF6L8?nu9K^9bG-mAfLRrZu&|r|i#&fDAD8)44qe$1{ybOfXFpAy|i2D>KZ*2rHs{ zK;hPg`FdBCw8O@@@_CM4W+tC;*IRPfuHg4d+RFRj><*0>cV+HV8f%3ZSv4Q2s9$l^ z;KZhhRVjaHARvCOkP*Xle>*sh}jv3`A7r zs-D|fE%CV4QG`<#Klu z-$s0w4{Z_9g5jp}LxH9s7;gDGM9<<=Nb^wyBhWnc%e3CT-0>6c*)`aC)q%-H8`-R` z;PHe7ir2d^u{&4+&si|?9_3P>y$2M{cVq%kAgU{-ELF)b$JY@4TFik_GpDZOOVh7zpE zRB177nU)k0un(Mw&Yoy*iA9g1q z)XLV7HZm>J51rze2D$vNrI*lZKgASE;cby%H9Fh}p>=_=i!WukHlT5W1h+CTV!m2y zCJv3co)KtT#ZmIq0Ih-6EngrlklUmDC+;Ft?e*Xz2_wq`X2RS;;@?y-*=S*=A7dx(j&WJ zS$g9N}3fOVhv9=a4gBB3|}Ig!jZYT<6pUVr%rTNO^<=xsn9-ah9Vd{qkpY7}?R zJu&!*hXtibF!)j_*EKP?+n%xaGYM9rj|`n}oMM0}-4yS))rK1n77xy`#ZG?LEg&+N zf8q@^ZppFqQ54&k78UB0G&=`$YCj&812l)u10}l%+u<0GBYB^T;E&JN9dy6>k?3Z| zy&Zk%s)LvKZgk*B`AC^3b4s@-Zx~h=H9FOgdlP->Qg{9>S^Zx{1UV8wYXmh5@&AbS z&nGCNj0%a-@?Z56|4e_5x&v;mtL7@B9vF%B%O2o3&bQ3QbRb0Yuig=#d>8>WcNgF^ zX{v2xkehju(viN%)b?NP{z2glBX5qC9;Uc&Pt~N6{Z-HLrzopi0=)QlN`>7$Z-8=z zRued_!Wlx95PFV;nnmBniKW&=Q+?}tCjVf$RV|T0Vl^2^TgGDl_`~i~megxFZ1V&h zntTr?C9ey#s05KOr3lYKD36u(uaYe&3nAmX-0i$cbSfMkD9Cyf{k2tfxYG9vWZEK6 zANL1d9)4v=F98{u0QEFM*70;ht}PA;HCz>>0{`Z)u2axOk>9-ONq10E!Idp<15Tr~ zp@1Gw2V`4wT7AcXxq>`a&v}2ZR78J>oMDgzGc-+LFY|kVOh|Ehvf&Xg zuICDJRUd&89l8}MJCMUl$MNiRFFo+G;iznU&Tn^{9USXwL87J0RP|TXm26p zK5!B?^gtUhonW8>4rwc=P0*d+3KoS6-b`$^LigSXLAC zn`Qp~_X=Mx^PeA7a!@=uiY~K%PC_*Y`LXBLeee&YGrz7TOd&%;{h*%V3)|S9uA6t_ zySZk$yqrBa$p%bsx}0kJPsiOFNK!wY>lZ?VwuNS;Z$LHC4ZRnF3kyO7uT?4QN0^Tq z$v-^6D|8NeHkx}u)>Kis$miH!hp=R-Jj8%w6mZ2|M@B}^(S5Eb%5*bV5cNO~d^&^F zJQx`VCp8|YDQEGA$xX3#`%*h=JLXvfa6LtknE1lUY#=MfG<*rsDDzvmr!qbfj% z$}!JC>|On*7q{|%v5bJ#YoHzg+3A3=aAR7RI>zKs;_)jdXO+1Dg}?|RXQ!zA_mgZa&!j?6_XCM z6W@2^K=z>rpM;uzK<@o51hD9KMRM(K_y6a*h9TBf^Uc9OSLa{ukCq9@UylEd#Glpt z^NC6gX#3xVbbl`Y{sg53q%j@FzpwhAewjhPYLCACs|4+zZ;#3W^P*kgzy0?F{^%Jf zNRUMI%Pj5lKb1ck5f7v#3;o~zzCbKY@@~ezt9Mj7!lR}L`yB_NVQFvsmKrxV8aB9t zV|}T1Y0{ZXUlPyjd+cfH{72VHMWqbp>?W@EZ(}>q5ksoaPxG~Je~Hl(&wf{}0L0My z(}Y3Ch94}yMwxq#gTg+(fOGGv$!5QkY3HWC9eI56sIsVMz_~)`M#kT=g$GI6c1_CpZ9X~O*&VT`d^!_QpQ?0l90*n~2 z5b8Kn?*;~**9+K?4y=6p-4JZjEx9`9*`y+S4Fi>T-8r>bBM2Yx8N`6^DTF*j2oun` zK*2o5QZNv5r^1DTEN6v|8F2bd?0#}pL?Ze>X7~ zdE!qn+x)3eXh;bg6y|LE~LqF$k$2;eUD|kUU!FF zu-@V);HBQEh+Z54O7bmXm>JZj}Y&bQEJN!1Llhvz*|lL3T!jG6w4yaa*3D(=>{OC@o#>TQ>ZhIEu&q>Ocfe9Q zPgNJ@6(;G|xX1t%XTIuOX_`@~=T1Tn>K4`}_)_EH1x?6s$r zjRoFJolZ{pdI8q)Re$uBK3f~eq}I@wjdpH)^5P~q^i>OhwKDkVT?s1zNvsY1@&XK6njaKK9IBPbG95mYOPgVwFM#)zIRF{ z?&oL!{apZ4&*pqqnQ9YkYka?NAxgSd4~)t|(HyAZvTikEe`mk4i*$=D$~N_SqjE!4I(-RDl=TB65QaXS%V_jjqJ7IcD-2Yu_=7Z95*r)iwEj<$4!;0Lji5?6v`uNhkL0Pj z>nCeba|u78xr759*9T5O_Z`1Rq~H)>!`v1-33az)R746_(;~?PIJLnO8OFT{k0kc% z5pm--K_-ODUP=H6-+e1+h#VoCEMS9XV=9vKN!QhiuBb$8hNqyaY2tQinQE5g;FY=h zt8L1w(=MREh&nUCr4FF;*SL-rfh-DMfaT^uQ5K*$@39-t%r2{xgBZm?#uMisidai_ zWt+fb0Tq#{`}nQ;ccmi<Pu(U!#>0u(T|}N*tSA>QRWjgocL0<~vy(9m&G7Gm(~l zb}xaRNBHFOa|V&}4#{8UhLCd45K~3H`4`NXY}<^LQRc2OiEDH80msFabGp122!$9#n9>*z zfEp~s2MF7u!5!dInLX47@wQo(=xXT#*!J{zn!Z}8flKq#u@XH@zeL#-Pjbk>6GWI6 zT;wrryW!QF6!LMs661KH?PbJgrWRhXACZP>n3b-04ocl)@|dVFYuY}(Y9-qR#bLs8 zpG>AfC1NY^pfTnA8P=_Lh&d%+_f3L+V?phq$Le?cz*uH2M{N5Br5J=z51#zEy2@FR za9MwV*s#Rjp-Xy>MaJ#g4dr|VTa++M5^OP@-NO3fXnnXV)nrj%zks)uCnifJxfn&8@bW8eLY*%$T zb@W^(xsYQ9ki|A2d;+$!HH{(bDK&RqKuiNSHjS*P(RZEQ1w<{2&aoM)C;HC&!4Nvo z{tPN4rL1bHqtoCIRy|XFZ7;ROJSb0bbUXXD1~NN7vB1pA zlZzvh+8wb6y=T(sHCA%l9xI&&CAk#36PCOVV~WL#!2|JdqGTjOqBqS z2<^3@Y^zPzI(dlhXywlmCJgj(B)7UPN+;7J9M+$NM&Mu>&7ma;uxzcTJB^!cUL{cL zi?z|lrw$QM1_-DhyDL=%ur(j4N2FnyJ+-;;?j*5&mSUWAnmw<{H|L!l2YUotTh%_< znSH2P@S|`{bhArztgRPr2R0uH1)HvWwqud2BWX#$&zF1XAn(aL%7SwJP8G@27{XrH#cF6R=LZVO z#|)qp7*^MM(U-p4Z#R$ueLSbslh$Yj`6YkXfwO#}BebdM+-}moJ54m54B_WgQ127R zGu%sZMtBq{rXO<`)?<-=bP2FU`zUFk@ZpAH*=g~t2G&WLYA(Q8Kqy6`&Q|P);`o zQIyQi6FW%S<*!Mb^!0_iwKoxnOKc92el@M2(uMFfy%4~}- zGh1JL#*li17+&1-Rkf+`;NY&JYMssS&LX8V5}SAkhqA+S<~W;&qFWUmj*V?EK_cMK zoW>qP2h&uFUL#Yd+UGzr!~jCvI8-jZ}0hi_&0O(@vr7yKBGnFjobVXa4>&SvR}0(3C=^D%rq*F>b2`8Re9 z!$d@0cc>QvH)f+NhrsUhwx+o4S#aYFCkO*T_&SEu?kDCmpXjnuE;xteE5tG{6Y192 z$2JunArK=s^&^q}&-l=ETVK^^5k*io%x^bD+h2LT%bYy`U$}AJFm^si+Tx4i5gc^A6ZC`ml@5bNDe0@B(71{K7(zHI_YCze|QXl*lbEv zHD;5auvSA8B5eYUVjHDN=~3U%)1HTT;{YrG3Zx+sBtg_u2&*-DIteq0^F@2Ib(W1jr(h{_!!z>rLw+!emF6^QsU78U&7$VJ1~ibU|bb@f9kZ zOs)XAO%L0VKL>do9nSPgx&TC-?l0mCe`76ov-BK}D4*=r#o%+#-yg?}_`FmsQ+lHNe4U4{>0-d)+~Ohk=4$xEviqEtucV!1Ex>%?yH0hpaRcGv`(7#0h*4jn z8-hGUl0&OszrH;zKi@RD`jCu4ca)(;nl8U4W-#Gz+wsxokbtbZsU08efi-9|@{V~* zis>T4-x*Riwsyo_c0kyoPk0eT^y-F{ANQrvaG1%QqwUZk_{nZ30NQ`2=>u8;1Bq+1 zix;@F%49iOp?8HZ?KyxhQ}0Y5B@;hfcq`9Xq$diFi(~Jz*jrA;RUGR_f9; zn3p+E^6x|2DcZL<)?)7z1LH?L!evu^VGN}_gGV?oyyZjQ7wVL!%{JK%W+4(7`l3FQ zx_r!BO`Jk@rsEx^54b$D5k4Dn4xuHet-|-#>P>Af&TnZ{K@nYi!zI4)sA-0XNH^hn zD}Rg7tx7C;#5eXpUtxlEU1|qEP(zInB3nUoZoTmIx|2t-H~=<90L!}O@vvVt!8eK* zDZ7b^t^Iwq3)R2uXz8l=}(05GfeTDe@6GKzj?Rl zg{Q_{E8q0FQOY!MU^2vKw)9 z36~dB@Ug$0^mtA8MLWolLyeGsn=co`pdzwSkbSfi>Wvv1&!qQZsSS$al!$PE0OC_a ztH>XF+TN&1%Qd|_nB=wmEw&1b6T99SSgm)03xKC26V7*A0gfP8sY(tvYU~;6OB0^n zIbb;YG!96@*oQh(Mhp41U#|5}-~k{vY}n|?5o9F|D`J5007CpdK-C}hi*JX$D?xGG zBrgldkOg08fX|?On|L?x%XrzxgKvODos=C0h1oU^vTTE3?Eu9)Z9(a?^bR0(fj!&; zz)c|PmE-Vhf z8at)1y|FykvB-ep65Y!76y>Ub>Vqk8+$MKFUb4P01YavbCr_}a)>D#>9QldX#8q5BoX+S124Xu0YA{>m6i_z)HU~QOWroCpf&G znfxdmMU&dER_HGrc;51wtx@)2OJQVqIaAIlAh(4;Cy3PsC3gd}HiijCfZ+Sk(`hH~ z5<{;f=(RJGr$nknCx*>DqVSI2AAev^m^=$EW#hy{PzeVA&5HrOiGdDrgkOplLu$ow zFDyO+-%|p7ye(QUK&!r$q6}bf0#*1m;82TJtJLbs*0ja>cr2ssfEu(&S72-S{Qgl@ z0L8PtPrA^m_FlLTsIeDsf41KXe-uq`PEM z0!nv-ihu}635bXw9SR6acS(v8(xTD|(jcL9h#;UyNWAk}-ur&;{XBbrKffREc*j`7 z0l4B^XUsT`W6m=#{_1wr-y7vIbstyAI?c*!mvCn&z*%3eGl`*?jF8D@r z_<6SLK7qrR=BA5F$%D?5wYu7t1a~$bDBA2}3D1<7@QLpEQuhIvVqcI|ZrSjw+48wb z*fQl>=J7${A=d3EP+|If9V+rVZBVTvq}((qEZzj%^f|Ahq(z9>H49A5jbiY;3m3+> z*pZXZKnV?@Fa$l!0|x6^U$`<7Q=)*xYvBbg5JQwLx$qjymq$>4Hqdk#-W4vRwa<_! zwz2Tp^ZDg?Y7!bUl=75pQbE^{>Rv`|cCG5u0#*m`y5STzyb+7si^Os$F+qyLYY2V= z3ABpDXQEQoN*DXyv?8h3Ukw67{^X;1NIv?n0IB{@KI-`;y$V6)2q^lT1|gwdQCFsm zC?L0eL2WdcO{5}r>$61()A9|{8cw`6(X7&qkKEjr^;H>YUs#6=5PYT-Idb`oQ}Pfz zOSQ7B_Y}(g;N%||0l?Z#0MpFOvY`->DE<@nw_h!YM6vQJ{wbO)a%b_uOq~tgC|!rV zx{Dv{t`TagtB4HC-Xqpg>sJ;RlRbbw5@PYTn#m?gLx)SO2%qy0b^=b_6Bzuag*aq5 z=rIJDV&!)IRz8a4oi|6$mt_}_J+^{YKe|}!pnw|McQfPA{D_7!XF1u2#?N@?S8%U+ zHOYnrJ2?`hTA9&n1r338y6l95SWskx72VAOylKTp)eQw2$yW!(vwz*pSb^+%SfPi} z@Pn?Ueh3uh8uYl4oH9_NsvPY59&+-`R~|~=qVJIlv`islYbyylJpg?g`RVhnuPA8W zULA2BDNW{!eHJYji^QO0_rRI;zrI)nI5=6S0yr~O>yhg7RhE=|jhXc`NUt78b-J>k z+{Y=~ls7q2Hhd?W>_)V~q(UlmLqcA#%v9ddIV~m?ydISogS@|=jkCBuB!f=PKm&YY zkhlU_4F2PczNN+AgJ6!n@ToH0tCGH+IK6p>4a1CO38Es^d1ecYubksKcoFHi5?{5k zy?e87SBALx#}Zcp(rSj=ztA{Pb;T3(sAM%FGvyj6++{Mee10EVq>%ad~hd3zjat^tFyyEY1gx3Q49&XK? z683*y2m4h9Nlli_1}Og;_5X|?2@QbTiu(UP^#8xpi;g3VW{}!yIc|i04YBK979eUn zmYN6zi1yOX$}=X|4%`%h6e{jUT9N<0qGb01L6KK(TG?LZ0b%uu%Lb^|*H*-DeNI~+ z)Rl)_qr(n}E-!`G|9&NAOU1^(Uf9|!9<)dHLLloI$T{MyQFQ;^jTd;IA*Zswe>nV2eywWIYhvxVUC0l5 zXMq*@2A-=C0yfQ5aYKL}?RrvxxUs6V>v5trUj-rqEZPc)`IiBWFveu$5V!)=-uLLN z%u`6BS#C?J~hK5audZVgf1T>Hb{@T2n#Fd##DfMs~CA><9BT-|BE#63R>BZ7#v-tg1g zkFq~LfIQt}#MJ&Fqnug4Xev{Nj*j+OgYZ6mjXfx7`UJh6piR>!_=Z5RcjzFbPI zdQZWmz*~^q??Orrz>~v<9)+eH#>ZBOy$n1chcM*G;->GvZD(~Nhpb7^iXK9Pt%f+s zSLIOebt#UM_<{c7!QPJ;sYM#%tdpmpAaM33oQwgV`j)IG=4+5un>@FYQ<=PKnE%%C zv0||F9F$hAG1Bwi1KomSX{J1d{rh9k69f8Ns$cksSasZzB?5u$A%+K0L5HHW7z8KO zd}iF|&O{02`D0>VDFhfut4r|2h27Ul5~$X*ufCVSWQlWF4`;-y@yrbl*mq>FTA-sH z^n_alE@g}jKIApPA9o?t6IUhd`Pm5B9#znwq*za1RRa`M8;HF&RNzl73Jx72rV*Te zB(v@+;%vsm4M4lRXs8QRAc3@96jiPqK>_)J`;Y2RP%}Ynzxw6%{mt37DNFG~!00W! zK3%8e)Tof+%vLW-xmDukTs#jMgJ3fB?FuR=1+~S9zN9OnWH*2a?9ki$xwI8fb@`VQ z>*JI|z7DC~!zDc{J;bMhf#{eZ0NC>=TR~1Y8DU^J0COD)OnbtsTokv2=@1Af&#dL* ztf+~`prE$&3OffBa}{i)QL7FHLpjf@6pvnD8lq~G)hv>QuZwc%2TGMu7B0+F#SVOR z0=j4kFm`+TXFZU%>U!7!kPdcEPk|Q_nv0*em;2{HIc^*>ZouujK3PW!UjTq*O`*0r z(VQ~C2VPCON8QcREGcqfcY!lJYG4|&RArFHlY0>Y^zG)~>vvbnUM1wt_|vu7k;XjH znOUD|m}nweE1Cl$_bb4&DuOJ;-m zU}KmgQ7Z7o&+7XJLlVT@{LX+%Q2;umCud zlkpDBN#gVpE(dUSd6n}e6Yo?6z12esJKKBe+<5T{!S;ykxWLdo$k)3M?OFfE;^uK7r9sNrYBV5T8+_}M5}dr98zDkxG0pFqE3gB~R+R#|g`QF- z?SfB32~1Nnw+8Ye_VVYS_9rm)^tpK?i0k`9F(c)XzH+7ic=gS~Y~qY$Wjea7Y|50B zylng$I|#}KJ)E@E6f6A!M&DgXw64ECv}soDSogZ2YV^Z?KNZU>_8W+C4>TZjII#@2 zq~MLq5s{cp&ktqPLr?v&%OawOAy0BQUDlW00=IwA_VY&mlFFDLU#qy!hK;6XwUl_a znl;2|V(T9k9e#R)<@{CvEhzlcfY)&BPT(mstA2TRD)=h{lqNw7!>JD@Hy(QHCw=^8 zSh@xz>dyUy)Fz@X%?#O37pN37zt_i{E-}NnSdJvZx@TPN4`qSO@MTV4 zpXW6~3QEA6Or}@PC2#+Lu-*?cV4^^tOV^JkvX$^(y-L}>W`C+~&#pU$UU*P|`!Jv% zH5^IkXcX65Za-7^y68hf=EiO31DN;*HNK&r9K=z5K14lz{oFWwrp|QneWgqdM~@Wm zM)usm3h{5^epGbq7!aD_wRCaeXUWBw36BQM;YDX{p!P49Oeg%_8PPAi_^eK9pUO2X zwXPxI0`Lx}pT11akm2th4U6N997omj=5Bf;JjMx!dlC(oGrcn$#mt67E z(}>FnPOxJI`uOWFlX;Cw`=Sjhi)QZn=9CklXjk7LPnD#Ii$Q?5hykn7gV4Iy~JS(p3#H(64j9?)|*Lu|D;v zN``vkg5VqK!LvM5Z7)(Iae!`6+*TD+I5rQMNb4~Znikw~Jww2YqmJL0NzbT!8Owr8^mH)ChX0OW4m7$gvc>^nbB&tHJ5h|(0H8J=pB|_=D zXRjb60GUdlMlZ&mCWBdtymfv&7-LVffQz0n9A~+4uD}ovmH5=-rtABLJRIwTj!Q)`hW5z75_>`jVy43C}=6=-Yu$hLBH- zWO7lHmdX6tHWr}oznU)5%N5iiJvmPqeRAiD001?JiYTqI)RX9s6NcGz%e7lSpMlDs ze4D`|Fa6Y19uK9535=Lo0*6bC8x~;eE(89)oo4hZpzMt~H~6<57+U%8z9L|fon5Nuv3Q`*oXy?y0Oi3YT^ zC4ssFIo2+!T7Yl2V}F{2bYTP`6_8I>Q?X-WX?LR*5nN^t@N{jZnQ8Yr`Vb;0OAT+5 z+}_2ntmR@|TtkZ3&HAA3seBfsVJoWSC7x{Vnp3m-P?3JeqQgP$AI6xB0NSNn5Rf$d zc~|9@-a(;5^{xaaLwH|?JNNsD5yx{CG*aJWVE3qXX&)lA#To)zFk}f$c*56*{jdOdz+-S`Y}~QK~k4`xUCv%U)y2g z$bYla1P7UD>4Jl2j&_F|f7EVjVdy5`ZKvueYCbv{FDEEKa?&adR&){%E);g*7>7!~ z84$oaKSBadYu^VTTKl-92jP=??VKnO^Kk}6QecCV9Ylo)SS()6-3r3hTi$J# zoIbt04uG!LcWBh+Fo-N@pe#AKtA7!ld;HVABSQNAvs8r1Te)OX;!pPOjr%);7{h~TCCn85Tz0%8;MPKKL+-z~-aC)ZYgssfGxnn$|z?)N`u z^T_g_7e!Fb+>14g_YokB5_2F+sr$$(KULb_)D{0-lx@!w{dqfb_<0bC^m4L&2B;Bk zl%kOGNk;!-Ze%GmlEJ90z9ISbi6jFDLg>;Z*-6hq%?!Vfix9S6xU*$@T+jcnL5;oF!2N4cA=2s@+AVNb^g`FJD}WHcSU0;v0$E0S6Z8|w@H;M8 zH|BZ!tfS7cN$Rh>)sGCU9i}oI93Jz=E$n;0N0X|7BV0o=Fjl*6ZP0pb-&7Si{M^?k z-c)tz(OF>|l4!97)w?~Vnc((6?F_W z`|t$5PuCcU5bSi*gN{4fF!<*<$mgE&TNYbip!>2^&e~dST19i#>o9jvg_Y}|4jVe;xABfL!NciG>_P{Jp^yOUoIy12iin}D{|gSf1g`O zd|bzsMbEhDeQ8i?&M&K`=*NuZ^WD`^KBgwm+-0y?)=j?Ec0rC(czCaW4&(+y2XCtT zh6Q;~O?3pd1Rg#{=Ih#Vu)-) z%+)UvWP-^+8usN*^KYzk*^O1 z?QY_Z(Ba=&bT|d3V7$7z-56lJ0cb4(#CF*vZO2o7Cy_?p;4c6T{S~8LWNJaoU@-v+nHO~M2*dk=V{J-Xla+4=r z{y(Km{ZuqUS6f>nD2Ho+!MZn)rw-VkvFL<)2@){;hZTXkmNeJ5ga8^ft<1o-gP!4) z7VFN@%?|;a2CPa@hJ!hb1~0zr2y8Q_cE$lD(4hQW1fuUbFq1P4R^U_haQTwyjy*mg zbu|usSzz9I=b$Tf)ai)J);LFlWJgFj%zr9L2>f;$#3W@8lnV~QhQ9y&h^k=0{FO^J z0ll~rw}#>R3s45tK|l*<%#y*-wyVLewFkuC=C@(H zdirU3HYsGgDiU_y(y`MB=N)7TO!}>^z5!68iiS6H&GX#_%9`YQJ zUhNo0{O$*ID7062dwjE%k5(HZ!cA^n%vE8`Pg{V<6Vn35*X;#S?OqC-A1F8v!CNCe z2!5Q@cy3X*1rsdM1|;-LL}#ERJcxBy8VP;S1Ay8;E|^0jCngXJ+wPEn1N4Eila=-# z5vQhEK)ui7C-}%zp7F|`!ZiwtWr%0!BX~_;+QRH2p#qd3I68g>w<3Q3=pb7utlX@+ z+h_vb0jZmD=xbS|N}_=pHShw4rGQ5!TKN{4Ul1X7{iufEhH6?+kJ44=M)l`BtDLvt zTFEOuTcc=qm=~Ggi;U}l+ys(7P^;ViHixn@z@vFw3w%tWrU5jHuH8I^KbQNjA+RO{ zzimHl!oWL1hTXLo%mv80l*NT$aBrXcrJZjT?4Ta#Ii~da z;4JcLfQ3#6R5mp4Ez0n9_VCS54TbD37-T$;>8|_7+$Vy1fOk1^z=X#3{dx);C!Q1N33lKg!jfY za*r_17tNfDGFvu#tt2{V48@YPLcZEM>gr~&h#VqujdZyBiv?UMbqrI7E5O;$Jy+Fk$Pm2p7Fyao{qD;XYXFk5o-aeIdGT6~g9SauFTI~)>oH8xHRF17 zNxr%6MoO19t>qfV95~%vdxGoD19R60c-{o->+56|57>8_ioP2yrly}=ODoWO%)G(q*+10LZ0ua61DF1( z693h6#VeDz1GG!un{xdqD$Zl?*7fn*fYdjeeED0!4@AasDR zoRBsjFVPyreSM{rSDkTQeq#?$gh^H~t=jvs6lSE2l?b7Dka+IyZpfFu>{#v03vb*w z`B`!)N#xubVOzo@UGARyAE?E#W+$PcGzzStqeRPU zqHAb(olE_B=RpG@ix}c1Y#g!r1vbVd*#z>gJtqW%F^m>Kj{4E`wKDehx$#=HZ zQ~A~#CSzzb6ElycH<|E#Uf0K4UpIK4^R&&Wn$9zw^0eG^hf>pb#+ABM$+Rn z1v7RT0rNcEn)9M&0J8L;Mm2UR){Ce5oi{+b4td&{Ekg~V-BFhWfu7>LB^=GG9YEMJ5z;A8$L;5;6h=~} zla0F677@SEoK3O_RZzD1_vhu`E(CX{Y7Jbz`*S()7Oxr_Ph&nO2bVr)K1Vs?fk=2z z(Dz$I&vXYK;yovGmsPsx&N5U2t|`2eGNlDC6pP=_i0uUNx}qD!k)&-ScqwDMpGrc0 zBy-U;WaF=&{r&}~b;3%8MT$Mx4iYVuNUB*s(5gDgxq(VNER1|z%I9<&mLF*j$=MH| zs%}vBT0pc8h)a!d(*;IRhgek)^3Kr5Uha{X?HPzR_{?iGqE)MFU1z@Cx^`F=VDJPV z*}qwYpq6!f5T}S7-j)sAgSagm?yvTTweedxN}gT&Z|ZB?-adT#j>#=?c}@nD-xpBx zo)pZ1Sd{4fLJJ|QMxSt=ph|V$Syw{FS9*mRxPq#YYSgq^>Iw?!)JfgByd;$w$z-eRdaLQ6fDYQ_1;rI3` zerAeaM3~an*bH`>(XJ!N!=Jn>>0EA=NNL2)OqYF%PmuMr!}Z-lZ@PJNl}yVyfCOT> zt|z3)Uh@w4E=Im};v;kMxMs32o{$?9#JgVTelC*Nnu9G-&oI_SF0g4SYxjYsD#_JH z42vr+OB!^Hb!4+-mUi#EZ}Iw=IZs9Dg{$IQaZktjk#J&zaY)2nj`oNM$H8owd0fSU z17_@@yKs&WTx_7Ip3_LHTDz=G=K-Q z?ItaN)Np00VF!TnIANSAm&w}AUu)*dv`;{HqxS zXS*U~nUQfz5RP4#cKghNjdFfn2SLE1A7O1+?Cy%~sp7|7B|bk{W7@DUVC}3;aA{Ku zSV50%*#P`Y84z36$$I%P`(9!mTkDKtb0DX8k0ywVUxL0#rnRnD_6BC&KynhMh7L9`2o3j?2`z3hxt1NXhv4KSl{i?or_ToCm|h#vVJ1?_Uly^d~FgJJGXlEZWlY zIPwNi49xH-S?F4*Go{J$v}@9oy6ogjnYr(Y%T6Z+vYpux&s=%^{i2_xEF}ew4r|{C z$5+3n?@B)NOzTi-KgJ@A4G@4W@FRqA3G)jr!LGmpzGp(uz8lv9{yshTYDd&gPGb|O zGSB=(;sW#;l$6I}!YXV1In_rDK6zATdE(s%9lOU&wl3WfETmhDJ@i|7Q?v$LvGzL8 ziRHIANFH-)Mt;-XoD;^Ox=reo8T_f6aE(Uy?gn3|*J*`MTNDqrEb38hHp(QSPoMJU z-?$x+DLKNXkGEuj_ci|2@*Om?(gy-hQxlT4}08`3oDG|rC2^)xPTHB4a1dZ zE)lMWjfZ~ERw$F<)49L5aAsyldG*A%mmX1y4PT>B*~G>@HGh)D^JEO=qhPiv@vaF6H_i#ckh!Auqua7 zZ`pMXP6%@G$eRqUmS#fOA;XpX){vIEP<@2-T-?M7nc8@-JQVI%Bz}=Zbq=-~pQd}< zeTzRSK)P_ZwVm`pY-O-eKqm_;=N;LPsE-m7#2WFO4^!FX=t3tAW%q#fKw^|lfv;&f zJh7@2^EO;x0+uIS*T%)9D9Y9EeA{ZNxmf)Ejn%0DrZVH_a1)FDYJ*0SAWQKe$b>$h z4cwYQ%4-!Th`bBN9&K7;o8OSd7?v`EKH9?5>fJ2JY0afTcayrD4*mt-b;ikO_gmj6 zr=I_$=x{@6CV)|w_E=X{n?3!fZ;ZtJNMI^mxI98CEh{-{^rZx%PUd)TXs%fEhD1%q zooh_N8fB3V?yVNLkDV&+{jzR=j-(9lhAHl=GyZ+U_>ZKNuVwquXHZYSfA5kv^`2Wd zXDoelfn&B}+DY--1jtVO;;zX^R5`lQ1iiiUk_lzyJ0D$H_Qt*;jk9CfCQ6>`TQgZ) zQKaqX*n7ig%Y7hjdKX3-b=j)QWy2CX%xse#3AuK)T2yp`zMs@VA-}~2%;Fv-Y~hro z%eIp~j982RfkeMMkaOSZCNjRo_dwwP%af zlyUFOVGF(2EKDl%^L|8*H3m5}yxExV>@d+HWGm-u=x(=k`*jFSL5_4FlG3yAOY` zrMUq1fo7*u9O#o7))LBd1unn?RB1FW-s%&H8?8_HE%y(9@6=osvF!{v%Ido5h zy}@Qi_HE+TN3W#!*SI!ayfkUnzr6mgm%b#cq2BnYy(C+AHE#OoP2y+Jt+DRa%6 z@BE2M+d+6Isl%^;kV#p#;XTjQ03)K>I)<<~WV+8F77ImJh?)CO1iNet_|a4!_uVM2 zV=X-CfwA{M+0>T8p*AqPMJRExtW9nvs1ijzId zshiZ~H_vKEoru<(Ql_%cULZe-WM6_Mu`A66%YrODo>!I19U=A+yJaNP-K=czOoiBm3O z>}R9m2?VPezHa2#5J_4*;8KSu7R?I-9Q-pgu>p4o!V@x~5ylMZK0rj z?_m1Y_17t9g;xu`zkl|b_XosAR`5GSA?^=iJ=ni~_bHBuPhbkCFqPLUXoi1?zmPWe z?kn`LO`ydC88r=OM5VkZ+sE$i(s9Un7Dy0)ZG3;V3+`S$0i#CMD{X1SdkKT=EP(aVeNAR*0?>V78G zI6tK>`cU&RdqkQHgn@S$#<8gYyHIp6mXjlmg2wI4_SyEYMuptS9%yC&!5wS#+W%hd zue|v_k}5;yDnp+P1OcdG1w_DzohJe*auyE1aIFgvP86_ADRA z&bCZ!gZj*e>Zdc;n{|%nGZ}JsIkiA!)&!;dUO2M8b&#ubX=jAgt3q!&l znB`Y|g*s3wDP=r?(>i&*GghHn*+e3xzB{o7AL1Y$aIlOUV;R9BBzSVpAAQ^=;Rr)< zb~kkS*9ulW^(AjBTGnTC7O(!903>f2o10kA-~wCb5YOd41;)GH8$dTjMSxKA{bP=( z%Pc9p)9xxHm_V0_aN|Wy43-r5ya3#2O;`W|dgX3yXpohbN|ltrw49|lLhAAZ;54=X z=GBgxU$IJ;#_bLzE#9uiLm>#=SuTj{o+)8HQJU#`d_~_!>h@Oy+?4DYJIHHi+aE@4 z%!850-TMfVW8HB*#|(bUdbpsqGckoJD>VH)eVBVm9~8TB0w$`14k8FUs7?z1&4$#vtXMM-zJMe(+4xHHmk#RfaA3 zH4Jbd0j;5g^xr!aIh{z5$E@eyM}aL4<^bA0$h_hbh5Yu1605(R+d!dqOfMG4rG(g`bcd(X(64nV z^7@A)H-M&(N@&A|H(O4mt6yI-61~{6gXhn1F0=p_&i!-qSL?v&wFUqvGKW1#VKTp?ed&R`ccuO+#Ab%6Rz@Qg7}B(5{CRT`3{ zLiir;6};2eV{_5@bZp87275bD1x)+s+T|{cK{$nS1cQ(c{#o??j^Mv6LKa8??jY6R z3J-8y$}{-P#g)xCG@z`1^E0v1(g>nJn*07fjtw8R(|0VYogw?VY{8a#cAD&dqKLf~ z*R|GsC_Uvjt5ST@- zxvuYzruMdY+OyyP>mi&T=R10I;3k@Jp?i+c>Ih%$ybf(o&2UVa(Z(FJUMX}2vA(x$ z=s@06Gj~rubd=PJ=RX*EVR<|z3B}ggXnyEdOwQ-rC1NzS+2y9R^*ywQ$Fa$4bk{_vv6So!mT!4A(&= zpjYyb>%tBjwbUoD+NOR^Za1ZV|+>RX1W<<>b#KO^W|M+8! z-^vs=>aa1D#+}-t#9=((2Y2p_mS1LURyK;oaQ07zi4X4KK;eln;uTwLaFx4BiADh9 z?K?ZXxxr+3V^W*kUA5qy5bG+wgkbG!cS&@H@(3w~`AfT=NYKoFf1VfHM*@E#T4j*) z6|EAQZS34PZT74CaRs}%yEGFEXUQG*Ss*)eWabHXmBgDY+@z+@cHN|e;i`+}3p4IX zqd4w|KjyZ7wrIC1)wT$9pGF44yn$;X!oTsMjun~lXKG-bAT}6PC#0XWWIk~P%Aq&Z z@meUk1S3kB<_FrLc7a_oZ>;4QPht{qubGHQ5U*G32Q)?%R-DKKYC5Jd!Mi@DOj(Vt#uwd1B|8aP{w zq51P~3REc;FRJmx{@Z)}&u9X084&9vbS8rXiS_>DqlXaEAt^l~_W%6f@R|+;iTMy> zkId_Te4ss0p6xdtC7}N6Gg^dTlQ4M9nEuBhAb;#j1E3;>Eg$_~pW&*(oHjQo`!^{5 zXC7?Gke+6M{}(DG|Oh4#|9-l3?Fg%3S{OwEk-&-V@6Q-(F2QS>^7Id)YORjV3Lz#6{y8N4ha%~ zX-T7i<>+Az3p;&oOMLGmrTv7^_tlO+e>yXiaB!x}_e9cIgQ^Tx~ZXs=$ z494(~3>h9|bb(NPQwmXJw&emQEyt{w7@?r1Vb8s6BY2K4g-vU7#?Lx&LYho0d8O#r zVO9TFkORaF`UhX`hi9IER!U4kE)2t**r%a6foh0l776kpe;!%FFf#_9&wjlKs1GUd zqRa2nOAwUhpN{}MBsD;#YEb^SWgS9D8M@zSzW=-}0&vg6Zk9ahB&_-0HkVEiOo!t) zoa+CwIVmczkTtC65{>(Bdyg3QUeOEEp$Grv2`wntzLhPUgUbJ0-hciqh5!QX7j#bk z^N;`h`6S>tz5l-_f=r88>=_<7879I5sGxvwaC;6#rHPqv8xGX2S%1HUf6)p-=H+-K zPkceJ{5$0Od7~lm78u=0L05iNlr)*-_Qx2KlOF1SeRf|8*6X1v&P#XxUbsNc7SUle z<&y8;fB9cOJmr8Dx&A666#10@^uSGl2@mA#-Ttpjb{UrJy^9<%*MG4G1ftmDt^}fP z<*KM7PMsM?ti{q79T<&ypJD1hTSrTi?mAhSkk%% zw`;9=uZ`c{{%Xi;>Lh6XF6876C{=+3Erl2hVxePjnZ$MhZA15dED~P;i0!Odwqgjx zy9)ra!p&*a*KE{0W~7KoNJ76)#EzRQtw8&P3@MLxx|)GY$U3-2LH+gQ+g1f+FgAE7 zkHrr+`g7WVRC@5>L8;AiLMP~;cpbQ-U+Yt;%nB&Vq-&QbGcMdqVNnVVBV~G94=54{ z((1vb0%AM&l;`CX0&E>-G956^41OHRh9IyOn(cAta19eYY=QbPxI_b~VuC3N6%Spy z%~b*X6#ytuzfR`jGW8@Iv!}99$O-Bz0(`gC=VW58K`l^k z0)%lAuQC7Rl5!Mv&qG><9GpNXe;z3@{^&IW2IoA$@6oqs3$#B{xX)+fWZL0mMVY$LNBNKu;dCs&qTS`Y8`IdLT}eW@>-##`T(}#FC(749H_=P|2FZr2 z@Z&_xequ!|MCZjp4a|aTU`crbsQTpg>_CC$st?1*bL!}Q7pN4Sgrx-Nhq*>qU9+rw z2remtPN@e#sl_svTfe?AlgX68#cN)3zR?{>-P@p7QF=5_8$$s5LnZWew#jLREEg@+ zNmfQJ!Q@^|3twUoZ!>;9*C`zh^!UJK%JNkb?3w6yozQ>vY9+L3_||*=5eB2AM0|K_ z>g7SqVDSRMwPbspmOjz#2K~n3I%IV+1Hz*zeUAVIfIGvWVRH1oqq)vIXyTjQ;s+o= z`q&X%g2Qoq5KK&8gAPP(bkE>#FQ#eCaG0ZNCsY%j5!?@=*TU8KMB@6R7O2p+s59Ok zf?;?WSmRbLYN$=NPWKeyd`j2}rE@FiO(Te0SnJHGP`?%N36Z?FS*EhVU$y^-gC zZ1SSNrr9_qIO(q7mNt;q*xx5$BV12Q80B30n8}pvsk|nJzOSupsBeH&Ks;Gf3@7m4 z+v1?h8_Sq~p@+NJn;x~{T%-p|nYf~lk;xPUq%f58YSSVo$2H>rSR$R}dXpDdy)q0qW5Kx>3*fj zf51t=j>{@paBruzEt-~cN_-jm$(f)18sJ4tKv(^v^@{D*eAh~Zhnhxud`+BNA5pKq z^3etV1Cz7FMTCek#!Uj#i+!SZ0a-@$eH2)^a>J6R@3ss*+gD!8WjK%x+zi0wubMM? z?cc<8RZ<*{3(dUVL*3~}L)&za5~n-V^-n)hqF5MvaZQ)9M@E-3^$8d&FfI;PsT%V^ znL4CQ72==;RbrsaRhb)#ef+73g`IM^msxa7i-W2geM~IFl|Dt&j4{FgzO6D^K)ny$ zOQYyV`y7*@d*Zl1i8ZeVbDjuEH=s~2pl}wSl+nP%ZM8U5(Rg`iCF*eO3n*U!8javL zj(7olY;)qNn;@4!w5J-3+$vaWbYx6)W@pjTa|z?;!@4Cs>sdG^SC=SBjAt+|PLK?2 z*r_U0gD1$w@nhZiI?PLz37Qr?2fb-Fn*wbI*CAOZSf}NPmzSPBytg^4A~sy}(nu{B zk82No8+JT-^@aQS9Y+58eUVGmj%2?G-)R6|{c}Sb>hukw~8jF*_R)Fc}A$Nzj-Xvpcw7 z#zUtnF#S@-A-$4IepHPhe86{p4%#13i@~YjeOA|Is2ET@TAm41!g;VeW>pT<_A~cu6c@|3D+pm&+ zhl32G0`V)Q74NGc-f9`Z6?d650dy8hFdFivr|y+re!(y;kIK$3>OMUaR%TK6GX5vO z)MNZW_E=VxNNk)FKCfo~n8j~X79JlXq0Q!FL%B8MsNi*QaY?DsV~$* zHNWSigavWaUfrx143sMNLJ#?^Ow!_~lXv$l7CWw=t%jg{gA!Q>+^1JD>tK)K_MHMx z4AtW~J2|PveY8uQ+_yvP=AwgY0A;s-asRfLrUL~OO#-lTXUGGqw*eCJV!Lp?{cixx zL7|gNG48JGK67bz&n5#k7HT&2{hfvrGGgFxaB>zeC-zD{Yl6GQpctd5NFMooX6%1U z$PNja`Cfp!9H^chwbRmbvTyU>+DN3k>71^62({gM3ie-_31P|$At2hy|A8Y)Yx2^T za6D9+$?j7Lpq7y6(vJ_-eo>~mK6g2R`-9@E;LJD@Mw+L0$mo)K4@!xF!h5N@Me0B2BV zmT~S)g`{>wk>KaUPRVS(?i+Q3DLQzA=W~?6{!e|BodUM%>3LJ2A{@vi*OT^OKY*k$ zOZDNY@b{?_0@A7o{(fg`K3lydC=6PywhdZm_T_DYMVY)cO6JZdJ*OzFyAl1nS5n;A z(DnB-3JBQcAxBsuEl$8tRQqJ;rYBIXvjwQA)p>11nC6)|TO=_9ipWmuUvG^|8uB@f zvY-d!vd?NOx_0!%&8sc29r|;z{cGv}nP!zm-Suro86uB*1)34~T! z3G*k3+dx-~w0s*k%peuSju{slTcH+n9J1|t{3(6@G9F>=_voN4r}E-y5U;cZnmpmAG0vbnFfp` zz!b0t`;qi~OIYF&)_|%WE^(7+XM2Erz_$4@gzedW;_Z@qOIRfER z5SFPq0e^#+&!hs&*sTD1G^^=oW?pEjjkF_y3mM+tYGQCo4}!XTyWX@r^PO?5%>3^2 zompYE(}+$%0R+3zmg*O+FO!*W!6L3~WHe<-WnJ-N z0~*bLhjQTxtxnVr|qkEOnX3582o7rL^tvfwNm7aORjDLfG!QdOEvW3br|f&=FHZH%0y zMZ#s0tUc%6SHSUnch=VgWzX4tNP^uEXVMo(P2Q^Jlwj|ZC4=obu?(BIxe>tJg|@^q zC)-z=gIvx{hlr3o?zcGo&~(z`!8;_*XfZ)f^N7E$Ibz*iFEYE$w!$RKL&Z=C(do2@ z9-a?-B~=Yhx%TFU7T5erlkq0mEABL_mg~CKM9@MEoq=k432M`HuzR$aY%6o{@CVFh z-nMW!S%Xj}MdH@c`jkvV+~VE3EALL!vUNP9=h6t$b_P-$gjj0n($$pLHm^W9b596s z&eF^DHfvezeg^CjR9!Yw*jZx7-gq@>M{FgE+p>|0@FiaSPAm5<- zS_NL$lV_A%?=wHqj@3Aud(I%dsige1_EVyFHp)Gs(rv}8FWmigDlQ0-VX{hxgt~sp zTE7C>9K(s$Po_c;_&{fEkP}$VR@1j6Mv|c`r$OQd7>%@TyPE?YxN|z%;oiGuB3fTR zE*4YZAz`P+MaZDzow<9E;63n|4E<{)5C>n!glNW{KU?s8F9+*&e4nq;N`ZX1L~o4m z9v$rUrpvI!Xqyh6e>Mho5facxa!5@KaPjG6Xf58i;nOx0*ZLZ;IOdH?O=fH(6$HTA z6O+>DELtze%wB~cA>{qVeOAlykT7D0`$8b7O=OcoUl1b@Ad5yM#{Dc1Afg`K9A*CD*~~hAavqtMLs971 zU76!~2sUh~Cl%ki>?O6SsZgHP+f`Ax!vv(-_45G^My)M5qTEda)NC`{>Z`piU|5{su~blqlKT7{cjHCqRycoRi$vmY zrU3qYw*YHGpZVx__H^CX-v%Cm}x)A&&%6EJL}beVWqwP{|c7TMNiL$vGpYdFJ=QhP@~AGBr{kl2mHVNnK=9 zr+nkN%pWPLAq3S+Ik!`=KBLiuAtSPPG(c(cwi-UIhZo@DVd& z(4QiLEal5c^BVPyR?u)08s)((=kX|a03Bd0{G$@lrNsZ11T7rXFfWl7QhA~%E&VU{=dGN8gz?1Q!$ueLqQvN zF2`R{JV0Qm7*M5EAcY)iqvo9=jC`$c0@O{Xh( zYh5|HrKcHR?@&5Lm{fxlxO6{bxvwuO>T|Fw^dO>7ma6(APC*doNtR9O)3X4MoXg9e zoPbI*^8F>P`X){yo$z&Bs3(RhHnC@cpWi^7L$Ti?6?JiDS1*cQqI^gYB$-=DkX5=u zKc#bq=MLOZi6uIWOjje#6)?aSS_yRpBJo?qZMsw#94vqpR|kC#fZ1BEunV=B3LHE# zZP1MeDSQJ^VosJ-$3&;=QH-ut&M|Xj`bp=Lr_TjaKzOxI<5pPc1J3hLHyK%I?`h37^mvZSt@;e$@!d4>uXXHriGgh&DR%(A^&M@3 zPA3SsNLb;f1|cWjysZm*5}3m(D6FNFL~zb2{+_y2%eT zvsRWYwdaZ+hzbsQvd-h#%OOaFW+H+fNjdN5)0_%_W)~h$zy<7)A_fU6uCy|1UGkaj;~SlkIJ5sWE1f?B zowag~ZSGVhAZ_yD(Mm(&52gVsiOzWnr~gcbWKt|pe?B&D0jI8uJ{FJF{VUh|r%Igr z64qIf&B8T+L$a9MPlMobMOimkEow-d>za?cDay5F<|!wTVI*S8SI?|_xDwv(yER$+ zW_E{!C;2=3`d#wt;-D?LLMK~OyX`z!@bJOY&vhP!fZ#yv8(;sX3)M4o^{Ks&4LK@2 zwmVs*u(K>wf?A=OT4<8Uj@98zdK~%h#lA5}nsS9i`Lj`rzy&cp`YS`Fg%DJuCR-FE zGJujS-&mHWllZP(*&B87bjS18eNhftVPtAa5$0#}ypum$(Qy;Dr1yidgs3S(1gPm` z$U5|RY#_YYho(oF^7r{gmAA$rV9zcT4a9%S8lZI+B7?9lNY9j6@GgIHal0m9QOi8b z&~HQJ=J8Uls_jLeAFR3u!t~VbB#r<#XcC9jc#HcE|M_DGs+S`wXH#3+?F`~5pHF9s zSxDWdm+@H^!Zx@r?LMyt9k|*w*aKO=5vZ3kp4@sD43i8OL=-?qF+gctW+vw6)la#E zKh{nLQPZWF2&FvndPh2?8;(WffsoWj&F=#qnEfIDq8?8ulB6eaf*Bi_P{LpsEhkYR z1#8c&OiGf!e*`rBU6w%o56W0YRt}d4qNb0~IkCwaxqSSDd6yH{!Ga4&No zPWE-+>=nboi=pD7;LKH)PCL6?d(QEa@A=+WgwUl?=^QH+naV!PH|RzNv4aL|{(F=k zKDBc~Nlh>!rwRq^FXwmci-+02tjOW#P;ToeA1hh7=kEj;Mxqo434(pzR8Bc2Kztl< z3c!da>HYkAhL6}r%zb(4lvg-e=K#dD!Hd6 zVb*IMC`}FytXKV>k|d%D!Y%-`(%rs(!xO_Gh}MMDHNPOVh7l3&))HfYPA7lU=KLCV0TxN3;o1jMq z3iBbDZF=0HUclN{sLiA+O}3r{I8;)%INcm!AQY@e&Oo}hZOhn!_>ce=B+l=2&Kb40jrY7?&+C5f`+n|cvM;s&EB-H$x}`Jv0m}?8 zPwfTO{+UzVqBa4=AhTpKAdhu9_rOx~J5?0VVwZteBW{Rzmf9ESbZ$U}*lZ>%JD;DrQ6|Ke)lBBsPl%Oo@os?d%v9#vJrr4u3t%j8F>Ua1F6AP|gBEFY$Vl<)ug=*xC z#(bK_6aNluk-Y~zC|P)g#}M_sSblLGvCjnSM^5r{r2ft2;(Dq0V)E|Qu5VBN{W1H2 zl7U@Dn4o8+YX5oUDY{a5(g=DEo{hsBKi9P1;4;6V_CbF5=E+DuR7`T#js5u4*Pl99 zXmH+|3q=<$IgoPp+tC+<#fkCW531$PAW|EUK|T4;?2pYnt0Er2mxk*> zqlPE^87w0oxLhU}279q#<;E8aOn*^Sgq1Ai_1%U$_$d1I1b8z?H!T*Q

>qd&Qx~cKaDi%@2sG0elMko_oO!sN4z=YP36p&C&MLtp2hPVGUQ<9gKN-C z+|9aL)f2OAR52CaNsw5eHJ`

r*zQskxWNV;c#^NX5Ru!a@>#4odlXRLh<y)6e9J1CIs2;Q3~hQ|E=-oo&qw+?*u>{j{@RdMx7RH@V-Q@QGwX%a z1xkCGV0vR4KqgS1_?v&z?+ty9*#6Nt@M?BJK?xEgVWLxUkQ8Sg$Il2)8yv_NPbDhu z`>uvrKWXkSL4fkquH<-&1czQ@-m=1ALOw-t_bK{gdDB8xT*;$drqH zBi$uqV?O>>j$$^4Y+lV`k=~|j3G7ZI2CD9bU>Kart6b5}qMQ!C5{R7UsX)m^s=K05 z@g}~eoAPz?JD|RV+Y~$R{#L8ykg>}~K#ieR01%0V; zyYx37vm4d-gwVs$DF>@%$uti_xaNpa%PqVy&ImE8LN8Ib;cu>3?D5=1;;gMYn?>!? z&Oq@qzs>3ouroL_htG7HRbc%#7FUi!HLv$HxFDw?ZMH zeq~I;>zCYJ#fWB;c0NPzxQYkFpG$YqR87dFuvmVl!>e^(h~f?Wzpuf;QXUM)N|uRK zRd~xEUD~pC-*T5Pzggu_6Y+zkg%n;7Zi=o@@+)`TM`s;2rx-&g8+@qf^Ox-#Hx)$i zM)(VdKqkTmeN>v0?$meGVVlXp1CAt0lUmN#22nk z9RMJLYTDipA)!7$OMPq5dR4I}YAK@!YwOH{zH~nWuM0ri_j&2T)T($Hfo&rlG@n@L z)0YxX9#j9T#(85uMoE^$m9{&4?JfhmCmKwSX@b{$k-=Sat`~=UubYFvMi`e|RLI1P zV=9|PN)oaQ4n@jsl~pG-%iG~R)iko7s_?#CJS+LIWm2I2tD2laN*UlSq9OKcd%p&k z%sv1PbvP-xNTeZXR=dvn{+d33gW>SL<*QGhG2jBJAc@C*jQ_>yA1}R@1w~d1aWM`~ z4U%pJ_7UMo~cmR^Y*|jV;v! zVP|rlUQJqDVs@*F9WIfpxJ34^gwKGexQ0;V#HjQJR_bqbmy+5`UqX%!HoTtsT4Msb zdEAG`UXQWj#Wz0g&IZF%_wz{ue}3HNvMfxSV@7xH-m9HOymGOcc;!AVXU(2@aRyjG)G#}r;MA{Zg~*0Ajs zI%k2fUCB#@O~8|&rZe{8uM%CSJh%}3BzK>CB7I51X9~9{W3P?0Pj{QessM8LYWvZk z(4DAS_p6*%1xT0s|vcHFWNLGcHTfrve-H=nGfTI~ESy{!dgmEA7sQ0dA+Y^}L}I%u)x*2>Rw?=5?D zYWZAEcX=V3F@bm2nR|1SL=UshJw1lz^VZ;tr0J=%B|jSsFquB1#6-+ zo=`EUEyWju{*$r0Ggn~;+}Je&2h^Mx+g$Bf@=N`2E$hzN^t{;&*D5xZ#H=;Ae((?v z_FPRMtf+UUpVmaX9_VH)aPXHxJHhD7rmNrN?ak{;<*W@XAF6YL@|)@=6F_P z(0qqcv5k{*EE$@ZK6ZXIS%QM-<^P;TqbY~i!p4dWgT$$^0wd9w4PytMBAYSqq@AK} z%ccg(Fc?NLf6ZT3@oHHqcWg<01C`S1SI>cI-l%au!bO?Ze`@(Mz3Z+ISYdAhQ9}L_ z{b(!A+mc{;B3Ptg#*|lDIqWar-#_x0l__CO{^!9OR6FYb7!c*}Khu8;7IFbI&|$@Z z2W;;l@M|8wP6uI47Q9vP*#eLV=ArVhI@O~;iGj;yh*5$*m%z;Nc4dX2Q~dng48I#o zo=bRIC2%=g7-gWPEV4iO3K;`&ecTz+jPn|R{=U5S>=^{qMM1NWLs4Qj0={f~oxAQN zrL*_|$wH&@_1uxFp>pFh6^$RPC@>bMH}9+INbOP>@c}tcr+>#J!H@15ph|rD9{BI` z&p}f{K!W0u?fnjCOE;R6$40R<#i3&UU=Tj2Nh!_8Ux6GwIl9AnEM@JScUtytGiRD0 z^nSAQwh*UFnNC4}#H_0WTYnKZCf9%OdjCuoIQ!MFnn2b)ajst8=ht}0F3tSOfrYSd zqy4?^BC+Nk#Q__TuoQ|IHef>JUq4$w;^4fsxf<=-9agNbD|9{>z%3dD)(Ma3*0@+`0QOt<#n>)g@YNa2U zi1SU|kw~NVDN+(xm>V(L7uRSIC;Bvv@*?c$KtlmCeol<42ZOyF$ElNi#jFqoG2VknpMTWJ;%I%<}nb zQn1?el@;an-6MHHMP*Ge9$~+itooJTFEg*zp$mN8y349eohmG)=g-MK@uuhSXwktk z9^RW&O_6bf#595|k|vQeh6oRce@1Z~AE`$jU)KL)03K53Vk?D8v#qOrLnO4*7Kx5^ zYC~C^sI<)Nb}%!SWP9H4W=8*+jx)V&m*OMlL%vdAlO+&Sllf_r9PYubFT2G|qa+1G z`z#VZ*Bex{o_(+T59D;jBjs3#cPw1SUmv8OIJBE{+sDrM=J)Xzhq?@8fek|f@Wupe zeZS5eS0*1d7CBpPw2BB!21fSRgpzVac&*V^e|E?UlYvR zmDWStoe(+CP949{r9#Jo9*PI;@RP1K{~d2UpP_}o4oa{FYgFzktb)NVZ-j#q9cMmx zMia3LXV*V6Xd-v&KzT9nuUBS&``xUS#$U!8Z~d1m!VC-QxQ>q1U}90U-8lvkvjlx1 zEe`*S>51aTJ!sz-Q6BT!7VU6z!TA!hFUNI`Qp&w2spB(b6Oyu_%=m%bcbkq7_O^j9 z11$;jAi;GtUV9gKbV6yM7M`B%UtZ^gaC3=G_mEywuSS$vBqYxlfr20rsk~?tN;Y42 z(St{k?@#7B-UdyCj*YHy^y*bT&jBnS& zBASYfnhn@!iJLy|dk5DG*sMV98L&>`$zk=)cMwm9m;$SZT0UdKFH>|P7%CPq6R&SI z#8`HZ`Y`G>Ih$%QPEKA6!~CA#+kV|joM{es6RKXtg*`V>8vRp98{LQrs;JH!Q)kP7 zYa0V=t-u3?g2u?$(zlWsIW(UyapP}{WsMbce5{SNagm_WaCl-KH{viPFK-gw4dvPP zqvjZq$XRti>-C|H&u(&WigCWo-RM^#mDmPr-BNcuop1rN<3+H6$LqAe?X%D49D}L) z!M$f5qxi=9)i}w{TG1lUVPQ!%KLG8#wpR1Ji4vdbfNP;v0kw`z-)Hc(^R|N-Nvgt2 zTmHL>0qquZN@%CAt!9Np2+8kzSSHB0%Gwhd|2kuTypCQ^h`Y^Rs$JF{8 z0gupL9Z&Y_Z%d?reccan{c&1WYV^pL!)vbUSKFzDC}5!QTu<=*rTF4j(K$-tSjEL& z=95&Z*(3dIjg)@hucdz7xPEOu=*Svedt0RHAo4a71^hYA&2iV_%9ENRAzlS^uN|n( z;`v^GYXRBeatgDkW)8+}bsoH$B)3D;eGWhWeXFqR>4s)%oE3Z>|Jcfw)4kGxHaGqg z6!>tVuHbODEF|8ZXlqiAw_TTCsd-HE){pPPObz#O9ulk(ioep5?N#UFZ{FX_1JL^N z>K0msH>e*r2IP^UL86*-$+H_*bXhgfADiHXhrTL#*2kYONpm6-GpP%w5+?2@7rrwj zj}_^*11GrYDxCZqn7vuwgJo$PR$ce%g&>7R?jo-Dv)}Wc;`Ke4O-fX1_+-|wvtvo! zji{=HS>miS&E@6FTC{&XINB1A9yYv70wIQ*44=hbbOj%uZ!_R9|97c-&xBKtfpjzC zyJM5u6VaBb)r$U6x|_1+L{|Q79Bu(B_(2gR%a%@VZ{jZTnt*fF{^TR^Aof0g-&}}n z1Qz-Kk)z{nNHk0MvpQU-1lK>P+rL1?Je`uL!I^%~o)h`+mtHoxQIj+-IS@^CNOlpq zCl>Y)hGFQmZ3T6I!u}pOXtLVb^e`dA_zP@%6hU3Q;i>7rGq)v8l{w$b91ZMmw4!20 zDi^3Y^0fI9!z6Tks5TQ_%j-k9_s_P>&~xv=iMc<&Qn3b~%uh8Md^Mtas3(9E6ph1d zPl%OYfmqGRkN*{f@CZ;nSrK+-EYffmu3Y;_9fr?rdcTQ-yo^eus%=Kd3fuVB3hp>c z3*Hw2-J$S~`_zusjCAp?^S7o-R6RUn$L+GF0P5W~84CNPvQV>9HB5B8#X4)yQjus) z9#8?l2N2{KCeRPC@Yrli1sfUxR$jg6)H+OIYjU=ynTTd=kY~S<7}M?M??|^nS@60f zzg|_T9+p#l?xKZdTtCP6@avu6mm^9c1@nk{cvtRytoa&Q_(qqDMww~h?YM5R<*KG%8_n7R`#e{Q>z>@EZT%+ke^Sd1Ob=MJ4{OmducyGUOS>cj#8 zyMo-O;MSYQ@5pAPx;{WcAl9Wgy>uQ`$iTh~w?iYWw@D2LH;+x{NU5|(v@pfh97Bvr zrpwm;uZvl9qWu*C_;|XwFB8&o_RVw=-sDz@l-Q7_%vik*A<}iVQ39%bo2bh3y)V#1 z<@h&IBaNZ)F91oFx@rnFi>(-FTJAMVeoMsD@=k$wHzS@me+AU-yJ=sO(RGztU{hY& zEW6zG9OhPlzXa@$BX<-6wbZ$?*Y7+?IS12P6_o@(w4VI8)01!*-UWI3X5qXZlozZQ zrX@~K)#7_)t4-5XDE8iy687X^2J#TOj$GTm)A(}|wA1dlsMo{n-*J^G=1b{7LeaV* z9v6+A%JGXT!t+IEj}9di3NMKm)5xuD2I4^F~BSYvUzskp7IT8V{u2D zzrB8N28X}hVFeh+fT6oBukV@I(UiGC5p$HT#M|i9X%XW<%6tx#_TPwO%I*0gPn;w| zdsi0LM@xby)EO4xf;g;t*2$Y@j=Uvg2zeldOuOv&NC~S~PMq2C#r)IK(6UGW^fJ8)$O1|`lguj@q+HRy4Km|SvCoX$ z=B*t4nEJ}hUct%2bUXajjziC~zHZh3mx~c9BWXUDf1zvZuT(ap=B!cUY>aIep9>St zlWT3n1GZrRyx776U9l+aJYexAN4e~yrtMcfcL?f}k7=GJkldZrLxUq_<{y{>ZxmqR zMfVkEg-hcIz#F&(>9srDD&L|7$?Ucbb;b6bx0&;9T$R0_jehP?jhD?>PuZ=m0h>rS zauIPa9J_d=3-`j9%MgN}P`;%WfuFmkbo`7ZVmLlfdHnrA?lF6d>$XNWCSAfN3RtgT z&J;LHWv3n|+J3Na54SQ$oo!m(DXkE+W!Js-LV}6sH=zI%d%X#1)uPh`ipJq zLyykCdEIF{GLzO_Ru_@l`QdARVm$Om)@sa;`6g_ukwdMyL2KObY%7kn5u?pvtf*V| zvBrx{t4g;0`nDmq(e{v7O@KP$j9q)h^3Aa!NJ4vQ_dJVwbq=Qq z6CBrMG~HEJh*%p(KX{TmeAbGJiAs8$B)y$B9Gy~GQD@~>0rYl+2VPHH-8oau{oDVa z1(51+=wJiH+<-UzNwohWHMS#rzBKsdCFHws9?;57FAJHmYr#yO{JY!p!C(042!~U| z+V@6FeOIQuJ-_b0My$jBkX0QS#-Z($!knX;JvZ!O!IKK|!$q&zg>5Rr+9$Mh-h>Mn zu49vOTGPvmsT%!k@{f=)dcJfsqHc;vgKxKm;osP8QM1iVE1jtyAPocqxP^O@e;fon{Q~YyrmvLrKKr5Y#8qUc zI-lHg^H4@!iqbq@q<`eXLPNB~)kX=p_>nNtQ25i-FM#~num=9^j}U=xzH{oF>4I}n zjkmIX%;eOOtw{=F5Y?D|?Lj$OT#1adOLJ)P?j(1Wmwcg$3?0JodWqCiuUU^M6iUYp z^IQxMVEu{A>QQ=ngjL}g8z$W{G2T|9@^~a%r^b{gkqV1qYgSw`89Y{TduZJeD(rk- zxV12Tzv$Vt+Maz*ti|N9Q+l$(ZQ|*!z8@*xu$2CNJ&ay z#o254pddJ@6b4+`ZZ^GRBwL`l*eM-`2&RIx)No%PF7i9!vKx#S@wI4*t=2o4?_r+3 zQQ|9%*KmK&kFIYwxNnbYbe=l2Rcu(Um$u7#q+0AjZvVZ!r}-=@dF;qR4L8|gYfG#p zfat~+8m2v>^p1-zY^{7~ZVy&rZt;E$U&Srir&`KjeTzj(OmCNUHk;;dD?5MPy^gnP z!LbKOB|eo8#&M+@RlvN7sv5UV<9Y#$MJ)yE8vjzvXve?5e~`~c2@f!s(JCy7ci7{H zImCYs#X_B9s?^VuI)`;Rjvl4QNehGLlCjW%O*-Je9b+8LV&x?F%e@<1bp5 z`MODdWe3gk$*}P+^)!H&(EJ>-TPZX~P0i`hIJVXR5&{V1tH;@4fC+f&OVg9q&RQbh zC*hSy)6uHuHSBi&Ah4fmO)@?-QktDFY@Dh|yvQtlT7ei$*h@~ z;{V4xIG19dr>Ote-u6oeUeYG%JGa!z8C>X(-6|`hMu&QO^x)Usf1%ghSI8YnO^f~v zJNz(xUcfUBrbA}zf@;urO!mQ1BZcC+CKrO0+`Q~c{+dZ`NF~MdH%K?b^)rC|6{QBU2 z7T)E~!wWx&BDZ0^-@J7P;MmB-R%5$R<)?Kw<7LF3y-!=GJP+9hKH@JL*X=u427_|V~A`yMp_X|Hd=juCTAG11-uc}@a59j2C(Z&OKgOtb_-T6I&u}AkvXz#V$9_`DR z$l*kd8o~XYr(`v)aXaeVMH-mhzL{5*`*Pz%;^!6ee7A6K-qHDJps6VEILZy7+sZL5N^nxeJ=b)L(;FUF(&Zs zXXvyGhvMwLGE$_Jkwz{-6>UuTvgXNUsVml?%XZ>-@pM;}OKO6D`ujUUE8D@LMry6{ zo!YwnG+vC}?Ul&U-@mYTD-N^`G1*IhW~&aMob5IQprNH@5XB0J5jz?hwXf!iFU>Yr z9_HrV?b|QJ(B<<*eK8jV*Gk1QQV;Lf{ic{vc#5zV>S^63@mC*qe-|!Zk8V<`GrZV6!@$IwI1 zB*qV=A7!4hO3J-!`fj$xIF9K0*N0$W07QeJ?2p--o;17y!U8Dn+Ll>DNc;;>0&xpv z`mF!4bacV!S6?ZueL1tp0c3^G9<2aR7sjp6d} zRT&tSUP3?)$Bs6%<_thWdwk($2Sckp$>mR8t0+G+FW;gy`0}+$&v~;6cIBmT&`3a+ zhW5wZ4s8W>k01<0cGtcCDPN(>g-@m8TqFq1R4F1n>}K77bC9oh*d$;KNCdvRAwied zRk{^2`;q8KRW@-?6nW7cw;rc#1qus5$f3dD<+HqmM7Z2&e`5cgc^)GlO5qX^C74H!N}u#!P43nh zyu8hDdQd<;=6-J7E?`5ifH%Cc-2+>uj!3R=8@(-XoZHspYpO=jo2If1G7;z}tnNKy zag8A6w*Lz{A<=%j41o}DnDdQ@jD*kZ%wz^7+m%#iZo6GTu-fCu#&B#iQ094&eA#G$a+Z6wLOkW`8{tMK2fr#UI0@RQPC-oeQg3j&X4!mCwX>GSpi$ylw44AMIs{Yj$8aDlN-in!{n!$fNNadG znxWfrmmkud@#{T<1>?ZPU`{HkC7P)4~P|u*H&)&F$9vV){ zx#BwspD>;hu9Uo~j@3P-GU{FV9S$C)7d<6Q4j{-J>Ag1`?}CnF3d@LX5vY}>`$9HW zl$X~lnufgjHdBxqXD~9C9B=DG=qDw`bvvzHEAU_fiI7OatU_yoWMZNi66y5Oi{LUF zyC^-f$RnRKOaqhR{Ft=1G+@@pf&zhyMEIobY4N8qTi-u)v!5-yQ*a&6ijrTuXxHCF z(?H2V-^S~wFpx}U+8tT3ht_CioX6CSBA;1r2yU7v$K%i3qeFKwIF}cK${Vpr-5Df&?+HEBT2C#+nGEM6d%17uQ%KWM8gd0`0>|aT(x9?D&o79|4Y%Jua=Y;s zbAz?SOE?j9ee2W=D(24bL7=_qOUHeOfwK4f=wW|K07`XTqj2Y}G8dLl6x!&nfP9i0 z5sG*O0RaEA#f2D=Q30;~In@318O+q^QH0iz9;owOq?I3tz&SRO|CxkdJx5$K_nEh( zLI$f9;5J9vcmzT|L#NVXHpo%`Wj?FwUh$I&CeASGNxIloom8hv`hKONdn*a`9ygf5 z6n6fcuz0v~UL=cbNoVc93(FXbR90;fKSA=yOQd(z;$2ZIUjlM}-Q7Y4E-yWIlKQb! zO<2UjraA(C-wE0NMNmpFBO4yJsvgXN$~_-MMNl=NCgr1BQ|totRCUfRGe0_h^7!;o z=ONq^`DywZxu}njFR+bw$|rYvbkcA9@!6xj=RloKb=0=5?(cOYQ^Vb1vzep%?!Lg} zsf}U7Mkt1HgcRHp>?w|4&!hi9FXCw-Sfp&n-s)|Z657b6u$dvxx|oH`Ipz?A2=)nE zWF{ejv8s%Ru^XY=K?XZRczi8-f%rcgIq^>U8CBX5cleKAE% z`CtP4BVe|q`+}?9R`^&|9pbXHnFKN>^WX`$bKv5Dl};ky zftq@|CoG8n67L#x=I_UPp~*5?#Sa%My}^!kjkv0Y51u*Po@vy(^ao;%qhu8 zUzG4!ZF_y*6*L3&*HOAwII5yhT+qQt1!TFAZO2YR|-Ww_v-meGYo?* z!e75D=R2Urw~N6liG+q!PnI7O%Kf6b?j~O2iQS}sy;Hko-^>t8j${~wZ0rx2zrYs{ zoghvA+C1VcVcea*75M)H3cdYVF>4R4ISar6R77JYsG#~HYA=GZq&q3^WTeT2$Kj@q z$$`$zTAJzY6B*}ex8LmyoGmg2!!m6+LjLyWZ#zd^&rTCUFmY2QnGL@qUfU1?8!uM= zFC!wDAq^r(Yg_?RV)4m$$MoD)`E0~kXZ9kZj{OSn;q5~UJd&QnI6qkJCO5ULK>cTE zJ77hxskfS1-6pb`kP#PzYTK`exSh{_f;E)xNn2awxIv`eq(m=$YWY+Uw+6%y=!^dv zJ2NsyDd|VvfpNaN{cBz1J1=7VT9=4exOv&*8IF>vDz@u-SDDaaCOVj}9Om^&*phFG z^G@Zq6dwW4Ffxrj4s!D*!$8tA+iOyCz(o!krC50A6T^2!H7G_?t(u z@*yU2V{vjIvv&@2J}yehW;m1P1o`F3FRU!DtEf$luxt|R^lhwSWkf`%CX`WJS*$g6 z2Ei?lDX)7ecr5n`cx<}d$p*(ORy7)lqNy$!uaE%Zd8 ze36>ABd{(KN`yKXt<06r(3)TY&I-*nD<0NRdgEfz2#u-g2{pkd*_J%})pD1l6F8Bo zE$OKEHsYaQC98L<4GD7Vd{3yHlw|ajE!i`YcQs>V1wIR-Hy?mYw!MmI#P={*!F4BF z?!!B>XKDG9B29ol<>?^+t#=ccGMdEx-xO*&vOxq-MJK+#Efet1MZvc=(~CM(A$U_& zMSHnLQrj|8vVj=qX#e6DOxK9V*fkYH+b>QGt42$E?B%QjB=`&qmHusz;`gI;g_(Tf z#M7W0aumnYm{N=W{Uhe*MTJ1)hR?ZWhuY4^sdEZFeBN*cB5Il=`c#7Aer}s2I2UF2 zHJWhV$Z>a|jkdG@*_sk>OAZ!`fUt_OK)ROZn$gS3X#X>=PCEN@G+|Y^`>**(^#$Q-rX8&(+!DlD9kO1S z*VkmPq}k@kN{X)OU_g$adLJm^PN1lbM1Se#DGxBiB7dXM>5tB>^@x#@tfdQwS|y)x z;)PKV7{OdT4`APS?)!q|bUy}Y#cBB3Q^~Lg!N|7LDXuSlPhcvs^A~<9{a@UnKPy4_ z!~R~k>My2z!~dF(jHbT#RQOs$qy5;=`B2*omp?oiKjZWTX0nsE;Zl4fqW)io3qQs4 z%JRln?fQ4htyAQ)hqM{>iUd`JQ^3s=Vt3oX>I#oOY~ znEAG;F|U9#r0W>dVLxbRy&sNPVzJ~MF6XAD@L*3F#h7NjapW~pCN4M}wmq2MKHG7< z(W%3x2$$oTqnym^3HI{Fx!F(U-J%1NV%)D8mAs}(A&$xSoJ!bQ;PmlV#@WBwmq)=3-sE|wQru6EUM zD9?Mv@;V3HhTB2r74O`$P8sL;8)c5DqF&^+(KF4E9V1#v>U%i_v?{klQf1W8Ho)cI z;R2B%%+Chc@pZ4Q0N)@go~nW&i0qw#g<(#-B=V2s$B+|s&NKZ)2v!ew!8(iXw&jnK zm{;>!w!F!W`j+!MAXe&IkcK)j$)KFDV$OAA?*|!+9S=M4!YS>PAEM-a!$%k58K(vn+8hNnP$VDDSqLceT@z>B-w>Me zdrQxRMjw%8xQc;sK;!Q@frJ2NOOo8xt17zZnkr{xYSFs3)lNJTC7-TYGQZOGF07p1 zRgS}Z?xAO#p1iIEyAyNy&@uPJvI!#-5kFLlwm2#CDAE7&c2*->M(RDbwk6?8-YM`7 zi(Y2#(X@%h8NDsor91nvKVs?x7{oqeWImKjmmROX9)q@0!e_4zF+#var}V$<7?vEl zM%)mQ(wgr>*Q=H+b>ziT5SS()rAIxN@BeO9w0!eXDJ{<-EIY=OXuK4$X35BR`-toJ z)5fjVHA;E?Uc-W%^8nipg9A|QOfSl}>f!v(4HGLs*Zn$GySJR;xk?Qjg{R`(cnGxR zhaAs4@#KkOcir-y03K4SIIG$NrL4mQ#r@_HtJW6>UgE$-&!Ai>_*D5XE(!MkGCAt- zGUX`mAq2k8DNw~g#&SQCpk+?7m5&n<^FxBXUjeUg1|b+~rB>zjqHLFcogS{qVuG!b;R3KmrTgSBf`=LkLT< z$(hTsZzfr6s}G~k#Q!BEH^e(EXA+0#15LrsFWRM*Fr4* z`u4tq@y8kwT8#_qLLU0SKR7D`UJ@eaO~XTO92b#Mu|4o1JpwKM>dv7@hu_NEkmsmy zp8BE)7bLm&$x3hf3*O;J7CQ-w=N_0WH6u~hTT}V zwiO+J(NtYq%q+Ddzt$ia?@2j+(Wc*B#usli=7OXDKU0dVh2$oO?MUM3cZO{8C8_Y4 zqE3|sfgf;GEA8`a_Iuq^EW3=k3EqZuK18RDKREFS|6%uRak@7asJYsZ#&H{6Jla)R zPyIM|$k}GKu7~gnFeiwfFxJJ^3_QiJv5T(n1C;E@Q)+cBHfi_25L4e>4J7HA1K?fv zk@^+0q@H_t_;b8gEgv(5mDBAc{(15rhKz2cjo+1G^zLQB z%pewuWXIVR3b*gqsAm7z_~mAMvAf?Nge#tnb~f9!c*kDOQEGQdUA9kLf3tT0F(8{0 z^X}wqb>fFAAVpE&oJ-zF8PDk%Q99 ziwJZvWP6L1*IfuK%*E|RT1Q74n4UXE^37^L>b6C)w96+A8N6!=N(xWJlT5 zBi{H>HSQ!cc{|w;m_zN}6jdka_S2EIgpAF5clPVfeaV(ST=ONP(QhINGlx|_B=;yR zp8tV5;f4sq$R!F+eX|8oha#U`j1YU^J z*08P85vVBCUFP0#ze23*6Nq9}Zc<}JU$U;mT2;O{V)R^O{(D@)0%DLtSAfZp*c!cj z_UxSl4S~kLo9reIH#d8kFg#OONVC9CNeLKAz-)}YS!$&OK9oC?wW`$HuVg!{3qNd0 z9*a$czwH>mqWN4rQ8x83z9}`XsMT-!&!K2Sb|#3A6tci#gvyiuTEaNKZ%G0O4`pgO zho(bC8oG4m+&=PMg{GD^H*iSL3^LHysr}wQu}eNMuVpU{PL`NBo5jU62G27FBf^MdwQ45*cp_q!rR#?MgTiZ+}^ zjNAvfX@Y;9n?;aTlq^!~-#J-o*KtUociap7iXHc3UjQTSuXQ+b?3e~qg%dT^EyI7r ze4*9N<0=dbK=y_-V&|KfSodD_Tj*rv%p%nm5xGKb&nxFof|m&Gllo|++cMcDCnnqu z5so`SnF!Q^d=()oD423=aU%Z(f}`Tt5kECmFP+dXjk!eJ7C@Y*GDM_%I%`)YX)U6q z=JGIml&^mpMnf1jOr6ERg2hitFDK%ehq$DyQlhAUIaE=S50)KXAo%bivEZs-)&3WZ z_!ikb;=LQDDKD$d{;GrFQQ~6PlvVZr66Mt&B)%*z0Y?@GK%lnknDfZ^W~9v`yHv zw*zLjkB`nAX$YJ7heV{E-U{c?K6~|aSOT$J3^{|NFGRQ8D2Z6yzSkVRBIp<$xHTfYSdW)RTO5&G0S2wy`!ao)nm78>K( zCsPP>+0NhbOzD9UdUaiSzC)3-eX5|0zx$HT6eC{Clxa( zz^hD=aXVvr)_l`VgBB#!%~a$95<0}SQ2y5!z?{&oV9<_;M9Q=H~Ge*o`#XF3^78Xgxe(Bq~-2EJm!TdHlBzGQkoB z|D}O)QSPldx3z&kQ0|Q_aj&GGj5esjILMOJ>O!1qF2)vymd$Z`be_AMbA&jTbzDwab~76JtLi2zxg3*)jVVQc^^C8_wU+yf@bINZ3Rv zND;N9A5U}%?`U6zXYrm|Us4wrwa^6L_hBlA8qNdbvq#{KKzLHS`#oi+YR}<60_ZEd=_TGVmMsO3Mf0XPMrUSUZ%*ERSa&`A~$zvi9#aV6paMmASJ7VC-VAA)X@Qg1}nJFu3UUEk2*sT7o z^}$PfU^<}l&dB{9O10GbsjnAft`DWT-3QW~z5#mFbBJhy;!8jN8%7A;JiZnc#>#(0 zSkR*D32Qn};ED2~=&xWIZ6;*i9pXf`2{HAkKz|K#kiJ9L=B<;m79~H#7&>N6fzBE% zN_+t43JOpuruFB2xn&{}|60t~`}RPmCH}OUDMU}ZMe__5+P!jE`A1@4`N;@LTQB<5 z%kvcmygJgKf>_9YK2x)ba0#w<^Szk(7>zI8pDg4ptmlyn9xKF(3~njuRH0UO)5TcXlOPt3k4vt(LSLT1lz6s1enT*sT*axt){hMIIE$6Qi`af!Hf@5+7d&r zatzbmdrKDKN=nT_nPS_OeZjA5*h83yWE3?4n|;N4QOUDf{InVeQrlwa?P>a~z>$f8 z{Y>2U744YutzMnnO;LTDtS^&o>#8e3<< zPcAXrmG8^Hw*`OAgt@1++D#U@PK;wC-+mDmfIXXY*z;`fYC~0y6?r9_+A!YoMMQ6XuwjztYVnQK5O$RY?7Y{=)L_(n2=UF&}U^FT(lc35K}GcRD9Go=(FkS2>Nq$Jd^4w0i@652xLQ{zI(v%1+te?J%M7LHpQsk>zaLY zv#=7Sz~?U--Fh*tql;!U;Y#Ek5LWVMS7SH38qkL2jRP2j@e%q5T-+HH{ZkuzeJqwI z5$KG^q=dUYK)F22IBO$yIC3w`k=iRgm+_&r5LeC=W#^x-%le}wHY@v^VIs2PKqe!f zV?vn_TngWl#W!@lbaor_G^~u#x3zKBNvl)y@?6<>O+@%8w<=6kjnDuUcroEe2etm$ z<}%Dc?f)cv^X#aDP)564=a zvsTVnX{jGjxlYZNEkNkFWcBf)NfbXPcBfJjQ3~;q76O+6iumg^=#7%B4r5d|V9{Iv z=cYwS2^Ji7NC0({VcO#f$RU_uLkU{thcU@H<^K;a~WJSu&1BQ9| z(PNu|(!iO$uaJOGi)ejwV(Ly&U*R;>my%3AXKnr(U&sC0_W7qd=uHpOF-bV+?jc5e z(lmEv4&wAbz`QxJS1|blmFiq-Zmfaw+YlQd+}=`}3;@M>IayyoV*<)}p(i2<44hxT z%zqj=^SWKO8)8~Y2nPr0_>iYsh&Q?3`x;w6b!}Ch+uVY%eVl2)VeefzT1;XtfX4fC z96xKAZO1R(cg_5xTl5(9a(w1m7aA1kc~)J5BlNh&Zy62f%xtminoJkm-Jp^ zq#IvLt~%PGRR_wl_Hk@zu5fDr6L~5gdo}&-<9km5pF@rlwGen~o#S7nDiYCm_+Ye0=1KDn>5km$GsBJWj*+vo%2bCMIlvm%Fp);ZPdW zD<^=F#W~gsDYVD=q<8a%7AlF#IleIH0Lc0Wk59Ug33AyF>aY}iJHx3$k@J&c}Mt2}5!-l~DTGJ}JI z6F>AzSB4~AJ8G#H6CqR)*pNMo{Uu*VH+64xB!jw;usVThw9n{6Fh+q--_lwT!rA!` z2$UO;?=BLDB?-?X-&V<&Vwbb>`Z_EjP;@`4v&Z=ffhiqvs287>T}^2J>7?M#x7Nya zyuuy8JK_~DN_$+)L$6$L{tCo%_$uhzl~p8TYU7{Ijgz#gS%HEEYl0|WK3g$n3$rlQ zdmpQth6#yRI85|enQm|njt+J2GYDGb|AMd=R$ohBgNgNgOUBjBZDO2r`JAN`IQp<3 z!9622_8PP~ALsc3Hn6Pl0e+kM+tv|2fGkqtE?a>?a3fe8aj$>L2V1hN7}Sd$P#}ku zbbj}E9tp(^WX%Ikmat9)&80NfCPao`(l6B{HH!!A==F8^ZA-xR5ZRkv+F@u7fZLO> zNuEh6T{~>|7kPWcWR;JpxgjkzpM~7Bjt|*B!oxavxV`&X5?4Rmi(6Ma7wETu^qUhB zL$^+d?@k!M@;!T5f-8BuI7dcS23A)&O{*!f?D^0fVZzZmX3k!1_uAZ_f+vs%kAjZ6 zVj7dHOy?PD?-cZ0F?-9EuMd}3)NR;~CIL{nTjf@FcwU}+VUP%!&)u5*j!x{sM0Sg^ z=+f?CZ*5rC-gVnM5#N_Prz~dcO6!{#HpG)*e@Rsr5nb5WQOxdgZ>B$nA0=!9)>P^0 z#lV6kgH>p_Dt*p%a2F_k9lo4CtAoPaQfi>fTH?s#C&NG{Rb+17ku=#k*K$Cn_p}j9 z`BFGiY&vSFiVpFMZx-;!GDPgjb>b|&**Xf=&WW6R3*Emys7zD~SpfL(WJ9NxA)d=P zh5q>q#Epz!^rLTnky=tHQBI^+DbLQYPp`{p{rC5>ClLz+zSVk>PQQnbx4H})m(IOB z4>vDuNgAb;RO(O24ZAlysFP%L%}x~fG7CR~-26%8fFW7Fsp&mT6m;6zmvHRPv!a5c zdrs^AO#@syciEuu2slt}AYk9fg<2TO=cvz!Z3})&%Vut>^6lAMY_3JqvLTt#LmoF{ ziDkYP*2H6~^EX5kvGqr@vo}&u1Kptnc}9>Q5kdAKe-<2mY)dX-ur_-CE@NL(bJ|mzm=99#;=;9G?LO z|3*j}_SHIk`K#sGmkZ2y=$|-R%wnOR>z+1hzj(Q2ZG^|rb){V$ zmPbrhXZ4yw;l;e#0E}x;Z$4E1a!^WAvqks*7Bs;JmHDsKxQ>jmho=rr(9+(!!K$vh zFfe4(^_6>ZEo1|X!nrB^6W|RV+DOa*0^^Vui#e$B&8Brb`u>*E(o(d;6kEMEPIy#v zw8SBFR}y>&)!fT0GEX}OzQCxs_xE~n+T(qs4}SILAS3^vYhtIM^H9Ai93IuIX)^PG5C04y2{^nvu#W*=nPk(DSu{TiPxBB==Hb5ky1t@!NTx#v5 z4XbB9Z}E7G>g`^tx8zq}vblLWK25Y9K8^h4DW6ixlqN^*KISy&+qYt6#q76VS8nEC z-xSsWJFbgcd*wojV?QJ)oLoy|^Bcy;`?+K?{a<4d-k44)NTr{|BJr94#P ztK%1uCl)#!oSD-QkW3a|-%A;?Rih`8yN@V;Z!s<5Un$gs{;gbwD0a&|>rnV6tjHL0 z!wx^49&}&1p9}t7=BYAzGk9>w;HF!KIR$iH)bnt@e7j}X>bRpwAr4^|711^v6{|l} z`xJc3Z>xork?Kr|^)-JAp2cG7==CNcr@QE#;9_QsH!JH?SbLp+2*;sqN?@{8L^ah> zm}*wGUOl)P_L`GZ3X~y$kWv;d2F;TuacX>;433Puj7DNQZq&@kA{`ZbfgzvEI>m01lV zP--LADrtFFXJ;2ITRGd=k!&f(TYehf=V%eyTqJ%yIKJb_%n;82qujVhUur&ajhC@d zs>P}W%!09q(Wj{Kilq;S^4Ep*`t`PdKQX)4-1HeI^ID$Jy3xPS`cALKO*ufv9IN5w zI_pbLJUKoePdf$Qsdg9D;=s0NZq6%++2MAd&=m1|yv+0H_`xJCW19Oe1>nAbg|_H571^hz12I6Mx~?+L_H;j(Y$s3cerJnWX3lHcR7- zHqa9Wu1)*0BJ+7JhQoadZ-*xwIZtbo`5VkIMM+GIJ`xfw`?$O@f>~IyD6VeJQ!fRi zNaALU0d$imY}}w}wss`nj`{!idJk}_`~QFZoa4kHaqP{p_a>CGXJr(X8B!W{5+xx! ziXxPdk&!}+N;+mzX;E4_RzoV9D%J1tc7Hzq@Avcley{&^x$f(_@4GnO@7H)fAM2U! z+I=el$DVn7;fSAmvV+r-_fK6_`&I%4Eo*UvY3yD+z={Dou79v41(pDoa9l?>5p}qj zL8PpL4d=9&NoM#yer6t;X-^rZ-7Z-sp}5A?*gXoMqT=w~u^Z9D2E&5C|2{r$uvJd4 zt77*U7{*n@-JuG8B9x&)0&#m#KALmpM2jm~SoSnp+AA&`?R7MdO<#qs2HW&)O*IFC zK8e$Bcln#lBCnzOoK91;2mNf4zQh@$Nbb5tyPO-+jM++;1`;igF!ymXss_Bmt_>oa z6;`Ko_3)JaR+7(0|2%U&eI7aq7UzbVE*@Ih+q`0`Hw{NytB@7ff`Zeg^LD3FbdawFw4w=N@CjmO0$4px zRqWo(;E1{dD}I1;B@5c$LFzwn73&R!HFUmIg@~FH+eSn8?8@CFnWJ;udHFZr6^?&5 zP|%emz2yp-y&#TSbf5OquiX86i5&WPFPj%PQQfz*(s!}r28njhh&+;zNRV6)t5&43 z=g~Sjnnt*IK|k0tgK+%+$jUN8@20YX^zv8kKD+cZQZxFme;{Rj0%c26FodPMV8R-} zSNh>vmEvu$q8I&Se0Brvev(FG{;5f{bxWk06HZ!O2z)9oWE13eF7dBj*D#E&&9_E7 zuVckI%M>n{9PO2*Y7gjhGCpTS9o$%1r~jRFQ7Msx&8(H6V*)RNc{(TK)-POe3~K~O z5N_&#t;WkZt&>Pej{#Lv_v@(%W%~QZPUPdF4I7W5JaS~_IMdU|m1A_Uha}$CnG2u2mwczaJ>9${j zAJsnr{=rgu^}|-P!E#sEjKxK$4La3bR!G(6RnSlo?}R7Va#PF>_}d*I zN_4HT^|>6mU`NGt91%b)PYqH`?BxT9)(zSbzOt(!&lc;h26+M3w+`w={8!u1;H$&g z_0!;SgdaE;>2;so<`T+}zy-*(fXE!spNrl#H8k^b^RmBo%Qyzy?i8Dxcj0|?*=RUs z45-Y*HD22$xG9Uo4d|J2%gXf)P94RzK1%E>{(4CQ6X~S_pkm)xTu3<10a9_DBNZQ{ z$~lvGYY;CriFmol2a(XaWl+0!GHY5Os%Y;>{Vuiw_G)=j6EFBPa+)^G(@?RojTqLR<-}ThY+i zGwi8&r^=L{K;V*DZHzm8TkRR<^HF@=A{!X93$zph?2Qix16}6xEm!eRc-8byM`isC zFg~lF@LlO@^qxI!fBCQMz1lU(#>~-1z|4dzl_u0G>xF|3e#3Hzck^dr7wza#E{#hv zH%p_J9b3ZJ3kwx@-#yy(TlE^15BpY&@YD2g2qtkCKSXm!Eg`V8z|G%L2;D$cG`%C- zn!hoN^r=ip+m|qv+Bqq%Oq$yQ6x_(Mc50v)$0N5vKf_wwF`4Fq z1KSr4J}(A%4PA*PLGN{|@8WiE^lQi3ACXmNbON|;JjO&5K=*yo9Jb-Dbkc=;D#o;D z#e-{}g~km%Kfmb~d(+IqgE79wN<&TyxTeA>vxax`)gR*)YCnl}3Fm8+QVy<6`H4q( z2`N93{AeS085owy5%!ChR1<-vS2>sZ##|mP@Z4;3p>_#T?J*RnZ^ZtMuK7`2{9Md0C8ZOK?D>cJDJOP*~*Kw%}i{tkW-lYz z5r;3?o~(!}Jm%c7x;~1Rf)^5~A?SLD^dVP8v8LDQRyS13ziJ-K?L2JGEa zM2i(;!r(L}h;$y0Ehx*M&f{wL-h(I$DA^NI;Gx}((Tq!g5d;>S#7vjn)4=7}E1Y&A z!m0PwhwlM>dOYE(6szkuMzV==E}R3Jot#3EuWg%ZHcuC5YDC-a4t;xukiWa_v#uP1vttXj37v#gHsp!B3|onJ!Kt6t&9=<{DK`Eo@yXAQ3s_w4;Lup^A+FWzZRK^sFi z-*gi1!J=@TZ|s7IN9Y1wtsC!UCayPgYsyL44t{^ZRapvrh?nhRG;4+V%c5dBouE|WCo8t$i76*%JMnwAf`NAfr-dC3? z9Mj`ozdi!Cu9aIg`DV;LDE7OEMo3m#-aX$(iOUwy2|hsIX)6M)Fy%ueP9J)eEa@0) z0(ySF$n@(gLw=enS@Z>N(rAdSWf%1Ahv8|_B@@YHH?8BVcq(jBeKHJ+{RzhxsHWqp z<`Gg3WiYFh`?gDjwKf4{d9`@a^@T1$!djiUNJsf6P0H%nvZO}#oJ*k&$Re9Mj;}s5 zfQN`@Yq+#zg_VOnQy8l_=)k7(9GpjIsR(SphtB2xQ+P8xN4?22-eMHm+2j{zaGX5g zSMK}6BdU{d)Of=>RTKiak~)2A{Ed+2vmq>KA3tq55jNP@caqR3ja_-BSmSebvE}Th zvX>aVlz#mPw!+S5MF3@Kf;$VbXR%Vzi;w)4my}mk~4Xc;41QEA0gVwo$@85EUNUkGz`MdjVusQH~a32TUIH81~Ei9*?;MBLeog}ei* zE@cT(IrnXsss(H(h)|~fI0T3Q!^2dP?9~;F5UGz8nJ2+#3bu8KAb9HL3t6FnB$!6e z9*fCKLyd?(t@rXtj;s~yzJtbczI5cnJGvt%g{Oz-c{-Db7pzEL;y5~Mz_av7!XVY! z!u|HS;3s+?oKJ7v5l1K-vLlw9QG)LLH|HE~OjN=AyX91u-+J)&`uO7e(^!VMXxve6 zbFjwQeS=jKhPlIK1gpssF|lK-H`FK_a5t=kH_ zT*t~vx*$vB(C)qmABXO5scF@t=12+;H_$o6+`yQf%<>k5=)im@CC<`iZ;FZx0Ra+1 z|E{+Lt=^e7!+*$u;YM|_WdH+0yDm&&p4^BQnb4X?FkLb#f45F&3aJib11wCle*d*9 z>6=_5)JYybfgrx)!UT&Z_Wb_(*!AZr5FY_KXujIpE(2)o&APdV*i6%Xi#FMIB>?AI z$m*2@aAO{Z((vuIwt<7x3vvnHvr)gKIL0*kdn`F~^wFh0hO8T!CMe1>`${Y`;HZ|n z9|LlKb~fZ~9gIA;mAmzLj+Vb-weiRNnKOAl+T~-fepAInmOxH!U1P__ee4@Z1Zd%BhmZ0kVo*h&(>RwT9fr6EEdzpF z`~3RuM{BMD4@HXE+%%iyC{g4kC&j|Y;-GR$NxsgFf31~AVC5vlj)?p{c{0`QW(jqA z7YnU_j?~u_Fl_$Lig`!G8;o{;o4O3C6re-S5rj^-Ukdj4A!lupp_+4qR>legI$DN4 z;1e++4+oo!hkCvjjmakY<>VmICDH*{zyVB1qZeTpZopmIGk}(8tl{F zvs*F2yBO}KP%5mmAL zn}UuXlQdF|BGGZ+&!uX9#}$k28dqjaJ-v9^egWxej0j~4=cS8hC|C7TvRG|qRufRO zK_8z<_y)jspqn7crGceFwFg26Tw1aRls+9dB(T%w(rEsZ=ocywf<{8MWib~b?(m4Q|&sGWT9q(3S9J+chZIk2|k9E(s zeS#zWz+p=7Hax~P&nEqO+g&#ZxvzEI?w)=i?u1SXpp}Qx57szJaU#6vhI^G0;z99p zG?H(f#JZtalEg*{n20+cV_^krx#vILZGjf#BO%uwz?<@E>cCka37&}XTY5mCJ*~@7 z@&p79Ipm(bRqkP|fya{Yl%g^zcO_eT*%BNyBm82;vnTyc5K_m{ZR}$MVwW=fIG2es@r*b@2D}8$CU@;l6kdmG^ zHBftmE_CS^DxD{m!Hbg7hrBA-5Kpx1*lkt>3+gXB-KL!&_-9H{(y z=d<=-dvx%@e=y-p4RTjixtrYX*S889a7=6t+i1q0Za_BnU~qG}ufbiT$$r|aSZ`kJy+xjmpptW z?-_=h1`?V04bdm5>{WNU&MS@D)0HN+oax6ou)4{*4DZs{aG%0hq{m2QJ`j0PKu)0c-=kiF7?7^eP zn~ZD-b;_f)>2|M;Hx)Q+Yc{ z5^uM!ZW$MJmY{$JDR{Z=tz`9`bF2CI8`E(`3MWo9V(H{6PQ#T36~9;ogL~E5HQ7^T zt))~sCDJYJlF?~t3HKk?4G{4J$dqqW+dgZXwJJkBW?VRuv2y@w`MjJAOXK|{{^Piv zo!%_*VGZZ=-s`A#X-vUU~&aM%Oa^H@UW$rRWL>JJy)&hH<5?pGtrvzNXOy2$iU z|Eo)=tc-+hw{kd+8ZVKrIj`lELzN~1W>QbtuUE?MaB}h9B!zqB`o6z^b`8hc(K4=& zIaST8c!`%R_-!xJwo+ON zn(7cMkHAfasOy=sLtQ3BC9#eIB{@PIp8@%~e;yhrOs!ABQJP4dlgA0GTAB8&X=7T8 zlf2p?9CxNU8J)hu4p`)>d4>!j!qd0o^D3NO8SC$>7V#T zCCgi`4a{Fo+uh?T62@UY^M&gZIMMs61yvVC5>Z`tn(^q#SE;I--Jfz`A;2Ek{IobbdwWS6(3DDt$eYh=AUU31M)9YX+shir1%>$cTy^Z5uEj`>@DcsAHfuM%3VBuxJ4kl25GNS~Zt>zc# zmvd#OVV*du;!24x2=DANhW7FC*%vrrKoyA;`NxKB;q1-w0R<+cF5ww#e6@#4|EBU% zh{&yj4oX~o`>sJ3RwRN%QES_nE{(d3=ALU8 zRuST?At(`{wTXII@^gnsdR-|gDI;_F>wMqdJK6bHYdW2ZxH#txD*aw?fTF`VFmrD6 zoy;z00i~7cko_?>vqWX(;o!K4J8a`mz|RbR&A`ZTl<%UL@P`+2Qe{}Qy>E<^{r-&H zZNznd^*Xp&Tl6X_(!db>lo5NV)6lffmc_YkYK1 zHV0GW%C?TysmGJ4k$E(r7$+EnX4`aTKm=H?F#Niv4&VXJBgN~2O9(i}7+v`l-``Yg z8*<4e!Z~-!F`tXVTJV{e6>RBXY=|X(j*y~6Z&eW`O2qPb>#FlOJ`IA0?q$$w&Kptz zv2azdsQfT|ec$!Mk&BOgpIzReTv9XEm9gas@2;zprpEeRS?J_@IC6&np> zpuR#};9>wa8FXZHLeI~dLKs1jMZhx5a^Y0H=67;`fn?mo7uUnbRLn^>5-ujzv<`ds z1;WX4ISHA>CyH=a5#Th2LCT-#Xx}Wy9G~20-9;)jJN*DAQ*3zAZ)>i+Hk+9hwABFW zUV}z~hzmIa=0ZHlC7imLi6ttk^XPz9FLn7Wi%nOaD-{8oM!n~EEWvyF%6U(;`j5D)C4V?qBFeJ# zbq}!!>}1udYgR2gqdU4 z2yC|_$*g`*I|-R+-h#M`Yuxswpf_4zw>7zz%N^tUQpTX~^VoN%*1wU)&}1Lx^)tdLjyG+MY#{!JD2M`{WP$HUNn4 zyxWAD$?_po9(ii+lPJvxd?VCE4JZ#xaBsz09M2OVR5Aah)IrURbkq*^lvM0db+?+F z{->*r-f^-tqo4ygAf!$Zpy^MLwR^~Ji|DI>o~*6E=ZalrwA{q5I8;+-VIOTo|7FE+0P}_EL88hUVLo2?FUNN zM=LvAal21BV%uLmD>uNt$->lDe(cuc)?sCwI zzP=?~R?SLq>Ng*Lvni^ zd!9OnJfzlGw4G|J(d!r{qW}5}mxl{^=bfAfw;o+_nWgg+zT#oCebAJo?@kq^Ef-%B zrP4Yc`MO^UVWK;0X!fsL&_|_LCZ0T;{CU!XM6RQ)NzsBm&PJg^+R=5Pd?Jd|PH&`v zTj$;(-Fg9q)|_Iyy46M_O!9K3`iH}%8I>o4Kl1#}3s#nV15I0D^U*Fnp}JHL)sS+_G>fQEGO z2b==>`WJeg_QJzE-mD<$W)XMCPb?;`LBB79gKv)kE{aA^Oi0=pEYF1^CO<|-qm~T3 z+(j8$G3%7{i&eYoweiazGd%qFiU|0kL2IWxb~(f9Is7M#AoM0?KgI>VE;yB3#|Np- z8emtLP?ykfIY$%02{CTNxbbiQsW`^WQ#PhH3J^YsR{>&aES<+OM{!zbUe`f^OiY#R zIkwu6Zr;%X9Sef3eeM~DpHfnT@5!0={4 zIrW2oVZms9&)cMq-D~Q>}!lpm;?CR;t_lOGx!$vsYGw!H8IxNfAE}@aWMc;+b z3|2*c;H+F(6p6xO`mn&qJ&;xW-wSdGFZ zvAZOj$K494Zj6NL08tkPXWP?*qPms~a#i+^Qe9%2K?M;L8*#^|TJQaQ$LNsM(nodf z>Pxy*+oEL!d0%-#XiCx;qj#hp4G}YV7x(z*+iB~F-rbq>GGG%|2{BBx z(kZbtReqbbAJPbrfB|y#(mhpN$9VldsHC8^GkkEkZar}*BLQm^ef`nf?G`%~5_TUW z)CR{!s$A>WH$@E`Uh>17#1ldD&|F=a-oo)w&{dAT`|4iNBp<1jgc=z`14Hw9hG}jm zsDJDddRqQAvH0-P!fCfb=wql~;X=imNNA8Ob}%`xij%>hz1;9J_Vo!uxg&(o;)4N! zpxK*iZ3UP1Qm4l?4u3#MXcYAR79}ShJ}DqM24!-*LUnf?YSC}*$%i>|NW-KMDzG4IS8R0kET{>d@KqPdkyCl51EW zPRjIEf14a!5rqzSCw|bfs8@<<0_!hq;nVB7gCZ#0JdQ5u&U%@*IK(sU$jP6}`EvRr z!fOO;)%IRFaYD+T@ZTc>NsLU!O@6oXtR$(IN$3F)jvG)a1rT`9nXy|7iFqq_n* zopp2L3932CWGu(Tsm!lZt(i&~obHLt#xl@na*-8OltvL$Irt#>&Sw{P_XE{^NA%FaSRa5Ni`pHMFn&V3(r73 z1GBhAHD6hqFDGjZFxy^u$oZB*k>`uhlH|pEuH{MU#@+A+2jPP;dY#qa>`P zvBdz#XbNQ0chE32!#{(|=mahc9B|nc_)Ze4IW`L)^g!49JSP7Je430LF`fuyYw%SM zDZkwNUCIfzb{lI27DpoC(f>ZYXVms+#57yk!%(rM(Nhra@SJhYEVwLKeCg+_uXoV% z0r&AgE7V+!1&SJH_SHt%BBI0-;X+9-zdc<68NtPe1J!M)>$ULtc91&|%93c$XT104 zg%%kx-Q`e~!e&3Vvq_TgsPWKP_%R43oXvi_5k5C6dHiLqYX$pUA^oF43j@Rdp*7Xk-YU>Y$}eZpkNX22F5&F8y|s5$-|mmf^5(X}i@4DJ`e-b)GU z{vatck@x=D&AyO+tYhDppm%}A? z0)1HW##ZyEmrxO@ble|Wo6@zFAhQEvy}Q;jir;8MVfN=06QT(!9TpU zm`HejuGZ>z&WrGWMs#6mCrkd)VS`ZIuwSa~>o9%Rjh_x*tgg)grOHuC#_fg)IJYC>gMHeFT32Ps>eTmnt6 zrS*)@)kp7q<|@X8a`!H6Sv8h&&&{~&WK`De!h_;^>ADvhj^&wNpwiQ@9NLj)wkDmd z{2c9CUsR3KrX9cPnB&)`PB9V*2e0P^TH@{AG|+?_8u(Eh3h}`n#*X9Vp&uV4BQ=)H zN?oV)JrjLGrVN zmqyoPdzXZbBON-Q^t=p($#zpt#M+2H+Bg1YW=T?(_UETZq=FG{k|K9l?9ySJt7B@# zYjZBCmWSfV(5DjP%eM`xQOW7FEd#l*$SOS$JeDENoDDBg!|+(^R=u`WtbpxIo zSP0}kbS^mvHnLpTuUcYhv@v*7r|&R0VS32)@i1roLr>U5nM*t(iGdyjZ4jXrExNI&6pM|7+)Xu6v7BR6zy)ki)I zC@N5MM82YOnN5}7eB*%1xC(5`=Q;e_W=9X8sXxiKTqa`bq_TGHO48Zz>5wm1X>7~J z+fmHWil zC3GNZ6HagPaSp9t3GNHgj&WXF9A!hSNl)73#5Bz_{_$>CPkxsI61Uc$!G-(N1RY!DU38&?ovgFF`J69- z(>>55@^ZRor2T-R_{_=!!H=-kZEQ_0Fd|=L2E0wr7i^BJ&%4*~(`aN<${9>^!1?_r z+xD*U->2r}q$W7>Yc*1Q{3dPA zx^KJ-0TnpRUp;0Py+7}M&$G4uw#3%er>hmr(KFtrty!Ph#KUCC&tSXnk6vM$=wR+j z(dnBVaZUQDf}NCBW$SuIWo;L=?Z>O**s69>Qro$l9a`t?KFr~e%?yP}6E(~)RBL`( z8EXY__kkFvj6=xfbBew~lPL2l$_u zF8@eLL>vFcef*8`XmO@mw&fL3-4s}-b$Jf51JNnpYTJGBi;@7v>D}Pz*M^2|^-`(9 zKV#<1pBn#7GHlK0e5b%FAyK%0)A-82AK7#4TiDlUqG{e-pz%*&Y8x7fsY8k4$%Ulo z`_Ol~`szcaMqlvgFAS;FSP6C>NzvRud;J@vq;%F$`CdfS;X$5tF2~6{uNyv2XdF*1 zv}vhrLqA%pnagYsS;nC{2T&TS zP+8|#KW&Q6o6cj0EU&!kKJ~9o;urt)$h+FT`IY0SNi7(D!=fD3b?81BwQJ~=qk?4R zgh{^*@jt45_&4d!o_JuY^^My&5gk9#<-JO)BEHL>uU|U#dD{DHLOqapzGtiB)({H@ z06F`b%*;ibD^qGepqdiiDs%fwkjx-0eY6fLzs0JeJ9>(!gd}`Bp*Pt5L&X$BH*r&) z1xOq2=HGb8QvLM-m0ryka5-=I5W{IwhfdV1G}J$cpQb@J?Sf17@p$2Fu^|4uYgc4j z{y-1&-C=WHV(LfYfR4KY{*Gx%AWp{{^oc?_aF33e@t!(cQn$OS>*K&Sp0;kckRK|% zoa+hfO}!=(<8&m+ZsetVzSRd^1V?w!_<*ccD%0oX4rgEYBc!!rxrPYe_~LG=P>LUL zu3=Y`89XQE$ffc-tgp|&1>$nfrxa^_RJ$;?X{I?8?+S#8!M=s?H8oJ_4{ll`F6NsWlgs5TI@N+8} z-_^w=?pjE5W~_QaF(4AW?~nw{#nc>9$LTv=5)VX0%wJ7%Z(43E)No+BndisXx9w;8 zKW$aL2`5e5SA(?uK>+tJR*pzkxp-g1{p9- z&8g>z9zk)uM>MS@LYg)7{jk_Z$yBCCgNiwf;pu-$n2wKERmv&Q9VvWbW9Z`BAiVxOSYcVq_WEF^F-Zwi!~`sR(I&RvO}=}7;&utv?MGVB{VyNsDNLnSpPPrB(^!m18RXP ziYq;y2qO99&Jm~rrZrx`{`g}2?%T27Uqw?>Yp-+%ytB;UO@@ zxGnZtc%g)YxjyB-;ls2^f@>}%wRxAT6c!r+G6g`30{K$#Na89^)`58VQRiZUnFYOw zW-Gv$F`~CNg8gf;qIr{{k_(zb(4ggQbqBuQnQ+@k>37vE7j-udy<4PzV`a9_>&Eg^ zNBI+GFof$pc8>KY$GMp3p)VRY2^%8r@m|5TpG=oGu&0sQHo;I62gAD7!~6RE3uw*` zu-Cax3!bS8ApCWVEkrC%9&E3=FThc>!ibBRq12FQr$7Xy6sGQ}xH-f3C9QHr$IsM) zYmGOLwdO)Am`tU=9|R~|K?*N`Dx|F`pmm@P0rdL-H_HL*X9m zV}P%S_~3qvyT%!o$M;W{M8xc91RM-|>$VDcp(go6G*c%9+2Vn{0|ruCKata~r=fAX zl9*a#(Kv}l@yE#a!-dCQG{ym8G{sK=KUtalCF_Vvj5$Rdu$9wTHw6CBj&_KygWSvX zS=nm;oXrykYxxtJHv*#TxJ%k+6myO9okYoIdB;SVKem?$nh~HyG$gMrvNX=EU>vLF zgl~6?B7?1FlNdVA`9K4Nq!0qPOI^HNy=Cmnd{bp>73=kVgPuT!eE8ty->u&emr-M7 zF1W7yyLy}`XKf);e+N%0Am@pkTI+e^D70W}Xw2Gi&@0w=R<+%DxbdcIY$MDK=T=MT zwf!#r{LJjhOD;hbrgr-7dcBJ}{N{Q0IAr3Wx2N1skI{8IsT<8|I>QMrm5BLqAxo=( zKU%!R@~NjbSA$U^F3$yRk+V<04lXZ{UoD*43egn;nB>ZNf}0m6qLMqRy5<$$G9A{< zCp!I4Z6f>NCuPZyZ)HP0&6JQ3>wZ<5*NQpA7&JFsZkZUJuUHwpoNwsjb`eR3##0bq zbgX%Y2$so*@4=d&IZtSSix3zvUq?k0=LXH82)-A6+|B~7R;d%>&p-KB{G`Gueo&Ip z*axN+cEUX)3sMK!FCCwLAKWBGZWXX};7HGAQ#gG7#GD|JA*M_@DlrB+;ZeEG!qUKi zAmFt4vY{~wPeWyqNwd9AHXYWoP2DoKUOtCmkp%Nlr;y;*xHw3vzCH0Kx8=m>#c$;lu0sW_LK6Y^zV0Ejd^gz3ey(x}NbkhF5q)$s-+Zj=!&)Np57p1(CUCdGI!gT_js zOJ4Bc*Q9Hhuyf8jGias8?Z38BR!B-u&ZV_4MeB$kon%AOc1_mRS}(vJCS9q!=~FUm z8Q)?s7gAFGB=>PsQ&p{y~qHwPmgU#lD@BjTzfV%&rjQ_I;|MQ=T3?jnJ zz6^Tw|Kq#Q|2!2><_Hc@aMegTl3bTg&6CT4@C65rZT4h$=AA|L19h&V4$N`(qrZvl zzW?zB4_Bu6m|w;_+{4U^6A21*It3*5XHy=d#-Qb`;+_;=qwp==V^jb0FM`X|=2?Dn zb;vv|Fl5LiaRZ?IpP=}$`bb<{3 z4-=g|f{`ZPO+9B4P0j(@Y;&R}e~)da-uQFd0~0E)t5S=1ur1xT4m{rxi6L8YSGRvX zrRn^H$sXhNqnP?Sd3x=XGli@SxNetOR>X5Ac>|5-jSTtk;>LYD-bRU7_3!jt^*d@t zJhFdL{mb%qmAQ`>)gL)WNU-K#j5ke1{M4kJqVvB#Gh>!2Iw%qMs`$PFS^xOMl`=;j zHc4<&kCKdn{9+>w{`mmU_*`D-_eYzn>de0$C@n-I&%QkN)J^`cW&h>GkSouBX1ssN z{^-ztuWZqEA5b4yIWYshO^m{O*JV;#6#XLEu3mdQX)8WfuXKCnpPwA;oFM!9(gsT( z!D;RBVu|$yUz@K<{|WJXb~?m~+>?KNtjqVLEX(k0v4pX~!cLDneA8DF?<^8v`65|M z$At%ir2l-w%lZ7?oONy)GAX3q4tCPto207$T8`{s{^X9sUE|BPrt?SNe_M5k?(y@E zCGuiNM`G5SK^c>PypS*SF!gfX02I991Uzc&- z#-4e7k>p6Yu(w}f-Cq~X%P4S5#YBjoeSe#ZcfW=~O^y$7vR$>JmK8=u;p3Ngu-TQ! z+Wn7j%^-s)2`c`p{s*Q7MAi5G0w%A0DAw;V;LXCo7%23P>}v8dr-ifGNz3Eg{FAcw zAD=K>9MyzLNOvs~ApZ9j#eV^Mj$NwvrFeeK9@c{#@eXo>dUHPa>Sx`iJ@3ptnYQP0 zqWnK8d&D;}pXStAX*he^1pjURds4GVR{ipa4ikS*w1}@IP2P(4dnB73KO<~DJ0F-6 zi->`nF4{qiqWnL<(uhX-fia+CtTgZYsar*6ppmOFtiG}?1%p65BdMjC#EH|b?51! z<7q^-kF82r8YfCi0sSnUFf+rPBoOm$&8V4~D0;wTQ2{fwS-907q3oP`m@ri@M(+NuW7N3$S2j^Q z|Hs!qNkLKBpphFOZ;VHeh7N{TdlU*(FdU!31%PTZgr|RGX-kL+rVb(+1v3ubNAAQ= zE>{GZ50~#22-s)Q00?wy4bkNA}pQVX5~qsT!TD=x*N`{WY~h13Lx`GZ=& z<77jlcPmG>6Zl{D`;kKc5sFZ-6A{vZ83N7bf55JGO7DWYb9#TH%|=S*NB>a2?C;X< zL|*KFe06i)(|08anCt2?8 ze;>Hgu?VKw-?{Usl_5g0<|w!tQzt|IO1!1`s|UqBRxvYUcJxXzFK}|3H1Fme+8d|- z4gc@Ot_2hSUh;IpM4deSV&$R@`s*KMiC{530$746BvL%vx7cmIgx@3{Cj;0N!52WC zbU_XvI~3Zv!I%nDY2$DfGU&#)B3ud)Esv2;iLcBaO!9(G<~rz46gwiCjW>w#X`xYo z^a_kJ_Y+175StIyB6@oi!sHrM(+$_Yc_IVST4rbrlYx1Tj3Q;&`W6ZH zYku{`d*SzTBz`a2zkaVMwkLVF4&*6+`@5c=0u#<)NgJtgk>=Wo$(JaULK^H(*7 zz>ABfH$*3%bA_Z9zCX_d#>=axt|SR_BF)%mf*=v0Q&IqSS;a)yZ=- zTosHP6ldLqw#TL7KCo6QhvjjgU%rCg$~a{K{o7SUE+KJw4gM&n>g!{RL%41dV^+hu zWH?>1zl3Pcx2)pbTa|a@5pNntAmMmbFgewa@)a%cZpAb3ZJS-q90koLa?QU@bqo zdZckJYTJV4p9I@>A~3{!r`vr@W$v41lZ$izc7uhbb+lTG0|)#z=wGW6op;~795q(! zL>(Y@(~|c_lL1m>_0c=BA$|{)bjF`wSya~EyLJleebHiecbC!xA&ZH7$08;swXZGa z_iReLoOtnxw5`pSEd2+xi5Z!FbN*5O`%imXA2n80^d29M`+Ns0B2ze-qny4C5y63J zX!=$b8@ras84T`Mbc(XdsW<46EqIPb#8uw%;51GuVBvG|a$=i;c3S*Ed5 zBYs%2dh3N^pJP~1@Kn~CjTawc$fj7sm1VXHG_%mB7Bl-v0 z)XlBpe?)JNbr_wqSx1>9P6MCs(qFkW=AWGZ{F5_nJaG2OmiZ@lXEEoXYxew;%Z-?* z)4b60X+81e42N}GvU`l(AAQG8%P)tBk!z>Kcx~E#AWINgx<_MYB%Ah-B{{~o!mC4n zq~^R#k~zrJkuGEWP;8If#ut_Q_VpZ+32PPnBidupTEO$NU)YD!@2>sez;nFK$V7C( z(zN75MgXK#`eM_r+l;SF@LB20Zz3>WJHu2A>41iD7HPE-OGOXmE++51wrSx5rQVGZ ztnIy*TThq>C~#%-))PxpJWRAs*pot2yYW$DbSskhcZoQ_XpjqLzra@E(pkv6z(3GU zK1AM($vmAtTZ=0LuzS|~-e&|%jDQY@H1-IgukWTW@6TG+)Bnl1yo=WR4hj}6+FQM+ zgqjrXdAgb%E3t?A@1IyJo*lt&lg>-$&{AZ+jEbH5${hNEJOTZ0gpJV@@v6n9m%fnp zW$9?&4xc_w$lI%SM1)t--VjeW|EZgx(kylEpix2j=*R1~j>~#~(hf4+a?HzU>1C0v-34qNs^o@PBDm(3#3gK@sKNI zJGakPyDDsa4O#|)E5_ge$~&=yH#v_5?GNVeB`@E@%3Ii=V!C%^LmXq|hz3J`2YnA; z<6G%h?3W%duDGyPV0!6r95#A~gT?qtoy`_0x8ti<^NT2#hz`WTJfctkzzrT@k| z)VS1GslI9+NPYx{zcIfr(k_U*l6&H!g5TBoXSlp_b*?hKDCRRU<%!&}r{~9s#qDQo zXRWRJsLcf(MtVV(F-p6e=eOY3Vf_KEWNHUTzMWf#lY*`JqZ9Ln5Dk9$7SaAL#exN= z60*1lF_S-xk-XEx6R8rjO5&N%6*27}yR&!|TE9U<)&=$8b7a(NEo8TdIRp{!z~lBO zFHcd&djvk6ZpHcyxpG4#I-Dlzg-r65JHFa2@{GHt_gksfyO{?gv)ciKN@EaU!QfFLFTJ{L@-q5l&GRZq|H@Yk- z%CFb_vitZu-OS_%r|pMI!7zcZD3S0ixd$kTsk0Z3laIcw^+ zaFQ82f64Vfu{Y>ZZF z8LeLPmquMY>>VH15);m$7DAEq`Ss=Pc@!M!c{csjaX(@_Y9#l_s@2vnhRzC*=PNO% zrO8)RY?!lx@@9Is?bG}9R&1SpeJZCbzryq#bSAmWxt&*UcJq^2M*C;U4oPgjfDVMa z%I`}%8=LRP14S6{`G5~qm{9)2bpZyoHf_y@1l4&O~(8)^jlj=JVW!f-q$VWpW)B<0^2HS#Bo&k zj(Iw!r+;M@OGsqtdOPCK>Y_b@V_W{AXmwxa_}$VCnUsZ3iRr%uKh7zZ?+1(IARzry zl%u_NW5(Huds;HmxR=i(;NqQ;WTtIi{AMn_uA{t|q}z!^|v;+|t()X;|R3qIUk{dWPnw z=LHxJruR~>UPHYGXITDuBETV9H+J$H3-mQX_O~W$m%(0DP;E>&iGkz?-U5p;5n)Y~ zbceUTf;ww0{7p{;Qr3V1(sBkX!8r6nnyDEIt8b`wlk19q+5pX6Qu$oQjo?UXJs!&sO>k3VGsu`(6@p|6q@?2t#%$ zN|Lj>48cu@m+qXT?(njvAmy|i)!{Txj#SPPW#j~W668QpEj}fG4x@uj0FvibkMK?> zVC)pf6JMla6eaRx*W_(8eQ|k5^_18jlK&;5%mEiXq1bTA=E)I`#foLK#Oe07t$fHr z$#3N@Mx;`qAXAh*RMPtS!3&&`lBExn^8S1eg5x5RI_7=5h|x4{PS**&ZvGx+(FZMl znsMT^#l)-XaNYHyY3H=gJ{L8my6!-x(tSG_jHsnJT!6gy8>^jd?f|zLL0mS__B=h7 z5HV)KR}k{n!?2(~QYF;y?sJ2y;F2sCK1HiRS-@qVNK79HqBl!ZA<0A#o!+0HxB6#~ z4S^|(RpTYnpTMX@a9v+K+lM(u-oA#?Q)otDend4{&i7ReGLoRFgox={c`sIB!Xe4n zZKg~YLhe92)5O-cSjrkz$uEHFXKTX6sK&8-Z+CXu?S1yg0XINBiu2)N7#$c8zcRf+|{3uaW&Hl>N ze|nHKS37fxW2;N@_-rq8qb;m>Na^0toTl*DDOYS`k37{4|VYce>xVxYnQ#qBIYqNIOPKT_7 z8}e>d7jc~v(i1Cf|m@88W%6^Ixhz`*Q>G4!99QHX2Uzpe6T8lTNfq!P;(m3KpI7h`sa7)^gptVK#7J0>nuB_xSv zQV@1%`lJWq4byx4`;*gqw>%G>ziDGST`to3&0Q|ne@hJ#nCm`>FkrD*)-lFaG5Lo! z@|4s{&CcpB8i;oRdC}6fSvz!fle+rZEas1psp1#iI_?vX#FvK?h(SxK*v)N^>SJf8 z{dtcXb0>-O;1Ypzgrk1S_eR$NhG~?n<)aX1Y7>#hp!Z=gMCLxNbSYKtTGONwDxooR>R5 zul+1E=15jRJ?-ySWSCh?(_8rb&jW9;JxeVJ;0su3{76`JHMQL?=4!rCn6Ci=U~aPI z-G5amD?9q1PkU)By8QNXz62Qqe!tJ%y?jsfL}7i}{eYJR64H{dTGf zy8s+L#r-$KM_aevGhQW-`uvrjJkWA}C-)i}dRY|%G}NaEkj#0kGpm35#s?EDzEL74rQHQX z9b{o^zP}^CUI&E|9Z_{{SVs~Nc)Q%_`9g~YEGhO+cU} z-S@X`SUndj^b+hk0BLG_ulKXh&rOI_oqsUDpY>#T&I_p5R4$yy#bVHi!X9Tk(}Y|#0;4?p#0SMAVSZF zp^ev)9(DD&SRNbmnA5z`Fv(m_Lm+|(II9eHvKKk!L+0*)li%0$^K=R^R?r9YTVUkC zcv&&khBZjRLdHvD9(aVJ*6M#C7+)5`uYO)=*ovqOb1KLgQ|%OW z|K~G{X;=o+(In;x^Sj%yCV48`an9c*5a0`KZ;&f!cPb#pB>214_@AykV5VwPh(r{1 z=TPctHe>c>-r)DM-#|zUBd$Km5}%u}xcsPg7d1rU)#M$;$WOqe1Y#E5lF7|=+?&>6 zK!;>jifgmGn?`R*{-Gf@!yCAkidv7!q3}%Meii#Zx}E$NaT)3sP2G`v7MqstVbA&o zM7RHJ#sdz!9oqfIIFjSeUJ_nz<2JN>-D?GhbCK;LL8Ecc|MlxJA;s7TiM-3#V5K&> zti%`2HKQufnhPJXKJw}bB>d%s`ai!b{0jzX*lW?GU`o`V)%^DlmJfk*^j{ZwEP2o; z@wLG5mTw@^Cub!m(b~W+C8$(E8)xLZx`l& zh(#HvDo6MI%ZYQ)m!py|v#X%axDoiJ;)iJI_X{9=c<9hHIy8~-eQVX996^hQjwOYL zK^TJe`^PpZG#k^4gp?Tn`IrBC5q`A7vW&!)jS%|t`~Q5E@DUQNy_NzL@Be<^zkjcl zu!Z8%Lt@u|ed?cgAv}j3>A@#ui}l}^^yfvm&@j3XKJJT5lmGQk|NSHFc?^mr>yS>u z|Hsu~$-q}#*W7&UxslWG? z6fHk|cko=Y|gRBNOg- zKZvW{H6G1%D&n%Ekw^B-*h(yQGsy@3Yc>A-%}vG-2UACPP-*P}u4Ori0yfsO&pJX6 z7s9VXt9=(Hw53DE2MyDPBZ&GnvjJcsF=v--=0Yug1XR0#4w^fSdxI)hgZ5(OO`MCX zCaYZ=79LU`09XQq?P{a0RxtNDL{0BHT#nD+Iy{C>F<0{4)1MuZJMgYR;^7tz%|*)l z?bNM8S|Q*(%87?E64Wy=^LgPjI$;xi{b1UwACxE)LpP4IA@QvZ#Wn>vuRLes zXg)MD!1%@1{dD_+t9;gNptk{k`L`fB6n2{)AEN!Koj}TW4@|&NLm*C*lFfW4s;Arn z7am;NsoyL)Z5VD-&wNN`SoCROoa`a`zxOk(7DiZp(o=(b^5E8gS=cSjIl@Z;WS)0y zUE)a6*df1dp0H2tLY8D=s^QsMH!R1gENd8ia{dyX$2%{rG8CQk=qpw3@Kv zAmbEYruzpngU*5e+YuFJ4X%8LKF=K($_xFG=Iv2F2d9CTdv`UrkGu20!mS@qsk?>Z0B zqL9!0W6;f!l~2Mn9F3*W zQ;5{?(Wix*`(f`YlrpPj`}WHi9At#Zc)0WVvs;Ww{UsMCjoSZ2R9XyFh>@;H5;=Ur zM(hRZ7Xz#O?4&q<^}_*D_W}l$kB=Zl*1sZb6X87gI^q?--7}qZk?i<*HD&Bc?%Mg# zhMoEz_0k>?dzkT+RkxFR)4qw+cgvpMfeAJZBl;$e*G78qMIa^GENR$@U`glgwOx%< zm{R5__8v8IsH1T;@~lrn30K+G^*@VdKNfIhT!0$#^sqk|rQIxuEzM{gcllwBDTsPj zx6M)xDmS2$U5C4R4>tjI}COFEfwk`a>g z{me)zd{LHJiB$GWrMi?_mRye;uYTL%%=#1136W?Jzog@JQpMJ6?*r-0+;K^0*3ZUh zQ8!DLB1&w7RkXJ5eb*(`6lLqO$!dKo+mT)e^LTSLFAOhv&@ib| zSIoAoA5_ewUUMyK^ygYubh_`PV@|eU#za(lD@4L&ZiQUKMj#-RKV!X<9;-~^3S&`r z?*|@CI3()#`(A2C5nIf)6%(ZJSj&E9zK>wXzQ6iFsy8f+|9G8#e(fVu4>)5dNUWQ-s>^1krIxJe(#Mpe96X0LpwVO?=Hr3A6xw8F`+N})z zB??&uOULQ+y9z^AMAh;UmM$!^m2&5*5_w7Mj0(vNX%3uYXT? zN{(^`C1kD1#*ll;k>R+MGT^$NBv?hGgDKiucSlb@uV^bE=V^QLm-JlZ*(OtH#KExfg6S2X{qv{fwy3l8U`5wV9^c0Wq<_s9$f zM{`kEKVyL>8pJ-*PSPPk3X{47?8H5S6>8@y14v3sjeb>SB5nj94P{7JR8%)zkq0)a zVVNb;)uQMtR1JGTNf{TR(%jvG6Bl4o|2IU(`_MGh!nw^6nM><{aXn(MM3E-(O~)+P zLED?9)G-xiobCfp+<;r`kGiXQ3_JHF$5l@ld@@AzbS<|%&DPyZ$ZFX&U@+wjPuHN` z;S1OL^&IqV;~KZ^S-r}fz;&L@7KDkT*|O@>dzxK0?Y?v?^(LLed3>%)gY)ET?Rrk? zbo*}cP;zLo)}uVG3#8u6vMpvaad`n4DMDf|tDY6U5P#HM%Ehsd=Z{e&j z<`=tcHMg!Eh}gSypFDf+{Icp-7j97!38`}blyVGQ41dY;d~C(UOBi|)94vVfJR7k= zqJ^(oT0`+2EQAX<=-O*{e{QNioW~bi7 z1kTy!+tVJQn6dj@g_&6tTiF*Mxa|m;eBrt5-hvLHwdC#WXl-nr6{p8W-X)_FgUnZU z+qX#L7-H^Vl2dw%i0)f1-zDOuTixaP`oz_fxptq)i@B4dx7wf&VIqVX+!v<8)1=!s zmTq5v4Ip$rhubM>*aw5;65l?1I!r+cvSn6d>B(&B)hp;`_oiIZA(X$6+{ML{`5I&g z(}JQTIMuKDu`wLk%a#?w#~*h6FvJxSI2cF+%D;iHk`>4 zT-WVb=`woP#PP-P@L7*RUl4xxn8`bN94>DYa*m!uW38L%bEd}oK+?K>ol(`|u#7`V zONl4knndgBQ>h~yPSJsdTd8`tn;_MyS)z6+lXk!GL)^e(2Q!$tj-hsYCLqx)>wc8< z*J-?3Z|W%tx11A@+Bk^s)GkF;UpD?>v zNy8Q+pBBg=z5!|F_39Ik4jO}9xxTs6gPIVQSV|{=IK)0oCG2-QK)QCC5$vl?)@BPj z=4yG#Jji}rc-R;fXw@)B**`5TSI5O}B12y3koguQPPSo*F{qyv9Y+vB3%uw@GstxF zsi0F)9=c9sdAg|->HZG$601*0#>;dne&afQfm8}M@&|ut19^To5D~FvwV!Bwh-;fSO$f3 z7fSKCV|`BhsyGs2hKf|U&0NRx%Y-(j{!>&jSS=Tq&m)suin!eZqghLW*k9wOQoeI7 zQg{+1v1S+h>t5HHd=EoNpyN-7dn0yBH zI^&))rBBT=xNhMTeH*YMu;t!_vre`zLb%jqL=aa)2Pw$lFS%hlXL#=cLOY5h8GRFc zl56-kdmrE9XQcz5#hL9dL?@32Os@VupPQ2hBi#8(C!4EjzaVJx9e7r%VceC-zK?~D z<7KAcnCm;0L+S!t=Z?2j&k-{izBb~1UDP6j#CI^d%FjgIUaRWbafXvM_Um(*mJZXG zowFu`f@LsXdVUKKww!Q@_L)j=JIbYL&S&X|vi0tP0sh>4?fxOlC;Zr$}r_gtllDHlW{p; zDV7Fbo3)#Yz3FtFDYsMk!CzYx!cc>x>GpdzWl~sN{hTYA6R}+Fg?E>!gKbobyb0}U zp>a)CnbH38+!@u3$7nDgGM)#{EU*nPduBUrRaeUT z<6C`wrZUZaTHbX!Uuy7l^HHvf=+g^^J2+96>}2ZHY11s3JT>L=Elw|pILR?bHP_rn zg{b@swX1f+1eDhx_&WAaPIticOlR?fP1E$*`XD3{9K{@52=C6*j^E-z7)!d zHEj9uB{E4aoc_}X_r86yk|c4WB}|tBZddI6ZlB9x5$t+7bx=fgsrgWe^0mT{fy+`# zuh@yw9X~0l_F-6YmQcI7*rFrhiQgv&pNV}B;;6%gTO z8a1zW(*AJJt&1Rwy>9!XZ753Z3J7D|Q3 zPjydHr|^z!?c{Ij%ukLfD{tNK95uOp{YPRsH3jik?{69l->Sb~`?R23-M-venM}Ho zBt?wF`E1R7#)%ZS6bC(;z=xA=wJp`aj$_kT+q>4SCjCK&T+7K&;rz25BAkHR1zx(i zLKTT_fhLYPJEauD>eE9XgV%))j@>*@2z;kK6oThM1&`3l+{0?h_kh?r0`|ObmTDGm z!1$%|rs)rp$~bRG6m^hHjSa;m#!05hQxr7Z5rslj^OVUmLBlWc`mg^;i~R!QNp!Km zh&_zx4nTU?)OO5re7`JYrLH+=>wHx z+d%)=(8LP4Ffr`otoQy}Xv;--7)q$b*qpjPHd!z8z5fK;@<&eeA5Hh9>1IT}#XS`a zothQMbnbMw*bI1n$TQQL2&z!cSE5*yN!>#m7HV3jE#P zVnUcYR138`Xj&YdACuWyS1?Odd0ZICp8Yxv5k05Wv=(5KLekqG9+vMz*J2ZFC~SC6 zFz+M*zcS6MPZS{P1L5z(fX*mL@8@VMJOj1zr=gG@Qxf+odTvZ|LU_=PkzeZ=Vs3Vd ze*0XfbKev<5`Q(|7h!V=7NM@W!IWA26S;6_Pm_2PQmcZlH^ttz69uqlHVTJOi=0Af zG~N@6na#O9R+`b2Eh%+&Q1f;|g)HjE(XBflAq5zQRsrW~m2*j5&ODci|IdZ+8rP{& zQ}!#%1%)N$HIK<4o{OjQNxPozU?uxns4-g=Fs5D)OsWVP(D_q0bod{}$P^(+=*jiWhc{q1qFo-v|qX zp0Ro0b`>5K>1@^9ym(2YZh!iecJ*1KFD;vymgIarMIK8~BjD^s(YDAyu{^9kRx{o(d4_~2t zc*Wa&MME1z92S&@jqM4HOFh!z0?pEoE}+YXWO`ns`-1qi(D=o&d-syi=4(#G5E;WK z0K`BTb_5+LAw0LnZ|$gw)87h4@p^sJ#P5_)HFO+xHu@|^uuPt>cK^uN(P?-9;pG{X z(GT4#f=&sNWxD=~1S0bxU{5^NjE-4X&30}Eva1}SjoSD4;++zAKBqVw_1^WIi;o@H zPZ!_F%fgWVtbY%+$gg0L4VYpiqW`*HiRkAs7AXS@$2&A}V`S)|-J4Dg&Yo}9+|}Hi zJ)h943W`?qMRKYnMe2j}s?uit=Yw9MsjYFFXqYSVEYch5iu%Elt|7JXpS4LMJ}5iu znJ8a`UboZd`myVKpJaj45pOYtCp`TXYPPk6k=*nEWpf{~!4JlZ)ARb`2_~N+%fC%5 z(b{@!&b>WLGI8_Y$j_biopR+L>}o>U1sj-pJh)6hz2_c&*|QUz55M5c)!Adg7YV(O zKNk70EXI^jksejanYsk^VnyJOebg_HdCnrG>ECoiA zYsaZH=3D5bfCXOZrnX8YQyf0{F-@89!}b0Tb-YQlY*?9-PzcE30fwH z*~#W{qCB-JJHG3Ka?(X5EM`*4Fzm>zqGyf#Q~PvWe?nByCy$bDVS{o$JWn-RNJjn+ zIEVC^7h=pu3{&>cEBgBW=zZLjorWDi*okBBi~+?&zFvZWbU)9FZ5#2)Q0=Q*U{DK< z#f)NXRwyXVbyCj5F%8{W{ZAir=VX8>1n@E1N8|bj_+x`^47_< zQ-d@&6O|(^Z?n}^-wPV=32B8`M;I-rHsaAvN> zqGHt8xocJ3;YLx@!Q^Q}f!4Ev;L6Kh~d$8jlkT=g773MGt!F*bZ8{!!*9Be& zLsR!C-|h{FXMmg3^<|Oj6#SgVTU`p~o+!4H!h7&8B5f`}Um>i47VKmajt(A4zSCq6 zzh8G;KO|l&FE~Wsf+-D|fvFaH+2+<*_Sqnln6rbyv&ls&W62mGLuN~Hs| zp->v$s(vQ|yLgb9l*CY!WCt3VXd130w@W_06dDTcO3AFm#_Sj-IcPaYAh8rFfHPZ_ z2)|AIa%0_weZ{OK+I5}>ws!yP1E)_5lT$kSx`7B)%{uMrXW1!jSe`Gq$+Jjq@P%r- zr1Gg*BM$L8F55w)jce3=pHUtwL`R_J z)57zKZc_44K5ooTJ`FmX2{;@seaU>0e0PyXCkt+4#(210izx}}xkS+PRq4~-<;Hpz z^Zo7Ve(8ruyB&5?uY&Inlmnbn{k&4fVfM<7?~i1)zA%(qUvQ6wjVCiJKcC1_v>26x z>&YO4L+DkT?UBs)RW-Rqc?Vc)g(fdW5AYrnGq7HWvOWPgVbiE+8f$p`dMh_AUudM= z!AWKFcRl^i*f~k5Lp*$ud9=W=7Y_3M*~AQ;A52z<^Jf|5WBhlZAHSD+m_V@rk2^D8 z=mpDS^H=Yy>MP!q;@X&QDN+*tg|%p=B2+sM!RX;SOwDBC`^u7$+5*+-LuAT!qtMF-}L*!;Q0Ow5qY3%sd|?DnlTt1^+}%H z6~?Yz?r^Ho z?SN*wwkY8-tEx2ulcI~&e&@zt$HNdV-C_Cq5SY%^*@Nx0R{r7^kuNlQ!5kwh_?k%; zrQMQ_z7ENy>~}D%P7-7ZhYRcj1vmH)x;9vBUVRL9d!<(w2D*xGzL#+X<#^c-lG+u8;VieG1u$M6P9$H6 zsdeT5z%LRY4NND#m7^4iCYDAqd6_`BQ zpO7jtfaz1F`K9zk9~9OIr-Z;DYCg9S5r>&5YVHEA)#>r!x_X}r8Ict70Rd)-V))v{ zf8yVneJJGDfx6H0f4}Vo3UJ%^#{~rb`N@C3iPQ_C7K;D=>l+x$!KB2yl1F>=!ep$x zIBC>&a+-fZyDh{R%aO7~f8!1N>LJI`gJ`>wpMPYa<3wVt8U)5({=K*x@o4ED`*mN6 zsQ$gyzyCB72jEdXbTJ*7e?rxHvKVcN&O=I(F&4cFNUizW%4FUVs4?*Wec=cq+7eT5 z#Yn=}^b^hD7+EbIj;!sq%1S6-5AfMaZCd_5;Q&H`(IT*L0}7Jy?WTR0sgy5MiH#a8 z0*6E}${qCc$3ti?3T`OmN6NfruSczMSQ=bvNF(Zc;*AK>R;3ZR-@OomFr<5T@41|K*x6vQ5{&{E7FzRT5aB(Em)hP%z zEC7Yqj}tloVhv?m;BFVo7ie33nEbhEH!HRfMsnruS81bii92@t)WE{cl)is$cTgo6 zi!U8in1W-bUVzVTe0bt;*(=|KCEafkUwq3Qd~|xe*KU6YT%y~_eq*ZAk-<5g4Ggm0 zJEZ#qtFhU!;Vz_p)h@VDo-IQqMwb;?%!fG=4ZeMlqkjR0DL57b>2S-j#m)&4_sNP> zq-xuS{@+Z1P6j0L`4H>@OS z#DK?twJu5IM+sMtK?$%S;T>Gdzz63EPcv%h?JiqoBIzJwOc>A`9Cv8F{y>F)s2WmN zuz@{N?hK23BzbnIQM0P8!j-)?3IJKl^_>&+way-6tCn_}u zf`^X}2X2zxlKAImHny!Ceq*}eRjFvNOc%#B=$ezMn=%VIEmv|sjD6%>ZGhDpbX5|? z#86s;r?R!35A)y8#qtY)J?*|^s+xKJDxXR4+KyAU;AAsZGks%6rVp>!aZ_C!fPSF;^myL(l$zi6k;Kx+XOFq` zB2oTWz1b(FcSm6wi5s33KavK@tl-n{8FpClA3j~vAr_Nsgsi_#S<%6XaSFs$dNv*3 zE+y=359?_7Y$htIV{b)b4~}Md4=Ul@t4!u^V$0EEmH?DEv3!M%>_yUdz{d+eqwwIZ z@*bB!95zBlMaXGM52Dy;H6x{nyRHgeI=uP>J@Ef;9PNRLb$vrCOq$&?K>H`C$cE(# z$AP9%?4@5;_+l#AlM_Ka2oFutj1?hP0+bxf(Q(M*=oL4_HUlKT;T2get*_pC(X%Wt zox5$@^orrHTTa`9z|!Fgyff|X2oQfp2k|QY9k8Ij4Ly-+`SnVdIz#Cl8=PDFhKnvX zCrr5_`hkZ~_K%lytwjr&z{Jk7Ibw$|PD$Ww1EzS16@qTk!NXL+!YP=EgF8?YC1|8| z5a%B7-!Sf(gkUCe@7OPzsV|Bxeg!Bp`3%f1cn^lL7;_4y-6uE{yj3fq5vcFi&F z?z_;{24MJ*{i6INAI@SpCctF+y%IIV2ZoL~aom}(yJW&`sL5DVdC0%=nCfO2pzTcOo{CqB?A*u0DZT(K#no*XV4+n^pRa)?T2z=Xaf}tC0~nmC5XD z;k~g6R~((Zt=|J9U~P6kdEcbdf>JYU&kF+Vh7X)<6sa#LrC%1I%TAa$qN}e7taX^VT#e*Rd}j#s(_m!^9uIzLL^67%tIyAGkMb|`WBpMQ05djH7Cb`Rp zqPoPoB?FH*elYYJs}YN((O{BqD|^G58JS`0a8di^s=si#+N3*Y{e&w4sdD8)^&l6@ zm$@tWh9|=P_F)f9YX4>s;IyYaH@fzr`No0v1Hn;4S@hSRb@(xK!zF+MGVOtrWM`D@ z`evR>Cm(QFGtrgAYnad5J+E+*Cpjx*5%M6z-~*%Xq*CBvyhBX$xW~8XMp+)P((R8# zldtg52-rvVninX?!#D=p)HAIvLUr-&Ybz$a3fTF{u$ksmd=c?2!{(ErlE{31mbIX` z>&l8;Y8X0oV=#@W-S?WquW^;&ckrfA)!{sK_64@PYG5#i&PiIs+N&p9S~5=d#LrMk zi*eTaO8xrbYBsYk2KKspJDDMc$_6ja<%V)TOHSw^vL{Q;XEayiDh)uir71KD4`N0+YE<|b zGN;;gMyYcl=xGF_JJ&-h(C#HXzYC25?_i!o6DLEHb&p3?`=_DzxHmrH1YZ~IBKjSs zm#ZUV>E>BtOm`J}?4Me2hz~OL;^&iJ!LFro}5L z+%1qaFAd)z5-+?h)-tVYO$}3Q#@`XyeFe50O`&C=ZCcH@ zp*oh49FLgtv^}0bR`+Ql7&f+8!5K`bQ{KTo>Prj^RllkC>DDKy=0He_noGGK;zZE# zoUdcN9JDt8{lntR``2PGKDvs^wX!ax8t*;wypZ6-e>@+c`tY`DPd7H>YY2-gChxHb zu{2sv*0njWbVM;3r|n6G$TxM zLy`v$abL-Y;Ygs_w@E6;`gy+_B#0#$l8@{3ZP%a=1mm+XvCRL;%P8R{o;ib*__SA^gr@4w+sE+_@Jb@6Ms#9c$*q45R!*{u|HL3Dxm> z&K`zK0%Ta6q!anBWPqku3L!kAsBoH&uH@I%k{Z*@=fZ?I zmR8=?UH$*(eOUSs~R5T{q6MLfH#n%E1q*>Vu`HEuzw6xBEEh=&U zCkcZ~-AYor5bPg$oz`0Ln0eN~Pm4-w(vxUOQ?xd0z1rNxo2#?1 z9L=1;|8|0Vv5l^t?fa|rYr)4$SD{uJ4NVd;I=@mD1SO#}HeLI#4CZ(3B29-L;e#Ic zcI@0IzC5}zo!;<f?Wnp~Vz&v1`X`*@$ z|24=`Pl?FhvPo_$T=8!U<~X3DN1fii*&jelVs8pSK7co_D zcD!QgAl%=;DcW|Vz^bnUxQn69#u8JEAZ(MR84LMBD-3L=VA419uMwEBNf&SK{-8(Z zXDBs|z~e({0Q4VsVsrklqf?S$rbWVanVwvdaOF(*yO|_BR_Zh@u~YuK@{Kr%Z3YHN zl@4`%gJ#o@CGNUSGlD#R`}c49a${y&!0G>Zx^kD}33AJ$`#x>*H51A}Y4q|V+%J&k z^E33=m+Wy>_{PI{qmtKj>sND_4iGSDq zds`PZVL??@TVMZ}kK6vVJlMx{S4$ne?q%2Z&{13WRke2o5mZYFvhN-~DXAP(CeDP) zhp)8kr#ld0kb>=Oc8=d3&WbA+Zxk=q2OtXa;Ct*>zew>?LZzA*(x)A!PO>*1sLP||a@<5k=F-dy^z=HXq@PgA57>P!){aJnwzy*9mmlr`|yZ{_`fzsKoOsNle2H`IMD z3OZYTP$J#pxsU4^?d}zhDLgtepXn)>$5_%>l%c{Tmo3eUdOl${GQWeHb)isg@DA=> zB=dE<7m=kBFFYR%#m<%(Jde^*^gaafO~uj|pdT`Ed5rqn2kL3N&&hExkJWxix~!?d ztoDrerL6$fYYo~USh%4%JNu8~$dAmyq=wLgFl!i$m8=j;i-BMi!q?ZVE%>!(*AcR2 zF)EXc3Pw_0?q}#xuRdr(6riBoIEVGuMnk9LlL^t+Jt0r`SiwWeU-;t%;Q9`#((H4T zs#lvxfpF~P9()p!AB+a&Tb1MINvcqB`PyHAwHWb8_!w}P#ob?P8;Nsm6&|-wb%m}j z{*bwrqUX8cNfqBQHWHKWH~enOdxkG+O#akU9!AYZyWKJ${~QNWMj zs^^FtXkO?ny=F@}juvtaO;N{Ih4MDAPHS`J>FK?ZTo@ZsueWQpp{#<^4ZQVbl8}QPvU-YOUmB@5}s%Tk@d3CIa#LPPnB!h=_q@35TiV+ zJabh5@PuEE5@n0&PVEgCG)nX&uyXlf36G!>ev0YZxCSwik?V{NQ2M^hQu+zX*Xy-v zMtAIT8iCBcWwqdG!Q&Vi-vVs~HKU48wK5nfBd|+bUu26T_~VYXsHIO8D*338k!$X+ za5(vQEFNCbw4L8aTi_Q*KnV?2f&cKwRY5kyu*C4rYp zuxUCV@73AUXsDJ2nuceJAc6Q+S$R}I*?pZys=HWj{=Et_CSWB zzTNo7)rR=lWDg@w`f)VM^$8NN>DPEls!kmVt1nvHbS$}jr+sBK;E^>(HS`D+z z;WL{kXdd~vbO6Nc3$#&#pD&BwJOGbg{o5?Sh`(VXODBjifx0+BOv_jG^fC8#6&8HF z_+wfsOZK=TKID|5qS&v_XM(a)(knn#sF1PRyT1q>lF_;EAfDbY?E_-O@$^^WC)w?7 ziIV%nIy<+Hj9$w!g8H_IWP2}FQPOK~4*lHuN|%X2S8`K)EOrBh#qRs-b_Z^wzjq=z z0(ujsK4eU)Ra;ylzqTs&O%H+Os;Xf&>pi5l%MZ4YoACc7TXkZw^YEY#NftnN3I$jD zVqm%eZ)?h&S!*0%XV=MEz_g9&X?%vYE?jmGIVI}m|2Y~LeI+e`nKE23Mbh?u#k*qQ z_5X+x-(h`6uke=RS3kI7*?J8d$=M@4FAi6@qZ&spM{DlcC~g5j*ve$Np>UX~QBzMp zN07Jgp^i@#Sxw7+djMKP5&WsgT7z*g(JD!6oi4@*Ldbl6POX#Qr$jpGRJ+TRs+O={3G$^&e-<+&fe_&$axA5TIIhYuE z4~!ckK1ar*DG%kuD=|eA$xk$t8t<~Prk);Nhuj{z#KWl^LsBdWg_#EO0lj)Qd^}o ziXht3xI)a}H^01A{usX3s-lMVr9|ElR>hUKkfQwvle~SeqE{#trm@4o_E)O<>Q^7U zgI?VkzRm8CR@v^%&|$8LYn7g;K$sJUq{!i%4|G{=H$l)JNhMeyeJ-+Sa=0AIV@ zbOz*zKR`1v>i8PHy$EbaIJoyghwu!XR{VO*ZX!>I)E>SCRS+Bg8~E>8K3rbU2G6gt zM??&uDv@h?JBYW^sN=nTR{it+RcpSdY+5zyF))dySg;%I#|&Mk9@}HeC-o9+>O28K-u&?okg_ zUUN777+4QaX+MA|#J9nBaCa@qp=z*H`7is991)@ z%(0I4IIfpoc&%}nq)wbhhc-qUwx`bcot#d=-zT^poZ!Y)+9Wqn;W;iI5~jG}~@$MWnV!>Gn5cp#uUdh=%NB=o-@bdt$$jTcRR zI@9wEGO+yBV#{>H?hJ;2kKUkxF@z>US(m6r!;Ey4A3n!dE=!v#y|O54>hB9Aqc~d~ z2=tMV*DP8^syydi`dbYaM<3I>9Bs^Xcr|`<_c^sNYTJ*cy?T@R7>hLzG)SL)vEs>& z0F@6s{??I5rU0r^#jLCaln95bX4(_Q23~=_OkTo`1OA9mmm^=pc!~vR_{a0^-Ux)y z23sHcV#)h!1F}k<=|H%zN;s5NI&w`0QJBek_5K^d6>=&e7ZKY|LgDr5%jdxH@D2#l zrrqD;!T5s5p6Jl1)@jX7m;gbL&{7Qq#_5P#7TCy-&{Z()s{Y)>V^!ckC%ya|lSI}t zYx=`!!f=*F8*}!?_$-EfY=M)bEKff!Ecs&Nh_=dSAJ{K>rZU6^J3FK+)_?8tH(Z6` zkZA0e^%VW3J9NsBIv+2IBL;Pf#rH)MtMjOg=yo6_+Z>P!{GnNX1Ee;*9I=7<$Dh4J zO`o+VR`PkQh`mZ1p!Ftj*l>zrVZ@4np6f31|I(-Lb#nLQOlkwt_}JXBqInzkCE5=aPE|&lCTJPW~;V)lHGQ&)(m^mrqg)64afy zXZO^mz%B!2Q489@{3k78Fa-*j3q&`>{)a4754C9Qr|$76ngp1L7)$|Etr{w}1u-(W zedsrtdv-Bp+}(|E1IM}REyO=iOd-GuQM7xA822Dh;0cXq&cyXYyPJo@d|@E}0bsHb z>>7%YjhsKA{2y>DVG68d=bH8gYC<3?1+y=QBOzPHCT+{Poj4ag8sre6&VHb|Qx{nX z1838&jfLBvDjkO)$xFMt+lBs6BACZ%BTNdXTSq%wC+4NPZ@k^V7eUjWZ zpQ@Cj#Kq9063U|S`pBokRh1yV3v%rh4q-5bs3|u*%!+4LTuI zRP)F177?V}a4>;gE`jJwnMZH6JHTd4E)Pk=o|p?_4EyWJu|+N7)=nP~YAgG1{JION zw?~ACb=xBl9m{12-ug5T0>6Wc;$Y1>Mmz@gN8n$!1^GU#ZI+DA9LT@Gi-G{gG{O+! zY8#k`Qw>NaN@zlbv#&%exPtv@Vc0gWpUZxP5Oh*vF+3Pbxtz2PGR=GFf0!Ib44?%c z^<6&eVTcy^u(%E90QBdKOm|N}>I}1ng|oI%vdPlsdhuOo5<;MHa)yC%LNH32@`i1N z2{e_!#h1wC5LnYTt5FVmu6aN*Im# zh~{Y;_M5M^gak-ZZF0n>I}V0l(nlr)Euj?ff6+vyyD`zdMJ`S0mc;nAC#rD99>A;$ z?fF$e-SV?kI-;+ZCb*(N@kRj@`iK)CXmb?pCbF95WosmGCsp|N4VYmAguI`ahK|{Z zKy!KyLne-)*~WK=31vc~E_vb|Y=hc=PTeG#@AJDfSpvKH>E`LA`Va(!+yW}JFA^}9 zRXd9?XbpP_Y58P&UF=Rc!`iU+E5S_0)HeLoI34rP2E1>LQ=Cobh$H z39>HPK5YSItNo(6-NWx!XTCy>XN>#-RXj*|d(Fu?z@qxY?9t(B2BeYBfl4n+u>N!R zt139FZ(y^2Z-K!lFu5`aK(7tNOW@M77rz5M)%rCN;q4M53A$CNs$NH7@AkU^+0HS) z$s1k768PDxA0JRFkM+%el8_;1eFP;<7!>3yyl%$bn9`+1w%1%h0luw*dh)X*;mMCe z^QoDdaq>z`X@icWe^o+EN2qeBx@=ePM)2t|Xf^s(vs^qtZ!rE1m#49eQ*`h3T`+xcxdach z91Xd)W_p}hS}26L3eGy%%2JF(z1OqbtDrkC8n$D3w~pIt;Kedl2)=bvd05xU&SvCq3~M)aNcaQg>qLb+Q#8*7v=VQGOTpHpB{VRXd>+%LJT#C2F+(AQ@e^N; z3YtxtO4j+F(dW$oR;^2X={Cqi8$l^SqK1It1P2p5ql8$rglhhaBMZ0q5uD&+8X~2L zaWKa(N+Ls?Wn_fVRx~0kde4TXNlr*5o7snrKqNl7=&|Bca)pdr^3TH^Lp-BG40#l> zRupyP*H&DV0KrJ=T^4n>St>rkBs8TGk~_Rj?NjWg8*^8pV#!*IX(*sSsl>zid_=NO8Y2iH0(=p#L@ z&}8VM|h28X8^KBGpduGK~l-VZii!MdrRPT?a`esRA{8Y&MZ zl1JBPO)*G3)Ave74t}m|Xl#j@mY`KN(G->7W5c`S3pd#(sal5tDS3k%VPooZ$RhF! z5(tH-RCdgoi`(F5AfK-4;0OAhudr#y7P@^2Y38+eR4k{GWljk5Zu&0{E-2gPWxTM? zCR9ua5iE)qpwsHRcqL9MECKV|hC2dXhWn{KVMqwpcyJlG_g;;o{S&uC7;E_;32zr6 zh?%Twge~0r(od}ZB^V~gUdp+z)9*#9dQsvfj_1t4Jmc$$J3-SRoGg%iVmA6s#VgmT)@)3&_#0|?eSZYrn@ zlBBkzSEy2IJ)b1YATa*i2cl8MWGu+p_PM>CYjcpCW=(w6#3q*25v+^R$@P06V?Wjak!q#vMI1Rg7ix0SC)6G{ieV~B+@ z{QX{bqi>`K+c^yzvATJ|gmz$GET0gNu<=OY=-IqeE*5`Z?4fn*b*chfvi6KPQFp*nrJf37BLX z8V|%w>(KVA?I5Fpr8u)(QjP>E)a@}EU_K{qsWk!0a3kYvtQ=Re;3Kw%*F zQ%Nld3I){gJ^Q_KYhCctW>T4kv;7v+=5*b6P}4MjYp0%~!Xac;q%~iO$a(**Thl#7 z7s>@YqCDPNQ!lnG#sbMO1*(hXkHo2%cD$T=aDwblI5@b-6 z+2Hd4R)GYfi(vBOJ8pHsIl}o<<~wkv`2n1Gn+RO&vh>ssB@#s5BEFOx{<22mh(&rr zQ3wsJ%g1db8YnG@nM#@`bL5rl#s>kM9v<6T#Ze^5r#D9zY7u!Oc-Cg96OL1Qg;>>N z1?Z87AcOgAy!h}UWhfJOgjrz|AICt3p?57Yy3=j(@I@~7tWx8-el`JhcM(k2A6B^yB z$F@w+|Gkj@ZJQBrt7>Wg0ZAagteCz`=58Ybo`n?*-SWFfNpK37*0q(-&NjE20mqcA zlSF$O5?wdw4CZ{p;V24W;${?n9`WbHkT0M`z|KO*HEa5v*wb9}IN%DM{0ZDYJ{-;d zU!9Y5@Mz)O?-r!-SXtrEZBohfJjvG#=eX2jhQC6JMVRv(V|gz})BPvSooTDjFfumz zqm$KV)QIm3cvQ9~-J;|fA}aS~c#-3YETQ-v+_GQe>TtnJjFFIT)|7$7^9nIfFAnh| zgJAkK_eKm-Q-(x{DAE`Qt|hzwhr6%->uO!vmam|cN{1lb4N8NgQql;Dlt@U2G$J7_ z-6eu_cc+R-3X+m4p|psgg2X!u_I~y`?&p2~fcN~cKkT#Dcda|-o_pq+nQO=>od%^a zrih9lng~(5S9rln-gK+>dMN+f95%)XuT&UMkwo128%U5LjPlbxEz^)Yz)7*3vyX{n z`R?fWp$pUEHEqcPk&$T^mS=~*WG6H}zpBnw4#yAHcZUUB;~RNTRY8fV6(H&F_MjW6#cm1|nK6a`wYnpHt>t)_ z*)lZJY7OYIr`MpO(k!wZIV7Mh-a<+;@eqFMZ73EqI0(sj^$j~}wWX_I^WnP%r+MsP zFEE3X=Ic_gSUAE>KyqI_e+4r1T#B5X9($7K7I%&D^xxO?L^sw+#}uJ#!mX)OF2+PH z69(hianWK4t9s{IT6F%s+?+P&tb8Sh(CN_Ql^EuFVrLU4sU-gTsM=I&{KTk(kBnJe zuGow1G6(3KovR8BPa)t;S=pv{N2WHL931w!o9XvOD9douF&webFlFi0Y?~X*XcxpT zT6&P9Dppk+U=*j;IKWv=m-NqYJg1`P?)I{JoA7T@`$a0RjE%k-riA8rQL`({2C6vB zasiPM*pCA}XYi}D+TTa%Z%y{GG=dM_6pk#bvTT?8mC0>B^tsVzFx9eC*6jb!kBGqK zmh(z@k3-;`-Xr($aa`h&c9)iEN!RfKrd9yn9gui zxS{91Q^z?#&`~O}xMIJCT5uQK;uP(yfB2Pp=r2eM5OCe*h6UyYG11f{IR)d9XRr}j zr`#7J`|$Efd@(^m#@BCIuIt$JQIs9R;d8^t4_EG9X5;BxbehMb_$wup3eAxz`%Ox6 z4>R7q9;iHwi+o0fmwyrYeGL4*(B_u%4a&cNA0pnpN(aAp{u1@~@A*+3spd9vQF(f( zCHsbs`xksI@qk9O(Y`|*N* zj9tPgCCBLcE&ZI;epV-Z|8F35gw6N9)Z%j~J@=AAS&`LHb?@iP_F<@OnXucT_+5>> zwkt8JyvqealATx??c^V5I{u^5r;7-TR})K7yiOUKPQhsisFs)1^@d%XAA?G0k14FN zR8rL4xXU+En~O<>t-?w^*J-jk>mg-!51^?hwG19o87t^&9}Df!P?9tTDF3uKy|M2!n0!!qxM|CO zjRzDlk8jRny9cBvJEm5L9=uXARWfk+@)e7DlE8!^E5R58T3i)VUHr9U)LPOFMZOb; z({>6GUt~QA=p?(`rH#mIrBQq)QI|8N1D(p~e50`Su3;64@;fY!E)J|ENLJT761_)d ztLdZO5eX?L^LFMaq&6H6f<_e`gmu`KwkYxx_L__|2D+o9PyH69H31z7vk!{Y{M9AQ zeQL1U4yh^i{u=?ulTSXC*~EI2b!0v%=+7C-=#&n_7 zJq76Dpfc~EPU`V{CnKO2{;>ZAsd4dws3BP9E8qh;gw+49#F#5jB&`FJ_Fg8u8hx8C z{u_|vpEyYR9#`GMJ!~jzKzuZ!?_~o}?qG`3cJSppT%4(M;Lq*N#|!|jnhjJ%2$d6`=W_h8;Y2I6Gucj{seGY z`Kidw^N*pDTMY^cG}E8{BWC}+FW()&ealBrj?8_5J?a}bl+5VNc&wHX*IQl%G((f3 zp&1~9uSkVDIQC-tL@pkC%XtWCBoOp9+jc8$0FJILTe#abD+owlT`v@o90Gs=NN`t4 zpY`rb9B#_159{&2LTrtU-%<-Maz1uWKSX%DxY34~7F@qZAyrJ08FyHQ}H?#h^m2??Z&pAQ|tx> zAW3VtAvRSPj~zLx_k*u00AkX&mD2q6ppU~fQhg_gs+(vVp(FYuN)wf>+_l_ED*zD} zUonD@nXzy5Wy~-~5KdNzpm{?!0cqS=KSvD|H6i{7ur@!|nFUA=B7xperDIzKH%en- zdSy)iddtECSe;qG-#>5kC0~AUH!LeN027}Dp`tXsjkJR76+~?CVoqIn%98;f$&{tQ z>_AM_X0a-vl=H%nU@ONs0cA-UTK9os#dt_#1$PXI$b*$Lgm#{2}VviC)GQ5yU#wqy(sgN0qxxQ3;eh%H3x!h@u zp3i6ubBQt#EGZzKGy#x|_10Y9mh;P3yl*J8w!m+2dMh)QL=^W0fIhYKdI-PF56WBs zdZ;zXO3^7N2jdi(MY0dG%}cQSb{dy}ca$rseWcHLh{??iH&);-(0a;II{2?IsmdAI znHGLbjIG8>o}}>yl14{a7*SIHTmvCO!iy&gTU$ZjKkaitd^^fS1R(NM0h^Ru`#pyo zawr#r!%dwlY`4&`FgOlO6_EdRj!G=)ffoI}Y@`sx_Y}%@(jR&%#@{fq4BTQ?*c>NH z)3P80E>qP9?A+;-hYVP-Pfv0Pjy16uy4#YQ4^i{Bgy}v%hNuSEOyHZ{(_`Fr9Q+B1 zX2UY~0dGY}Dl|x31BkVSAck?JAZ=bFJ>}$}(mY0=uEqi^(7OvgH=d&-Tlp3wSn|so zggG$T`04XoAcz^VJUw*nGRzEI>7uk!?{HDy2x!bt?!qt>z#& zA7Q*yLu4r?*!pz3CaPTR`m^Kk^!_Xd8x@g({p zHA2@PCvX>V?YBg+elL@%vG`sQH!DF<14PE37KVR9} zsKtRihd%#w=J@FjW!TPXS@H|ub|*v4BkyTbp}+5xeGtcw=F(uO!D|_ieTQ2Rijjk# zfHpyxQ)E$3AekFlw?M)N-4E5`DQG_#x5>4?iLehNX$p|cy{XSP#8-7E3jb;_8X7{n zQjO&hZ9;kc4HgA>=gFxml!uuUf4aOw4fQgGGE^yocD4hTj(}AMx}_Iu{s4>;;k#89 zm{C^Y^oORb zTR^kA3()ddzslE#9toLp!X6M3n3XzX)YW?TZf5D zKxQA3dD)Z=Atk-B=g^THG9Lu1<85CoJgx_~ZFIck2z*pSu#u6h0RRaS4LmF=Hf%}b zO#^=eOC!H=2d40oWxhWA>dHfXxQBqfxO}pg#q7*Oa1S88g!0lS6RsV@?yQ8*j!5$w zXi5MX0qZk77|F9-ZTpVNtTiTeEgX>u3u1WlHzd|NtRl)8_TG|uoTUEcuU-*!wa;D9 z=EoaoKM$L1!xs#J+?cIGt1(MG5X2iM_#bmMi3Pc_-cagS%z3rOq>|=5dilB< zsXB%eeFP~wkn*Ry}nhaPRJk9r7{U+s2StJc@8oG zxt0k)RvVs2^NdwBT<`sYjJep$1xy!MCm=SZstq@bj!X)f)5`IqCgxy$l`LU!Gq)Oe zd%KLx!(CmX;(tRPi_6aMxJk?zs zvBzf(xvKj35G)F^hK7Jy_}bUT@I_Kq3OWf-JthPI-p1-6t5Tixx6W1GWQjL8*At{a zyI6Y&o8|%ZL{0^QmPops}qi5%8c`*>7BPIx5Z6KVNkz?<%ZcKmwVCQ~?gN+oK z@?hCpBoh!gTq?tNsN^N2ouOCnAS4yCx%A9Fhc$=Hi|*?>-pO0<`3wagsmE#d-C80V zg3^jZ^-npaP05qJ`wKrb9xgsR+-9cu;RZ$!)a=%b44gU$;A~1-&neSwIhX|)4ycp6 znv`u#(-j|BOi%CEp-7PWqx-a#>`y&b3b5<60)_5fA7M`ffl?y5k|SSrdL{DFZRVXX zttG6tqQZx2I=VyFz*_lH7pI`4;rtiK=G~J5LRJ$|p z65V%ePLBpztvIX>;;OQ*-is29Hh#6-sTPFA2Hnp35}t9dUjhvg)X&x!{5pWF!q)ZL zskl!g4-%FBTCl|i!mt0iU=`L<^8AA`auWU5BPeedw>%o9g0v54Su<8PX>5~UrhqaC z%1Y5=c+!JH^J?hAUxQq*?{^HgIigcb^<3hmOTB?89C&wh*=n^o7msq>ZtOmqU}U+v z4V5@`2|}Y^;q?3{<;?#LTwa#kH#db$lJ9JF4L3P>N$lmYeI_Su#+x8y+=uQ}oMrb` z2bq5gjAi?BbRRp0aPJ zQ3Mp~sSE;BJ%r}mHc~BP5aOmN zDD=2AVEJ{@FNIYq>?-7N{*b@fM@CwXcYAIDO{XtM#T5zM{*G1UTVnQ&@L+M#E*|6h za&aHc8H5??W)!_cIm_a+ATY?qUT+CU1`Z^1X)A(=^(2xFQ&NC>|%qDq( z_X5qBiv2}6XQ)1wwxKyC#YBjXQ(P6la-f?xn0d+N{R=PpB|90!Ch?6yH-5O$u`?Lm zH=#OCs8dSCDbUHu(lAr&`*1E!s4H7h$I`l`RU^WBkyl5T7K5}6PEKtOE6XGKvbnhS z5zPV95+ndvB$_(oybhvE^{LTmc@F5rny8f0Y~3lRQRhFzJ$-v=1uOsuZgJB#EyfFk zo38Kb%z2g4l2wAp>!G7;k6dFCuS$@L+ddy+wMM>AS2}oZZ$%`)ek8P-VQ#m}$ZdtQ zCWMczNcCm;W=vI{__15T#(;OPueJ4+Lh8qf^WuY-L0op}!oFu1NOeQvRKC;(0hr1Z z;O4DXaLqs6YBvu1Zq17XsvuJ4vC;8s@ErkPdMR_Kp17{H-W|2EF?NMC4u`F19T7Xm zw=Uj0OS{kgKV$nk5+?Ry$y}E?&{rShxbl`8?WNZDCRg`(y-xy1L3XKEy5t`A2n^C7TzWnrQ z?bu0`YNOs(G<@XUV>^#!9FKGN#W#BS4Mk}rjZ3l=(Lz=v>P*h^=4VJ;ycVn4+RCTIdOE_u zs%RPvE8}?<*$*v@IB`I_?3umw@ZR23QRi@u5k;NGm`p?90bmNX8WtC=CpeettuA@3 zkKGqm9Hw8LQa#$O=hm1`u-yed`I~)ZckcBpDsE8wLXhVaN|+H(r|+r-A&Q9Ru?1n= zp>(8?Y5nl-{A&gs4T&6#hX8=soIXey(wzD_W<#Nd-2ZAZk{kn1-Ka%kgsYZ4j0aBU z$0#^ylkuU@<4so9%C=umiED39O!h){>DiF;;m#Clr8Wql;OTw2QN(-G!cEGV%#YyS zs%E89L#I%r*4;3NbS}ngL-!=3vu&B0jP>_-YisQJ$Gnv@Z^_jeXe%+kePCtXpx139 zckJ3&21*9-{szx^&>uHfc`#GK18odwgHBaATY+EW9PY4pPjX=6BiA(IbKu2DG7dcT zrxJ;o=DykeLF$l;H+NNmOGov^Yo@qTIf)4;1GREv7QdUegd%(00-9_O?&PX{NAA#l zlJT#onMB;@YD1Ub%CKdQf21ujQd|Mx|1Q7C_M~R_iMJxlWMuq<-qVuZ#!o!x%8zQS zQZ1;eyLL-{&4O^Dy5vo%!DCi;KR)Ip&7`$4aC9E2u4Beb@%D0dDbU^^eag8Zpjo*d zb-}id)D+ng3@oj!oJ6>?9QSexHy$wGC{4eA;i=yf&U(#)n=6^iZ$}F>nSaJO+8`N} zQ@`&-79p83i`uzi+3AFq+ezK{=YMwGqIY2o`Qcb?FN-We6qB!_@ehU6&1c0}5~CWE z=;Ja{h3`QTULlSF5X?7)8E>~Cc3MWRPh2DZO6P7<+;n}^vPaC@Q9`M$7^NrEl4sq$ z0T`(D1mX~-^l@_v&3rUuLyZV;zdT~jr~{_+$ngSm$jd8!MD;6kp_lB-04WrJwdIW< z#&dp@B%_LIs3fh8wEUsFLTZNWHKfYyiLUp-j`Q2cE!l{X4OmF@Gtq8wr)rkJC8FE^ z#(1n#DbM$fF~q(Z{j`F)5mk?cG40WF43k8$B3~SF@h80=r1IXsWUEX9 zQjlK#%t6E#!1|Awnw(qleV$B+h?qoA^lPVnzwjYEG#q@L-2s1pU8E7gP^; z;H+1R&Y0R2#Horpk47c3Y#-rDa>*Sb3xL!D{Z}qagHn;H0JXBuvXXIf!wFn8&5Qc( zjrkBPi!Gnu;K24dgPV@j_O&r?AS@~n?$)#1?IF*LLlc~~f6RUaX$g76-=^A4PxSY7 zWF+(y_{w2WY>^K12cK`e2a8cfyW!zMKc3C#w%;lAEd-xZ`DqIjPs~r9h9Je@-a;a| zikzFE)tGfoymrrWC|(l*NIP4m+y^$i!Dqc=1bNF~6Mg_5gIC-^Z#Y%XZS4f@XhS;u zOKkS~H$p5fK&HS8zHYg9#j*VRNY=nWGIbZ`Q;_t(gYVCd_JQrxQZnc1Xe!U8+hpS( z!w}dQA^>7ebBoE7_mM^=V)CVpD{McLGJ+BxLJdLws|jb5DC~mnSF*Qh%AH93B?GMq z26Y1Ecp}S^zu5j#8W7qt#+lGD=SM2JZ0CVBC2}K&Q;w(89k?DQqi6i`!oLjJSoEd~ zq$^F`$bBVou&nxHhzR25bB2j|;Vt`cgOt_JVS6sKx_v&^hLP@2k2w2k1Ly$MK)P)k zG`IoM1XD6MKy=Q7rxHn8^`%JEkUPjy)aK>f>nD-IpOXMHxMSH2_S6Lm0bC_L@uMeZ z>v&y4u-)VzGK}Yxp{FM-osLtXRY+C%PSzc7c}j4q_IAzKS^vgZ)yL|MUZ9?K%`X?! zy-82GpO{{T^vPgb5E{!HujT$c{2MN^34vU-t(%YK*amXUuRK_`r~+iguYX!mm0|$X zwiim1X1_mHrfr1wJZ;kguX`p70Gz*YNu>IHf;%#|*Jil4zdq~QIm8jh>_0wszCH@6 zMubUY_LJxrY+?moXePf=Iye*tG%%i1<>e|y5>*Q0kyh6{Uo=<-`R8Dx92+A6bj_OG zYX)>Z#v7daAN#FUPasN=V(|GB?Oopy8MJ$P;%km^{6P8Cc89|pbBVM1jhtj8$&nt0 z)qXBOO`nOZEiQ!Cx{7~hzGH{|sDNnbtHpdn9C3G5lyL(0l=`~3KFaJiz#ag+IZcfd zL5>37rN`^!9mCfR7ei;g=EDV%E*O^v=NpQYj*MT~`FdAZTJ95m!_MdML5^CbOh)Mt zQ~i78N%T6VNK?~b&AD_So|1mKD)~c+tD$3lV2MjnVNnI5pMbn{mt-TnW;Dt$y`k(& z^Lp3|NEVqbRKo)UVJkF-0v9*ePg7NNz52|#X7X3t9*Bl~;$RurBLpZA>~Ud;D}Z`? zU&iVluL&LQRa%;`ZB>tx#8DV0%eY3&u`Z`_Au__t!;Z>YBX7OZ|twRLNC zUqgYR%`QXqGjPRxH7G3BD@?yQkb`TIe$k8m?5#1VF%&wI8dv1typx)6PeZxsG(t57 zlqirOD{i7R1EezpfCmBh)NfLXM>uOV@34E+SC)mGSG4|8zG4U(h&I=@%|N}WcWgd& zKCoySF_}bfpz5lJ9RJAfQ?6k0Al=GbS`Pj6CU+dIrt2Lxs0j|ukdw?WL(>oxf#=?8 zE#o20m{RnHJ6^8KPjnUbENq%_IVi%rg0?Q>W9Un*l`gG=N^KR5*fUZB^~!=X+kob@ zK*A98BVQ($QdrzaGe9WIaS`aOD!zhKHjqu>c-;;(1IVOGH+T#C7K%eyzs)^j4in zRK@UX!35B(it~HqtzWlgYy1JB7xf#he)q*VcLaYT{Q@7vx1ResVNNZKJ9S2JYAsQ5 zSv@|h|AbjPPL@uE^{2J>%dm8BzIw$mi{P1>F*_`bDwleXT@lO7hf;eC-7SOMM~j_ zQ|#tITs!Tk^z&>1xwg9OVM~P7_0H~#Xk~ULV+%6rPMBSx*=fU-HQkr`vU#}#zQT3O zayhE_7lN>!p|H+4TA3a&4!3F1>w`|u{18n7thCaIBky5PoX~AUg{Pvf2|3o#53Is# z<|X*~O>d|?!>A=@OSHK9kUA0aT;ffZNS=e}64Dt2qy-}#Ze@!*E6)88RsvL#xkfv4 z{Cwp(1-2F|wqHC^>()S52Nys~^ApDOy_5h8kh?Nzf;&JmhY$EAJKw~239LD zvdV>^Qn|uAUlSICOA>7+Ut~pyu+fA08eXc#P5891O=R18=uP}Y!j6iH_oZdIXa}S5 zXJ`-!ta4?AV^O1s9E;#i7SXXVRMZ=yT;5vvoTVj2}X)| zW&s924}WwLi!xlPL)lMf8cdp^=Wn^~Za=^E?V=^OW;E);>7n8zUel1_uLf8D<^qU) z!RYqTJ*cI8fgTp1J(Rx{n5q_cm}F9?Bxeowk}d{R_7IE%@-1Gy(D9|tyT)>K1%iu% zuFqP3jSO9SV|WPVAtvvJ<-F^AwTR4UD~VTzt1UDqe)su^6nJbcbRWL33Fq@zq0dTv zM8Hy(S0adU@HYB|N`xJD4!-+<{!d2;0uSj~E^DVal6A}2`g(8rT*T*EGF@J#V=up2 z7)Sq8=7RV_LoXOraASt%C=c@T6H`6B&0*#ZN}3C+*=W)qAmI1I6}Kk+uX^nzC2UJoouVY#ezO?o|E1jftoAcVN$fbOvUdASBhYozksX@f*Mni!$jsE87oSozAxcwoe zjZ96o!a@mG0*@skJ?kz`dHK4XM_u>Ah6e`xKO4U<1`f)^no9){Vm)_@G35>l-^sOM zZ%4-~Qy(mI|18Z(&{b5q6+eJVvue?*Pbs3z4cFn*>Z3Sf@mD}QA z`dV63+-DIUvp8Gx!?-QWcp4>Ym*hA0s2%OBd{*_t5)~X$H2hct3oJ*$!L9X;C*OgZ zlEm!RBnbMBq-0^IMBBxSl|m3Gl4>+Gn-tShRq zm-8i=cV*e}eShck)!mr%2h3JiWo&OW@1xxBLkT#Ez}#5!s)jw)VCWYq#4gF6 z+16FCp3PQX8ZY~RP;7sFe(`Sf*+yZ{5dj_xLWZPS9i(t;?|foaAltg=WsngZp2&i? zPG$Bu#kjOM9@AY?1N28nP^~t}ey%_d&HbY3}UvRPEj%jeq_F3epoMj z`R5JKXQ^L22Y#j}ch_@C5ZOTe9Xb0QWi$*&w!xu{#|re`+qy(fLWQN_qRJOU&Iw38 zxjG80;RKgf7QE(ebk+nn;s}afXuDOrUH)sMD*$)RFZ(M%>SIG*4c`n1GtO?oJsHi* z@U_#ePVwOEt@`WyN<%!{J`3fTU&IC?9+EJ>)nYxHn0R?hs7cwUZEE|OFx#Q+<+Yrt z^JUTVylni>#{-?KJ_yiK-Waw^Z8r;HIz21kYq@VbP)5&6$W^K^Bk(DhK0n zvZ@ExibLqz^us0lPY|!J0t@K;t$yqh!Y&IfzPaiR`gg^gbs|glY5Qga#B6$p8p|%? zBi&naWg${QA1cl?>tB!?{Z>C5moePIWTaV=R6ipuVf>mCL6VUGP+HC4Bj4I%4aAkL{Ybo;HM6qF%x0fo zYM8Jk?Z*)_?$)9UURc%}foZ?Ty?$h& zFYTPCG3jpLKF^cuSl1+Xe%tXBmaaFv;MELq<=5U6WQe}nXD_j&IG2$f`th$Ch*RD) zupNK(;met?qcHjVug|6qLWFRDq0X9*t3Z9Nvb)~6uFO0S8A66+I(ofPY(WqHcTr)@ z98qiS+q}&+)<;^k7k~28E5|ytJHJO^UE-QDxwu)2H7$5~gu`pEkGn$8DUx{b+VkF@QmI!_B^4n;p;aK0inG0dTV>p+Hh zB}i)9Wa<7`c{ahl1`UHHaEtxb@esj(>XckFJ{bU*G2Ezg^x$_#Gal)u_3_Cj#t@ zRn`9WaHG6WQ&ze|C;2y@)~$F*!xiRxtIN{t(_Q~ku!1@Nct!v7ANmaTJ*;_v7>j=L zfuuN_4bG`+XTM}t8?>9~t%lD$Mt0yop9h)XZXa$uf)*thVc@$S!nWz%&kU$uW`!|{ zk4~o-|LZ4CH;jI)O(1n=P~g*xzuC79WFwx?33KKBJMiJLlEPN;nwxIIQvJuIk#!=V zAw;qU3?OYfZ5v8C5f%XmF2w`6BccQZJAx?Q{_M}zI=~G&L`Xa|Ve##1o6Jl8H9ut%q@M<46Qz$_55gs{ z1Rm2%(5V#2V^vxft?jU37X2Snvq%^W$>H-ZS^xG-(lS7ByI^1 zm-w$$Jb9gODL$wde&tWTe+m_{bwv8`@ppILf7kr;eYho%A>|O9xjMR^~KnyF!0YDz4tqk^$QKh1s zaR32(>>Pw^v3r!&7&%BxtdidL0aF5SZ;~gVtF-J+`in+Y;YJ2H;?Lt{M zAuSAQftp{72@WXwn&JF%E}3|4TV{?984W+Bj$|{;@7P)~DEeh1`FIDMGEaT$iW&)6pQB}k zX|&MTL*_vmz?d!ex#?LbW&BOP(OjO!=yXEh?;Qe;^EnG>8sPwJyWv$N$KxcrU)2t2 z2@TK{xqP@lGmopXi_T`YspVcrc<)P~EW)V{el%CEhwwE*Vcz-^?>_-TyK>lXxuP+&K2(mO=+epJ%vOHcYVx z^nJrnO!Iuwx0QV(F-ifd0BCA?e%E`K)a-VWp7HHtP3>EYbt7HOegD_DyRJk@HdCcD z$L9sIkqzALN`BmRf$mC$G}tUW8L+-+A;lSzx*2O3k1KJ8twhR>6)XsW1}_W$l;5RB zg_9F>mEfc8q?jVVV;UepP=g+ZEMrD7yr7bUODp4fJCHI!`^dKV)_&<(HvJ8Plb`9b z@c_*|TyCNbIR809608m@tye#gJfF&$9Fp~!ezqu0&}YInLKm9up`do`-nR4d#_g^< zIlYLrfe5A&2EZzfNO`O43l5xEC62$=p3c`T+;aMm8j$&0wXhwgpl{<%9WqhG7=?Pj zH5W@lQ@@`(8O}vQ77DK(E^BLv(vzD+Qr8HWw<4h-ndLHMWgWp?l0za=cU9 zfiw$plHPZ;)U#rQG8^%gy4kFd*Qgn_`<^+iegD&o1`A|JzO#fvP;dn;$qhVaITlKW z7P-`qwOK|8DrY=DFnpMEWp&kRE@C0(>i9KFroKpG894E#7?1@U4#_cfYj?OnVWQvY+~rlHDQX9nfde3O zyIH1;QFya#zUf4EuUzxt8Uz!I-6ph-Z6}(oWbRlwpc2YEyM3haKZch+%0Iu_;52LAN5F>-LluiK$(P0 z&{i+u2Bl}t#rV`_Pr(zTx}Eu0yjGHh3N?K#pXm(#O7jJ>r-Y? z69U-?u+mV?=ST7DFKcQpj!~A~Ii5C4y=3RD*b3bj6=LGY+iQlS)pl3!J;f^`AYQIE z`#Q1Cs)mlnt;s3P?s2|^~kawY2{yEq?-TcbgasC*`pROj@qM7H*P~_@%{C8_0AZM zEFZh-DFsr3rVV@{qz5c%D^QoiU=27nGN-r zcE%DY$k5rB>4vXR&W857KSR$^S5h3?uq>65m%imh+zjPdYCvuc>?<@;2Ny@nV8 z8eE2)TN}Jd3|~wvASqWD&tbwws04ujj3W}Q@Lnn$_YVM*BE%{pW6s1U($Rz6m%{%f z^{#3%U(Q-nJP8+6#7@_k57KxClrSuuTfpN`r=jJEn8*IoTs91WPLpnrcr(;2GXW<7c{;9f}9oihqTi6V5u(gX^dogRF}` zHX%)Lu%gh*t~68fE@EQVT8LNVVqS66SAAS0xJWd8Y!^yKqpw78IG5dG5Ffa9y#G?Z zI9&A^t(?ehxl^*Fi%q1_-m8T(QJk*z03B&10tWDzI(~<_4hUz~y$G*wJQ)op!k*L1 zQPl_iB3B2={*lLjR>Q9ya=rGT{oPpJi>wy+c2}>}#stt?_9l8SE{+Ua4bI2 z1HTGr%55*8{AhxV;3Dfcbwp~X;2V!{PfQrNKPWNnmHZhK^(u#ahe)yQ)blk;hE}+A zPVSJ#b|@#=e@wlfq45;%e75GBKUOKY z&9Ix%3@hA%qEoN45Ucg?*C2nB4pe)=XoTz72X6Bqqur~lU!$Zi=8W=P2doj<3e8xJSF z0qcq!o)fd+pOwp)fkm6-+&J9gFyiAHDEU(Hj}V_sbr1udXw1MD^PfgsZ)HvQj6WC6 zDG$ECVNDuYD2nWod6S#gpN}kDH+o-w_tPye9~s-pzSHYz@Qb$-P3C_t6*etg7SdO; znBOtM4%zs*2>$sdqMNWousoq25$TDjegbfOITY$nl-jXoJ5g_u*c~Qmi3LBFnA?X- zufgm&b;tgVgb=_-7#XYDLa(&`+x5N;5@4!c8G({&H%WVj{^xj!?#siwWgeDJ z--BiHD1ITX|K}qqoFK4>>(T{Jgibf6r=DoSe>;01p_V*vxHgz?@SeolH z-kccG|1qZ^4A}2aXY7AAz^oICK5rTQiAOA5^###~|Fvd)4<=gTg6YkSQc;Ef`Namp z;BFyq&3^qU_x5#CSi(mSe*Q~%GGXe(g~H;=B;ava@s_8a=!5?dp8vTdFT=j`YYy8S z--rKkSOy;>t5W_>e6bOp7*+lf`p>NY`in;nS=iHH5`uKunkxUfd_*ktJl5~a%SX-% z#l3TgMm;0qA^)jx;mh9#;eSq0#&tBg&QVOWopchI&~X3P?th7Lz2uF~ot+*2R{pvS z$qE<|ag&$0=$~KI<-&_~Fvy*Bv~rE|h85hklNtDD(kFlUMu2=xLm?FuolLyD=kNvF z3E4JVJK9Lq{~6I=k8+X1HFsZ3x=bsah=hoR9mIj-3C^Eehgi|ToctN$oN80-H>(P7 zm+8O=cPx87Xa8KZE2=aG1(tlJ?)kO_Nr+TUl%~i3xyS27$5QpQyWXX;sZ0zYtloP? z?tOp0%PG-^e!j6tTr7nPS-02W^O=9{ViX}r)pxY>!WO(2st`FBx&Afc&+)T_!Q#3n zTi%@B+#^*~O`1^(K+$s~B9b6@YyAB%;Cakyv*P6ewPO%^;6rl$Qb z(8@OyM!+`oEAamz3NEm1HK`5PBN5>4UEt5^KSz82c{?z#y&QY!BSUy|MvE4ah5z*) zV*`Cu>)2E!hKT2bRrj@jtzXcIP&vQBcPGMjDu7XQ*^pgO`eS0glvu|K>2v<5*hmC& zL18wD{;7~M)L?4ZqiN|k*kM358sj$q5@8B>!XxXccrO#Y!wU60^t=>5YzI~d&i`_jj_)^qSe|f4?XSYHoZ{mP>E(aQm@+9C zWy1#!{*%*F z7+$Fr!3Ft0*Kitc&R+dk_x7iV+GlnD2`=_O?|{D~9T1(B$0;8Bf7MxM;fWndde6{) z1T&?St2@C3{&|rkH*EFl;@Hn=EBJ~IVPgEh7clHtdWN7@${y%OOfBnSsC%#kyHbB{ zhotXy48X823Vu4d{QvfK|CkyYh9w~x0DMp#Lp`>{soYQ;XNglu*}jkwOAtE$WN z(Pm!v`f4B!BWM$~{^jofP`Zc#$})(A4W7ohw(TN!McI$7`+v@2NdunHpX8y^PDDWZ z$}`{W|CEC+l(DG=1%D(<<_k+h>!>Y)Bzb*TNm?2IC^jM z6?q$T-!-a#PW2~nwB6qGhc*d9tv^+U}>XrhXxnb%d?L^Y5xRUU^d~S2k;_zJ!+oct$Qe>4n}+8!XT_2I zQWI05xw3|?uI@{M4D6Y(b#)pwpLZ+Lm)`ZWO83^Z>4QU4_obn}b94_5!Pnlp#5&dNng_IfE6`-X!s#-5gq_DZYiXwAhT+^l?)mv;`c- zvi_$AkYxkufR7pPhA!Ts@ppQm?uogGvBboQc*NVPrZ`Wa@^9_|GutM+_U^d`EjVCH zX{TgPeLj;TeAIc02)lb6udpx6xwPcW`^(++cws$(41PEH*woDiJ5Jr%wgI`#SVt|q zhIVZvrwQ+}$kG|B72=uoXVtWXRBl9tQDx>S8qJk7=;yo!^!k%zP;sUB?Ew=#RRPb> zu}lQb2+!Y+DwOapejvMM>^nGAV2hjB&O^z8d+jr+8`I$|Z-xIj4@J*if%DGi>C@c9 z12(nwVwzuGF?dUe{V``Yd>mW-V0C z$|)i@AZO^(O-`*&CU&6d^mik~QR1lR7zvwJsyJGi7KFK6fVuAFPT>%W+^;{ z&n(Mmu*`)YAM=d`orz8-Pvx2YE9w?J4{0N8G?oO-cWmSy8VB^Vid?J|KGI^LWwb{4 zfr_Wq)~EweS4Xh)Gs$A4#jtTSD_DO7#-Oou7L@b!@ySO`sO(;#zAAPmOD&^JmVu7y z_513NemE{Go6~mYASKRYZQni3(=H}%w?Ua`GhYitBx#pP{IEl6wJGVm6UlvFcRWiM z3Yb`sraP;j!V}=Qv{25_M(l91@m)8<4sN8V{z%OIQbQqV_{2 zJ30Ls3}g&L+GTyWmFI&L9m5R>2z6pN*9Dy-jA=Ux7K-nkzbI(8M?z*v`de1^8D5ff z8g^A2D^D7{Y$R^sQt}vH{1O-MJ0)W}w&?QjMM}e|_5BoA$j8k#UDhHDEvSI6p*xmY zg4cB}^O>_8{@!IV^q_CmH;96%GwozJlBfD_hyB7o!&W!HJuT;oxk*U)E?YQN%=FbPa&PKi&fn4*#W`xqws<@&v#j3QM``V1d52^r&1JX#&rBnM-LB%VQ z09n5cOnBdW2t2OxTQjQ1F9BhtdWd0I?-?dP9piIP#AzkDSxgp=CMaY4!tRr6cNNKp}E)jty8FgwHW89_JV|`;QEjIZ6zk&m|Da&s7 zQ*XV#5jJM>-dmX=+e@LlWJ)3&oTh`F@jP~eVUprvACwKAkrRb)NInsYP*^BX$n*nu zT{c#oCYEo=ds!ZA37mj63gSw;{uFEmj9)ASaO~(9)M>~NVW`?3irpznB)=|q6qF$= zKj#dMjX%!odJ%uPDe3zI$A0xOY%V)5uZb^r{=|jHyk}*f9nKD=k!qEf#GM?bSz&tfCb9;TM~)zCvAsIB>Zl zxeJhJPkk@1s`&bNz)>he?=G?6G#TSGLx9eswf;(sRhB^~w3ip|GEGy4QrHjre)-RXO7HQ0)cvQqIQE%V!HWL0OwTniZ|F5Ad|R=&+TNYtk&*O2 zuOoXRs|Auh1Gd+~^jWS->=J8Qs!_47cOcEO zzMgw}$)8W=mO`UDqs5)6zNh>&?aXN#YV~cjm3{!A&Ao<3^-O2@gY^|RqmaFXij zQu8&JQ%S~g26rc*-Usw5btz50RZwZR56Tps%+MZ?KtmwFhd$4Da(*v9peFRgr)BM6 z>e;7RFskEA;p(Y6)keW9O)JYNO+ZNAs3?)R^~@NSjfrg?vV2^AeV<;V@*KyCV##Zg z)oO&Ro?Tz(MZJKnWHYo;-uam@meH4Coux5elXG??>C_C}a{|}T=e!snvE=A2A2!S8 zK)7oQ5^CtwLn@wyuTRvh0-mD7tXHJd1jWk{zyyQPrkg&-Uamd&oB5FgW6SzEsXH^2 zwr!#qZX)Py4;?iN&r-ShSmDW!##1pzn?Xvz?z_lGS6tTK54j>i)(H$-=q>mG!nlwc zadU!_(kqoLX=&Z<^wYKSF^;fPr03r4(y}29EcD9y?*|NN(z79zwJ@^U_s_TW6TzBx zL94Wr%>k>q(pt;795o(x+2K3KBl4|Uv5C7Sx5?1X^n8=S|Ea*hz{~4Yx;;YH@f}Dp z((`3)OX2s`THul>xvx$L9VsA5_eg}OhEL)n!n4glR6jlSK}%EUEt}7&teUTlt6T&V z*|_zw(mf_`F1d<9U}@JZL3aakt#Db|-JaQb|DMu|ODXIg6~~oYj0i_(;&ze$Bvd5X zoH+LZ3fG68J%LpFASl1?&R_T<>nnm@FW{H&gs8W%#m{2AZ0>f&L=jck{+_IkL58J` z^Lt}NUC>`!Rcmf?eZYg9IM*stNm_t~1GRY#yOc}}s&GL~9|)BK31 zyXJ`vfbe{p1$Wpg?R+O^g^bN3zJsy4)|zgnR1@6^MetEne1gs+B^;0>W#blmb}BmW zfwhG&TOzC4b)WW*oxEvc(dIYYGU!ZPa-5o@*C0<(%Ew%kAbt)5?|KfdnYQ}Jd%jv) zbLN?LkG`jhacs?BOn_OEpxN%oXwmVKYR^fZeRQR=Obh4kf%DnXwn8g`J0BqgftxI3 zTZ1q@rMpa8n9Zfwto`s%@i~%3ViHH0zY2n4zqGxF!hn$n*nm{*U{eC694+CSSXFo1 z)8pJpAw}auJ48wJ<)CDS*P%91&n=}~y*n}h#gCQtXm6R94N}+KkqU+6`jk^8#gd9X zM^gI|bKaW4iu*@%-nR=QsyGqcf$qJBK<}{XOG_~!es?J~@hQhk7)0`z_T#g^IO2w; zx*6GYL6@syc6c5HiY7do6E}z18K7}}GXQd{SCiLsJc#wnNHM^dBTz|I(mZg;CzU`; ze_)0F>qEy;LHYWxwqpJ!$0BNgA%4GqKF^qboifrFe;kpwXFv;4-vRqNLdyevr=A_{ ze-W!reS;M+3N2Sx-e(lC3~Hj_O0f|70JJp-g9{=So-5npw=q(<+QUo9?luBD%j;Bg ziuja>wKF&zcw9~&(HpSD2R_ejv_nP)$L6kwC3dy$Uvnos%6r*5cT}nUb#nbeG(=$z zCnap4S_wL$pFI+XFyvTG(26hXpbaW}w!V$~D5(BU+J~XW5@)e=N-SKk+o?ERjTorw zpjE~iaO_NrP5lm$CC7o&CD|F?^!nSWv$+f5U(V{i17}|isD6mNzleU=9W8rw&3Y2l zr`w8tP71`Kd^OP0Ih>Yr5*_o3k>VNTm;)fMM-gQ;i{V}eVW#qn13IR2bWLzC?O%Sj z+edQ8`bA5%)$DV2mO%>CL(V41z;XXI57*ECY45$msgC>q@sJ2*?p32S?T|i1F9RO+0-@+dz?>m^gf1X-H=ie3)(!Fp#Fh=FuLFRr0p8vO zKK^FY>t;%1gbUNWm0m7lda3l`KcBt#C;&WkFL>b(A7@@8I9-5zo&I#v>&sN8?{qnJ zrN~DV<`c5OMjdfC3ajJ({D@?A(0XI)u2k(GgVx>a*)NatrMiHzTgU=sr_YXfuTT(e zJ>P!*E}tqt70~F|huf`nGtUuETRbIm`_7UHa&;M$pO6JKI^#wbOae$pkVlCo!0O(4WT|UV2#eYdVsSH)C&+IsNd#e^J$P@YPgn)EiBR1WKNE7 z6mZUA=d;@@@ynjN3mg`=eU8@3o!=g-rCf4wDof9?pvVj-I1A{6 z9$BDdvG4D?regBZ1(vsKlesTO+bLX4SsYP(vYaEP6cUx-g8MwiylhtIN03)o#0 zA&$7e5LnMb`1Fk?ai`iq!MAdB`d0AKQ{}#=i=CF5D8|u>7v{K3wN$u;x#sP;vUR^0 zKC$8MxZ~H+qY?65+FIAGi#MB&3y%*rEp4i8A)YzY)|*n$#juMx%DbzbXZ;~2v=Me1 zZFGHRO55TaDvhLH3EDk1ADYwUVhGkUfZ%E1k8aP)U0gQda6jA!&2%@1yMPcrwl3bi zw8#TgFpWcdsCFWB$mumd>D^1nD6q*2B3_L9efIkxewo)9G!Iy~c#9vCON6&(U366BGwf2S&oQ%pb;* zXY$+5pGtgp=J5Jwijab?C5FPTqgBz{*ZNFj$9Yf*;TV-h=(0&04ptJU(Rst{ACGZQ zP*SIuLAkjYrjx5AWMqUWnd2W=is|O-+0Xgv!9LPGGb4{QA~(CE1WxfMlsvk6wcaJ# z#QRhh$8c~u-JwU)e2mT+jou^99aQ_LDHfL#`CYOEC$>=;nF&U(4N1SyoWY3}L|*#} z3%iTpWN8&fM87mTd?(LzqhV=^&VZE$S1*&YdUA)#py(^0PHxs5IYdpYWvCBUh_nc~ zC2f19o2HcT1nh|1PH36X4f+0V)jEq*lvn}x1l^p5 z&WcS53HL(XP)tk|ESP!hCox(={db;L^u-dc?xov2#JTw24l3q&F$$>jZ&Psmh7oYR zm)qE!K3Bx@TyGZiy=Gh|Y4eLAnQ1J#=pfI+O|*R)w4}wIgiq=&jUHF!7ah3KP&tLt zV5Z}?{eW6k!$jdtMZ3iVLOE&aCpvdq8-+}%g%|JZIxY3^MH*zQ`>oquosATTJC2s{ zxSh?VpdNGInj(R21H;!_asG90An^=u+o9o_LI=aii1tz{0g8mHG=!b*u!64b0Zgb| zsb97l&M4qg3XQ*&DCQXNJeFKbUcP+1R4M^r=B?W5R`cf>gRp)kyZoY0rOQY{o#oxP z8hRqV8TdrL7b=flXSf-ClxYY`c z`(G(_F3~n4FMZCVw%fN~L!f+AQ4%qEa>3~MMJ*GVxRQ8u>;&blo$;+jA!e)KgKJ?@ z6{8;kstjdJ6ndegu$q}oyOt%g;H&%RK90=6hI?#O*lXFPl(;JfYyqV5>XR5t! z7PmiYcyL6lB4iX0pEMRR4Gp9#SE+0=SW4}Vz6`M!%xQKdHOf1_pwNnrv-J*fN!e^7Q z-Z&Z31J#hz0Ud8v#n4avtzX`Wcdpor7|$p#j(`665~@D_?+q7Ce6!v`7OP z$WHT~7srY}2(TO%gN_>m$yn;g3sP6RL%2Ah5mRp1l4PU6;xL-|;|{r2R}KxdW31wc zC|`_M^J)nSJ~1b9W4&9}al^HHT!2RJ6Dp()+J0C(>L{D4q;11;(90`(R+u{tUxkvI zXrh;GmLE}5&t%)kT4|7L^tGu_3#y>>MVbtm^=UH$bg_jO=xAd;ad)@Hy=t-jZKrFy zp69$d7>teQf)<9#?d+VcE}>TG__FyJ6J$8+@m<1w!$vi+zE$J1zS1H&kKJ*}EE==p zd|d6r=4GRd#X_I2O}{XBsH04Hv*PI|zbkc&bNP2w_Xl3ioIb}ytBVZYc_+)ejCidW z_26cV)7>}pp zC8e&lM$3sDB-7p}SLUKak?2KWd<=w5EBfgUe8npY(H=$3-Oxk#xAKOV5r`#M>{H_J zebV(cHE~>l)^6fmNSfMJ>i<+;( zvqCL`jVGU|aLjTa3o)4YjZkAqv`x9Lp7Mvu-<2P;mA!0}RO`5QbtW0EOvAr*)Y8Rk zGIpLhkS4;)BD_L?3eW9Gf54Vt`of*d?^b7cp0ll?k_&#bpt`LgcvdGuo9pZ*8|7Rn z&BNuRSqB1-vc_*mT&kc^))lvKT@*uSU1lIB;{VB(kgr6|E2bVYt7k&jsJ7Kc#;=f>2Fizu9kN zCgNk|Tyo5ItVUP9;#;lHYm(txQ?5%AmP$AB&ueP0UeNUan0!i7DT#(o+-Wmr@!gv& z=9!ZHMkv*E&Y;>1-fh_Pd7OS^BhjK8Y7qzhz*9N8{4`lRp6MoaQR2UOpULRE6I73~ zuqd5{B!EE)7v9eAb2V{kM%8l>!SN}21Z<#^E~)DmshZ(^btc#7+WC8Obe`qw-MYk@ zKz}re+=X@3xKx06QSP{uR>hb~Lv>CUT{h#x;%8sSBNRH( z44)I|$yw}RA@B#6Mad-*`-#d*kW%oreJFks*XL%&70on{eK`i>2lL~KVw~L}a~(Wy z2P-L8adaR5kXTP8?JkU}iHOOfm~~HX^4op5Q8#L5TGr+d3Cgjvl}hwSckcFd)vS5k zF7E#{?{b{#_@zLaHysQES#6FbI=52q-rx+C3mv#CJuVt3b28OVa$z&JGo1cuqPUJV@FTMbq2Ky0cuhcRK2CPL|U zwwAV;wJe4So-QrAh0Zl5L7Z?xgG-Wh3TF{sRl;IrMNs~U07o+?j_$Az=nZ^%;KE9q zbphRt%A&ciT#*1LDxvfC&jN-(T!fyjh{WKmOIM(c1;w1ek4H-C;Y&1l-6{y6N?%Wf zD3RZ2X57W(alo1DE!frdRn1A8G#TkB8^$>0UnicshHh1;b|8~*k~)??!ff^JUET2z zwmN$cWRxBtdwfLDf`wTx;gov>hD)Ot8)DJr_-Vzx&-Fgb-vN0k&BtpGS++;QBPZ$x zcOGNIbl`p4a33U1H3Q~hvM*?sa$mKSk$f)r;l>{=$L+) z=e~3E$dL3;4pkAmMs5LfBxxu7tLWq{^x`K=ZV6C{Qqz1QuV>EO$4Z5 z8=&7ow0dtEs5Bs$&)~cr&yYy#8GD_W8Tauh5F38S_wb)UX*c z$%$3-xT~{QMsP6A)n5AgED~|gg$mOACr>hxxu0OKRXcZFJ$*4IIH#Vf}X6V|@7(1XjfDj}>8jU6%@$Utm@0D9=9U{;! zodz{v_N#Y6<)QU(f1T2=KRuH;gjp0_Q;JVvCrIBWiNzM)$uRTERh&8+fw_|-iLMEJ z-!@M1M$&oA#w4YEs;&uIM9sj>kodL^nr~SLBU#Fbxwo^=0y{`T>uVQpTi6C80@Bd9 z^3S+mlyLy6{)qQ<>)w(`?qbXKi}ljy9TbEw^kr6^h$#f2i1ER1m~AoBA}klHj1v2N z#b|OC!uyet(lE@a+JL@-@kiInn{+Y<;igIYvlEO$Z(BH|fp6tkxrO(tX(=2}4Y=?8 zm;@j_$hV3cnA9HDufv$7bmb~c;OzfUpIP7p=**ok78)rFUSTH}ASy!taT6ScegRag z7irH!bSi@PEc$r%1K=bi{_@G7PmHz=mVmEHqhSQqWS`>uk-inCHHZo0;j&cgBs>zb zvYBSK+~3f}`TYC^^e~t2)F&C=2XUH15Eom3{+@=DJK?(i?+)e`QJAa)Yvz7d2-CcS zUW^koeicNTzE&a~>$g5IZ)p8l>im>oO=D$p^DnI@v(X=07#@u+q=<4x@SLSIqUh^b z3H0@LV6z57;5Pex7EuN}>yH_Zr{n@PzZ0X%RDnF%({Lxj1x~A^hnfvcEJongoTt0y zniT@{mcx%^3iUu~=jmp1EZKxR<_6+DcuoEi@X@ej@^M0YeJRZE5ftXrPVVXeeu7uu zZaf?|0kg1vB`QiIG1^`3dQkVLz|-KZZNe7f6P=3Mh|QiEeeBWBV2gI-PXoF_3n-z4 zn~a{k^wkAO>IeJlek)l-K+`gDvmMof*`uDDMsxzGX8i)jH8=N0?l^2BDFXiSx-N^n zk2E;%!j3}Zg6dQM^rEEb5{!{YbYCU&!r6+h0-V|^JK>AsU4(ga24#W61dPi$;)+s+ z7$0EZ2)HJ(-NvU$GaqG#@Tlxk?l3uiUpUr~yAw8C-)v8b3KoylaR-_SSUE#}WJ~GU-vl;D|>VpVwuY4ZwL%lFuzW50N zqRDRC^ot9Z=)MEpmHDRW=OXt)Tk5A_r~p|lrmY~37+8tj1OzZp7wwtEWv#u)uhn}Pzhde# z<5cojT)-{91Ejc;nr{%QAdb~km8cIE1C#>B8Ywa!=-%fqxgS4wbi##x0eoGYEoH@3 z9-Y)n)Vt&<^KY;A4I~+QL%$QThl4RL-S2*$DCz7U^7l)8K^VZ~?x2LMXo3tC)hY1& z0Q5?$$6=R7cThEw#NivsQ>BB-n zLy*>P(lmVE$l?LO(;N%V;Kis%EzBw%ivt)qx9Qf zHTT;cgMAM#4?6o+fjq(457|B-!?5?^aNpd4t17s-+-FJ9^1to-Dwsvl`pL6a^Uyb+%7~yggiJITGi+Zv#fR;uHmy;fg?Yp491PXm$f% zm^_~ZYbZ-SA0|>XXPvp?9x{-U`Rpq#S zJ#0il*TwgyMYAKM54^y=y#4CCu|f?Cpde|^(nMMsniv{3l8Ot+MtFew8$sOFk^n7l z#GL>h@dW)fS#0Ce#)Rs7>k&ij7u{@R--C0m2@K6$J=;2O=Q;J>dq`|h3YGn zKL|Gl)^r6-D-s~R_WUC`_f_(pMNmDX+nqd;b|*k}jL$3;C;)ac#t!i{LdF{7p!;R+ zoAioS!MtlwQuou#SqR-Li(-E34?b8gvJKa(3*W~42J*1|0@#@APqPJWfim2nbXfga z29_*@>6pjFYi~?;#eaoF@tH(JlTDH%7BTKLhe=CBk*4rzd7mBN%{7AQ5na`STbZ-n zA_uc{FES$rNdbSXKp}1aoyVB3WrlDn;@(FALpVT|f{KM8UHEptn_Z`br%4ixnzC(c zp(A_~Ms?rM-*Y`(?=>`ezEtDPMT!W<1_nPMiAES()c>plj6k9d*NaLx8^wM_UO06t zM<;S>(gdE~H2*o?kKQy#ql3;2GSnf1=^ifcX?iZRmkof=yX)#$(xcsPm84YlD}CB7 zeyjEHPpdh^LK7u+!3;QZsm%VopJ?f6D+5E+^TN*0k#S6HlMRn-72XlsK+2Ax6ncI{ zlZDLd{jL1XKgW3%I?U>GPAe}pxL3R9mj=ym9Qp{g7z=SUK{8tt zUd|_elS9Y%CJx#2=!de%o|kCSB)qSCH=vC$NK#xp0Su|-CTR0Ers}LAScbfK9vn=2 zu@H`X5yJrS&Uyp|;^c~n5nf*K7E%^x-7Y*3C_$S3y*8)te3cIhPxMUMeUoz%wGI%X zlXCi)J+OFF`fB9VoEUA4!uO=%C(;Y4>YBFGc9RaW#Q{q;3SgYFlh{ZhaqoWTHQz0I z+~=SW6asnvU1GB)?o{QW)4J0wGT$br*z$YO*)7#1oZU;?fiOkdW9UTX1tEBWK0zn* z0Dw51ckQBOz7&AqXBz5y&m_vA8W34>{BXeB`B8fX{V{A8XO@f*V4|*!JzS_LOiD=vys~$*>qeRiBews;7u>1rp5TU~6NvZ;aL!bhWTiYpLc{Mt)L)t(~Lzv!IK$=97pSQ}M~Q z_XR2>wx0#DgKN6+)OWd7VQbq|Nq)~QI?D3MYEwP5z6v>{+$M>Gy63l~Q_;eOH6xQR z?pE(9buZN|zVCmw*zwtA{k*U_s=e!FIlydUXkXUIHwVr+NAMO0cIKR)Z?ZidJ>tNB zGCJCt%nDFtC<6DL3x1UMX*`d>D9`9n` zwzwto@Uv#$BxcF^;i~a8u57dkncj0k0(AZNk!l%q#cB`oIWUyF@zAEenu)>9n;7jg zms1#!d2vMd7aYApXrWtwZyKHv&nX0~gm%^PZ*hNicVh%MR6?W&s4tr&vDg6ag9^RgIUpz=EKZF89%q(ITH~`)zX0=az zWc4N~giLInUYR$ZZjJsm?=VsytfZNx-l4CNiV_*J{kZ7=`Mt|K;MvnBEX6cwrH9xe zu>mstV|!<-h5zvhZnZI0L&mhIj(h?kqh03mMqM(mG#SwxU5c|89lZn!Upv~yy1sZj zFOlKA#Y$R|3N*q=rUt%Bv}|m^inTnbP8;gqvPzR&PczQnJiERj>g-h3E71%#H3}=f z0Nijp+xOk-S8nW8?r6`E%Vem8{Ql{{b2T3UF^Y0PJ_orag;zoojNI4j7ou!9IA1h^FjTMR!&v7Kw;*ckb2Anu zqe2DfpczO+lC)a4R!ye=_=hY(e%B_WkCX`vqUbjhYub3s*TyTzDP>*-#BU0BlgzgT zT|`o=B4!*bv+bt&QEv;5hRx)S=VwpYY4`jzdOYFNQb^{mzpJsA{$On`eEYAk6bSkb z{<-NLGPL%vtrLOVHJGxE0mpd@lCB>B^}!dru-~Th+|y!6I7BQu+2A@{V4`q1BIvYXfi;Gnze!Q{JXtj>KXf%gi zT2^>mn{yF_yVFxk&CDPCM1>%DtQI0Ns%4|U#?HGLZJ5c<{Z1cw+k7asDJ)Aw$ zZfbwA8JCtGj!A8T+enuA-?!1TMfUIqRA5o7B@N5T%8z{J&v$;<7V-N%U=wG>L4h!2 zz=EltAf%N$$D$<#2~K6IaXzY8!p!M#_MXsd(4wJ_Mo>p(F_GbKVn{i)UB)L~x9)NYzx!%*{`fT%*fJ>>9TkZYdh!libyBEk2Ns&fi3O%LJd5VyF)7Zl+dBh~D zs9&><>vJUOMt{8S;4l&xLV*e-+yZ50Df=o`c@tM>zjNsLca1?m#kwX^@;!gNfQzQI zYGb$7wCwj+cQ@L}zHco5n%*epd9c!PaJc{BFp?bUHI(*AznuY<+Imf;Jyh-+e15!= z*bQ-qT#8hl;roz7PjqfA&S>aKXxf`2n46HldMa>z!?2-9t)Oa@2I|>4#sW^lI}gp) zPy>aUsITt};vC}8SbVZ)g09I6Nk{Rr38Usm-}iwd4m!-Q7AYV-GI09^!Uj>Sd8EuH z8*zG9e@{TxZW9t8f2ndC6@Mep{$USa{M-GmYw3z?pJ&0YX8v->c`F?oAkW(TM68p> zyCMZM6*=Kv<(1mu&=Q4}P}7nkl>SWp0HPc^_OI%ps8n?|&pWuHpL#YHMb}Hzf08l6 z{4!yg!UR4G0;8B-hTn`}vbqc@=y|omst{><|D^iF1iYgJlAHE*p?dd`vdJTrIOxP* z)EPH5E^*vqze46K}YJt2T&KWOXl($WosCXXq-~*JB3q*IBn?dAPZpcn(8p`6m7$!l!st2x? zRTLb-^9+(u)_8LRqiYdTNP7EX2I2aTfuNe+^k3J<2{GongL{rHDrs~{HBXlw$!s1X z3r%vWeSMB&8Z!K6*Wv;SdCLN?OCm}+cZNPs1HKD6rxyl4zWem9!Wx42@Zww6DaP`* zHorkXFJ?VH9z4T5LD?!nxZGKTY#RX!@kX;2*yxJ?*3T2S?uv0n5Mq6k%kh`UyYV2+ zE#K$r2N19hfBW7v-UE>RI{_M(5KZ);m`4x7(+iBfKLEwJt3DCw7y6fUgm-Rat)X5n z6RLU1^0;HqlItn-ClOMA_3N=YOJeY1?3!r#RfC)#PLBUj>ANSdY@!svRnv!hqxQn} z;XKrX$<3c6ew&0X+6jcJl|KsED~i1@^{Uos-PS9?lvW zbxGA`0)K{KGs=Bi;wcQ6V-q(;{SqkTsd_wVt>w_Zq(m67Yw}%}Df4r8!;?mU0C9Ah)IqoR}k;)4bbK`_PL!gl4Iv@GWdskWMOXVGo?8 z`G`1_>^8eoeZVw|S^lntRzwpb^2YjYxO5(@8rf*->j|ehjgQXTK~QIh&I+rwReX;) z^us7TA~2JsC54LDj#qP|iC%imJo8HUDk!lXI~gun^qKsGmV_%~;a0CeGHxW4N8|xXV;ZjE5I3;x54UGQtlf#1lbZ4evBwno~zbL%^NY{udIW^5i@-NX@4am zHaUq2f$k4UaYXdk%*~ug_0I=i3|c9Sry%aKmKxp~m`wPd@iH-;2p;~aNyJo941U3f$=t`aCvbD>z<;F{g1+b) zK7->eglr-~y}5+eJye2=e)(kM3xf~NJ_&QCNK5(im0g^xWjrxCtT&@cEtB{+j-_L+ z{3jD#T>!q>-MCtI>0>+Mp6s{NdHNBR>&TivPG2X}0C~2?a1PRjJE-P;)qe{(=s0B0 zho9T!bTSus!yoyowjk0$?kp1jNedwUABlW7BH)rHORcn|giHV2ccuocUd4Vy5UBu9 zN;Upl87NCfM9v2eUSTrKt(NaxeOc;a`589jPz1^sJet7S+yv@Uo8Dw=rZ?^fbu2s7OF%AV2e}kasWq;@%pTW|2vpoRg!K8Sso>v@ zy=Gs@cn8-D8~cmmz)v*DMqC-3=yIL-rXX1M)(5&5_h^nL+;Fp&_O<+2Yo2nuvJb?_ zbjdo;vx4rPLwaK0U8tHUu`5ph*d2SQ4&aTMC`h1c?)WP$u z{FYw?Sis!gLqiZH*h6~s` z_H5{&VHJwJ{MQ9>`+j*?0YnhsQ;DK)4xJMR)d`MikG(b)fc?iS{UM2cm?!jYbXvxY z@~z46>U^0>)VL)=W}rFk!jj?DF6+t)6ogq@|LyF^ln~Ta`K1?cbf4d=bhdg|qVBmJo`Es=R{~jy<VTo&eN6A+uE=U^aL z6go7mcRf=q066$Ow-KXUz-LplO(lZrS(_TMDhlKpF7UfRLW6K6PB?YF0Xj1|4FQSY zI<@8Fwbw%W>T9vo%{e3|r7d8%cH*Zm4AYQdE7AyWD>hUn0%=pkS_zq4!0>Nl*zX|L z2+_-9ZrKRwfAdg{h*X~IIV3@$zahJhI#fecOA+;JJlhcLv2(;=V)O+VTgExM=j`5I z2a*xIx8N(SMA12f5z@z(u?MH#!@XJ*DMJC)Klc~p0SM_0N`=@G2lGPJ)<;Fd=fMl? zk>tmIZN@|yk1vq-Z$r+}TZ)LJ0;8fA{DBZo!Qa7eW+O7j2NI_p9hnV{)fyt+1?S~^ z#JlUEDZ)YJZ}UU6NmpRnjn~|H;SPk+mHZq??f#qBh5Zc8kGWPlW$Fmbn^GTxfG@;q z93JDeYNu#dsn{iN!)$F9N;5c*zs3wVz>Rmt?y`{j>aVxBQ2nDronPNdiAA8Xhk+ zGKQm-RZv^5eO0{ZG$4if)MyGLO=QEP!eHF23`GGP)>rir>Pj7)u0X&^F1P^G>_uak zmYyLO-HX(0{w#UHPG3aYrV2wZJk5?)^$D6*!NKVXpZ_O1#~fgA4%ZLNqUw+r~VzzBIN@23H1cSGkOtiBrbK1+b59YjVgT)IFMG|l) zf=&fcS7O{hvZpUB_bW9#KSI1G;thVr-W|usz}gtY7S$A~FH;V-Caw?akv`3yqETTK zBJRu^(9$AbRs{me-<`liVw}3j^^#94&sl_f8PZk_ni)M_=lFHb5~G-gkpEr)Kw9qw&(538)#MA>g z6>(w;_p%du{Cq@-k~TUF)5Y6}0kw;U{+ipi%;=FCVu%8~9H?xBLY|1;*Jm3FbT;2R z15}skz5asn$sP&T3B$wtzlIK_0#DFCU2Z2ktu+&v`<1DolHhYH+CTHK(}j!IH{G@m zM0D@Ol$9h>P6@P$B#HdEwinY$L&94oZSj)5vxrQVN{QS&UvknWc5s)kQ3kLW>;8_0 zFse`HSDa0K(napXTgHSK;#A&F{a}s907N@c2{$kA&ilFYlVN@yvCkvmSh9$97sPF5 zMNWH$a-IyT#A}+MRygR4-rsOC0j8a=a3d^PWs635)H~}YM)~f?r4&wVhP~@jQNy=& zsVtXOS86ic%s=va1#WL{&R;^045sT@#D{uysonyWc1P^gr01J$Tk*2SYjv#y982{= zfyS{~Uo%+J+x%XCB^!KxXUp|R*r!7si|pr%)3>0)I&YZHF_wJ&Xni@^jE9qM+YjBt zuArb!m79g~C1+|30itOtS*N||MzTiI(M5P)C<9?kk&Ki@Pz56lPa>(*Jtd@)tWQ)% zMgh?=@b+D$(-gXVbN*aLoB`8dNDak?^|dfeGhG~LjF9erdlSl-`>Q+l6dn_NmUs+| z`GzxZx5(#xLE3s8z>t%~-U6Ki_qZ^ONfUDhN77OOiSh`d zDT|r1h`>=^C!9oG#)TF0T?}luA%EjA#vLuU;wSG6xY3(G1zH@lia~VBQP*$@#}`0TK4^9;*{UcZ@!3OoD=ht=^9;+^p@&S@(luI%8#XYR%}xf6reNlaAQ7@WViOS37sJ$(uM6Op!UO-_Cdb&M+NKz!V5 zQRQ?t9ll|MP28Qz{tPET<~?U=Km43zvO6d5_7_5roBz&441z&F?pB&y*P-Z8dmL~c zBGrEW4J#bF#tlzwD%eI^x`$!NrUO_jfn`u<-?B}oQFGb^qA8rg_Wx>9LqG;@k?@6DcfTgkmUW9 z``#Y6%J`hsmW#&4iTLwngx94J^F!6XOPD=_Xj712B2aNW#ZTNIuWH`S&#JVoYwq-- z-EyP~wUcHq&!XvXrqhf`JJtVYz5nS_NlXIvt_L+&umSIV>8QXyVxT9w@wJ&l6aqyg z@eoSko!HTxh9}Y1+{(;jcV9i5$0nlC_beYQe7y6H*xZ0zac}yipdPu_KGBzROuQuY zr?Yl_4>ZzAHR@bgJ?di_5K$Ym{kF4NLV`LhL_M9OSPB5^ALVu1{)ze=BOZ})Vw2x9 zISf-#4g?i`>2YC=i5gqq*BTyAt%9>@b}?d-60-}-d;mEgJ#Q7hSCKg%4;h-*^)x?)(s0Gli}~*veHK*cILj=~K95;2zoZ+gQY`jN+=NV{;sFUJ)K zswn;K;R*l9j)~(kjbdopT)1i70<3uc?_wWIb$|==RET6P?=8iT$j&9Zu1uO}rQ6(S z>_&d3kz8rHA{Xh-gExMwFITK_*@`&ew)wK(+%=5djQ-qU!z|#Q9CoEJq_Bk!+oxbo zB1s{2Kb#A%6sFUsj+2zF5dB3kdaccSg?;TcZ4O-p9--sxq$rS{cvo_-1nRr!(zj98 zyd0e@M%>!ul38tn8Zx@xmfWl6V`=`$ZqthkR!&jGv|%W!L%aECoVTrp5EyK}Ha^Pd zyLPoDe&`X(y-ew?2m}B55KaT3P-4E2>xm6@(Uj z;gNeufHYU*k7KPBmEwP)`fk45_H+0-Q>8dO7XOJJ5$#aPrilPupRU3kj19b~35*}xa##4Xf!IgdG0Qj#2w8WW0LIU zS}((}0sv_6h_dqLHVTpHtEbPtmWQ%^plVB^_lYc|KK4FHN73g`f1pqPPDi3QiE17I zNwWy;z9`)rEFJW3VmMGn!5-{hWw$7Oq{6WH7CJHhNi4gJ0_m{J1S|#bEeLw$e z12UL1#9hgiSOL~(rQmr&`DfCL-#DdD^aOzdS(px$%VVqnlUDW5Hogu$94P-r)M2im z?3C3(#^&4b^b$#hV06s1VdY~6CAxAgF_vy(P!yCsPF>UqRl3adkr!1De2-ZQd3_}o z>rcR=vYy;OF@QJQi=s6X*yvPWcaiXZwukFJy)P=3EJt;7Wtk!O9VWj%mB=ZpxYKc` zqPV@&1wrS7H%FLAg$PcLGvrJ+3VoF48BOA=C0l0`--Y8O7s$0hoXA=3^B8T*t@itaPziTqp@o9(P}RurfsSYlGj-?`k52D9iZEK3cA)JP6|R zoBd~G*su3DcZ)hV(pzHU4P%m*)*saO3;Z@}6lJ_(^=oiLXYuejqDAW-dSo+O7ooZ` z2E|-PH($YVGvsyXe5ON-(#8}W&0%@YkE2#?W`Bmmp(9m*rgg&-^C;KmVhiJWCoaxS zw%v$=IvoI~vn$aN- zxEvx)c@5t4-?WZSA&rL1Jom!>wH4z}+zkjxg9y(bLjU@M&^kH;p@l4Rk?Y^Jju@1o z#1eb|y6Rt=RY>b70512`g_zKPd%0B5I_kcuHvI3d_Y4`*IvP>u%>H++BQZ!F09YtT z`JWH^pC|hNZs{Jja$MpkK62#DnRl% Date: Fri, 12 May 2017 10:00:12 +0800 Subject: [PATCH 09/13] fix image link --- doc/howto/usage/cluster/cluster_train_cn.md | 2 +- doc/howto/usage/cluster/cluster_train_en.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index 03246d8f37382..fe33a1f2814ea 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -20,7 +20,7 @@ # 概述 本文将介绍如何使用PaddlePaddle在不同的集群框架下完成分布式训练。分布式训练架构如下图所示： - +![cluster train](src/trainer.png) - Data shard（数据分片）: 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用 - Trainer（计算节点）: 每个trainer启动后读取切分好的一部分数据，并开始神经网络的“前馈”和“后馈”计算，并和parameter server通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients）然后下载优化更新后的神经网络参数（parameters）。 diff --git a/doc/howto/usage/cluster/cluster_train_en.md b/doc/howto/usage/cluster/cluster_train_en.md index 5cf2252650d58..b4ac7a9b46e9a 100644 --- a/doc/howto/usage/cluster/cluster_train_en.md +++ b/doc/howto/usage/cluster/cluster_train_en.md @@ -20,7 +20,7 @@ In this article, we'll explain how to do distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the mail architecture of a distributed trainning job: - +![cluster train](src/trainer.png) - Data shard: training data will be split into multiple parts, trainers use some parts of the whole dataset to do the training job. - Trainer: each trainer reads the data shard, and do Neural Network training algorithms like "forward" and "backward". Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training. From d854b1b1391e1692f96bf99fc181ae0a3ccb094e Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Wed, 7 Jun 2017 11:02:57 +0800 Subject: [PATCH 10/13] polish cluster train docs --- doc/howto/usage/cluster/cluster_train_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index f3a962949e94c..07e238c0b4126 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -282,4 +282,4 @@ mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh ## 在Kubernetes集群中提交训练作业 -此部分的使用方法可以参考[Kubernetes单机训练文档](../k8s/k8s_cn.md) +此部分的使用方法可以参考[Kubernetes分布式训练](../k8s/k8s_distributed_cn.md) From ee3e442923be03965057354dd6d5aa458f88f469 Mon Sep 17 00:00:00 2001 From: wuyi05 Date: Fri, 28 Jul 2017 12:12:09 +0800 Subject: [PATCH 11/13] update general distributed training document --- doc/howto/usage/cluster/cluster_train_cn.md | 38 ++++++++--------- doc/howto/usage/cluster/cluster_train_en.md | 45 +++++++++++---------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index 07e238c0b4126..46ea026fcfedc 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -15,7 +15,7 @@ * [检查集群训练结果](#检查集群训练结果) * [检查模型输出](#检查模型输出) * [在OpenMPI集群中提交训练作业](#在openmpi集群中提交训练作业) - * [准备openmpi集群](#准备openmpi集群) + * [准备OpenMPI集群](#准备OpenMPI集群) * [启动集群作业](#启动集群作业-1) * [在Kubernetes集群中提交训练作业](#在kubernetes集群中提交训练作业) @@ -25,16 +25,16 @@ - 数据分片（Data shard): 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用。 -- 计算节点（Trainer）: 每个trainer启动后读取切分好的一部分数据，并开始神经网络的“前馈”和“后馈”计算，并和参数服务器通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients），然后下载优化更新后的神经网络参数（parameters）。 +- 计算节点（Trainer）: 每个trainer启动后读取切分好的一部分数据，开始神经网络的“前馈”和“后馈”计算，并和参数服务器通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients），然后下载优化更新后的神经网络参数（parameters）。 - 参数服务器（Parameter server）:每个参数服务器只保存整个神经网络所有参数的一部分。参数服务器接收从计算节点上传的梯度，并完成参数优化更新，再将更新后的参数下发到每个计算节点。 这样，通过计算节点和参数服务器的分布式协作，可以完成神经网络的SGD方法的训练。PaddlePaddle可以同时支持同步随机梯度下降（SGD）和异步随机梯度下降。 -在使用同步SGD训练神经网络时，PaddlePaddle使用同步屏障（barrier），使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大地提高了计算的并行性：参数服务器之间不相互依赖，并行的接收梯度和更新参数，参数服务器也不会等待计算节点全部都提交梯度之后才开始下一步，计算节点之间也不会相互依赖，并行地执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台参数服务器上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 +在使用同步SGD训练神经网络时，PaddlePaddle使用同步屏障（barrier），使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大地提高了计算的并行性：参数服务器之间不相互依赖，并行地接收梯度和更新参数，参数服务器也不会等待计算节点全部都提交梯度之后才开始下一步，计算节点之间也不会相互依赖，并行地执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台参数服务器上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 # 环境准备 -1. 准备您的计算集群。计算集群通常由一组（几台到几千台规模）的Linux服务器组成。服务器之间可以通过局域网（LAN）联通，每台服务器具有集群中唯一的IP地址（或者可被DNS解析的主机名）。集群中的每台计算机通常被成为一个“节点”。使用不同的集群管理平台时，会要求集群节点是否是同样硬件配置。 +1. 准备您的计算集群。计算集群通常由一组（几台到几千台规模）的Linux服务器组成。服务器之间可以通过局域网（LAN）联通，每台服务器具有集群中唯一的IP地址（或者可被DNS解析的主机名）。集群中的每台计算机通常被成为一个“节点”。 1. 我们需要在集群的所有节点上安装 PaddlePaddle。 如果要启用GPU，还需要在节点上安装对应的GPU驱动以及CUDA。PaddlePaddle的安装可以参考[build_and_install](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install)的多种安装方式。我们推荐使用[Docker](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)安装方式来快速安装PaddlePaddle。 安装完成之后，执行下面的命令可以查看已经安装的版本（docker安装方式可以进入docker容器执行：`docker run -it paddlepaddle/paddle:[tag] /bin/bash`）： @@ -119,7 +119,7 @@ paddle.init( ## 准备数据集 -参考样例数据准备脚本"prepare.py"（位于`doc/howto/usage/cluster/src/word2vec/prepare.py`），准备训练数据和验证数据集，我们使用paddle.dataset.imikolov数据集，并根据分布式训练并发数（trainer节点个数），指定`SPLIT_COUNT`将数据切分成多份。 +参考样例数据准备脚本[prepare.py](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/cluster/src/word2vec/prepare.py)，准备训练数据和验证数据集，我们使用paddle.dataset.imikolov数据集，并根据分布式训练并发数（trainer节点个数），在`prepare.py`开头部分指定`SPLIT_COUNT`将数据切分成多份。 在线上系统中，通常会使用MapReduce任务的输出结果作为训练结果，这样训练文件的个数会比较多，而且个数并不确定。在trainer中可以使用下面取模的方法为每个trainer分配训练数据文件： @@ -169,31 +169,31 @@ test.txt-00002 `-- test.txt-00002 ``` -- `mylib.py`：会被`train.py`调用的一些库函数。 +- `my_lib.py`：会被`train.py`调用的一些用户定义的库函数，比如PIL库等。 - `word_dict.pickle`：在`train.py`中会使用到的字典数据文件。 -- `train.py`：训练程序，代码参考"api_train_v2_cluster.py"（位于`doc/howto/usage/cluster/src/word2vec/api_train_v2_cluster.py`）。 - - ***注意：*** 对于本样例代码，在使用不同的分布式计算平台时，您可能需要修改`train.py`开头的部分（如下），以便获得训练数据的位置和获取环境变量配置： +- `train.py`：训练程序，代码参考[api_train_v2_cluster.py](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/cluster/src/word2vec/prepare.py)。***注意：*** 对于本样例代码，在使用不同的分布式计算平台时，您可能需要修改`train.py`开头的部分（如下），以便获得训练数据的位置和获取环境变量配置： + ```python cluster_train_file = "./train_data_dir/train/train.txt" cluster_test_file = "./test_data_dir/test/test.txt" node_id = os.getenv("OMPI_COMM_WORLD_RANK") if not node_id: raise EnvironmentError("must provied OMPI_COMM_WORLD_RANK") - ``` + - `train_data_dir`：包含训练数据的目录，可以是从分布式存储挂载过来的，也可以是在任务启动前下载到本地的。 - `test_data_dir`：包含测试数据集的目录。 # 使用分布式计算平台或工具 PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务，包括： -- [Kubernetes](http://kubernetes.io) -- [OpenMPI](https://www.open-mpi.org) +- [Kubernetes](http://kubernetes.io) Google开源的容器集群的调度框架，支持大规模集群生产环境的完整集群方案。 +- [OpenMPI](https://www.open-mpi.org) 成熟的高性能并行计算框架。 - [Fabric](http://www.fabfile.org) 集群管理工具。可以使用`Fabric`编写集群任务提交和管理脚本。 对于不同的集群平台，会分别介绍集群作业的启动和停止方法。这些例子都可以在[cluster_train_v2](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2)找到。 -在使用分布式计算平台进行训练时，任务被平台调度在集群中时会使用计算平台提供的API或环境变量获取启动的参数。 +在使用分布式计算平台进行训练时，任务被调度在集群中时，分布式计算平台通常会通过API或者环境变量提供任务运行需要的参数，比如节点的ID、IP和任务节点个数等。 ## 使用Fabric启动集群作业 @@ -202,14 +202,14 @@ PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务 ### 启动集群作业 -`paddle.py` 提供了自动化脚本来启动不同节点中的所有 PaddlePaddle 集群进程。默认情况下，所有命令行选项可以设置为```paddle.py``` 命令选项并且 `paddle.py` 将透明、自动地将这些选项应用到 PaddlePaddle 底层进程。 +`paddle.py` 提供了自动化脚本来启动不同节点中的所有 PaddlePaddle 集群进程。默认情况下，所有命令行选项可以设置为 `paddle.py` 命令选项并且 `paddle.py` 将透明、自动地将这些选项应用到 PaddlePaddle 底层进程。 `paddle.py` 为方便作业启动提供了两个独特的命令选项。 -`job_dispatch_package` 设为本地 `workspace` 目录，它将被分发到 conf.py 中设置的所有节点。 它有助于帮助频繁修改和访问工作区文件的用户减少负担，否则频繁的多节点工作空间部署可能会很麻烦。 -`job_workspace` 设为已部署的工作空间目录，`paddle.py` 将跳过分发阶段直接启动所有节点的集群作业。它可以帮助减少分发延迟。 +- `job_dispatch_package` 设为本地 `workspace` 目录，它将被分发到 `conf.py` 中设置的所有节点。它有助于帮助频繁修改和访问工作区文件的用户减少负担，否则频繁的多节点工作空间部署可能会很麻烦。 +- `job_workspace` 设为已部署的工作空间目录，`paddle.py` 将跳过分发阶段直接启动所有节点的集群作业。它可以帮助减少分发延迟。 -`cluster_train/run.sh` 提供了命令样例来运行 `doc/howto/usage/cluster/src/word2vec` 集群任务，只需用你定义的目录修改 `job_dispatch_package` 和 `job_workspace`，然后： +`cluster_train/run.sh` 提供了命令样例来运行 `doc/howto/usage/cluster/src/word2vec` 集群任务，只需用您定义的目录修改 `job_dispatch_package` 和 `job_workspace`，然后： ``` sh run.sh ``` @@ -240,9 +240,9 @@ sh run.sh ## 在OpenMPI集群中提交训练作业 -### 准备openmpi集群 +### 准备OpenMPI集群 -执行下面的命令以启动3个节点的openmpi集群和一个"head"节点： +执行下面的命令以启动3个节点的OpenMPI集群和一个"head"节点： ```bash paddle/scripts/cluster_train_v2/openmpi/docker_cluster @@ -282,4 +282,4 @@ mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh ## 在Kubernetes集群中提交训练作业 -此部分的使用方法可以参考[Kubernetes分布式训练](../k8s/k8s_distributed_cn.md) +此部分的使用方法可以参考[Kubernetes分布式训练](../k8s/k8s_distributed_cn.md)。 diff --git a/doc/howto/usage/cluster/cluster_train_en.md b/doc/howto/usage/cluster/cluster_train_en.md index 08cea4dde66e1..d2c8c9ea18ceb 100644 --- a/doc/howto/usage/cluster/cluster_train_en.md +++ b/doc/howto/usage/cluster/cluster_train_en.md @@ -21,23 +21,23 @@ # Introduction -In this article, we'll explain how to do distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the mail architecture of a distributed trainning job: +In this article, we'll explain how to run distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the main architecture of a distributed trainning job: -- Data shard: training data will be split into multiple parts, trainers use some parts of the whole dataset to do the training job. -- Trainer: each trainer reads the data shard, and do Neural Network training algorithms like "forward" and "backward". Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training. -- Parameter server: every parameter server stores part of the whole Neural Network model data. They will do optimization calculations when gradients are uploaded from trainers, and then send updated parameters to trainers. +- Data shard: training data will be split into multiple partitions, trainers use the partitions of the whole dataset to do the training job. +- Trainer: each trainer reads the data shard, and train the neural network. Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training. +- Parameter server: every parameter server stores part of the whole neural network model data. They will do optimization calculations when gradients are uploaded from trainers, and then send updated parameters to trainers. -Trainers and Parameter servers coordinate form the distributed SGD(stochastic gradient descent) training algorithm. Meanwhile, PaddlePaddle can support both synchronize SGD and asynchronous SGD. +PaddlePaddle can support both synchronize stochastic gradient descent (SGD) and asynchronous SGD. -When training with synchronize SGD, PaddlePaddle uses an internal "synchronize barrier" which makes gradients update and parameter download in strict order. On the other hand, asynchronous SGD won't wait for all trainers to finish upload at a single step, this will increase the parallelism of distributed training: parameter servers do not depend on each other, they'll do parameter optimization concurrently. Parameter servers will not wait for trainers, so trainers will also do their work concurrently. But asynchronous SGD will introduce more randomness and noises. +When training with synchronize SGD, PaddlePaddle uses an internal "synchronize barrier" which makes gradients update and parameter download in strict order. On the other hand, asynchronous SGD won't wait for all trainers to finish upload at a single step, this will increase the parallelism of distributed training: parameter servers do not depend on each other, they'll do parameter optimization concurrently. Parameter servers will not wait for trainers, so trainers will also do their work concurrently. But asynchronous SGD will introduce more randomness and noises in the gradient. # Preparations -1. Ready your computer cluster. They are normally a bunch of Linux servers connected by LAN. Each server will be assigned a unique IP address. The computer in the cluster can be called "nodes". Different hardware setup may cause some issue when using different cluster management platforms. -2. Install PaddlePaddle on every node. If you are going to take advantage of GPU cards, you'll also need to install proper driver and CUDA libraries. To install PaddlePaddle try to read [this](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install) document. We strongly recommend using [Docker installation](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst). +1. Prepare your computer cluster. It's normally a bunch of Linux servers connected by LAN. Each server will be assigned a unique IP address. The computers in the cluster can be called "nodes". +2. Install PaddlePaddle on every node. If you are going to take advantage of GPU cards, you'll also need to install proper driver and CUDA libraries. To install PaddlePaddle please read [this build and install](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install) document. We strongly recommend using [Docker installation](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst). -After installation, you can check the version by typing the below command(run into a docker container if using docker: `docker run -it paddlepaddle/paddle:0.10.0rc3 /bin/bash`): +After installation, you can check the version by typing the below command (run a docker container if using docker: `docker run -it paddlepaddle/paddle:[tag] /bin/bash`): ```bash $ paddle version @@ -81,7 +81,7 @@ Type the command below to start the trainer(name the file whatever you want, lik $ python train.py ``` -Trainers network need to be connected with parameter servers to finish the job. Trainers need to know port and IPs to locate parameter servers. You can pass arguments to trainers through [environment variables](https://en.wikipedia.org/wiki/Environment_variable) or pass to `paddle.init()` function. Arguments passed to the `paddle.init()` function will overwrite environment variables. +Trainers' network need to be connected with parameter servers' network to finish the job. Trainers need to know port and IPs to locate parameter servers. You can pass arguments to trainers through [environment variables](https://en.wikipedia.org/wiki/Environment_variable) or pass to `paddle.init()` function. Arguments passed to the `paddle.init()` function will overwrite environment variables. Use environment viriables: @@ -124,7 +124,7 @@ paddle.init( ## Prepare Training Dataset -Here's some example code “prepare.py”(located in `doc/howto/usage/cluster/src/word2vec/prepare.py`), it will download public `imikolov` dataset and split it into multiple files according to job parallelism(trainers count). Modify `SPLIT_COUNT` to change the count of output files. +Here's some example code [prepare.py](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/cluster/src/word2vec/prepare.py), it will download public `imikolov` dataset and split it into multiple files according to job parallelism(trainers count). Modify `SPLIT_COUNT` at the begining of `prepare.py` to change the count of output files. In the real world, we often use `MapReduce` job's output as training data, so there will be lots of files. You can use `mod` to assign training file to trainers: @@ -151,7 +151,7 @@ test.txt-00001 test.txt-00002 ``` -When job started, every trainer needs to get it's own part of data. In some distributed systems they will provide a storage service, so the date under that path can be accessed by all the trainer nodes. Without storage service, you must copy the training data to each trainer node. +When job started, every trainer needs to get it's own part of data. In some distributed systems a storage service will be provided, so the date under that path can be accessed by all the trainer nodes. Without the storage service, you must copy the training data to each trainer node. Different training jobs may have different data format and `reader()` function, developers may need to write different data prepare scripts and `reader()` functions for their job. @@ -176,10 +176,10 @@ Your workspace may looks like: `-- test.txt-00002 ``` -- `mylib.py`: some library functions. This is optional. +- `my_lib.py`: user defined libraries, like PIL libs. This is optional. - `word_dict.pickle`: dict file for training word embeding. -- `train.py`: training program. Sample code: "api_train_v2_cluster.py"(located in `doc/howto/usage/cluster/src/word2vec/api_train_v2_cluster.py`). - - ***NOTE:*** You may need to modify the head part of `train.py` when using different cluster platform to retrive configuration environment variables: +- `train.py`: training program. Sample code: [api_train_v2_cluster.py](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/cluster/src/word2vec/prepare.py). ***NOTE:*** You may need to modify the head part of `train.py` when using different cluster platform to retrive configuration environment variables: + ```python cluster_train_file = "./train_data_dir/train/train.txt" cluster_test_file = "./test_data_dir/test/test.txt" @@ -187,19 +187,20 @@ Your workspace may looks like: if not node_id: raise EnvironmentError("must provied OMPI_COMM_WORLD_RANK") ``` + - `train_data_dir`: containing training data. Mount from storage service or copy trainning data to here. - `test_data_dir`: containing testing data. # Use cluster platforms or cluster management tools PaddlePaddle supports running jobs on several platforms including: -- [Kubernetes](http://kubernetes.io) -- [OpenMPI](https://www.open-mpi.org) +- [Kubernetes](http://kubernetes.io) open-source system for automating deployment, scaling, and management of containerized applications from Google. +- [OpenMPI](https://www.open-mpi.org) Mature high performance parallel computing framework. - [Fabric](http://www.fabfile.org) A cluster management tool. Write scripts to submit jobs or manage the cluster. -We'll introduce cluster job management on these platforms. The examples can be found under [here](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2). +We'll introduce cluster job management on these platforms. The examples can be found under [cluster_train_v2](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/scripts/cluster_train_v2). -These cluster platforms provide API or runtime environment variables for training jobs. +These cluster platforms provide API or environment variables for training processes, when the job is dispatched to different nodes. Like node ID, IP or total number of nodes etc. ## Cluster Training Using Fabric @@ -208,12 +209,12 @@ These cluster platforms provide API or runtime environment variables for trainin Run `kubectl -f ssh_servers.yaml` under the directory: `paddle/scripts/cluster_train_v2/fabric/docker_cluster` will launch a demo cluster. Run `kubectl get po -o wide` to get IP addresses of these nodes. ### Launching Cluster Job -`paddle.py` provides automatical scripts to start all PaddlePaddle cluster processes in different nodes. By default, all command line options can set as `paddle.py` command options and `paddle.py` will transparently and automatically set these options to PaddlePaddle lower level processes. +`paddle.py` provides automatical scripts to start all PaddlePaddle cluster processes in different nodes. By default, all command line options can be set as `paddle.py` command options and `paddle.py` will transparently and automatically set these options to PaddlePaddle lower level processes. `paddle.py`provides two distinguished command option for easy job launching. -`job_dispatch_package` set it with local `workspace`directory, it will be dispatched to all nodes set in conf.py. It could be helpful for frequent hacking workspace files, otherwise, frequent multi-nodes workspace deployment could make your crazy. -`job_workspace` set it with already deployed workspace directory, `paddle.py` will skip dispatch stage to directly launch cluster job with all nodes. It could help to reduce heavy +- `job_dispatch_package` set it with local `workspace` directory, it will be dispatched to all nodes which is set in `conf.py`. It could be helpful for frequently manipulating workspace files. otherwise, frequent multi-nodes workspace deployment is very annoying. +- `job_workspace` set it with already deployed workspace directory, `paddle.py` will skip dispatch stage to directly launch cluster job with all nodes. It could help to reduce heavy dispatch latency. `cluster_train/run.sh` provides command line sample to run `demo/recommendation` cluster job, just modify `job_dispatch_package` and `job_workspace` with your defined directory, then: From 3cee5481a5a93ba24c13b54dd3b3e0ad27b0ea8a Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 19 Oct 2017 20:42:17 +0800 Subject: [PATCH 12/13] fix sphinx compile error --- doc/howto/usage/cluster/cluster_train_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index 46ea026fcfedc..de90f7dcac19d 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -282,4 +282,4 @@ mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh ## 在Kubernetes集群中提交训练作业 -此部分的使用方法可以参考[Kubernetes分布式训练](../k8s/k8s_distributed_cn.md)。 +此部分的使用方法可以参考[here](../k8s/k8s_distributed_cn.md)。 From accede951d9ebd9da4cd15df19ec10e844d0b4e4 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Thu, 19 Oct 2017 21:07:56 +0800 Subject: [PATCH 13/13] fix doc image error --- doc/howto/usage/cluster/cluster_train_cn.md | 2 +- doc/howto/usage/cluster/cluster_train_en.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index de90f7dcac19d..93c5544bcfa91 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -22,7 +22,7 @@ # 概述 本文将介绍如何使用PaddlePaddle在不同的集群框架下完成分布式训练。分布式训练架构如下图所示： - + - 数据分片（Data shard): 用于训练神经网络的数据，被切分成多个部分，每个部分分别给每个trainer使用。 - 计算节点（Trainer）: 每个trainer启动后读取切分好的一部分数据，开始神经网络的“前馈”和“后馈”计算，并和参数服务器通信。在完成一定量数据的训练后，上传计算得出的梯度（gradients），然后下载优化更新后的神经网络参数（parameters）。 diff --git a/doc/howto/usage/cluster/cluster_train_en.md b/doc/howto/usage/cluster/cluster_train_en.md index d2c8c9ea18ceb..1e8b4d54b9ffa 100644 --- a/doc/howto/usage/cluster/cluster_train_en.md +++ b/doc/howto/usage/cluster/cluster_train_en.md @@ -23,7 +23,7 @@ In this article, we'll explain how to run distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the main architecture of a distributed trainning job: - + - Data shard: training data will be split into multiple partitions, trainers use the partitions of the whole dataset to do the training job. - Trainer: each trainer reads the data shard, and train the neural network. Then the trainer will upload calculated "gradients" to parameter servers, and wait for parameters to be optimized on the parameter server side. When that finishes, the trainer download optimized parameters and continues its training.