From a2a5ffae60043b9e7b446cb746b1dfba900da27c Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Tue, 3 Apr 2018 21:42:24 -0700 Subject: [PATCH 01/15] init checkin for aws benchmarking tool --- .../paddle_banchmarking_aws.py | 393 ++++++++++++++++++ tools/aws_benchmarking/pserver.sh.template | 1 + tools/aws_benchmarking/requirements.txt | 4 + tools/aws_benchmarking/trainer.sh.template | 1 + 4 files changed, 399 insertions(+) create mode 100644 tools/aws_benchmarking/paddle_banchmarking_aws.py create mode 100644 tools/aws_benchmarking/pserver.sh.template create mode 100644 tools/aws_benchmarking/requirements.txt create mode 100644 tools/aws_benchmarking/trainer.sh.template diff --git a/tools/aws_benchmarking/paddle_banchmarking_aws.py b/tools/aws_benchmarking/paddle_banchmarking_aws.py new file mode 100644 index 0000000000000..5e36f827a877d --- /dev/null +++ b/tools/aws_benchmarking/paddle_banchmarking_aws.py @@ -0,0 +1,393 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import json +import math +import time +import base64 + +import netaddr +import boto3 +import namesgenerator +import paramiko + +# You must have aws_access_key_id, aws_secret_access_key, region set in +# ~/.aws/credentials and ~/.aws/config + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + '--vpc_id', + type=str, + default="", + help="The VPC in which you wish to run test") +parser.add_argument( + '--subnet_id', + type=str, + default="", + help="The Subnet_id in which you wish to run test") +parser.add_argument( + '--security_group_id', + type=str, + default="", + required=True, + help="required, the security group id associated with your VPC") +parser.add_argument( + '--pserver_instance_type', + type=str, + default="p2.xlarge", + help="your pserver instance type") +parser.add_argument( + '--trainer_instance_type', + type=str, + default="p2.xlarge", + help="your trainer instance type") +parser.add_argument( + '--key_name', + type=str, + default="", + required=True, + help="required, key pair name") +parser.add_argument( + '--task_name', + type=str, + default="", + help="the name you want to identify your job") +parser.add_argument( + '--pserver_image_id', + type=str, + default="ami-1ae93962", + help="ami id for system image, default one has nvidia-docker ready") +parser.add_argument( + '--trainer_image_id', + type=str, + default="ami-1ae93962", + help="ami id for system image, default one has nvidia-docker ready") + +parser.add_argument( + '--trainer_count', type=int, default=1, help="Trainer count") + +parser.add_argument( + '--pserver_count', type=int, default=1, help="Pserver count") + +parser.add_argument( + '--pserver_bash_file', + type=str, + required=False, + default=os.path.join(os.path.dirname(__file__), "pserver.sh.template"), + help="pserver bash file path") + +parser.add_argument( + '--trainer_bash_file', + type=str, + required=False, + default=os.path.join(os.path.dirname(__file__), "trainer.sh.template"), + help="trainer bash file path") + +parser.add_argument('--pem_path', type=str, help="private key file") + +parser.add_argument( + '--pserver_port', type=str, default="5436", help="pserver port") + +parser.add_argument( + '--docker_image', type=str, default="busybox", help="training docker image") + +args = parser.parse_args() + +ec2client = boto3.client('ec2') + + +def create_subnet(): + # if no vpc id provided, list vpcs + if not args.vpc_id: + print("no vpc provided, trying to find the default one") + vpcs_desc = ec2client.describe_vpcs( + Filters=[{ + "Name": "isDefault", + "Values": ["true", ] + }], ) + if len(vpcs_desc["Vpcs"]) == 0: + raise ValueError('No default VPC') + args.vpc_id = vpcs_desc["Vpcs"][0]["VpcId"] + vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] + + print("default vpc fount with id %s and CidrBlock %s" % + (args.vpc_id, vpc_cidrBlock)) + + if not vpc_cidrBlock: + print("trying to find cidrblock for vpc") + vpcs_desc = ec2client.describe_vpcs( + Filters=[{ + "Name": "vpc-id", + "Values": [args.vpc_id, ], + }], ) + if len(vpcs_desc["Vpcs"]) == 0: + raise ValueError('No VPC found') + vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] + print("cidrblock for vpc is %s" % vpc_cidrBlock) + + # list subnets in vpc in order to create a new one + + print("trying to find ip blocks for new subnet") + subnets_desc = ec2client.describe_subnets( + Filters=[{ + "Name": "vpc-id", + "Values": [args.vpc_id, ], + }], ) + + ips_taken = [] + for subnet_dec in subnets_desc["Subnets"]: + ips_taken.append(subnet_dec["CidrBlock"]) + + ip_blocks_avaliable = netaddr.IPSet( + [vpc_cidrBlock]) ^ netaddr.IPSet(ips_taken) + # adding 10 addresses as buffer + cidr_prefix = 32 - math.ceil( + math.log(args.pserver_count + args.trainer_count + 10, 2)) + if cidr_prefix <= 16: + raise ValueError('Too many nodes to fit in current VPC') + + for ipnetwork in ip_blocks_avaliable.iter_cidrs(): + try: + subnet_cidr = ipnetwork.subnet(int(cidr_prefix)).next() + print("subnet ip block found %s" % (subnet_cidr)) + break + except Exception: + pass + + if not subnet_cidr: + raise ValueError( + 'No avaliable subnet to fit required nodes in current VPC') + + print("trying to create subnet") + subnet_desc = ec2client.create_subnet( + CidrBlock=str(subnet_cidr), VpcId=args.vpc_id) + + subnet_id = subnet_desc["Subnet"]["SubnetId"] + + subnet_waiter = ec2client.get_waiter('subnet_available') + # sleep for 1s before checking its state + time.sleep(1) + subnet_waiter.wait(SubnetIds=[subnet_id, ]) + + print("subnet created") + + print("adding tags to newly created subnet") + ec2client.create_tags( + Resources=[subnet_id, ], + Tags=[{ + "Key": "Task_name", + 'Value': args.task_name + }]) + return subnet_id + + +def generate_task_name(): + return namesgenerator.get_random_name() + + +def script_to_str(file_path): + if not file_path: + return "echo $PSERVER_HOSTS" + file = open(file_path, 'r') + text = file.read().strip() + file.close() + return text + + +def run_instances(image_id, instance_type, count, role, cmd=""): + if cmd: + cmd = base64.b64encode(cmd) + response = ec2client.run_instances( + ImageId=image_id, + InstanceType=instance_type, + MaxCount=count, + MinCount=count, + UserData=cmd, + DryRun=False, + InstanceInitiatedShutdownBehavior="stop", + KeyName=args.key_name, + NetworkInterfaces=[{ + 'DeviceIndex': 0, + 'SubnetId': args.subnet_id, + "AssociatePublicIpAddress": True, + 'Groups': args.security_group_ids + }], + TagSpecifications=[{ + 'ResourceType': "instance", + 'Tags': [{ + "Key": 'Task_name', + "Value": args.task_name + }, { + "Key": 'Role', + "Value": role + }] + }]) + + instance_ids = [] + for instance in response["Instances"]: + instance_ids.append(instance["InstanceId"]) + + if len(instance_ids) > 0: + print(str(len(instance_ids)) + " instance(s) created") + else: + print("no instance created") + #create waiter to make sure it's running + + print("waiting for instance to become accessible") + waiter = ec2client.get_waiter('instance_status_ok') + waiter.wait( + Filters=[{ + "Name": "instance-status.status", + "Values": ["ok"] + }, { + "Name": "instance-status.reachability", + "Values": ["passed"] + }, { + "Name": "instance-state-name", + "Values": ["running"] + }], + InstanceIds=instance_ids) + + instances_response = ec2client.describe_instances(InstanceIds=instance_ids) + + return instances_response["Reservations"][0]["Instances"] + + +def create_pservers(): + return run_instances( + image_id=args.pserver_image_id, + instance_type=args.pserver_instance_type, + count=args.pserver_count, + role="PSERVER", ) + + +def create_trainers(kickoff_cmd, pserver_endpoints_str): + responses = [] + for i in xrange(args.trainer_count): + cmd = kickoff_cmd.format( + PSERVER_HOSTS=pserver_endpoints_str, + DOCKER_IMAGE=args.docker_image, + TRAINER_INDEX=str(i)) + print(cmd) + responses.append( + run_instances( + image_id=args.trainer_image_id, + instance_type=args.trainer_instance_type, + count=1, + role="TRAINER", + cmd=cmd, )[0]) + return responses + + +def cleanup(task_name): + #shutdown all ec2 instances + instances = ec2client.describe_instances(Filters=[{ + "Name": "tag", + "Value": "Task_name=" + task_name + }]) + + instance_ids = [] + for instance in instances["Reservations"][0]["Instances"]: + instance_ids.append(instance["InstanceId"]) + + ec2client.stop_instances(InstanceIds=instance_ids) + + instance_stop_waiter = ec2client.get_waiter('instance_stopped') + instance_stop_waiter.wait(InstanceIds=instance_ids) + + #delete the subnet created + + subnet = ec2client.describe_subnets(Filters=[{ + "Name": "tag", + "Value": "Task_name=" + task_name + }]) + + ec2client.delete_subnet(SubnetId=subnet["Subnets"][0]["SubnetId"]) + + # no subnet delete waiter, just leave it. + + return + + +def main(): + if not args.task_name: + args.task_name = generate_task_name() + print("task name generated", args.task_name) + + if not args.subnet_id: + print("creating subnet for this task") + args.subnet_id = create_subnet() + print("subnet %s created" % (args.subnet_id)) + + if not args.pem_path: + args.pem_path = os.path.expanduser("~") + "/" + args.key_name + ".pem" + if args.security_group_id: + args.security_group_ids = (args.security_group_id, ) + + print("creating pservers") + pserver_create_response = create_pservers() + print("pserver created, collecting pserver ips") + + pserver_endpoints = [] + for pserver in pserver_create_response: + pserver_endpoints.append(pserver["NetworkInterfaces"][0][ + "PrivateIpAddress"] + ":" + args.pserver_port) + + pserver_endpoints_str = ",".join(pserver_endpoints) + + # ssh to pservers to start training + ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + print("kicking off pserver training process") + for pserver in pserver_create_response: + try: + ssh_client.connect( + hostname=pserver["PublicIpAddress"], + username="ubuntu", + pkey=ssh_key) + cmd = (script_to_str(args.pserver_bash_file)).format( + PSERVER_HOSTS=pserver_endpoints_str, + DOCKER_IMAGE=args.docker_image) + print(cmd) + stdin, stdout, stderr = ssh_client.exec_command(command=cmd) + if stderr.read(): + raise Exception( + "Error while kicking off pserver training process") + #print(stdout.read()) + except Exception, e: + print e + cleanup(args.task_name) + finally: + ssh_client.close() + + print("creating trainers and kicking off trainer training process") + create_trainers( + kickoff_cmd=script_to_str(args.trainer_bash_file), + pserver_endpoints_str=pserver_endpoints_str) + + +def print_arguments(): + print('----------- Configuration Arguments -----------') + for arg, value in sorted(vars(args).iteritems()): + print('%s: %s' % (arg, value)) + print('------------------------------------------------') + + +if __name__ == "__main__": + print_arguments() + main() diff --git a/tools/aws_benchmarking/pserver.sh.template b/tools/aws_benchmarking/pserver.sh.template new file mode 100644 index 0000000000000..ddfe2f9d3167f --- /dev/null +++ b/tools/aws_benchmarking/pserver.sh.template @@ -0,0 +1 @@ +nvidia-docker run -i -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file diff --git a/tools/aws_benchmarking/requirements.txt b/tools/aws_benchmarking/requirements.txt new file mode 100644 index 0000000000000..5c523854f28b0 --- /dev/null +++ b/tools/aws_benchmarking/requirements.txt @@ -0,0 +1,4 @@ +netaddr==0.7.19 +boto3==1.6.21 +namesgenerator==0.3 +paramiko==2.4.1 diff --git a/tools/aws_benchmarking/trainer.sh.template b/tools/aws_benchmarking/trainer.sh.template new file mode 100644 index 0000000000000..70aceb8814420 --- /dev/null +++ b/tools/aws_benchmarking/trainer.sh.template @@ -0,0 +1 @@ +nvidia-docker run -i -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file From cd31c12af0e639ffdb6e73d6a8bf6cbc136e585f Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Tue, 3 Apr 2018 21:46:41 -0700 Subject: [PATCH 02/15] test pre-commit --- tools/aws_benchmarking/paddle_banchmarking_aws.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/aws_benchmarking/paddle_banchmarking_aws.py b/tools/aws_benchmarking/paddle_banchmarking_aws.py index 5e36f827a877d..f6512d923ce6f 100644 --- a/tools/aws_benchmarking/paddle_banchmarking_aws.py +++ b/tools/aws_benchmarking/paddle_banchmarking_aws.py @@ -318,7 +318,6 @@ def cleanup(task_name): ec2client.delete_subnet(SubnetId=subnet["Subnets"][0]["SubnetId"]) # no subnet delete waiter, just leave it. - return From ad4bef711d159b922696d510d999a25bda50c5d4 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Tue, 3 Apr 2018 22:20:05 -0700 Subject: [PATCH 03/15] move required arguments together --- .../paddle_banchmarking_aws.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tools/aws_benchmarking/paddle_banchmarking_aws.py b/tools/aws_benchmarking/paddle_banchmarking_aws.py index f6512d923ce6f..c63e4b0742e4d 100644 --- a/tools/aws_benchmarking/paddle_banchmarking_aws.py +++ b/tools/aws_benchmarking/paddle_banchmarking_aws.py @@ -28,6 +28,19 @@ # ~/.aws/credentials and ~/.aws/config parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + '--key_name', + type=str, + default="", + required=True, + help="required, key pair name") +parser.add_argument( + '--security_group_id', + type=str, + default="", + required=True, + help="required, the security group id associated with your VPC") + parser.add_argument( '--vpc_id', type=str, @@ -38,12 +51,7 @@ type=str, default="", help="The Subnet_id in which you wish to run test") -parser.add_argument( - '--security_group_id', - type=str, - default="", - required=True, - help="required, the security group id associated with your VPC") + parser.add_argument( '--pserver_instance_type', type=str, @@ -54,12 +62,7 @@ type=str, default="p2.xlarge", help="your trainer instance type") -parser.add_argument( - '--key_name', - type=str, - default="", - required=True, - help="required, key pair name") + parser.add_argument( '--task_name', type=str, @@ -316,7 +319,6 @@ def cleanup(task_name): }]) ec2client.delete_subnet(SubnetId=subnet["Subnets"][0]["SubnetId"]) - # no subnet delete waiter, just leave it. return From 2d324b62a40b2ebce7325d4a7822225bad7cd505 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 5 Apr 2018 19:05:27 -0700 Subject: [PATCH 04/15] GA for creating and cleaning instances --- .../paddle_banchmarking_aws.py | 189 +++++++++++------- tools/aws_benchmarking/pserver.sh.template | 3 +- tools/aws_benchmarking/trainer.sh.template | 3 +- 3 files changed, 116 insertions(+), 79 deletions(-) diff --git a/tools/aws_benchmarking/paddle_banchmarking_aws.py b/tools/aws_benchmarking/paddle_banchmarking_aws.py index c63e4b0742e4d..68285406c461f 100644 --- a/tools/aws_benchmarking/paddle_banchmarking_aws.py +++ b/tools/aws_benchmarking/paddle_banchmarking_aws.py @@ -17,7 +17,7 @@ import json import math import time -import base64 +import threading import netaddr import boto3 @@ -29,16 +29,11 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - '--key_name', - type=str, - default="", - required=True, - help="required, key pair name") + '--key_name', type=str, default="", help="required, key pair name") parser.add_argument( '--security_group_id', type=str, default="", - required=True, help="required, the security group id associated with your VPC") parser.add_argument( @@ -55,13 +50,13 @@ parser.add_argument( '--pserver_instance_type', type=str, - default="p2.xlarge", - help="your pserver instance type") + default="p2.8xlarge", + help="your pserver instance type, p2.8xlarge by default") parser.add_argument( '--trainer_instance_type', type=str, - default="p2.xlarge", - help="your trainer instance type") + default="p2.8xlarge", + help="your trainer instance type, p2.8xlarge by default") parser.add_argument( '--task_name', @@ -71,13 +66,21 @@ parser.add_argument( '--pserver_image_id', type=str, - default="ami-1ae93962", - help="ami id for system image, default one has nvidia-docker ready") + default="ami-da2c1cbf", + help="ami id for system image, default one has nvidia-docker ready, use ami-1ae93962 for us-east-2" +) parser.add_argument( '--trainer_image_id', type=str, - default="ami-1ae93962", - help="ami id for system image, default one has nvidia-docker ready") + default="ami-da2c1cbf", + help="ami id for system image, default one has nvidia-docker ready, use ami-1ae93962 for us-west-2" +) + +parser.add_argument( + '--availability_zone', + type=str, + default="us-east-2a", + help="aws zone id to place ec2 instances") parser.add_argument( '--trainer_count', type=int, default=1, help="Trainer count") @@ -88,17 +91,18 @@ parser.add_argument( '--pserver_bash_file', type=str, - required=False, default=os.path.join(os.path.dirname(__file__), "pserver.sh.template"), help="pserver bash file path") parser.add_argument( '--trainer_bash_file', type=str, - required=False, default=os.path.join(os.path.dirname(__file__), "trainer.sh.template"), help="trainer bash file path") +parser.add_argument( + '--action', type=str, default="create", help="create|cleanup|status") + parser.add_argument('--pem_path', type=str, help="private key file") parser.add_argument( @@ -176,7 +180,9 @@ def create_subnet(): print("trying to create subnet") subnet_desc = ec2client.create_subnet( - CidrBlock=str(subnet_cidr), VpcId=args.vpc_id) + CidrBlock=str(subnet_cidr), + VpcId=args.vpc_id, + AvailabilityZone=args.availability_zone) subnet_id = subnet_desc["Subnet"]["SubnetId"] @@ -211,8 +217,6 @@ def script_to_str(file_path): def run_instances(image_id, instance_type, count, role, cmd=""): - if cmd: - cmd = base64.b64encode(cmd) response = ec2client.run_instances( ImageId=image_id, InstanceType=instance_type, @@ -222,6 +226,7 @@ def run_instances(image_id, instance_type, count, role, cmd=""): DryRun=False, InstanceInitiatedShutdownBehavior="stop", KeyName=args.key_name, + Placement={'AvailabilityZone': args.availability_zone}, NetworkInterfaces=[{ 'DeviceIndex': 0, 'SubnetId': args.subnet_id, @@ -270,59 +275,94 @@ def run_instances(image_id, instance_type, count, role, cmd=""): def create_pservers(): - return run_instances( - image_id=args.pserver_image_id, - instance_type=args.pserver_instance_type, - count=args.pserver_count, - role="PSERVER", ) + try: + return run_instances( + image_id=args.pserver_image_id, + instance_type=args.pserver_instance_type, + count=args.pserver_count, + role="PSERVER", ) + except Exception, e: + print e + cleanup(args.task_name) def create_trainers(kickoff_cmd, pserver_endpoints_str): - responses = [] - for i in xrange(args.trainer_count): - cmd = kickoff_cmd.format( - PSERVER_HOSTS=pserver_endpoints_str, - DOCKER_IMAGE=args.docker_image, - TRAINER_INDEX=str(i)) - print(cmd) - responses.append( - run_instances( - image_id=args.trainer_image_id, - instance_type=args.trainer_instance_type, - count=1, - role="TRAINER", - cmd=cmd, )[0]) - return responses + try: + responses = [] + for i in xrange(args.trainer_count): + cmd = kickoff_cmd.format( + PSERVER_HOSTS=pserver_endpoints_str, + DOCKER_IMAGE=args.docker_image, + TRAINER_INDEX=str(i)) + print(cmd) + responses.append( + run_instances( + image_id=args.trainer_image_id, + instance_type=args.trainer_instance_type, + count=1, + role="TRAINER", + cmd=cmd, )[0]) + return responses + except Exception, e: + print e + cleanup(args.task_name) def cleanup(task_name): #shutdown all ec2 instances - instances = ec2client.describe_instances(Filters=[{ - "Name": "tag", - "Value": "Task_name=" + task_name + instances_response = ec2client.describe_instances(Filters=[{ + "Name": "tag:Task_name", + "Values": [task_name] }]) instance_ids = [] - for instance in instances["Reservations"][0]["Instances"]: - instance_ids.append(instance["InstanceId"]) + if len(instances_response["Reservations"]) > 0: + for reservation in instances_response["Reservations"]: + for instance in reservation["Instances"]: + instance_ids.append(instance["InstanceId"]) - ec2client.stop_instances(InstanceIds=instance_ids) + ec2client.terminate_instances(InstanceIds=instance_ids) - instance_stop_waiter = ec2client.get_waiter('instance_stopped') - instance_stop_waiter.wait(InstanceIds=instance_ids) + instance_termination_waiter = ec2client.get_waiter( + 'instance_terminated') + instance_termination_waiter.wait(InstanceIds=instance_ids) - #delete the subnet created +#delete the subnet created subnet = ec2client.describe_subnets(Filters=[{ - "Name": "tag", - "Value": "Task_name=" + task_name + "Name": "tag:Task_name", + "Values": [task_name] }]) - ec2client.delete_subnet(SubnetId=subnet["Subnets"][0]["SubnetId"]) + if len(subnet["Subnets"]) > 0: + ec2client.delete_subnet(SubnetId=subnet["Subnets"][0]["SubnetId"]) # no subnet delete waiter, just leave it. return +def kickoff_pserver(host, pserver_endpoints_str): + try: + ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(hostname=host, username="ubuntu", pkey=ssh_key) + cmd = (script_to_str(args.pserver_bash_file)).format( + PSERVER_HOSTS=pserver_endpoints_str, + DOCKER_IMAGE=args.docker_image, + PSERVER_PORT=args.pserver_port) + print(cmd) + stdin, stdout, stderr = ssh_client.exec_command(command=cmd) + return_code = stdout.channel.recv_exit_status() + print(return_code) + if return_code != 0: + raise Exception("Error while kicking off pserver training process") + except Exception, e: + print e + cleanup(args.task_name) + finally: + ssh_client.close() + + def main(): if not args.task_name: args.task_name = generate_task_name() @@ -349,37 +389,25 @@ def main(): pserver_endpoints_str = ",".join(pserver_endpoints) - # ssh to pservers to start training - ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - print("kicking off pserver training process") + pserver_threads = [] for pserver in pserver_create_response: - try: - ssh_client.connect( - hostname=pserver["PublicIpAddress"], - username="ubuntu", - pkey=ssh_key) - cmd = (script_to_str(args.pserver_bash_file)).format( - PSERVER_HOSTS=pserver_endpoints_str, - DOCKER_IMAGE=args.docker_image) - print(cmd) - stdin, stdout, stderr = ssh_client.exec_command(command=cmd) - if stderr.read(): - raise Exception( - "Error while kicking off pserver training process") - #print(stdout.read()) - except Exception, e: - print e - cleanup(args.task_name) - finally: - ssh_client.close() + pserver_thread = threading.Thread( + target=kickoff_pserver, + args=(pserver["PublicIpAddress"], pserver_endpoints_str)) + pserver_thread.start() + pserver_threads.append(pserver_thread) + + for pserver_thread in pserver_threads: + pserver_thread.join() + + print("all pserver training process started") print("creating trainers and kicking off trainer training process") create_trainers( kickoff_cmd=script_to_str(args.trainer_bash_file), pserver_endpoints_str=pserver_endpoints_str) + print("trainers created") def print_arguments(): @@ -391,4 +419,11 @@ def print_arguments(): if __name__ == "__main__": print_arguments() - main() + if args.action == "create": + if not args.key_name or not args.security_group_id: + raise ValueError("key_name and security_group_id are required") + main() + elif args.action == "cleanup": + if not args.task_name: + raise ValueError("task_name is required") + cleanup(args.task_name) diff --git a/tools/aws_benchmarking/pserver.sh.template b/tools/aws_benchmarking/pserver.sh.template index ddfe2f9d3167f..e6642c2db496e 100644 --- a/tools/aws_benchmarking/pserver.sh.template +++ b/tools/aws_benchmarking/pserver.sh.template @@ -1 +1,2 @@ -nvidia-docker run -i -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file +#!/bin/bash +nvidia-docker run -p {PSERVER_PORT}:{PSERVER_PORT} -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file diff --git a/tools/aws_benchmarking/trainer.sh.template b/tools/aws_benchmarking/trainer.sh.template index 70aceb8814420..05a7d3b91db42 100644 --- a/tools/aws_benchmarking/trainer.sh.template +++ b/tools/aws_benchmarking/trainer.sh.template @@ -1 +1,2 @@ -nvidia-docker run -i -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file +#!/bin/bash +nvidia-docker run -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file From aaa642821e5d85a2721abb750084de070a14ab2d Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Sun, 8 Apr 2018 12:52:14 -0700 Subject: [PATCH 05/15] adding client and docker files --- tools/aws_benchmarking/client/Dockerfile | 7 + .../client/cluster_launcher.py | 374 ++++++++++++++++++ .../aws_benchmarking/client/requirements.txt | 6 + tools/aws_benchmarking/pserver.sh.template | 2 - tools/aws_benchmarking/server/Dockerfile | 10 + .../cluster_master.py} | 198 ++++++++-- .../server/pserver.sh.template | 2 + .../{ => server}/requirements.txt | 0 .../server/trainer.sh.template | 2 + tools/aws_benchmarking/trainer.sh.template | 2 - 10 files changed, 556 insertions(+), 47 deletions(-) create mode 100644 tools/aws_benchmarking/client/Dockerfile create mode 100644 tools/aws_benchmarking/client/cluster_launcher.py create mode 100644 tools/aws_benchmarking/client/requirements.txt delete mode 100644 tools/aws_benchmarking/pserver.sh.template create mode 100644 tools/aws_benchmarking/server/Dockerfile rename tools/aws_benchmarking/{paddle_banchmarking_aws.py => server/cluster_master.py} (66%) create mode 100644 tools/aws_benchmarking/server/pserver.sh.template rename tools/aws_benchmarking/{ => server}/requirements.txt (100%) create mode 100644 tools/aws_benchmarking/server/trainer.sh.template delete mode 100644 tools/aws_benchmarking/trainer.sh.template diff --git a/tools/aws_benchmarking/client/Dockerfile b/tools/aws_benchmarking/client/Dockerfile new file mode 100644 index 0000000000000..812c5d4bce0ad --- /dev/null +++ b/tools/aws_benchmarking/client/Dockerfile @@ -0,0 +1,7 @@ +FROM python:2.7.14-stretch + +ENV HOME /root +COPY ./ /root/ +WORKDIR /root +RUN pip install -r /root/requirements.txt +ENTRYPOINT ["python", "cluster_launcher.py"] \ No newline at end of file diff --git a/tools/aws_benchmarking/client/cluster_launcher.py b/tools/aws_benchmarking/client/cluster_launcher.py new file mode 100644 index 0000000000000..eaccffc204116 --- /dev/null +++ b/tools/aws_benchmarking/client/cluster_launcher.py @@ -0,0 +1,374 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import time +import math +import logging +import copy + +import netaddr +import boto3 +import namesgenerator +import paramiko +from scp import SCPClient +import requests + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + '--key_name', type=str, default="", help="required, key pair name") +parser.add_argument( + '--security_group_id', + type=str, + default="", + help="required, the security group id associated with your VPC") + +parser.add_argument( + '--vpc_id', + type=str, + default="", + help="The VPC in which you wish to run test") +parser.add_argument( + '--subnet_id', + type=str, + default="", + help="The Subnet_id in which you wish to run test") + +parser.add_argument( + '--pserver_instance_type', + type=str, + default="p2.8xlarge", + help="your pserver instance type, p2.8xlarge by default") +parser.add_argument( + '--trainer_instance_type', + type=str, + default="p2.8xlarge", + help="your trainer instance type, p2.8xlarge by default") + +parser.add_argument( + '--task_name', + type=str, + default="", + help="the name you want to identify your job") +parser.add_argument( + '--pserver_image_id', + type=str, + default="ami-da2c1cbf", + help="ami id for system image, default one has nvidia-docker ready, \ + use ami-1ae93962 for us-east-2") +parser.add_argument( + '--trainer_image_id', + type=str, + default="ami-da2c1cbf", + help="ami id for system image, default one has nvidia-docker ready, \ + use ami-1ae93962 for us-west-2") + +parser.add_argument( + '--availability_zone', + type=str, + default="us-east-2a", + help="aws zone id to place ec2 instances") + +parser.add_argument( + '--trainer_count', type=int, default=1, help="Trainer count") + +parser.add_argument( + '--pserver_count', type=int, default=1, help="Pserver count") + +parser.add_argument( + '--action', type=str, default="serve", help="create|cleanup|status") + +parser.add_argument('--pem_path', type=str, help="private key file") + +parser.add_argument( + '--pserver_port', type=str, default="5436", help="pserver port") + +parser.add_argument( + '--docker_image', type=str, default="busybox", help="training docker image") + +parser.add_argument( + '--master_server_port', type=int, default=5436, help="master server port") + +parser.add_argument( + '--master_server_public_ip', type=str, help="master server public ip") + +args = parser.parse_args() + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + +ec2client = boto3.client('ec2') + + +def print_arguments(): + print('----------- Configuration Arguments -----------') + for arg, value in sorted(vars(args).iteritems()): + print('%s: %s' % (arg, value)) + print('------------------------------------------------') + + +def create_subnet(): + # if no vpc id provided, list vpcs + logging.info("start creating subnet") + if not args.vpc_id: + logging.info("no vpc provided, trying to find the default one") + vpcs_desc = ec2client.describe_vpcs( + Filters=[{ + "Name": "isDefault", + "Values": ["true", ] + }], ) + if len(vpcs_desc["Vpcs"]) == 0: + raise ValueError('No default VPC') + args.vpc_id = vpcs_desc["Vpcs"][0]["VpcId"] + vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] + + logging.info("default vpc fount with id %s and CidrBlock %s" % + (args.vpc_id, vpc_cidrBlock)) + + if not vpc_cidrBlock: + logging.info("trying to find cidrblock for vpc") + vpcs_desc = ec2client.describe_vpcs( + Filters=[{ + "Name": "vpc-id", + "Values": [args.vpc_id, ], + }], ) + if len(vpcs_desc["Vpcs"]) == 0: + raise ValueError('No VPC found') + vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] + logging.info("cidrblock for vpc is %s" % vpc_cidrBlock) + + # list subnets in vpc in order to create a new one + + logging.info("trying to find ip blocks for new subnet") + subnets_desc = ec2client.describe_subnets( + Filters=[{ + "Name": "vpc-id", + "Values": [args.vpc_id, ], + }], ) + + ips_taken = [] + for subnet_dec in subnets_desc["Subnets"]: + ips_taken.append(subnet_dec["CidrBlock"]) + + ip_blocks_avaliable = netaddr.IPSet( + [vpc_cidrBlock]) ^ netaddr.IPSet(ips_taken) + # adding 10 addresses as buffer + cidr_prefix = 32 - math.ceil( + math.log(args.pserver_count + args.trainer_count + 10, 2)) + if cidr_prefix <= 16: + raise ValueError('Too many nodes to fit in current VPC') + + for ipnetwork in ip_blocks_avaliable.iter_cidrs(): + try: + subnet_cidr = ipnetwork.subnet(int(cidr_prefix)).next() + logging.info("subnet ip block found %s" % (subnet_cidr)) + break + except Exception: + pass + + if not subnet_cidr: + raise ValueError( + 'No avaliable subnet to fit required nodes in current VPC') + + logging.info("trying to create subnet") + subnet_desc = ec2client.create_subnet( + CidrBlock=str(subnet_cidr), + VpcId=args.vpc_id, + AvailabilityZone=args.availability_zone) + + subnet_id = subnet_desc["Subnet"]["SubnetId"] + + subnet_waiter = ec2client.get_waiter('subnet_available') + # sleep for 1s before checking its state + time.sleep(1) + subnet_waiter.wait(SubnetIds=[subnet_id, ]) + + logging.info("subnet created") + + logging.info("adding tags to newly created subnet") + ec2client.create_tags( + Resources=[subnet_id, ], + Tags=[{ + "Key": "Task_name", + 'Value': args.task_name + }]) + return subnet_id + + +def run_instances(image_id, instance_type, count=1, role="MASTER", cmd=""): + response = ec2client.run_instances( + ImageId=image_id, + InstanceType=instance_type, + MaxCount=count, + MinCount=count, + UserData=cmd, + DryRun=False, + InstanceInitiatedShutdownBehavior="stop", + KeyName=args.key_name, + Placement={'AvailabilityZone': args.availability_zone}, + NetworkInterfaces=[{ + 'DeviceIndex': 0, + 'SubnetId': args.subnet_id, + "AssociatePublicIpAddress": True, + 'Groups': args.security_group_ids + }], + TagSpecifications=[{ + 'ResourceType': "instance", + 'Tags': [{ + "Key": 'Task_name', + "Value": args.task_name + "_master" + }, { + "Key": 'Role', + "Value": role + }] + }]) + + instance_ids = [] + for instance in response["Instances"]: + instance_ids.append(instance["InstanceId"]) + + if len(instance_ids) > 0: + logging.info(str(len(instance_ids)) + " instance(s) created") + else: + logging.info("no instance created") + #create waiter to make sure it's running + + logging.info("waiting for instance to become accessible") + waiter = ec2client.get_waiter('instance_status_ok') + waiter.wait( + Filters=[{ + "Name": "instance-status.status", + "Values": ["ok"] + }, { + "Name": "instance-status.reachability", + "Values": ["passed"] + }, { + "Name": "instance-state-name", + "Values": ["running"] + }], + InstanceIds=instance_ids) + + instances_response = ec2client.describe_instances(InstanceIds=instance_ids) + + return instances_response["Reservations"][0]["Instances"] + + +def generate_task_name(): + return namesgenerator.get_random_name() + + +def init_args(): + + if not args.task_name: + args.task_name = generate_task_name() + logging.info("task name generated %s" % (args.task_name)) + + if not args.pem_path: + args.pem_path = os.path.expanduser("~") + "/" + args.key_name + ".pem" + if args.security_group_id: + args.security_group_ids = (args.security_group_id, ) + + +def create(): + + init_args() + + # create subnet + if not args.subnet_id: + args.subnet_id = create_subnet() + + # create master node + + master_instance_response = run_instances( + image_id="ami-7a05351f", instance_type="t2.nano") + + logging.info("master server started") + + args.master_server_public_ip = master_instance_response[0][ + "PublicIpAddress"] + args.master_server_ip = master_instance_response[0]["PrivateIpAddress"] + + logging.info("master server started, master_ip=%s, task_name=%s" % + (args.master_server_public_ip, args.task_name)) + + # cp config file and pems to master node + + ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect( + hostname=args.master_server_public_ip, username="ubuntu", pkey=ssh_key) + + with SCPClient(ssh_client.get_transport()) as scp: + scp.put(os.path.expanduser("~") + "/" + ".aws", + recursive=True, + remote_path='/home/ubuntu/') + scp.put(args.pem_path, + remote_path='/home/ubuntu/' + args.key_name + ".pem") + + logging.info("credentials and pem copied to master") + + # set arguments and start docker + kick_off_cmd = "docker run -d -v /home/ubuntu/.aws:/root/.aws/" + kick_off_cmd += " -v /home/ubuntu/" + args.key_name + ".pem:/root/" + args.key_name + ".pem" + kick_off_cmd += " -p " + str(args.master_server_port) + ":" + str( + args.master_server_port) + kick_off_cmd += " putcn/paddle_aws_master" + + args_to_pass = copy.copy(args) + args_to_pass.action = "serve" + del args_to_pass.pem_path + del args_to_pass.security_group_ids + del args_to_pass.master_server_public_ip + for arg, value in sorted(vars(args_to_pass).iteritems()): + kick_off_cmd += ' --%s %s' % (arg, value) + + logging.info(kick_off_cmd) + stdin, stdout, stderr = ssh_client.exec_command(command=kick_off_cmd) + return_code = stdout.channel.recv_exit_status() + logging.info(return_code) + if return_code != 0: + raise Exception("Error while kicking off master") + + logging.info( + "master sercer finished init process, visit %s to check master log" % + (get_master_web_url("/logs"))) + + +def cleanup(): + print requests.post(get_master_web_url("/cleanup")).text + + +def status(): + print requests.post(get_master_web_url("/logs")).text + + +def get_master_web_url(path): + return "http://" + args.master_server_public_ip + ":" + args.master_server_port + path + + +if __name__ == "__main__": + print_arguments() + if args.action == "create": + if not args.key_name or not args.security_group_id: + raise ValueError("key_name and security_group_id are required") + create() + elif args.action == "cleanup": + if not args.master_server_public_ip: + raise ValueError("master_server_public_ip is required") + cleanup() + elif args.action == "status": + if not args.master_server_public_ip: + raise ValueError("master_server_public_ip is required") + status() diff --git a/tools/aws_benchmarking/client/requirements.txt b/tools/aws_benchmarking/client/requirements.txt new file mode 100644 index 0000000000000..9454801f20256 --- /dev/null +++ b/tools/aws_benchmarking/client/requirements.txt @@ -0,0 +1,6 @@ +netaddr==0.7.19 +boto3==1.6.21 +namesgenerator==0.3 +paramiko==2.4.1 +scp +requests diff --git a/tools/aws_benchmarking/pserver.sh.template b/tools/aws_benchmarking/pserver.sh.template deleted file mode 100644 index e6642c2db496e..0000000000000 --- a/tools/aws_benchmarking/pserver.sh.template +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -nvidia-docker run -p {PSERVER_PORT}:{PSERVER_PORT} -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file diff --git a/tools/aws_benchmarking/server/Dockerfile b/tools/aws_benchmarking/server/Dockerfile new file mode 100644 index 0000000000000..1593242cdc863 --- /dev/null +++ b/tools/aws_benchmarking/server/Dockerfile @@ -0,0 +1,10 @@ +# A image for building paddle binaries +# Use cuda devel base image for both cpu and gpu environment +FROM python:2.7.14-stretch + +ENV HOME /root +# Add bash enhancements +COPY ./ /root/ +WORKDIR /root +RUN pip install -r /root/requirements.txt +ENTRYPOINT ["python", "cluster_master.py"] \ No newline at end of file diff --git a/tools/aws_benchmarking/paddle_banchmarking_aws.py b/tools/aws_benchmarking/server/cluster_master.py similarity index 66% rename from tools/aws_benchmarking/paddle_banchmarking_aws.py rename to tools/aws_benchmarking/server/cluster_master.py index 68285406c461f..a4f54e4441618 100644 --- a/tools/aws_benchmarking/paddle_banchmarking_aws.py +++ b/tools/aws_benchmarking/server/cluster_master.py @@ -18,12 +18,15 @@ import math import time import threading +import logging import netaddr import boto3 import namesgenerator import paramiko +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + # You must have aws_access_key_id, aws_secret_access_key, region set in # ~/.aws/credentials and ~/.aws/config @@ -101,7 +104,7 @@ help="trainer bash file path") parser.add_argument( - '--action', type=str, default="create", help="create|cleanup|status") + '--action', type=str, default="serve", help="create|cleanup|serve") parser.add_argument('--pem_path', type=str, help="private key file") @@ -111,15 +114,25 @@ parser.add_argument( '--docker_image', type=str, default="busybox", help="training docker image") +parser.add_argument( + '--master_server_port', type=int, default=5436, help="master server port") + +parser.add_argument( + '--master_server_ip', type=str, default="", help="master server private ip") + args = parser.parse_args() ec2client = boto3.client('ec2') +logging.basicConfig( + filename='master.log', level=logging.INFO, format='%(asctime)s %(message)s') + def create_subnet(): # if no vpc id provided, list vpcs + logging.info("start creating subnet") if not args.vpc_id: - print("no vpc provided, trying to find the default one") + logging.info("no vpc provided, trying to find the default one") vpcs_desc = ec2client.describe_vpcs( Filters=[{ "Name": "isDefault", @@ -130,11 +143,11 @@ def create_subnet(): args.vpc_id = vpcs_desc["Vpcs"][0]["VpcId"] vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] - print("default vpc fount with id %s and CidrBlock %s" % - (args.vpc_id, vpc_cidrBlock)) + logging.info("default vpc fount with id %s and CidrBlock %s" % + (args.vpc_id, vpc_cidrBlock)) if not vpc_cidrBlock: - print("trying to find cidrblock for vpc") + logging.info("trying to find cidrblock for vpc") vpcs_desc = ec2client.describe_vpcs( Filters=[{ "Name": "vpc-id", @@ -143,11 +156,11 @@ def create_subnet(): if len(vpcs_desc["Vpcs"]) == 0: raise ValueError('No VPC found') vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] - print("cidrblock for vpc is %s" % vpc_cidrBlock) + logging.info("cidrblock for vpc is %s" % vpc_cidrBlock) # list subnets in vpc in order to create a new one - print("trying to find ip blocks for new subnet") + logging.info("trying to find ip blocks for new subnet") subnets_desc = ec2client.describe_subnets( Filters=[{ "Name": "vpc-id", @@ -169,7 +182,7 @@ def create_subnet(): for ipnetwork in ip_blocks_avaliable.iter_cidrs(): try: subnet_cidr = ipnetwork.subnet(int(cidr_prefix)).next() - print("subnet ip block found %s" % (subnet_cidr)) + logging.info("subnet ip block found %s" % (subnet_cidr)) break except Exception: pass @@ -178,7 +191,7 @@ def create_subnet(): raise ValueError( 'No avaliable subnet to fit required nodes in current VPC') - print("trying to create subnet") + logging.info("trying to create subnet") subnet_desc = ec2client.create_subnet( CidrBlock=str(subnet_cidr), VpcId=args.vpc_id, @@ -191,9 +204,9 @@ def create_subnet(): time.sleep(1) subnet_waiter.wait(SubnetIds=[subnet_id, ]) - print("subnet created") + logging.info("subnet created") - print("adding tags to newly created subnet") + logging.info("adding tags to newly created subnet") ec2client.create_tags( Resources=[subnet_id, ], Tags=[{ @@ -249,12 +262,12 @@ def run_instances(image_id, instance_type, count, role, cmd=""): instance_ids.append(instance["InstanceId"]) if len(instance_ids) > 0: - print(str(len(instance_ids)) + " instance(s) created") + logging.info(str(len(instance_ids)) + " instance(s) created") else: - print("no instance created") + logging.info("no instance created") #create waiter to make sure it's running - print("waiting for instance to become accessible") + logging.info("waiting for instance to become accessible") waiter = ec2client.get_waiter('instance_status_ok') waiter.wait( Filters=[{ @@ -281,8 +294,8 @@ def create_pservers(): instance_type=args.pserver_instance_type, count=args.pserver_count, role="PSERVER", ) - except Exception, e: - print e + except Exception: + logging.exception("error while trying to create pservers") cleanup(args.task_name) @@ -293,8 +306,11 @@ def create_trainers(kickoff_cmd, pserver_endpoints_str): cmd = kickoff_cmd.format( PSERVER_HOSTS=pserver_endpoints_str, DOCKER_IMAGE=args.docker_image, - TRAINER_INDEX=str(i)) - print(cmd) + TRAINER_INDEX=str(i), + TASK_NAME=args.task_name, + MASTER_ENDPOINT=args.master_server_ip + ":" + + str(args.master_server_port)) + logging.info(cmd) responses.append( run_instances( image_id=args.trainer_image_id, @@ -303,13 +319,14 @@ def create_trainers(kickoff_cmd, pserver_endpoints_str): role="TRAINER", cmd=cmd, )[0]) return responses - except Exception, e: - print e + except Exception: + logging.exception("error while trying to create trainers") cleanup(args.task_name) def cleanup(task_name): #shutdown all ec2 instances + print("going to clean up " + task_name + " instances") instances_response = ec2client.describe_instances(Filters=[{ "Name": "tag:Task_name", "Values": [task_name] @@ -327,7 +344,7 @@ def cleanup(task_name): 'instance_terminated') instance_termination_waiter.wait(InstanceIds=instance_ids) -#delete the subnet created + #delete the subnet created subnet = ec2client.describe_subnets(Filters=[{ "Name": "tag:Task_name", @@ -337,6 +354,7 @@ def cleanup(task_name): if len(subnet["Subnets"]) > 0: ec2client.delete_subnet(SubnetId=subnet["Subnets"][0]["SubnetId"]) # no subnet delete waiter, just leave it. + logging.info("Clearnup done") return @@ -349,38 +367,47 @@ def kickoff_pserver(host, pserver_endpoints_str): cmd = (script_to_str(args.pserver_bash_file)).format( PSERVER_HOSTS=pserver_endpoints_str, DOCKER_IMAGE=args.docker_image, - PSERVER_PORT=args.pserver_port) - print(cmd) + PSERVER_PORT=args.pserver_port, + TASK_NAME=args.task_name, + MASTER_ENDPOINT=args.master_server_ip + ":" + + str(args.master_server_port)) + logging.info(cmd) stdin, stdout, stderr = ssh_client.exec_command(command=cmd) return_code = stdout.channel.recv_exit_status() - print(return_code) + logging.info(return_code) if return_code != 0: raise Exception("Error while kicking off pserver training process") - except Exception, e: - print e + except Exception: + logging.exception("Error while kicking off pserver training process") cleanup(args.task_name) finally: ssh_client.close() -def main(): +def init_args(): + if not args.task_name: args.task_name = generate_task_name() - print("task name generated", args.task_name) - - if not args.subnet_id: - print("creating subnet for this task") - args.subnet_id = create_subnet() - print("subnet %s created" % (args.subnet_id)) + logging.info("task name generated %s" % (args.task_name)) if not args.pem_path: args.pem_path = os.path.expanduser("~") + "/" + args.key_name + ".pem" if args.security_group_id: args.security_group_ids = (args.security_group_id, ) - print("creating pservers") + args.trainers_job_done_count = 0 + + +def create_cluster(): + + if not args.subnet_id: + logging.info("creating subnet for this task") + args.subnet_id = create_subnet() + logging.info("subnet %s created" % (args.subnet_id)) + + logging.info("creating pservers") pserver_create_response = create_pservers() - print("pserver created, collecting pserver ips") + logging.info("pserver created, collecting pserver ips") pserver_endpoints = [] for pserver in pserver_create_response: @@ -389,7 +416,7 @@ def main(): pserver_endpoints_str = ",".join(pserver_endpoints) - print("kicking off pserver training process") + logging.info("kicking off pserver training process") pserver_threads = [] for pserver in pserver_create_response: pserver_thread = threading.Thread( @@ -401,29 +428,114 @@ def main(): for pserver_thread in pserver_threads: pserver_thread.join() - print("all pserver training process started") + logging.info("all pserver training process started") - print("creating trainers and kicking off trainer training process") + logging.info("creating trainers and kicking off trainer training process") create_trainers( kickoff_cmd=script_to_str(args.trainer_bash_file), pserver_endpoints_str=pserver_endpoints_str) - print("trainers created") + logging.info("trainers created") + + +def start_server(args): + class S(BaseHTTPRequestHandler): + def _set_headers(self): + self.send_response(200) + self.send_header('Content-type', 'text/text') + self.end_headers() + + def do_HEAD(self): + self._set_headers() + + def do_404(self): + self.send_response(404) + self.send_header('Content-type', 'text/text') + self.end_headers() + logging.info("Received invalid GET request" + self.path) + self.wfile.write("NO ACTION FOUND") + + def do_GET(self): + self._set_headers() + request_path = self.path + if request_path == "/status" or request_path == "/logs": + logging.info("Received request to return status") + with open("master.log", "r") as logfile: + self.wfile.write(logfile.read().strip()) + else: + self.do_404() + + def do_POST(self): + + request_path = self.path + + if request_path == "/save_data": + self._set_headers() + logging.info("Received request to save data") + self.wfile.write("DATA SAVED!") + content_length = int(self.headers['Content-Length']) + post_data = self.rfile.read(content_length) + if args.task_name: + with open(args.task_name + ".txt", "a") as text_file: + text_file.write(post_data + "\n") + + elif request_path == "/cleanup": + self._set_headers() + logging.info("Received request to cleanup cluster") + cleanup(args.task_name) + self.wfile.write("cleanup in progress") + + elif request_path == "/trainer_job_done": + self._set_headers() + logging.info("Received request to increase job done count") + args.trainers_job_done_count += 1 + self.wfile.write( + str(args.trainers_job_done_count) + " tainers job done") + if args.trainers_job_done_count >= args.trainer_count: + logging.info("going to clean up") + cleanup(args.task_name) + + else: + self.do_404() + + server_address = ('', args.master_server_port) + httpd = HTTPServer(server_address, S) + logging.info("HTTP server is starting") + httpd.serve_forever() def print_arguments(): - print('----------- Configuration Arguments -----------') + logging.info('----------- Configuration Arguments -----------') for arg, value in sorted(vars(args).iteritems()): - print('%s: %s' % (arg, value)) - print('------------------------------------------------') + logging.info('%s: %s' % (arg, value)) + logging.info('------------------------------------------------') if __name__ == "__main__": print_arguments() if args.action == "create": + logging.info("going to create cluster") if not args.key_name or not args.security_group_id: raise ValueError("key_name and security_group_id are required") - main() + init_args() + create_cluster() elif args.action == "cleanup": + logging.info("going to cleanup cluster") if not args.task_name: raise ValueError("task_name is required") cleanup(args.task_name) + elif args.action == "serve": + # serve mode + if not args.master_server_ip: + raise ValueError( + "No master server ip set, please run with --action create") + + logging.info("going to start serve and create cluster") + + init_args() + + logging.info("starting server in another thread") + server_thread = threading.Thread(target=start_server, args=(args, )) + server_thread.start() + + create_cluster() + server_thread.join() diff --git a/tools/aws_benchmarking/server/pserver.sh.template b/tools/aws_benchmarking/server/pserver.sh.template new file mode 100644 index 0000000000000..6fbf2c523092a --- /dev/null +++ b/tools/aws_benchmarking/server/pserver.sh.template @@ -0,0 +1,2 @@ +#!/bin/bash +nvidia-docker run -p {PSERVER_PORT}:{PSERVER_PORT} -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file diff --git a/tools/aws_benchmarking/requirements.txt b/tools/aws_benchmarking/server/requirements.txt similarity index 100% rename from tools/aws_benchmarking/requirements.txt rename to tools/aws_benchmarking/server/requirements.txt diff --git a/tools/aws_benchmarking/server/trainer.sh.template b/tools/aws_benchmarking/server/trainer.sh.template new file mode 100644 index 0000000000000..a83408733d8dc --- /dev/null +++ b/tools/aws_benchmarking/server/trainer.sh.template @@ -0,0 +1,2 @@ +#!/bin/bash +nvidia-docker run -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file diff --git a/tools/aws_benchmarking/trainer.sh.template b/tools/aws_benchmarking/trainer.sh.template deleted file mode 100644 index 05a7d3b91db42..0000000000000 --- a/tools/aws_benchmarking/trainer.sh.template +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -nvidia-docker run -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file From 07b31b80178a51957a1ab537445fef113c927c1b Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Sun, 8 Apr 2018 12:55:59 -0700 Subject: [PATCH 06/15] cleanup dockerfile --- tools/aws_benchmarking/server/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/aws_benchmarking/server/Dockerfile b/tools/aws_benchmarking/server/Dockerfile index 1593242cdc863..333523abcdb6f 100644 --- a/tools/aws_benchmarking/server/Dockerfile +++ b/tools/aws_benchmarking/server/Dockerfile @@ -1,9 +1,6 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment FROM python:2.7.14-stretch ENV HOME /root -# Add bash enhancements COPY ./ /root/ WORKDIR /root RUN pip install -r /root/requirements.txt From ca8af9490fc5d55e42fe5cadc638d8abb4a579d8 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Tue, 10 Apr 2018 17:29:24 -0700 Subject: [PATCH 07/15] update features mentioned by @helin --- .../client/cluster_launcher.py | 5 +- .../aws_benchmarking/server/cluster_master.py | 172 ++++++++++++++---- .../server/pserver.sh.template | 2 +- .../server/trainer.sh.template | 2 +- 4 files changed, 141 insertions(+), 40 deletions(-) diff --git a/tools/aws_benchmarking/client/cluster_launcher.py b/tools/aws_benchmarking/client/cluster_launcher.py index eaccffc204116..d713bc2b45618 100644 --- a/tools/aws_benchmarking/client/cluster_launcher.py +++ b/tools/aws_benchmarking/client/cluster_launcher.py @@ -88,7 +88,7 @@ '--pserver_count', type=int, default=1, help="Pserver count") parser.add_argument( - '--action', type=str, default="serve", help="create|cleanup|status") + '--action', type=str, default="create", help="create|cleanup|status") parser.add_argument('--pem_path', type=str, help="private key file") @@ -355,7 +355,8 @@ def status(): def get_master_web_url(path): - return "http://" + args.master_server_public_ip + ":" + args.master_server_port + path + return "http://" + args.master_server_public_ip + ":" + str( + args.master_server_port) + path if __name__ == "__main__": diff --git a/tools/aws_benchmarking/server/cluster_master.py b/tools/aws_benchmarking/server/cluster_master.py index a4f54e4441618..38d09dc869401 100644 --- a/tools/aws_benchmarking/server/cluster_master.py +++ b/tools/aws_benchmarking/server/cluster_master.py @@ -127,6 +127,8 @@ logging.basicConfig( filename='master.log', level=logging.INFO, format='%(asctime)s %(message)s') +log_files = ["master.log"] + def create_subnet(): # if no vpc id provided, list vpcs @@ -299,28 +301,103 @@ def create_pservers(): cleanup(args.task_name) +def log_to_file(source, filename): + if not filename in log_files: + log_files.append(filename) + with open(filename, "a") as log_file: + for line in iter(source.readline, ""): + log_file.write(line) + + def create_trainers(kickoff_cmd, pserver_endpoints_str): + def create_and_start_trainer(trainer_index): + logging.info("trainer " + str(trainer_index) + " is starting") + + instance_response = run_instances( + image_id=args.trainer_image_id, + instance_type=args.trainer_instance_type, + count=1, + role="TRAINER", )[0] + trainer_ip = instance_response["PrivateIpAddress"] + + logging.info("trainer " + str(trainer_index) + " started") + + ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(hostname=trainer_ip, username="ubuntu", pkey=ssh_key) + + logging.info("trainer " + str(trainer_index) + + " terminal connected via ssh") + + cmd = kickoff_cmd.format( + PSERVER_HOSTS=pserver_endpoints_str, + DOCKER_IMAGE=args.docker_image, + TRAINER_INDEX=str(trainer_index), + TASK_NAME=args.task_name, + MASTER_ENDPOINT=args.master_server_ip + ":" + + str(args.master_server_port)) + logging.info(cmd) + + stdin, stdout, stderr = ssh_client.exec_command(command=cmd) + + # read and save output log + + logging.info("trainer " + str(trainer_index) + + " command executed, keep fetching log") + + stdout_thread = threading.Thread( + target=log_to_file, + args=( + stdout, + "trainer_" + str(trainer_index) + ".log", )) + stderr_thread = threading.Thread( + target=log_to_file, + args=( + stderr, + "trainer_" + str(trainer_index) + "_err.log", )) + stdout_thread.start() + stderr_thread.start() + + stdout_thread.join() + stderr_thread.join() + + return_code = stdout.channel.recv_exit_status() + if return_code != 0: + trainer_create_results[trainer_index] = {'has_error': True} + raise ValueError("trainer didn't finish with exit code 0") + + ssh_client.close() + + # multi thread starting trainer instance and run kickoff command + + trainer_threads = [] + trainer_create_results = {} try: - responses = [] for i in xrange(args.trainer_count): - cmd = kickoff_cmd.format( - PSERVER_HOSTS=pserver_endpoints_str, - DOCKER_IMAGE=args.docker_image, - TRAINER_INDEX=str(i), - TASK_NAME=args.task_name, - MASTER_ENDPOINT=args.master_server_ip + ":" + - str(args.master_server_port)) - logging.info(cmd) - responses.append( - run_instances( - image_id=args.trainer_image_id, - instance_type=args.trainer_instance_type, - count=1, - role="TRAINER", - cmd=cmd, )[0]) - return responses - except Exception: - logging.exception("error while trying to create trainers") + logging.info("starting tread for trainer " + str(i)) + trainer_thread = threading.Thread( + target=create_and_start_trainer, args=(i, )) + trainer_thread.start() + trainer_threads.append(trainer_thread) + + for trainer_thread in trainer_threads: + trainer_thread.join() + + for result in trainer_create_results: + if result["has_error"]: + logging.error( + "error during trainer starting or training, destorying the while cluster " + ) + cleanup(args.task_name) + break + + logging.info("all trainers stopped") + except Exception, e: + logging.info( + "Training exception, clean up resources, please check log for more info" + ) + finally: cleanup(args.task_name) @@ -373,6 +450,21 @@ def kickoff_pserver(host, pserver_endpoints_str): str(args.master_server_port)) logging.info(cmd) stdin, stdout, stderr = ssh_client.exec_command(command=cmd) + + stdout_thread = threading.Thread( + target=log_to_file, args=( + stdout, + "pserver_" + host + ".log", )) + stderr_thread = threading.Thread( + target=log_to_file, args=( + stderr, + "pserver_" + host + "_err.log", )) + stdout_thread.start() + stderr_thread.start() + + stdout_thread.join() + stderr_thread.join() + return_code = stdout.channel.recv_exit_status() logging.info(return_code) if return_code != 0: @@ -421,20 +513,21 @@ def create_cluster(): for pserver in pserver_create_response: pserver_thread = threading.Thread( target=kickoff_pserver, - args=(pserver["PublicIpAddress"], pserver_endpoints_str)) + args=(pserver["PrivateIpAddress"], pserver_endpoints_str)) pserver_thread.start() pserver_threads.append(pserver_thread) - for pserver_thread in pserver_threads: - pserver_thread.join() - logging.info("all pserver training process started") logging.info("creating trainers and kicking off trainer training process") create_trainers( kickoff_cmd=script_to_str(args.trainer_bash_file), pserver_endpoints_str=pserver_endpoints_str) - logging.info("trainers created") + + for pserver_thread in pserver_threads: + pserver_thread.join() + + logging.info("all process ended") def start_server(args): @@ -455,12 +548,20 @@ def do_404(self): self.wfile.write("NO ACTION FOUND") def do_GET(self): - self._set_headers() + request_path = self.path - if request_path == "/status" or request_path == "/logs": + if request_path == "/status" or request_path == "/master_logs": + self._set_headers() logging.info("Received request to return status") with open("master.log", "r") as logfile: self.wfile.write(logfile.read().strip()) + elif request_path == "/list_logs": + self._set_headers() + self.wfile.write("\n".join(log_files)) + elif "/log/" in request_path: + log_file_path = request_path.replace("/log/") + with open(log_file_path, "r") as logfile: + self.wfile.write(logfile.read().strip()) else: self.do_404() @@ -484,16 +585,6 @@ def do_POST(self): cleanup(args.task_name) self.wfile.write("cleanup in progress") - elif request_path == "/trainer_job_done": - self._set_headers() - logging.info("Received request to increase job done count") - args.trainers_job_done_count += 1 - self.wfile.write( - str(args.trainers_job_done_count) + " tainers job done") - if args.trainers_job_done_count >= args.trainer_count: - logging.info("going to clean up") - cleanup(args.task_name) - else: self.do_404() @@ -539,3 +630,12 @@ def print_arguments(): create_cluster() server_thread.join() + elif args.action == "test": + init_args() + if not args.subnet_id: + logging.info("creating subnet for this task") + args.subnet_id = create_subnet() + logging.info("subnet %s created" % (args.subnet_id)) + create_trainers( + kickoff_cmd=script_to_str(args.trainer_bash_file), + pserver_endpoints_str="11.22.33.44:5476") diff --git a/tools/aws_benchmarking/server/pserver.sh.template b/tools/aws_benchmarking/server/pserver.sh.template index 6fbf2c523092a..fe2360ed20637 100644 --- a/tools/aws_benchmarking/server/pserver.sh.template +++ b/tools/aws_benchmarking/server/pserver.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run -p {PSERVER_PORT}:{PSERVER_PORT} -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file +nvidia-docker run -i -p {PSERVER_PORT}:{PSERVER_PORT} -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file diff --git a/tools/aws_benchmarking/server/trainer.sh.template b/tools/aws_benchmarking/server/trainer.sh.template index a83408733d8dc..89f405811e768 100644 --- a/tools/aws_benchmarking/server/trainer.sh.template +++ b/tools/aws_benchmarking/server/trainer.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file +nvidia-docker run -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file From d44895e679c86dc78e444e64efdf78e374851e55 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Wed, 11 Apr 2018 18:57:19 -0700 Subject: [PATCH 08/15] add readme and some small tweaks --- tools/aws_benchmarking/README.md | 153 ++++++++++++++++++ .../client/cluster_launcher.py | 6 +- tools/aws_benchmarking/diagram.png | Bin 0 -> 41741 bytes 3 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 tools/aws_benchmarking/README.md create mode 100644 tools/aws_benchmarking/diagram.png diff --git a/tools/aws_benchmarking/README.md b/tools/aws_benchmarking/README.md new file mode 100644 index 0000000000000..5fd586cc1537e --- /dev/null +++ b/tools/aws_benchmarking/README.md @@ -0,0 +1,153 @@ +# AWS benchmark testing tool +This is an automation tool for deploying paddlepaddle benchmark testing to AWS. + +## Features + + - subnet creation to fit just the amount of ec2 instances required. + - pserver and trainer ec2 instances allocation, and instance state verification + - nvidia-docker ready for GPU training + - Instances and network element garbage collection when a task is accomplished or an error occurred + - Test log is collected in realtime + - Web service for checking log or tearing down the testing setup + - No testing code change needed + - Lots of optional configuration options + + ## Usages + + ### Prerequisites + + - You have a working AWS account + - You have [AWS Command Line Interface](https://aws.amazon.com/cli/) installed + - Your AWS cli is bind with a account which has `AmazonEC2FullAccess` permission, and it's set as default credential. + - You have key pair created and pem file downloaded. + - You have a default VPC in the region you want to run the test. + - You have a Security Group created for the VPC mentioned above, which allows port 22 and the port you want to expose your control web service (5436 by default) + - If your test is supposed to run in a GPU machine, especially a multi card GPU machine (p2, p3 series), you might need to contact amazon to raise the limit which allows no more than 1 GPU instance at a time. + + ### Start a benchmark test + +#### Create training image + +*What to expect in this step:* + +*You will have your training logic packed with paddle runtime in a docker image, and be able to be picked up by AWS instance for training.* + +Training python script and PaddlePaddle runtime are supposed to be packed into one docker image. Use PaddlePaddle production images as base image and create the training images with the docker file as follows: + +```Dockerfile +FROM paddlepaddle/paddle:latest-gpu + +ENV HOME /root +COPY ./ /root/ +WORKDIR /root +RUN pip install -r /root/requirements.txt +ENTRYPOINT ["python", "my_training.py"] +``` + +***Please Note*** +Training nodes will run your `ENTRYPOINT` script with the following environment variables: + + - `TASK_NAME`: unique name to identify this training process. + - `TRAINING_ROLE`: current node's role in this training process, either "PSERVER" or "TRAINER" + - `PSERVER_HOSTS`: comma separated value of pserver end points, I.E. "192.168.1.2:5436,192.168.1.3:5436" + - `TRAINER_INDEX`: an integer to identify the index of current trainer + + Now we have a working distributed training script which takes advantage of node environment variables and docker file to generate the training image. Run the following command: + + ```bash + docker build -t myreponname/paddle_benchmark . + ``` + + Now you have the image built and tagged with `myreponame/paddle_benchmark`, let's push it to dockerhub so that it can be picked up by out AWS instance. + + ```bash + docker push myreponame/paddle_benchmark + ``` + +#### Create instances and start training + +*What to expect in this step* + +*you will be asked to provide some basic settings to config your training, and this tool will have your training started and monitored* + +Now let's start the training process: + +```bash +docker run -i -v $HOME/.aws:/root/.aws -v :/.pem \ +putcn/paddle_aws_client \ +--action create \ +--key_name \ +--security_group_id +``` + +Now just wait until you see this: +``` +master server finished init process, visit http://XXX:XXX/status to check master log +``` +That means you can turn off your laptop and your cluster is creating instances, starting training process, collecting logs and eventually shut all pservers and trainers down when training is finished. + +#### Post creation operations + +To access the master log: + +```bash +docker run -i -v $HOME/.aws:/root/.aws -v :/.pem \ +putcn/paddle_aws_client \ +--action status \ +--master_server_public_ip \ +--master_server_port +``` + +To tear down the training setup: + +```bash +docker run -i -v $HOME/.aws:/root/.aws -v :/.pem \ +putcn/paddle_aws_client \ +--action cleanup \ +--master_server_public_ip \ +--master_server_port +``` + +To retrieve training logs +TBD + +### Tech details + +*What to expect in this step* + +*You will understand what is happening behind the scene, and how to check the training log, how to tear down the training on the fly, etc.* + +Let's understand what is happening under the hood when you run above command in your laptop + +![alt](diagram.png) + +There are 4 roles in the figure above: + - client: your laptop + - master: who tasks to aws api server to create/tear down instances, and monitor training process + - AWS api server: the one who actually creates and manages instances + - pservers and trainers: training instances + +When you run the `docker run` command above, what it actually does is to ask aws api service to create a subnet (step 1) and a master instance (step 2), and pass all the parameters the client collected or generated (step 3). The master is kept as minimum hardware config to keep the running cost low. + +Then when the master is up and running, it will ask the aws api server to create the heavy lifting training instances who are expensive to run (step 4). And the master will start training process as soon as they are done initializing (step 5). + +Meanwhile, the master will expose a web service for client to check training log or even tear the training setup down by a web service call. + +if you are creating the training with client docker container, and also monitoring your aws dashboard, you will initially see a instance tagged with `ROLE=MASTER` and `TASK_NAME=_master` starts, then you will see several instances tagged with `ROLE=PSERVER` and `ROLE=TRAINER` starts. +When the training is finished, pservers and trainers will be terminated. All their logs are kept in master node's docker env. + +Master exposes 4 major services: + + - GET `/status`: return master log + - GET `/list_logs`: return list of log file names + - GET `/log/`: return a particular log by log file name + - POST `/cleanup`: teardown the whole setup + + +### Parameters + +TBD, please refer to client/cluster_launcher.py for now + +### Trouble shooting + +TBD diff --git a/tools/aws_benchmarking/client/cluster_launcher.py b/tools/aws_benchmarking/client/cluster_launcher.py index d713bc2b45618..bbabd98246599 100644 --- a/tools/aws_benchmarking/client/cluster_launcher.py +++ b/tools/aws_benchmarking/client/cluster_launcher.py @@ -342,8 +342,8 @@ def create(): raise Exception("Error while kicking off master") logging.info( - "master sercer finished init process, visit %s to check master log" % - (get_master_web_url("/logs"))) + "master server finished init process, visit %s to check master log" % + (get_master_web_url("/status"))) def cleanup(): @@ -351,7 +351,7 @@ def cleanup(): def status(): - print requests.post(get_master_web_url("/logs")).text + print requests.post(get_master_web_url("/status")).text def get_master_web_url(path): diff --git a/tools/aws_benchmarking/diagram.png b/tools/aws_benchmarking/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..9dd656c9b4719fc6a96eb3d68c796daa0aaf7b98 GIT binary patch literal 41741 zcmZs@2RPOJA3tmyS>X`MK2}zm_8td^GBXOHAu}sP*@R@BNM&SZ&mxJ;ij+N~>``Xf zdpz&2>i@g{&+}ZD`@XnuzT-3A@7MaA2Rd46R20k<1OxNu<(0Rhn@0Rdqv*&+BZ z@)dXD2na9)>R3g653{)>(pdVQwbiyGC={*Yg8-Z=8qdGclyU>lkE7OzPCU2FhocVQ zc^sb2^9gro@#0Gq`6I;uKO&7o_Z{b#&06e&nKIv2=2qt38!o$4d!gsu&z)cM&#q*( z%C4K!u*-O0?0FQ3{`;lKy>KdQDW&9REf1R8e}B7XKgMTw3y&s`COG&~c#2Npr-^xx znedBip~!dl{080UE4Nkc)csI6_^Ra~qMzJrAAiSVafC3;C874E2_1$I`EufipF^|` zw@^3|T`(vAy_vQC=f0zAerYm0KN8tv{@mf<-xA>jDisaKgMR$I?ZJNx;vfuLT1dEk zca8`y;atblf&F{&;%NBw`=|RE$S@>#0ZL7IWe*Z*;%j6>x zN=96nZBMMl_t&?0)gSSkwWuZ7-G3}(#N)fS(^+ILZ_}RQ?=#(H9}c{n`ta}D{O>)S4*S;ylAdpSF8jh|viYUcsL}%# z@xm_&;@WrDe-#9?rOiX6K;)06k&dSm3)oo}T! zlYJ%E^idXK`{ew$vM!OwR~v$DZtV73C41xi#TmW> z`nyUM-XQzn$+OJlLN1FQ)izQZb{st73)kt+r8-iMrk%~@7bfKnE>rECo=czjM7NUU zyzk~*%!;H~=*rUm9L{uZ;>)vhf&&*bFXWh?e{wQu$)wEo@r6VQUB**p`TN?55+(U2 zX$ZPb57|VP62yS5tF@p>t|c?6A_zEna-BhBxzp9p(??` zpI@GF!|35BKR@|=)ZB}6du=|3S=6>El$JZ=_Rm(l_B%Q~CR6A<6 ze@ScJ@tn!Y+7W%S{_hR@=->^%=VXm*9k(#pqUh8;uKhgUMgYCp*G`@qp>ZPnt;^Dc z`lo2lz_inXuP~(VEb0PWMrsL}M69D7&?>p-TN5QMXS#Frsyr5Pu31`1oz)9>Iy2Oh z!ed|K|Lk?Ybl3~hckf!Wl5c2@{I1!L6Cd(W7Rtoj|6na-RM@T082SrEPG>DA*?D@yDfMrTY_m^I*9qE|@VK5@P-VWPjrf_D{(Rk9zd^1Q*o=moQA*O{tJ z`s}Qw29M~(Bko<^+dsZ=UU*da@1b{LYtLJKp~cnO5gEQH@{lclz=D}^&gFAyis3zf zbBx+d>0eDnvp~t}BU{|&L8FKaZ><~-m}Y(ScQSFcq);s76vl;lEW}|?RFc2){kKOL z;b*HB(gIFG-jRtF1})lu_1A9|ZeC#3%>K8|{H=0Aa3!?KmYG4RSVfOEgWVG}AE3tSeu@O%F zqU0MIP%x_>jeGyS*@zb0{8ys5vF5+GhL(a>f;%W%pPi=o_vA1ke$p`AmA8ki-9nF8 zY-xmQpC0^I6#3!TD|m3V{-`${UH&mjG5?-a6)rATZ~2vHLd6yxpUc5i zEXRv#_TJuOJSxmfNJoZ@-c`{-SJ-B?6hwK31&>}@ezrAIi(-EqD{xU`?EQm+sd(=` zg^}jRLN6VM$o7>@cf@4%ALw*n{NUMapeG zH(TOA{Y>^{zZ==X4G;en8QJ;njs>0LwJQu(tqxGlGpsw(kVR+3jLW-loH0cVFbtsNAZnX`M1tMK|!rSWqP@)Gio<*CPK&piQfSqKpF)l|H>w_a#k<8uWz8aE)ClJ4J5)8Rcv^_H2_eB=@dFnF(ihV(-D zmGJZHN!tN{(g}ThbM3q1Bi#I*?8ic?~RY-KR354xb6reI;sTd)%0e(&)YC4 zv5BHr42{e;=gWtA+-ACGCTQe$Qm(DTO3*8|P^`;rfU@ujo)AF^NZabK_O6_0OE&*- zE@)eg!<(n#_RqM(OlO;*K4s{nDlL6A_f?0!c>^9@aBlo##5URcsgI&q$*uXWtf+4; zzlUV!$CK_1;Q{>R%C^LdcAfcNYBQeysic-U%!Ly3k`gv^4Q%)Ah-kKGq;t^aTg_y! zXK5#&Oc1;E2x?pIF%EHu#CX!tgyWjb)5A42iSoW?`@6q=_f67A*`!>fWB9al(ege! zZ|3@6sl9cZU5h3NnNmf0_Z44##IKvCW*TW@8oHY(dAm7)gmR+ntJ%YP`+n?~}}mTO<{&bjyk5Sq)6FHFx) zUk#|*n#t=dydo{>Iu*^M8Q=X=;N~hra>t@7i~uE!fC*X56}%tP^$NJAt9&q0oW>zE z947%&SH(i%)YETGmN(y8oz1f0$ula7>wjgHHe^f$B28%dcmgAWO8kz(o4xbqfHP=?jWw4znI4X(DTA}uXd0rn34O}rdbjXsUqX7Quw5!t&d9!*2g$38)$7Rb&Bm^z!* zhSPi|eWy@0=<}0PMFH+R8;e_R6CUOjE-K=oT$AI;zI(bIkxf$7QT(^#PF!YZ=6>xQ z5kedumFTh&zQUCLfv1_4R_#|si8_6bfUwEySU}2ZnUlx76%@F8P(4~Q7q3Q(M@3|y zS3E2b?CKKguq<#%G4UvLyWgb@4JQrFOIF7lS^;rgfLba}9`*#rJm?f*9#(Rxk(OL7 zm(=AAEpKpkflg{bqMWyJULmIbux?~iiS9EJ0Uo#6-k$8!L3X+NIU<^<`tPCG>+Z#p zB1VO#XDyrQ2X|~r`#x$um+>r>oP2(6PePaa9{SesOUs8_0WF#7su7)<=>^7Rovu&e z^KTvhGuw;CHdSa7BCb=VRhDdUT(X?7h3 z=KeaW^c-u~1`@;_bJ1!upJ1d{N5>mEe7or)g^f$wr{wIhTk72!C!5$B->St9;OrE; zSy{A!Je}~}+sumP*HxUP*=4UN}4hRu0B~yro>Sd;qWt9LJ zDa@J7P+vzqL977bO5*-$Op?Rs`Rw|D;8lN9mkBEtsGWxI#qfYMQ&~ir%M!R=+Ec(v^XvlD>5hzTg&gR#7wj# z7;#)$@be39Nm?Mm-xgU3A(I(8S<5&P%d+}?_p(5K^%#xp(0S-)4A8#wa4$LN{t6Px zRYLYJv0J3Mg%~OlJHGaL7!a4$8H0!sW!HOR83idocfVM@(rMxK+1ZdO>9z`}!UMRV zg83ReeZKU}92tVqE;?AQGFW-4+95y)NFcPF0fBRh((pa&c|XUKtkR-!$3D&o#aZ0o zgKgyli*)@6#*q&oi#PP=RN2PF;r8d|ch!g-kY*nDEG_A($^l4m$jdE~_$*|69RdBu z{~rKC_l5_DaY9}4=5@-Kq$7`3_yP}BA%&oH{$@4_0g(rPsSMC>T?cMccTJ*dI>0AI z6LS?&Jw|h*&1QdS${uJuiFp8z{ZQ99YsHVIzQe*rOMgBY!uy#3aHzD*wC02Bx5M|O zRNe}Qe(W4Vog)R%6)H@y;%DV~8z1dAjtmCVMWp0!jz`Jg`1173*<0U4td6M&od7Y% zK~m2hkn#vIp=z_Y-$xT%tpftEj`)=B1MdF}vd2mrP`k;IxsT$wBH`y?=e!k=QGVl z{^maH2$P#(?!a&RG*Z-Kjx!6Gzq%QLl%0L=pRH`5)|^zSD3}NwmgIOFAWvEfOGY6G z1|QZ6B6RsD(CZs*uGj1f559Ga0c3K1b*4wucn?$^TMbI%#z;2H?rdHEb79;hu?o|j zFD&8L2-#u+51v;aR+H%wo)HD2@BxyWdYEd?_xr>9+rE#4O~lw%U{2zo^?w3vH94J? zOdZb2%4>}(&a&vuzZ@XWdmz!EFTf@0GTKViF^Ay0wNq6329i!AekU%!oJg&UWS14p zZqm!Vz_cAPLI#6^d@YzGccnn?3bpHw9?~Tr3mSCBGMu{wz$o6xEJcCv+>_-$%!QC{ zAs&Y4Ym)~t6h0Gj`~H_I%Z;ET@u-B+pPBy%&3y936SGIdKHDuTvQ_PDC{QTUr-l03 zKOgj@It0dF*kVlpDrbRmvKYhLcrH9`FY@uYy+EF_!n=2vMeM}J$95uwa-rj} z6I>WVHM$X1ZM#fFK8r6r3ra_hVL`lTQdrZ6hevtg4gS#~q!8d;3V|_TpycfHk$Bd7 zjQhDFbaiVuRgDH@p|7mVX7BEPs{CEea#^dRQ?c)vB}!*)0e+5gtl2f}%+fvu`q}Gz zNB0Ba(-07Qc^J<&C*UR6XYx;>UPnV6Zw4Z_J3Dpe&hWc?M#nYddA=52x#%1z1_cXO zM>;C*Fd&g!E$1|%Fb$Y=XjWU^uz!5b=QPd3qo;bqq?(BHkxz5sLz)h;vgk(Xqz*Zk# zI3r+~e+d&jaSU0GBmA%&Bjc%=m5KHdZu8raNKe*%u8$M+s+C_KF=i2=B0oGqQ8Z3l zM7DkJ2(s<9b@CgO>yxDKh_yW&}A(zHbjju1m zcrh`Jv}$!#(np!~f!hHq3?fycwZs(c)O|>q!X!d@4e7G)GWrVjKFT}S{ntRwB(+ig zCC(QbkHdBkg_w0@s2_=l-(TCv#`$2y-`x&3Pyx7P=simpX@U$U> z=E>_@-=$nXS~DU9x=$G@Cwt-CW7G~;?aJfEab-~|dn~p>8S6UOx3@7)>_1@SiiiZibVJ6vRfCYtcd4g+seudy=RYBvLHDd{$&pF-;-gX|45 zK_ICqV?Re=fQIO@0d%kroO7SM_+N1-9 zWKP(hXI?u?&~@UXw;e()^CIqUPCha4x^CZ1(|D<7Z!51_xc1|q(tV1{vOL=dQdA?9 z?vQasSP<^;!To1800Pnxz_KPo-L#DRd4 zEKX4S&w4K1`1Isda+;+3+|wJ)v7bIYwPr!Cbx8x}LoabI9F@CT(30&H3$Lqrlj9~7Krf9|Gk-=^v;pUPGhD0Y1^uf~yfzr3+ zl;pXkNkq+i8tje!H_qv>?psqCKRPfjrP59&1Lf1tGZbE9VRctYQP~9J0I@5UKA)CDmQX(&99EW5COkU@I5F!40lt4+BIyNnD#XOr>CKMX4tTYp5Ox__%w4vMof%9 zE%@$iZ@~)~6S?*ASQg@Ud#Cre*MC0&J4l*@a2^&v#@`Q-uq9lmaYQ9Y$V&Oc2D$DP zT7$>KqY;Zcu9axrMnE{lz^C1tYjEB!yJo3bi11j)9w*VeJuWV;mD)5=K>L5Qsv(ym z3_(`+)cf}Qh2!X%X`!0kwS>$(h47vt^Wn0X!7Oc#{*vn<`KA?}WpY?6j}#Tp&vC-M zcYX~7tZ;}O>_j6dexp*$D+#Coti@>k{kw@$od#!bemTY?w8QZlWDSG?PY+g>?0#mMVdzuVX`*@;Q$=g44R5WT1N}wUUKnd3v2@l$X$e&Tg{cb38H?Kx) zdqgxM2q!7`f4c*;2x+xvp~9FnIxGG3H}eHi3UBVaYw+Cj2NT~LZI**eAqrh!KvLkN z+HrD}F@3)%hjQ@Y4iOr04hrIuQ2u@JTP6nY43|+pPT!)M*sAp2eht!uzQf?#N&Z73 z2htr8sI)Te?k$fy=}{D1$r6>Rl_Uri{%b@uh_J8nhZWp| zTq$JqMQpoqVC@}gN@Y7|h*8-Djr;~EJTG0R^%^&{l@G?`7&0b`N!K4KoInjct?n$f zxwvZE94kE&bR|hb!&&PR+IcroMf*o#wxZZce2~C63Ac z*)l1xdub@#sWphA_|Q1ri8fQrBmrzaS18Sc$%O%T9l=S>0KclU#&l6k+%o9+pr9XIoJLt!Xhz?Fa#~O=b*RG|S76L>c4FcY0=mMtFN#Lw?@$Xl8E@uGD zgjcdfUTOH=Z`nb#X_gjS-k^o=0b~$_*D8!Z$$Xxu6{7>jRSpORuD^!{nGS7Qd;ll% zMtu-D^CaT=(Z~k<5{4;$asEl1**n458r^?cfV=8BdM}iY^2;8ysUtAXnhImQaQS`C zVz*gCSsZD_*KFbW-rf{NVuag(mDo{mvd{JoM#Zzxw4!uwLa3(QQC~)pWn1eBS4k4*D-A^jXr)+wj&m9R zNcBwe_RaQNi#>iV3F7A;(D90p?yY_`UoT_%{JY_Jzh;7%`f#;3i6eH-0qS`>C#Y){ zw%M?LK7)8z#uX+$4>%-1TxcjU=q-5AOj54d@Ngd!Qayx`4jse;JuW)YP)dcUnzr^+ z7|sZ|#N$%yj{0&fR;RDgoXR&~vn)aVkgSd0e@0LOf~|4E=iL_ZVO*kwvwgYa$8aXW zm-p7@Y;5yE6Pj<_)_eF;D_($Wom%zvF%E&fLD_t1hM{ko*&_^RGJm_-* zpmXRShKKF&Ogd{BRs~XSKrSBu9D|8G*bUZ^zT7 zuQ*k;5A?EAaO(=#Umw-GG))dB+75Q28TU99kA=Lyg5wHS@B_fuiSLdzWy6vQs($S% zu^jabc!S2QOAWb}a zE!lhR*$6=vFqG*-z3>P6f(E%fqPE@9u(3YDIC(k;nwB25(%paarzRW)XhlVX!h-3f`eNCHxu2 zQ+pBf#GvohfoPn7U^~w>>w8UfWEuR855mxz5D{?}Zi72CgYN}h=fvf+jhp_gOEhvD z3bW42RcVhj^>rg%SvEkBo2f-4Sd0c|zHSnV;l{#sg2B`gy!+pelHMD0! z_|+oC(-xt5mWA05M5WKsE(RAj5)-Ahpf@}!Ekr_H!b6tu9%Qu*#*cW^^*4G8ub6O9 z>t$&b1iS$>EjWjtaaozt1sCyw>cUlduI1uMc(S*j80*<~N9yVp*hqCa>#3yEmfv}A zuXTK*G{H

JXV@Ru*e(4+y=(abQ$G`qSq*&tuTjLb#&}H$1^8DwG?m0$KYlb#&b5 z`?Zle;)w9TbDd%*&8o6*j@dmII05`M4$-~0$l046hN=WWkLgXcQ;y}3biszv3#8=; zgz9mJ`G5z5RP)RPgm4kDLusOAT|@bYcb$#cP7t-rmnH7 z^lQ#++Ma#jQoAES}A%uO`yoUR3NCV}EJrm{nYv8hK6ng>QMO!!WHzzX643gqo|MRKz?3r!9N%D>B;*|86yYr9d`_hs(R2=b zBCVbOBDt`m~fQbft*OZ zbW{j6c8S(st($#-sKmMV(`+~r5_yA|f0aL@dSm>)qN3~l1}w#Yj$9k(4m!pz_|mHF zmuR5Hke>)m#%+b}Au!CJ`)t|N5g$tHKzH>8JqVU@A+y7_E_!Fgq;T@Fjvb{MW2+a1 zB_1F%OGtcZ>MJO~!ca^MO2TpHmby62ppNEO3~h)r2MOz8Ww^hfr;pLlpgagRzI16;uPsu%oEmX76B>5Q;Hl0AII9eF(F`T zPDO*J+Ck*BqJz`{xp{-oyacPL*C8her8|w#mMVL~OHO!}x!&G%HJX-|;9ANLH?cjm z29Y^TIl{WjKk*@vB6Qdh+Ginss=p*%x)$Bi6L zn~DXm7kEaNWiRS%V(_gqo9s#N-Y~A$lgjiJA8h#V$cFbmWaEpYE54h~r$-w7(odfp zdv5TXjx%br3slWJd6cEwH7K1zT3R1_oaF)5phKtf{}eHz|2DAxSL`&+9@G|uyMb;0RX~iL2EgqGRGgo{|Af}Wyz-p^iS;qOBX|n_V zNHn0uhsaQl{^Jc{mmYjrkcRNz_!fXaVjlD(loJ6}r%W*uO~Ty3x8hV>m%J$W?N;DBTj0-^cIjPZ}=_KRT^x9>|~I63gm$y7RRM}mZobiSA*P630J zw0(NL;pC5}2P6UqO0*py7xmh&D@fMsJZwPj^W&186_>*6X%_(UFi)xpJG`zvLKei| z$a>u%ek+h(H>!ZiTzL$1?=z@y zDADJG-`wZFz0mc+s-J%>IB6)Z^v$7CWfM9bk{xjYh!On*_w(RE{)8^Z%6QNiB3+dZ z0OAHsz1h0T02ApkLirpwq4}Kk-g1pt;=@JPZV~pS^P=V$|I3$7lsriS^cAhZk1&M$ zmosQM)oL%&k4h|!eR!A`Ev%m%ZRcG8F+Cpb=cXSc*=CJ7>OVNa;{Yj`L}aU7)Zanf z2WHUvSb>O|$0DOBMd|~>hL(S{P?vTls#~=tb}-LCdL;%7hlI)pMkJ5@2Gdw`&L82^ z>MmxXVus(1n>;eqnm2@9v#N|ONLHWqLJ#5oEW~_T1Eh+>^2oRM&v15*uXEL;y6 zgc#F5yT2#`eO^t>VSqo`XDcT%f?w8F<&^20g;eWJ@R1N`@3OPuJpHXKua=$FB53w& zfe9Z3P``FbvWPq6@22jIBEb{bz;vvPLHoYTE#i7$r#zGKd>L0GbopgQWQA25yW$qn zM{hvDqQ+q_AADS=;y<>N-#<&3->;+1((T(cU@w7SKhb2P$qObd%BH()1~0eq`bhC4!6` z6!hW+C{d%O&erH3IkYZsNuOD zLw;8SN+Zt!MmH)jevw9za8Z7H?wt3VTAL(bl=>y+5C_gdVi=%msI0(CAa?j!pB;~g z*z|7hAQ5)df;iUrwS7Nowf_~8bZZ1BXqDf6;yZ_wD{lKrh06;h7K{XY=2^tvt@fHM zb!p411CxINIc_Oxd&PZ-jKl%$B+K>27+6D4uH;)W0C}8z-H)8C=M?P}&aJ22!(1{7 zx(iHPn-TW8R`vt-*2vE2BUs}EZ$L}myE)1orCMDPI zALfuVCJu%#JfqZR)N z81lRF8l#qw*0?XruA|e%O2m=%lzIN)sp@E;O?Cy#>1t6k4Rt_*+++rZORlR$yysqu zETQI*%QycHp`~vR#pyy5p9mRgsrbT+x!t~MFJ@ZlF5IboF@_=FXn)0CapZzUusGfS z)!5NQeV52aH$U+uPX%kdYAIZKT(iGta$h2C6$yPqhDh#OCzRn;5#5Yrg+$J`k(#NY zdMs(I0*D;f(M{iNsP>+j)}xz0gv{qx#ZT8G0%?mLXvHu}REe{^?Mv=*Hd@Dw_z|-{ zqQyCZX4yt%-zicWiqM7hARc>VS1*_v2$Hk!G7sSt^0`-ULcNSXejB1Nn*RQ6CH;O) zZ=EChHz~?y=jbPKsm9e_6(gw+|BD&y`TV*cK<%^LQh_jsJxALY0`x%W@%@eQs7K%_ z`Qy|wm@7t#UqN&$e#pNDZ|3J^-yIu_3HA_sFUZv;!AhoUyAh0&SgPUdJZIU=15*|U zIi}B0lnWCysC22F!xMi)FH{Esp6k9F$g2jWKOx6&3|SJHAc;y-eiiBE9)7=Mkr5-ta{7=;g}2+%V*AQ$QBAy$-Hn@9{urZVB;cNsBMmQ- zyZxo*`3j%iw?8gO=V(IWe7QlEMavM|y#(#L!<9C}Q{jSgdm3jHYdtgmwik!zkabhP zSlT}Nci;Q4^e+5aUv`n%l0EA&8eWHM{4{Vz~X zH>CAsoai#IwES9Oz^=21Sf|i1fXY?=nQ9GEFVOiAAOH!73U!^TaFmk;bu$|^(^+hk zXZfrCXL5d09gygI1<{6Xt)qSABq*m)TI%n0oa)(gfusVg&*@E+2R&mG>|sCJKbcu~ z4(&!jDZt0C4H+^lTW{~K+TQTKdy(y~X9p>xFmmv=@ zA_3Y?WC9)Y1eQpGfHb7XB9K`uw|{VsZW7G%C(w08H);TkpxXy9R(c4w4AA9oTNYwFO3GdRTz5Q;Mt3%=SEN%qGDRQ0MeSLgWsa} z{75OVo9UUy-}GcT6|>Ycb4db{24Kw?ahHZeZYMS31CkzeuDG}EP>`|=sICcMxhMk( z7jqh20F$@r8z)6XO8eQG6a&$1GOoqTFK^%hFXZZkgnn2Jl~Rr#GfoUM&eazXUf=v? zSCpfJWJ(RP$I&U&Y*JGR=>W`fsBjVSU@u}`Ol&NUpS5nsFwAGGQ~O8FLP{-P4h}QP zyD-b%^0x>XUSOhnoeX(d7^?g<$cFU4yxvG%H$g8m}{&UBG- zc%`h5)2!*b0#f2$h5&na8_CxDKq2-MwZzHyWD-XR-632n*tuvy%QmY}B}o~ws>J~@ zDlFgv3LJ)+h(*l4J(h5Xs11O%9g|{;f=D>CXmgodP~Ir$I%#LXW)}3Ky|PCCDs+ph zo|2YA@fqZEkd&Ni(4sr303Ui1=dBpK2^kZ-c?cSaSb5kmlA+$GQj`ZP4dbw?B_)yCHL@0vUvwOF<^ zR!2rgVPhmM`55%vnLtvmjlO}y=3qby;e6|eNtxMs(7I|50Auayk9~~D+Z`R=cor?^ zy`^?3WOge9iBR=|QCj|7;GU}_A@*zZ4}bYO+r7*pjNSul4h?ZGrB6TPh|s+?J4@#? z$kb-YJNYrxvcn#x5X`+OhZ&23*HR`IG<=2BIjEZ zg}`yIAZZ4M9&|ZCR`(445R6(_6TCwai?B1>ulu1YJj|f)wgzuB)HIR}wX9hX9z>Undr;TF&o#Ng@j|{MYt^1^e6W;JsD!sTc=T}0{mA22i(Lle;4NoF%Py+;S!Z@i0IP$HlPn}Lkovc>uc64Q+y2AZ@RT!N}6-RBUIr%8#+`Fg1q1^!iX7ELZ_$pUrcSmPs%!av-(pP*{ zjj$5$RN$qkGHw+5hdkgn*O36i?|}{%4)T<1C_S0t%O^q8kXyU<`_|1)BjD8OZn29N zb*e;$bPI0U4deC-x12MKt`ZCVfmXHY2s;u)0;ueKVLBSrx_jW(*~nUo(G|!0K@l;y z;v~h_EBs#|_9j5#LK>M{BSeF8b@#Ga7EDKvMYxg>N@FE^h1Xz>fGa7tC8?)Z5aTmv2QuA+ME)%LaOW%gF^yUbW9}BRE<}XW$zin z5dfP_r(pgN?Y%fztnkgmUm@RqpijArvob}xgrG&UIY2LRKT@n5duOU$F)9bid=38i zb5hWW7toxMR)V+cxb+P5uIa1Hp@$lInwHR+Ej`N&oGc8v@u(EW!(<64R{sub-Y`N||rjV^;!{Swq_&$D{ zH$Z@87fAUwI4L6UMOTIq8-h=hN1D)`XQWM|=ef7}bG6T9OH43EXJ8F6;5xx#^f25qM)aa(`uKp*S5fr(ZYjzP(?O93}aqoLmj zG@S<%dD?kGLiUUliv`U`;M7@tZt)!KE>1Xd`+YbcSD~SfNkPq_C$ky5LE3lFqMo|o zg5=7?A(?ci%bujY7rH+V()wu(lR}31(U4+NV~>gGT5w8xPWxE(#?fm>X)QyY5+8-= z)9fr`Xixvk0mE`s|u21)sq^r(fH31l7&^@lm zEydz8(Znfnjn$DL{Ca&sB6IQkoUmd_NBTY-rX6`Ji9QONPac#X|EQS=8G+psbz~Cv z`>EKi_{`*yC*{tps~|{nr+0qb#cBEzsM50~s0rz3V<1)8MfDIYbcT%2m^K!obudA_ z_;Zus43fa5;*?DM$PlOjw(ke5(M3(ZA;lpIYCO~eCyL&!#rGR`Czq?nC9IjVc0qV=GvWYme=~k8Q^E_AHx(dc|Zc!dRnV-g{Ybz3d-)cO6j^(uo;V)0uZe>0280a807}RNRJaxW1(IvypVw`6w z5uZnO2E07xr6$gNTFJ(BPQQ-mD5}M9&DZWk>86-ixX>c>Jzfjtqy)-}3v8g<`*!#A0$^FrU$$vM$5U z@_LrhXJh1jeIjDVuM(xPUstgrX4hD#^@|@3F4{|LWI=`7IW3mU4`~zap^J?bkONU& zhQ%C#f93#l7uoJiaz_+UEH=D-GE5iG>6xVU9#UhM(lgeHAn{f}VP?WhsSqN+*;Evn zV<@#%1JuZegN90V8^n&%~yP3pngZmdNkSU(x32^ER-g{^llH59Nc~HBic^?rx86Qw>T#tW^}C~%Y;R&{|uWL9EZ8+P#oF7e^6@-~HAYCu^cG0*4mf z(z?v`l^B0(FQDQ*8pR=B$Y9fltDQ#WL$=O74>@)G#`$_yr4|y&#$l|Xy#@w9CvH?F z*vtZ@zr&TGTjb7lWmz{F{S*OeSO)Ha7%P(cZiB$1YkAj|qdkfzIxOkV5;6JZSNj4JAbzrM; z**V-o+!Ii_njvE8?NVo}I}t-$C18&Xmy`7|IZ6r3C;ywo+EEl#dSQqec11)NRd3}s z6h!n_0VlyhHM4{Oq1?7b8UIp0dgxdjD&NePAC#YD-x)=V|I97*@pky^3{X)X<{Ota z*BQzUl>!H_+uML@F#!iU*FMu^7H!;*65m#@)zGCr6N%ddk6^bn3^}M2tzdn`Ld(;- ztw^Zq9%$%gXNhg^RF*-7(*)}Pycxt4HW^GA-E}c|k{xm*4BYs9izV0qjfU3i8Puo%xr~o%dD{Xae*5(lQvyd#CdfbU;UXZ!36MY}m@#K{GNeJ?N zb$txS^ZKKnr`7am=-#Mso;y83eyL(lhr$^=5jeLH6#Y|`^Z@?+`#=~oT8VpW`Kocq zSuZ#nyEtRUcV*y}m3DvSouU&aufK@B%BNR(&Gac!InM9#EKNe3bEqMN@|%{@t{=!t zBmi}U_+Lg{BL9*kZG>2ms=SBFUxQwJ9%LC2wcb`Z0p;H5c5}GziCdLXhkw@Cx(`O)u*_$c8haO9&-d=jS7HPK2`!46`5=sWyW)e-fx~ z&pEP#q<$?GDw0F~WM2mL)yUypI3!^3>gXx66Nt3|k&FH@P-Ao0GvoO4+#jf}bLcm{sHv0oNGp z+@6Sd2OGnJ8tFT~vX9k3c-O=wda)n1-oU65Gc4v3j}$?J74LDNEYT#kcp${hwfy_DzFYH4i6Md29f z5XH#PddLnAPRQMY52U~lCX=zsc<8j9L_W(xJ7HO%*}J|qL5AFGYdS0bhkw&c6La14 zi2)YJDo*5>4n+A~0@Q0AiLxbPWjtMs9H?Jw#QyFpQI^;KUhcS_z$XnQ0z+7iR&b)p zic6<26^}?8U@@**qb2eX<&gLBLi|3v>3p+y#-Gb};nQ-KU=h9TrD6Vcm?%4csa5*7 zM8h$Tf?|!>>w$brC+20WKdT+}JSg382n|XTa-dF;MEn(8_Jn}hI+`%nslg7q(!KLg z#;=?wB`rqIVF5##EaZ>QiXKy`5V%m)6hpw}52OZ45P+^qMrvx?x*wGQ=cHOm^RW1FR?4e^ zh|7GCqMGjfuxe&pwucm9QcS9rBnx>E1)-N*eS!0K>hE(ta1>LKLGT1LB{e;Tr-^D6ZQ21l?FMFrnYtUF~3X3ZIR|hRKBrqQx=xZJW;P%l_*uIHn%Or z#3Lx7#{J7@IQv~1iUl*D{kyD61>1^psW#c=2{TBA^_Q{3jy4lF$N27m`{$C+0UvoW z*Bx~Ks^KnOA)M*a&%dO*z~~==Qo8SFkW5p)Kql@xAAdAsJ5%GahH<$A+c=tj(!GNw z)d!y$!2Oc*24KrM<2KA;h-w^f+J;slM24Kv|7-;=-9Xqc)~*nPnxMtwxgeSPaAc1j zJ1r7~(mB6nzW>Nc<0o=)(+NnJ9$_s$)>Xj(m+qwTujUj3<@p5AtaoAL|KsItCVd#ZGBxd5qs}vP-L*PLXO+Wz=DrqNV8x<#ZIAbY{;hqMo>uR|Js{VStZKT1@2>XeJgVKFMUYwK5$*hFNNa zG^yv4OS{F-z{TxxeSCLyMjf!e>2zS157}Efu86f~ehjlTR!o>%llAv0<-XUksMkI~ zld>;R~UgG*jsr$nW3Zag?=mQp6eqT-2}s=N^n z?7)0)Z`-lXpTpr6IJmLM2hzONfUMHsOfVeJtGEayL<%`O0UsmKU7@;V^Nayzo09sT zM09mMEK6){SX)SfN8j`*u`goJfS@#I;8m`AKqnExVR7by%|}i zvp1qa`bCZ#tXARQIFD;Ue2DJo}h7`+kh;ab4$ip67$^ zf*062U7K=BE%u#l!H+4NbNoa+gc#J^#3^hY+XEKBidT@G2pK&dp*KT=vK9k+o0t(N zY@N(V@!AYLRdjsx)%WJuU;h){S8<3QNgS(pHbb}xugIr9HUt!;{P zsjJ!e6#4{GH`sxjCv{0^xZ3y7&Gg5CMT>s_G%R# zVn%)mBkMUOV00C%igZX`_hajzAZ_0C8n^_W9)VvbFHoK5_Q!t)OloskVAqHlx;!u; z-;9EFJeh|T;#*=A@ZD@NT3q=4WCAqh0H4#@^~_3~j-yW?+NE3|U4<0bAU<~L4@B}o z$6lsSt}h8N9>6BiE#?54YnUI@k~brQcd0Q$?<#@O<>)QfQ{ubTobebER7OAAKt-($ z5*6yH=&O}l?Vmd7DK}KStrp407e>Q>y zQ(5D|N+;j&kN*Y6P2=xJj{9QfbO*4dZn&lKTgt^)1su$Tzy1&$lN4db`UNNW;_Riv zX=Uf24#iN~AcY%fPt+mXEi>0y`W}$<%Xcq&{dHGz!2hD^_PkFV8ElIwIgpj)CToOtE5vLLB z66pfXZ}3^T(3wn-H1f;15>*R`7Cgxd$n-eN*Bgjyxl?rWM4*4AL z=^yu6;M6@6_@Q`r5<=`3yw^@Z$~bx;r?tVYkTel~{%PyRuM-Vl8qggVE?Sqn737?J~je8bb>pK4|W#WxDsD@+*C@_PPAuo!t*xqj8 zO1+B7lz6p^!!Km^ovHMxY3Fdq^x)w8Th!la<+-j1k)v+}0iUM>I2$)#lM#Q9GH>!4 zcpCl+F!nGU>7yl$f3xiEGH6~l^!e+f>`_&%jsE142Hpe~c?1O*R-PVwLU#ZWXgM$c zfj#gsl9D|?WWca18VTxZqCMxoCWqQ5QXrp_wq&3tl#G0e%l3qQB}?EDaE%50TJ0&U z6D5ZmpPEN>Xt_ByH={b~n5PKDvzl|a!YxF;SMK=kJvyBsje{~_xrYg6kq|3j=!@~m zjkSNXIR2~t#hOB$xKN>h_C}UbNyAGxTjTvU(2KbksAR4_;a?J zaGo{ozZIh!j7aglQG@?OU8>b4FOkFJ=y-=o$oSAN9?Fwja5iyu%HO*Vg`wT+b`VyW zu99aub}+U(ra}hr2{Kj&AD}A;#s=ABjgOCK#`5o%^B5L4j1$#H0dbSmC9^NTUf?ft3y2%lWNf2)0AN(p~|@kiJb;Z;OKmgJuNSpa9Mr6_3L+V zHhzT=br*08jn|5!6X+f&3QgwZISZh*zn&w11C znhUKbT8s~YM%gXMt4swESb17BpPc{rn;5%^VZE1MtgsinnG^O|;iARZ@AU(l(~2Xp z`|I<2Bou~__Wd*%ujbU`J1pyw`g!77#c=EgNh|RAf4mL3;W*e;pko<5VH03DDj*_G z^cen#w39$jB6m{y5|o~e?9;z}h$~NFHz04hGpJ$qmmjMOk9C39{$wAXi7}qd^e^Z^u%0 z2>wA$I{n(6@(-|3S60W{zS{3=Cx|;MT9qsU9$igF{(0#4mp|#>^8aRhh9b4k_a{WL z&x3e1M7Bp!6QN&Lr`GuRe*1WA44fTB$CLoybI=_Z@a17-Xd5f1#?#vD38lDwGHuaFl>LQKW9Ppa95r!7;hOpL__^b@2=MuTB>(UbQ>f6NW+^ktSYq zhROW@8wkXYmO7sx!WHus2m`Cwj_-tBLC^=4JCG_2-(!99Y_dC$M=m8)Ck7jQ&NP5e6w0zXM0w9;)usvt zzec|cfsN;beGo=*A|O2ok2-<|lLhhH9f1IA{O$w}((8{XihLpvkF*M=uCr~omjiXi zbIATz&UL0v2GoiV^w;P-zkNwb)*(JKE1H$a29G|8RiTL*ooXAkFkeD+&E zSWH?3AD-|CVRz_{raw+aRms(4^fWVWt`W9pDN!uXQcU-oMQ|W<19N$d zGy-t}(e@i-z{%U}4nPnLbPP8=-US&4RMpB z2<AoB*NL2ww2+EY7S?u8f=(DtD3%h|y(z#0Ts`Jw&vO@+6YDv8W1 z^&G;Yp$WK9ML^BVqio$+$CDZyQ}s!4$5yiDLdt7LWO2B&a&wGyt#D(d6QM^QO?oRq z50D&5uU#N}f#4Id%>lkH)v+TK_#&?YN6gKLJsY`yo@@yN69+^}3@XV2?+vWCXE)iV z3(h@@r5s$NxGnLvcMG6#$%m1!DPzj4{`;Kl5xP0W9^mfcOL0a-Ngc111yit(q$<3D zFozgNA*jY#;}a@zS~rCfD$Nkkv@}J^6n`w5PO|w_@nAHL+QpqPKvL&+=7%>JYAoUA zB>*(E!6{*++9OW_m=Vv$t@M3gU;0Kg^y**CZfIK;l49vZFlWf+5mS$qRVBK+A<|CW zR|3P?om`N5V_Mb$$f(s{J3ArS6fJlg`$F-v_cztj>wj8z6B{0Q9&y?P*o#e&l$WGkI98c7kM-|JgHOpap9(@A>-J_s?-H&eyhGY5eo(X#wS;;qeW*{7_kQb1S@sGj*E zMT$hW@Ivos5>&9BAKUn#?5D%kX6=V9*EFc6DHq0=dwAoIZ67IT7O%z<`r33w?NbFRq3p-_Ts|_U^=Pt`bKVdo-h%uk5&NPfF5G| z{Jo&|B}(q4OmT}^0t0c0bDja8R6PX)x#i|;qWvbYBpDxSMN zQX$IcmN_8PN(8hmxfCM%P`(j`_jGV>iWeDvC{~aKNRhG~uGlEZ(eQ{?+c5_{FgEyi zb`fUtIMo~(faM$mC7+oII7U-elFALZW?eZ2Uf?!#Oxjmi`b;(N+5UJVTVnQ&aE$}( zZ;mh3Sfa}6Z%j-_U!o0J;M@O>SW&6B7`FkYs{BDM+3MycHhZM;l&$pI8Bt2#5(K66 zK9$Z;qsQ>}JhvX|Lfv+KC)$~le4~R?b zf3+Dm+&?cnGIff3uF=Gm%b!dt1DVdG`r%i{zv|+5x0}GR;}%F1=E3y}!0u9IU>alr zhlpfi{d;wA2cw|P)7s4@!p<@PK@rOqQxeEye}vs zcAdr8Pu6~ZgL3(McDi?h-iTU%%#1~^VXgeRpW_WE@M zz%0m=&+rzQiw@#p4TpFKih#{iX-hQqv&``oh55iU-Gu0lkO!Sc56}Gxd`qCJ-mQ zf?44BzZBR7s7SA>X}!G*CvPe6*pek5SNhXik!q=6DB?HcC39QjpbeHjrA4s3=Q!YHYUyKz$1qoJOL;#3? z-a}CF`C$T0R)poV4J&l@3jnxU#}*uC&ze!pS+4sE;MKY-d(K+6I8!@mMw+-d54@W8 zUjQDnPW$IANrYp@nr-mWGp?-Vr?v>{`$9u2AwiqD2>PL)c;sbbcwJ0!%n8iT zw|?NdfD|PO66TM}xXti^t04oRPvJzY_W*{vN_zt()=0gIo!G_H0a~aJBt%}M&GHyv z)7)U0EVgV`R`d!CdY}|aPvQp_c0Ws)*Gh1iyT0Ej%YgVRXiZxbnH;EZ4lwdM8~L7m z_|Uxr=Adl%6MhS5fkvbur+CSq`H72-mxsi;3u?q@&RDQx(1I-@m;HHR>hHYkQEcu} zN3);~4uzrfL zn8KLWf;g4x#6qPCg*U>BFDR zk7N*&V*lmkglCT_W zFEILU21%(knq>(G;SF}U_ef^nr`+F|$g`eS7o7ABJkJm-K|4rDBj*d9xS@EGTcjJL z6UqvWb=W<2Uj4VpCvE1OFC6i|sI(ftzwdr`&tvfP3VJFXkUqX!FjZn|@6C!)q*J zNeIQEeK~GF2rYyXXX)yV+~$Lvo;jY_n5mDHEo3Pq;z=>aob)(7lWvIR9=055@1rbD z3;B=CgYgbzdI_}q%+83b(Ph{0S3+ybiB}VyCaU#x1TSb2Gk*dKHQ)X(fn1_CKlYKCleUA@Cob z4!@GJgV^;Qx>KSFzmQP`6}N9(hXbx9hdGuhhl!`;a}gwySACe(m|w=tMK|C&@qoLM z{zo|!)hsPGfjtwad~`%byaxFTP9>k{MaA1$H<@f{Pi4GVJW#phy^(HNflQsT0&U;` zGJuw5*+-XSlIqA)IKv~JEXnUuvpfR${;Ei0t4@m&M<-d#lfTm;9pVg3aV{aau@=Fc zWtKQC>ncc9{P- zXW6Z+)I>`zkzh$bQ-JNZ9KVmvpzN_txuEDk5NsXMW_|4vmw-j=p+bQ2F)*UKyn=2wAhGwQcjmu@g^WkT(*IB)f$*9 zVvj5$R&3HeXBzR`qYs-xh%RHqTB>>*$a1xd@yy#pl3bmgeE+3TB_plBrPF~Z^rVrX z@ycAYOM){2PLh0+0wmh29TEkhGZqyiF_`yD9p2X za2-kG&o>Wg(!R))=(o8MC2b@9kU2JJBR-HDKXhzSSo6#$>b^Kiq_wh4$|dNIfyUM- z0$(mt{+)2Z%Paci@vF9YY%RPyn%A2PI7oqj$t()92WTyz!qaD%E+@R*`Otc$A!eyc znCwB6^ZgF-#XJ77nHhxl``o+IBUVz49%ZmWsZ%GqGip^6K!YTqt7TTaYCy{}*Yq)nD?X^hkO*InG8((k;3L8VjfbTv?Op1a(CJvMjb^lFC>{hgEirs{Gn*XI5^v^&Twd@(9az;lL#qF-t5abv)ZA?Tbqufd$-2mU_%@4 zCWw84c+^Y0maLu|O)hg@SQD+{%l3<1V}~lTY-q%p@AxkX;@0r*gP0@ zRf`bG{id&5w=FSWjCl3-C3hwaY?yU100wV{-nl&JcXcHp#t)*0yiPU*H=g@kcG=7{ z@i6K!R8E;f?RQB&%_dZWiNLBXvxxsCJa;d}PFKpZ-VE3N8*B3Fmi@8q2gT>{G#jX$ z`kH1az3c+{M4nvA{{hZ_<$MPuQs9;0YquAS?{w~VIS5wtLHqHJ&h?1SaQ48N?pII| ztv_=((GI$oB27|6>Il+eAAUTI3!#jFV14GNiw%<3+`L0k+d#cP6X9m!Naut-2;{GT z`O8k}Eg=_vU3}keK|QFU`}bYm4&BnTLemUZUr)uSG;cj3UHgwA2iv9bcy}nU0q?)} zb_aU$M}ODT2>bPQ&e{X!$p~>+8+;xoz+Hb2teT#lODq zZIyHTiYX*ie=1A1puORi_aWN#8re#{)Z<^ZJG=X}zsoWf{knHjzdb4%_&E6}!_WVC zXKXR`_hH)96dHSLTF5B$W#(Rpr&q(~->!)Z?uXaP>VanSV5}eSd=1SzS>gjA zm^jz0SEh_%QTUJRV8>r8NsE41?Qh=UL1`;jce+?2h%2 zm3>v}I0oFeR7uPnDJo*nX`jhw_zA5mAzq~_gOJm(PbS?lnLxShjfD4=qy>-$y0)`8 zkxn|v1?9Y#!uIKr{PQqg$FRicJm0n9woXz6U9T)Tb<-p@;kU~E1P1Bd>u;3kTlo2S zc9SPI3TwAjtI8PGxB6r=$X0=yW8TAHR%}OYiOaa{4fK=s*!H6GZ@lilY;Pnz-^e%o zE@@}zPC5~{_`W|svcfO5#VyVPAo&3P;m1>@tfZRI&qoCQm=5jX8@R6BE&)$3`6u1y z<>1pipepI{Xsrm2V{au0=-n6pT{~A;Vv(dORAXs!UH`LUh$B_wx5}wnkv?|u3+bMt z_JzFSMB6M;=FsC>Yr~m~73%o;i>_98r%&~ zpV=qx*Dgz{7ne+ba}4#x7V7pLY}ppN6j46xOZ&3*#Tf;~d86|E8J>_rTT!>BgFV}r ze~KNi9~<@SI!p1cNVlEmPwcE~?0spAZHzaZJR+Xp`nSe9offITCgUkils=gHGj-Vi zy=wjAUZlXz^A|=3*e!$?=?KXdo+euCP_RtaZf@#@^HDriX=9p;r$>7m*|4zjS*rPC z5_bAJ2x*4|DlG;1%FA1a?@DYcL=}kxv-GoIT9RxbULk86cd*2iD0}0r%Y18~ft}c4 zdHen^`PuYUw?C>|bUdYBD;&D3>g)C#?mZ>mXV)AYFo?LY<^2LrJeb{UlSSF<(~bql ztaewEtuKQN<~IQsnwCLBqIg!jsdfo*I$W$b^&DxPL9=|oJj4mm05{%^+8qt*(|Oyh zNTv^wcXvqt*8e@0ezf*^G2Icm9}l%=)Jh`I7IaL5f(oLU0>gKb(bt;zhhJXCdN!Qn zV}=n}>~`YVoP*V|F115}|7#&|$xKg6!*mUi$$xWab5k}>m}Xz6m1=UGV`Yw4IW zu?Mr1=ReAyI-S~@97ZjF3{EPd!K55E?=RH-?ap?ob2cdGp(Z=I#jJoMy%ntBKsYO5 zt`&RV{#p4Oi!L{!8TrgHxDe)nb;{P)9`6wX0=^29TUkb13(-v`WNo-|67CWYr0E51 zy2PsF+OC_2h$u2Wd6jZ{nR1VyJxYX#E#?JX{bepWsYqrh3DtYc~5fc>}Zht z{kpwWQqA9F*B*0YD&p}6hrZA3@Y#Qp-o%LLUZij3kK-hk!`{+SmSu`hC!5!#?o-Tt zh7U0{oH9tckjolvseU7&e#;)XSaf`6><;ZARAf;ySNLArT_*8oamD=Z@L(fP0Wuev zrd|RbcDx`YxzcSVV{4qqa&<|{ZcSB_y5ebXr3-dMB6nkFF zvv(9}SW>YEYxD2sg~$+viu-*V4A>y6v#S0$P2_7bw#u6itNZDXfI5p!mO!Ro46nGf zcfLs5)=Bhge2D7?^#nIciN-d|-&N8*&DD?m%YATZzHXG1WL#)#mU-!2qt~U{p%X&9rGfpXzAET>xkr(rr zl(sQ@&?1wbM2^Kfl@{wOP9P^K32!DQ%Xag@n>dQDkV3KJ;vh8jUI#6EPtuaFvs5#+ zm}YbBQpX;98nzkb%WJF->QkVf|IN_1*$=8=a-d4VQDG2z2};*+ujZ(8tMK2j zk&D0H;3|uMM0z5vY(B#lQtQ-m^RTpLqdJfFRz!fmB4b29Yx?04DMlm&-=lc`e25Td z770jS`rb#O&L2nIdErCxRLd}~EpRw<6MJ_$grvHh$}I_}TNWdoB57j0$x#qwa?cF+ zAiyRo^}3=eRy*F%RX&m0xbyic=3n*06E0ya;xsVzANLst7kDjAm1=d6@&H{OSJod9-xAVvC{ND?xUAOq|#gMwPHeGqX0b-7EuFzkMJt zm~xw>_{wtYCVUi7 z+}UL6YgT8NxdAA4QgF}%jKvl{RghvVn;d#ZqZ7pvx95{r_G~B4rlAw%8>Q$gAWQaG z)%4R+@X4^%;@54>Y$hu2xWRJI>twnNoRvM9mGevX?cqD zZ$_!siN5$|hcgy&ws7Y#WQV-<7uy>LODBr+wy9+2ZJcb=DYWSFFE@(dcPkLIgNdoh zsEVevMeE;42OBDu+}d5HxC2Q#(h1o{#X=l^s;q;%(M$%m&x)i)(xvXE*IZq7^7vSp zQSi4;C{&N{X*e~X(75dX`YIkJhUB^K~xABh_;+!_Vbjkf4be=5J0{Q9--12vTA;$&~13Sn?t! zrzx)1f1_NGxDZj-t3N;<;l{2>cK8eTnKEyj&vfwaDC1^JVE% z)h%T|?hs|~qvUaJ`*=}}=5DZQt~>Lx{M(E(_l1GyJNep@K^Fcfvi7bV&yr;3PNS80 zlQXBey5YrJ){iX<>$E9)iWy(mmTzNsL^CW}b>Z^q)yizmHvASa7ZWsRDoOK6_{bL> z_LRcOf&pc&8Par%?~j9-*`|EzuU*O}nRdzync*R=t!ss*(TBc?J;IWOo-db|Oi(P$ zotG#hOjv$l9(nO^awYD@G4(x9Cj4sKr>2!IdWrv4OTsk=+6cEZc2g$95*%erNr)bo zR|m>_F?o?le|Gxh^BQYZ7e@d=#_3l!#kFv*?ov^4XW@4yICcH?w#!P|B2Bzm`1k7=5tIAe2aq-%d)}7Tyssn|1 z!O5IA>~ZoN;Ko{lSUi-xPTlpWi?Hug&A+#_|2Ri8!G{NK0>v?yl$zN1lE1Dd6C6ue7WbHNGW(S3_9OFuT0V68hPT) zdw5N)`mlQ8Xf!Xw8g+E^t?P%!H|~JQ{Ac)6bnbE z5f!iD`;f0@S>;SlTxu+{Q<&zjd>nqEO;h=2L1>#`hRoC@&jGy)Io}pJ4+wn38t0iR^%Uv#E;J^Dz@Rbzmb|EG0~+$ zp=a=-!-kRKwo16=VygwYo>b+Zk(zyZr%LL3(l_}n ziK4=HUPsqr^p#-Z5*o^S2{TkeDxMRT#3)OAV}Jh0Yy%nP&)%yiU$4aV4;WQ+~6gqmBlUam(q zap;?-e&!9bJ2PA6?%lnWTcA3&xBAmuC}q?>J8k^+QWu*xS&#|Jw@t55Xm6EK^gkiqT*mdOF$=mF85D-85S36zZqVaBn)X}1yUa+BB1o4E+QaW%A2I*o z7J5mz|HR@idi}`gviUL(3lnxL2a}eW?}vDYlDKe_EU`Fk>x8=J+U*9*o}-0Ra>w8N z$q{3@F`C%4^orH&LPj#upiju(bKNn-+NU~--gcl|TM?Qv;@d+xTKLDfe#CcF!rlN8%fw=1d6Sz={4ga52pQ1;et<@@BwbyXj(2XN6x!YuQlg z9aPka*<$fr{HB#>RQre6;L_`|56v8xQlpfPk7D6wB0K7JXIT0)(%i$GJ`w2dVu0l z;0Jziv-dvd??IU8*_g!rXr&MD(tDX^;f^2M9m-i-bzrxzy|Rui0a;HC(UbC3+Y?{T zXa%Ngs8r4==A<%E_%Hq)pCj4q%6BbRz^p- zjKB(&ep%SX2|C`cFQhA0ZKY6pFzRn8QR6Gz(8Rto2A+8cIi9OK*F7PwArzm&L}LN` zgl>81r>UZzE6HR^)JY-z@(P!UQQ!>Bij{yBk>gO$LAB1yXqilWCw?%9tUx>+O&?8o zL~ht!78iy~mSEVEiD>wlVkQhS6yDgO5^x`WV;%e-E`a`xgnRaOKs9Zno>5}(Zb{3S zkV-!J+)ON<(8`dei!6&$vSAA#tT+l!Jk&V`ajEkQbs@6%p`^g4V{{sGuBrgFD0`OD z^2(DwgT96PdqTe0Kyxd>CAvHeO~#3@yL+s@zr;e&5rNWJRUlhWEr>*%aH>A!sS2RM zwR+?lz9;-Gj(750t6?blPP)R4m!S=B!x5}* zmDc$C;^M2{9E-l_r?7kE!#kn-UpI(lrVjE}FR?%qG5vJ?en1(Rq6{9P9Nv_pl#P}L z3sO=#uE?6**>9Cf=`)B@FZWzuX3O|TGskY8^`lE{zdNYNl=_!0h;=UPd7^^z=@OXh<{uo5E0ha~Nm$K9pjJl-84NOMF8ZZ3MCj~TZW;PXep zxxQHTn&+1psHvQ)#yAve{Mo0#$XwjF=MMlGwK5DmKgX?KgL{ zHl{)yq5{#BDS^nR>8?jM9HD@WVNW8l8MqF>_eRp_p3XZM_%|k~s|My=Nmyb<=m;54 zY}KS69i%~<`1yt@-RP5~1|JGZ1w{zBm*KnK&4f1) z{F@QVpO-UrfHqVLd@Tc^JX7gt6+`F9K=c7=9Os|6Sz;`jxY(-ZTsXUzHU90Ocf*6o zt!Thcs_6cxH@a6goG_3MZ=sGVEv#d?3lSRrTP2UsXg_=h&Qx}wO09x{9IPmShsPQc!4UNC>Q%0YEo8Dm81+XUUr%yZ z_6UuT!4E=#-^ysx{|{TZ=- zCTRl}aD`5$HU3Y|-BE`>oi8s#A$gMt!<2~(!vf8}LFl+lMv6tId>K}1mbdPy$_1?V zMVN1iJ}OZ!rSHZdOY=$$@C{U6D-9qAiY8h)Aw1aQiz4*dWsiXmG0_8!MkVpB77FJf z(?{REx9`^-wsqj$2ZXS@S~d*mP%WorK#ZAeKhSOYwbLU#Ww$mS=Yl*_;)870{13;EZVl_So-f_&wk3(NgE=JtTeJ3jXI`yq z4ZPt2we9f=+U9~_lLP>D2Ph_IbtY2zqgmxLfMjX%+4zcASI# zUjF6JWX-lOC?Cs_laImwz>*{X@6k~GDBvo_-V*hTkAQ7abGm#xpcIKGK$dU;e(1yX zJ}FNij{|*GhZgSPcsxP_!r~Z4=}CHFAY9pw!5zx>KtL&jACJXK9oY>f0k>}qUfLqX zr8+mS-7RJ}7kFkHbl^W&37!r$ML5|R{yz8qT81Fx5_i99h!_Rt_5K@#C?TM#wC^Dt zx$E>?0>-H4Dl&ORSAG_)=wYKdO2O9hb3R^a^m;qp+@Mkbv?eA>koa9jB7XQ6(EHWU zT~Pkyg9FZnLZG{;+gG-ISds?&86Ja=cb8^HM|^j_n~u>b(QbZOcxjvuiml4;LSyp_ zjCxd9WauQ=$WOi*6hzB3!VV%-r0xuYQ<^e=^@b|yHCoUr)eZ7mBL#)9Xeqp~8$>K^ z=wBe`AK4$aCP}ITK@mS|I$2Hxa-YB`*u<>Gif=r~JONkPa|%m-(u`(r&{0yl3?BhWt|Cz2ylOzale_N03@o~Itx zH+I$=2|$VTx*uqz`>b%){Ce$Z5_%R&p-~Kq(!m2Z7T?>e7ht!Hz}9y(iIBm-USYcv z+j~p<;OyV&TlQTjuM3}JU>=U;M`XXdB(^{;F#;mzbvQoV>g8qbmwsjEt&W+l-5+%{ z9mOMaLWz_8a>_h`Zj`A~*3V{WDYZH?l*3vSCyNREaIMAv17|kx=eeu~Qa?nHJ`vh4 zlMhCUzLHuD!%Cy7AS^R&p}$T)Tms)VhFlK>L~8f|KO&sL(8p zr8j(Vl%Jl2ML6%HbYVFosgej2z%x?vsy<8lq8 zs-^3dW8L7ZzZ5EBBB0 zEsrEdZpcT>kdU$aur!hNQEKaU1bpZRqx`y~6>j0>Ex0lD%NMvOeq#~s(I*JDhL+HN zD$Be5F!k$m%Fvbizo++E3Dm3c7cD%f36^>0jctDgP%N`|#%yLYl^cB^o9V%*IiX`o zUP@)HTV&eH;A=gLBPNim20IwuqBLc1@Qg_N!J!=+NIbg07AWCPaM~c0IgNJcGtJ2u zp$bl+#&JR)-HsJ==Er|2cZX#8CJ&a%ZpMykLb!Cg2QC?BXduR$x&Pr)bf}ulGB-1h z{!Tnw;DZlLGI`t)hMCLrNqFZ=qKC)r-o>neUEw1N*{_Yrbw;M9D8GTG>sfWW*M=K< z$Y$qrC=#e|+o zq;gYQ`_=o;+g}Sq}S8I3kPrqGTrg63Rj2_g9c-TFnYzjm-LUud!21s$fH$w>?v~L zG34z|3CCBo81DV;(KgqD6coM;LiEsc19gTAb5<_&`UuBI6*|jzQ*8vL471`G1*2O z5qM4YNKY*u8kchjdY%d%~mU;qt6!|J}XBj|&j5-6^CQ{dRj>WK{n% z7&dR(-mpT<#npHo2~)Yi@dnTJ(2taZaM=DKXv|#mSj^-|4Vi_2{5TP#ltiCt zzf0O`jZ&sWpS;k1n@U3aWY~%fIWo_YW_lTYm_Rmi6$8-Vx(&MgSsxoxKf5x(_L0pQ zyFa^L$|G~-_Q*wN$j;bC&*D9}*x%3wM5PrKay)4O!}bD5hdiO~vsu^ae*_hwUkCCN za>2lX_~+z5Fg?#_2ftz(G6G#r&hk~Ybl^pU-2{4i3r)poK`R9wwnC10?`l_VM>pi9 zPWN(Obj9bJg{ebj=G5e1jv~17W;YSLG3?h z&S4gsxu514Vha2U>RE9}cK7z@z&%)8C-_UUsnUYVP5I@!=LI#JzAZn(m&aG{Ek<3 zda^a_LX?R((qYDDP4kJ{;vvPKUAj2}9cT&>NE-{la&-{A!}2eVtd|A+Jre@CyQbC$ zbwg-iXQ$sz(@e{5qqv3SB7+qxhU554ZH$jdI{nEyG~PgJ)7G4%XH=S~y>^sbc^rK| zz;ETSzjv5ANDd8?{V%R01j0bF1*W!hL2}7ZtXrXo4c@F}kj0OHDmbrzdcQEkzNY@* z-_z(#A?-#(E-aU`Ohd+r1<7#2IX8yvA0Z+nh96(QXn<>W9Vpn3jnL_FR z{mgExHv}wkVKAJ%#gI~yf;pfM>0=q~#9H(Pda-^GecD1rn!Eq07=Z!2_ScJUOJ<09 z6QT5|f-YdcUdo;MS3rnD;(L)Oo*%8Nt4lL`@~+V(R_>zbeBhm3z+YD+^v&)g3e8SRrOMPzAOFy zx;bGt5ZJ*$>N1;G3fDE}3Tn+9#l6L+>4Smx7 zom?G9yxbsNq%!)?AAo>VUYmvIT7D9baqO1ZQF9!tEZBoGY;@xyUU-!;Fb9DzPqF-T?oe||&}@|K-~RTTm_>k3UW z)KR4L$VL>aQ!ZtLyN=6|qZZCVEo))N8rVetGngvdG)-;npf~*pC^Nt{1>8H z(i=b??gU0eZK@Y;CXT{V6S7<$Gtcmm^01Od4+NDWA^Bco;fp947Z}h*o2<($ak|)$ zz~9LH6|wUgxsq&Rz6Z2D3t=q(J0>rZ1a*y2XSa}=j1sBIkQizWYZL|r*AcGXa{zLG zCD=jt4^PM;E9SKIKXaI6>HG27f3ZLZEmV{oL{NoB!*a!4Cf>`cHxux%w!azG*TUm& zG9>8qmB#V{^F|GYOH5}644x7wuBsFXeIXL26z~3L zQ{21EF6P{YKzFaiHGgp=!4CeqSKz}A8JpcuVOk7>MBoZ{h_!T9ip?5{BmZISY1LaYnYHD$ z9ztZB>~D52-q&Tz&B1e3;a|NXimru=X?er$flx%$ND{i=F0&kmOK=}fIp{8oKnG%4 zF6lfs@}H@he;9i;LWT+%m1a34Ig>yy$*y6k=&8z0|4O-E`Aqt3IC9m5ZFp}7?8}iG zg^fT`8*|aL?|-8RZ~f9A7EOBCZ2iMX6bYh(qB!b_asqmHZRF?KLA&%QK(2=GjDWH5 zpvJFs@DN-@Y*U;ola0;d_3pf)eEo8kI|~VeDuf1mWd~5GMEB@ki;^ioO58#802K7E zWuQ7YjTOl&)x8GwVD*IGpNYy=D&t{Dqje5fLG?Ixt!murq>fbU3>+?pk-84Kavd06 z!zr-q>KxnYAEhE_lRn%Q`g<-)uh&5N7}@7MH3J&eNag#s^*m4VRqT6A z#b8V_28b7E4(_d{or0B6Dfn<*qL2camo-x0-b;hEPILGxEMgnXWk!TX?ouFF(z8a7G`@j9vu5%wv-bTKV>%+POw|Rq=v|s1fbwnEC-~{XO@4`AF%+yga-??>{CO& zE81s00cSruY?bB#k7?9Mtp}XL6At<@qyXL?2i=#QlV?lz!7aWo{ih6C6Clk>5wwk^ zSVOzIO*|8sMX!!k6+z%_$3F(JwQJ#vIp8Q8VK_l&2%`^}eD}d-whWxj#9X_*BZiNu zfYtzqGQD|??roF+Y|x7_cnE$3@DpAM-W5GATC5fxt;MLQs8k`+7{wN9mQEFO`;pT3 z-_s?>B4?3O6yFUc7LRzZH||db1W?ByI*b_IOPQkxvG5yL26!x~r-=UW$DDu~#R<8ogW29mTkj9#&%;@nR(IL18Xg<`tuJzB+zaVFRtZR)JN8_*5y(4xf6@dldi%Y2$S~QiE_3fM73sezMl^IzqXp!QwJdjv7=Lj z3<}8+gEit+SV%3g^X0qABeSySuZ9tPlS#mf-&WJopnMEQwpHLG(mPt|?1yaU(uBN_{Vc|ZBSh6i8{0l)LtyRXl`-jNGBASU?yizCCg)DMd8jYZ5H+|)5m zGFN}|znRq}?#ekbes}Tpt8m)LqF9vS#Orm7Vrc2z4$sd`KG(a3bfAC-n^Hg0+v}13MFpl1ByD@} z|4G|ANG-w=5p-5r7@vSUyO^MpSbS}o#Vrho^U1`&>?8h@gnj)fp+_C0iXD~U$p3#1 z#VanT;VfH78eG0SAXsM2kCpR(^uM`Aumh{^cdd={LEBu06?BK0JEphpM`rHhQGqp8Wzz*t4ewR%wt#>|Z5Bih~W3au3HO z!T9GDtwz+T<9d;9k=ryErgW2YBGYr-s>EnHkiOFP%YR4WUG?uM!}p;%w0X&!;R70w z4b2c&uYAl~n~mms$=MV9W3g^kM`=z6s0Yw;ME}3Gt~{#AD~n4SF<}ZrAZi&FlMq4_ z3Tg)$D4_uo42uZ?QBYZ`Al9NH5>(0-s!RffMnED8f)oV70jelL#Ij8W#6bsS5wLUNU`|kVR^1Hv^efQcJn_|pJ;8y6dI-V23QPc47ZRaJ1 zuHoB_5cfxhbvJEIMoAdyGP?B=m`XM1uwSXSZeXjga4 z`_|(FhBaELox6HY$R4zVbzE(%fxZPAbQUMt{Gc}?bs1wtnS+|<4pLt>iHG%j`>k?% zI2tG&dEOR-W?U39l9Bejx^I)<(mH;5{!bKmNWWSSMGET$zRGYELb<~sy8+a9_m<39 zn!J@HEj^OF3zmw`nl{WWqFnpi9<-iv)s!?O)C_F*p?dI;?rAKl#(Mw&jmLM@*GBEP z`7|sBjuEDv%OGy#r~b41fgfQyPTW%Fv&A-ht>uN!k52WR*hvt{^|>mDc6aXCRKnv8 zOpNuN=_GU}v(O=?;w*;}47CFfrX_vW!DzOJWf{b_l@hUsXF7m4K6u*2|ZI=C{efohqC#^rr#NC15~M zIN{L_v7w=apSTCatYK^?%@JnNp2~!4Qy-h_mO=BrIoK(e_wXtgv&k z=r)0jK}32U#2~u|Nu>Oc6oeO>oxXDd?ZOhX;wRMv`o#7Awb4k1v>9dP+9$x7-*k@}%7Dgt`MSdO3QmB}^31Cp$uRE#V03lwjeNG5&BkH8DCn`1LX7P7`9q)k*v#?Wl*ef=9o5VeB zYa$}(T%Ff(`#NB^8?aZ{^e5l52VDKRBr-tr7+Oo#^V@jP9b7`NfkieKu+ILX$&4)& zX)4Q6f5Y*cYh_O@N*t^%x!2_Bjf~Cz9Al02D)r@s!Jw@21NE7)GvS=VJ6PVsjcig7 z2vBXZ^W&)i_7UE%#!I(3f|~oqbi+qx_kMh}$=@9F-n77|Rr$SU_6-p1a!ytKanZYSuR~HKjqMLZkxkshvsb9d$foq>D zFc}AQGHb`silbaye4X$fjw}ouxz%n}*Ye<`zoqJ#b+t6InIX)`e2QSo#Z8EQ+mP7+ z0d?U(N58%H_d_d&xtT?PEcH0q@W{cE+{lz&>&@=fZ@!v=%uy1QQ+2N=ueV#>Fgzk} ze7&k$4|(kbu8|0@PwRt*XX8~y4Z82g5^1ZV886&6zNcag1b!vMb(| zflMHSq8h_8=+>X)=@3QJ$oqG=HF7Y=;ka$8^2zJ?FxW(@L72*F{x}b!BT&Ez788~y z%+8Bw!beXxuIVikN@jKscQqmthWLp&%A)F7pt9bRx{0cu5EDg^dZrK3n!JKkwN_;T zdLx{FTve$0I-doR#Nn_m*1$>LO$&n?E+PR}%<{)UrPYT|Zb0_g@X_4k=?lL-zpHT1 zTFUTd66j>ar|3o-@aI6`>PPYpz|6lEzRc_t4y{^+Iq$}H+JWLwdOUL(|Gr@J%>ivg z+o*(cH1^<(n8y&H2bB|&hpHdgLd`>z|EV6*&&{{!^X2*0;4ClKKZzbVv+sNMDY{>G zDeh4^c&k0>L4!ZgzB~pHdj5xmcg=m;>1{4V1;+~&?Or^({O(J7i<41Qz`L-I#V+wi zkN_eCg6IBKScM-1y6#otM>Z?| zr9W9>u*>Ci`4vuq7qs$6t@0RXmHz_^R@RP{&rsU*dpECfO2nDlXI8<{9 literal 0 HcmV?d00001 From 94ad30e52b348d3f5917dc101c2a28dcf26b4481 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 12 Apr 2018 20:54:29 -0700 Subject: [PATCH 09/15] fix log service, add docker run command --- tools/aws_benchmarking/README.md | 19 ++++++++++++------ tools/aws_benchmarking/diagram.png | Bin 41741 -> 40790 bytes tools/aws_benchmarking/server/logs/master.log | 0 .../server/pserver.sh.template | 2 +- .../server/trainer.sh.template | 2 +- 5 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 tools/aws_benchmarking/server/logs/master.log diff --git a/tools/aws_benchmarking/README.md b/tools/aws_benchmarking/README.md index 5fd586cc1537e..dfa2a5f478400 100644 --- a/tools/aws_benchmarking/README.md +++ b/tools/aws_benchmarking/README.md @@ -50,7 +50,10 @@ Training nodes will run your `ENTRYPOINT` script with the following environment - `TASK_NAME`: unique name to identify this training process. - `TRAINING_ROLE`: current node's role in this training process, either "PSERVER" or "TRAINER" - `PSERVER_HOSTS`: comma separated value of pserver end points, I.E. "192.168.1.2:5436,192.168.1.3:5436" - - `TRAINER_INDEX`: an integer to identify the index of current trainer + - `PSERVERS`: same as above + - `TRAINERS`: trainer count + - `SERVER_ENDPOINT`: current server end point if the node role is a pserver + - `TRAINER_INDEX`: an integer to identify the index of current trainer if the node role is a trainer. Now we have a working distributed training script which takes advantage of node environment variables and docker file to generate the training image. Run the following command: @@ -73,11 +76,15 @@ Training nodes will run your `ENTRYPOINT` script with the following environment Now let's start the training process: ```bash -docker run -i -v $HOME/.aws:/root/.aws -v :/.pem \ +docker run -i -v $HOME/.aws:/root/.aws -v :/root/.pem \ putcn/paddle_aws_client \ --action create \ --key_name \ ---security_group_id +--security_group_id \ +--pserver_image_id \ +--trainer_image_id \ +--pserver_count 2 \ +--trainer_count 2 ``` Now just wait until you see this: @@ -91,7 +98,7 @@ That means you can turn off your laptop and your cluster is creating instances, To access the master log: ```bash -docker run -i -v $HOME/.aws:/root/.aws -v :/.pem \ +docker run -i -v $HOME/.aws:/root/.aws \ putcn/paddle_aws_client \ --action status \ --master_server_public_ip \ @@ -101,7 +108,7 @@ putcn/paddle_aws_client \ To tear down the training setup: ```bash -docker run -i -v $HOME/.aws:/root/.aws -v :/.pem \ +docker run -i -v $HOME/.aws:/root/.aws \ putcn/paddle_aws_client \ --action cleanup \ --master_server_public_ip \ @@ -111,7 +118,7 @@ putcn/paddle_aws_client \ To retrieve training logs TBD -### Tech details +### Tech details *What to expect in this step* diff --git a/tools/aws_benchmarking/diagram.png b/tools/aws_benchmarking/diagram.png index 9dd656c9b4719fc6a96eb3d68c796daa0aaf7b98..b97909c5fe78b59d0e636ff73c2ed3e63a0be722 100644 GIT binary patch literal 40790 zcmb4rbySsG*YC484I(YlNFxo>sibr%5)z7pq#&`8*a%1qNOyxENJ-Zg5Rew66a}P1 z>8`sr$Mb#fJMR7Gj4=z?VNjA287wL_!4$ey4 zHje%rG})DMYm>l8i2vgg!sXBXjP7ATI1<-VVTKCw2~@^Zkh}aaisP!pv3E)J_Rf%~ zz_O+XXZlmQul@6^|6AN;4cVLaPg&sa$)Y(t=qj-Ped`5rG{?c9NjE78E_>E4MJdkT z^;DSPqziX%O}gCscLA^rDif2X=S-ZhQuRN6$QX)V_={!ls2qv*v1(Ry>^=`KIi~o#83t0Y>}*~)3kk%i^Zp4Voayp&S3xdEDwNImJ$4v! z*Z?m4Q1JfWl^!uO>d@23R{!16-+d+Fsl;_fXw#ur`=8H$Et_9fr+EMDuRus5@ck2K zUCTGPHo1jk|31M&2@BnxPjN!f(87IXd=~7XNf<4bGl!VUlcK-xJxoL@)cY+dC9sOZ z3P%jyV{Z~7JD~z&8#63t2&Ff6;R<^I%OXNP8?#e{d1GacG5WtPNVQl975VTHV7oiU%ZInQ1HB`R$kzu0!83W|Rgls;FIZ zIrI#Qjc644fYm%qvKwF~<2=0g*M;RUa~Oe@W8D%a@kbJ?yl7DEwlB!GFCV_=$t(Uz zL^MFxJ~&V~>W+6ikeZX5dkOA|CHI~GB3W%)3|Mgp45G1r#R}yz**cf&G z%zcir{c9*Qw_u?zg`vf=B+jppnIi^iTiVV4<9K;M8RL)s4J?; s`&DQXAP#`C|z z82BDFG0cxt`#4N?N%4N@Uq|EuQa}7ytf|Oo1^>28C3MH_-?;Gc2GC1d{hR#Hir@t6 z7ma`ZBPxqHunrb8$Yr_~Gp^;=_o5h!eJ? z<$VFd**DNO!1xFRT@q$J47AX4JP?C1?R%)42F38}modB^ii2-bBJ*3SbzgzSScJjW z^D=zJ`QS{%SpKHa82peitmf%226!$swZ!0R844nBCHf@yDHt!#ROME<&eM`tpmQRW zxnBzVJ0`}Vt8(s8nZl6LJ+$4Rgu&wPv+s2=5tD(|49}#EqTd#gO z37vi21SG>11WPjxX4q?4f=5E78n-q6>T?)mlkqoykG6Gj;U?sVF|o{7Wf^&?P`MoN z?^0KKnU{2CO`7_qZ{bZu*Cw*OY4~07*h}!{QJ?%AA+C%cRD&1es}=CzQz~( z?{1h#z;5y;QmnU zQR~VFn53_PZMA6>(W(*yL)jWvc#S>u&5Z3cH4aEplDK*g!67Rs12dCwe98)UB#TyY zPAxyC^!LME26-BrgxJ3h?OZ41-4KZiIz3%`_19`yNYGx9VqZ_qarx%wE4vha{lO(@ zFySAFc6(>No54UaCX88Hye7!iTwb^1uqZb9g8yHUS1(^YG6n?J&_6>9jP{F1mQNYs z8N|^lo0wJDBrtio!ZMp1AWU?mgB?#5%3+7dHK31|AAL4xv;Qvy5!PCuR#Q}FaG0#0%QI}GD|AXYgCuBSmrs=NVWqM%lnFx)qI0|VOUn$P1wLFc zo}Q!H12?D|WBB$$|K>k5U#aAv_MBPiC=; zyDMA>=+v_Q!B5ggamd@&LE;ow?mn~w>0>!dBPkL*3!MGnsBx$UUPpz}g_dB~F=w#V zCK$AEEWGAkDTtr=3{);We-}jsIbzAZP%?bnvRGF1tb;n#MVY^t3>@17G-|k`<)Da? zhLhs`so0YrEQR!8H>w4Bip~E}qipI!BJ(PhpM1dNFsNoOnM~f^K$M|a2 ze-&b=S6Mt%+S#b;?8lk$g_NN*Y-HPohF#N(5}64P8UO)Iu(PC*!q6+Ijudr%o$Y>M zSqG%00pdv-2-$zGDDN@BHG!FWq`dVz&Rx#4@xes#K{5nn@fy%>#jwg#xQ%V-z}9_y z0y7%>{3es9c)JT@cI17jFxp|OEoty@L(-!&(-^^iM=Pag5w(jY_16@=wZDrrec$9$JAk50xi{AF%<8RlZ z5-}ozty{ApTTg{PRVtwEj|qf-dU2s@Vi+GTI8B0{{VwP2-FRI{rrM>OlZEpYLscLv z^K8WVwGT|a>N?p;XBa3}DCzM)_bujG?)C6HDQ&m(+jdax(o&;!R10xhSVP~!8mw7* zlIJv2lXT@P`f$&$LT4pf@aucl7Ab@$<^xj9i^kmk0|z&hKFMDc%0`seR!=;yp~s)V zIr&v^e~i7Bi`&A{kL{iwT0U=8xtys&GUh7u-xX1xwRkyDCk0!M_{c@b^9{bJiQsYH z*Ep;D3ZAc7Z6qI*33W*(R^N`}WSHdsGSDsYU7)9lv;TT>B#oEY%wZpA;h=gxUVMat zIPQomWGC|)6Urc_jK58@=4UJKq8$$Ueh_xzez zbPZ`G+Ba9nH9AA&T+q>N0FgM3j&3B*s&PVoX)(?P@A-pM{Q@|jT4QjSr#H0W3Mq;+ zcV)*KCfyPI5{9o=_orr9ful)2#>T6oKYRLtj+Nv2mH58SAE~!B5n}mzp)TLrZB8^2 z_@?Z?{rGlEqE9ur9@P_S?WO*$rS5J#(-{_*PjF#>TKR*m+W|yZEI-9j4`7K_2rjUW znQTY1u@|y0OlI-Q_}$ zWBh6P>z@jnX%FnxLR>Md4cBIK)7%2*aJd?^Xp()~zNT%)nj}dMxD9fM?+i?e<}OZo z9*ZH^$e#jmeOD6aP$ZKFuYaU#t@@V<$It<1y>Et((q6{1 zzg@@F{_zd7$16L{Da<{it*xzfx{uY`*Zu7#x2t1GUJ@Z+a^wqk$np9Ug^Wky%sQGk z()QD~X-b)?N=EN}1-&=uX70qYyZtVue zxL(`u!j+PJn8nk_Nt8aF;XhepI<~@9!(a3Ugws?I?tbLGYf)tM0s?=jQmaXrvi!VD zl+n<3i~K!ZS#L5lP|I{k{$n=CmW%u35r0@=J?&^@z5ubl9)L2_AOIYkk$*<9u!0iP zPopUo*if&KA@e0HpCJJ6Y)QhV0$F#)6@EP5wnRprivV$53Hg*4fE!7W04Au4L1pUS z0bDg>!nNc`WBn-JgNYayJnEmsMwJ|Rnf~v3pc4{;)$R^Dy<7wS@=Fusqd$6#7z&IJ zLEJRD&NM>@OK@wtO?L2Kzk%=)(tofq%m75&#+Ucf!{qwFi%Es< zC`%5laJ~MeZw+i6d~89kvC}{!U%5rh-Oq}P5id>T3(GJO`<9KkEVyV;lB`mX-q20( zW`nKHX4ihF%gl!aZ?+>0LCT^N2g8mF6_Nn<90Uvow$J#K=vAS64*{@IPy%ubniliG zO-Vo=4cnESBgb|w-7CXJzJn~WHgS+@O`^)oUMe8Bdg2<(gVdX+C6{BuC69S}7leyO z1ShV!5Rw^sjUgs$znlvl!7EJ|h4f&ofmdMrfMd{qOVO z1UrZM*TVP(eF>28u>)%(P9D+)dUZM4;Gxq_l9pGXT0CR`swu@pD?h)?Q3_+$RM`ER3DMeWV~#IZN`#Pp!X0$1J0x_dmP2&jNZ*3zGz=a^?}aT z=9e(2jus_#pfQB|&(Sh%%7QN#;W5o{%*^H+)e2hB4S~g%)8{{T%yMsXxwI>p#-J%9}#p zAt@qr)rn_eSBY}}&@!PO6LDd@#WKf1j?=9g{J31}G4L`v`J&efC>@}xP{Ji#$Qq-L zy~a^<^7slw1Q#Ph5!~;;YVk6vtxPYj?4S};HnnRwN^c*tnPH&a0kSeWn)9o#Pk5M;vY=D8f2UQwTQa&WbVYq&eo?yll>XM?B zO>|ROz@UF!lnoT7fD#2F`3O-6g2WDjYjvsqn{8;Ub0U$lp8!5_Wd;|b@9LsakYC_H zwm`?87Ah$M^f*J`sZSJ%x-M|#SfN_@p)66?s5aUb{v$A5FlFt=cVGMD3catwd}zrt zXOK#=Qdq)#?E)Lc;rDj4yVL%eU?dTqRAM?O{pqUWEGI`SjNj;72B~CvrC8unG%X<7 z*)5t%Ma=UXS&>KGFUN`u4x(=9)J%--Zc*m&0FE_mY_5-|o-P!Zv|(i?ZDy|{EW z6>1>QU7=AIeN56Jw3&9so~d^4|9*t24{_YynbJ7zihL|{EBEv_Or0o;!8CmP5o87DpqINZc*&H|qOm-#k{eibhS)y}^2VF74L%N6 z7WFW$eI(e^%1)0h#$Pa=(?p7-g!J7}?rxVAv_HD6?zWBna#xOVa@auCFlM0ulDWEh z`WkW;LB4>mV`v;0#fjF4YzrQkDxxF%n>Xl)r`ecx6@ry1>PW>%S>_}7E4}$MUUl$` zh;nRS*&By7k;u#X+OF;$KTgrQ3(=jz<=ijB(Kt|JGu!(u1r$AaI7a$iTAA=b!|lrS zorf>R9y>sKB|=}vy+n!|F%Zq?UdCS#rfhlts1amLDbB)&nm9R zAwn%d8BwJ)W;OdpaXW$3GHJ~lYSe&>0~?eA1m7Q+B_YsHw5mBVvKQw3*i2i^kgLZG z2W5TRjft$W`PxF$!iU07!nb33c+kOG3@|+|rNkgn`YWiJeSYc?X!&Qa_gulTw^$SU zQ|-7kPZ$?!1msICANX7`)0lZBVekye+at-rkQp}$b6(5y{-uI4?-`*X=0qCL+?T3V1d6z@>LCtnh21)gJvONb&$p2z=$I&r?@ z@b&HM#$DLw-~PSVCJ=df48E*DzVYOkho3c3YC4v|doQ3GnCFD%lgu$&%ZIw{8b!w5 zPjk)swq4E;;t6JlJ(s#X1kD;39(*)xY{iXlyKh;Kfm}IPc+y*Z8{)+onz-)}%$1*) z=PYw?n0TVfnn*Q~H}XfLM^_)4S^~G1Y!?%ximgd6wT2LcevV?pu8;xrng&iqw>USr+ zc-$HBF-rD4#I)IKvY4TG_q$W8MQ;L6-TpT_huV|f5v?z+etv!FlH=~F5jR!;?Q?Yi z!WogqF4rE89K_wU`lJZGf?b~#`pzVU5#WbM_{#7JBh`#uKZ zkug!-{LugkRAzk~U{22fHA=sPH?#@LCV$VovafLdQB{Lk0k_*c4Dbg47uKHSXvZ->a}_$L>oKz%Z;5Y+TH&bXicnZfj%a9nG#w1D@djVwjb$YOC*Dke`g3`APruoFk+abB08xy{o8Z*2-Fy zigfcg$Bdn3!Sh^%|*#-H@Z7~aGL)4b#1(i0iTlB-oEa1yH~*X z)N{&p@Ippm_F%V2HU4I&qYUPz^{iV4Vv%Y`yWDP^m(1TfGXRIEQ$zHJrM=JIs6kA| zhIIzQs$u^eoJifl&lehjXJFY+Z*_8=SMn2A&-y6eJyb&DS2a1mjvUrPccvL$SedAo z48aUGRQNS zbJCZG{90-}9`Z|QK&%~i57@_qeOO+(-%h`%PQA;Mrrlw+i5dqQrPs8fUVE$AcVpRX zMvEzAj^@dvE>3sMy}F08)N8dgB)q!UMcK)EnTfD|;+OhL|^r4j)$=_mNt^NlLzK1s)>t?ZcqKC>YG*szY_RviWSZIcf8-p-G>ZblM*%qH@MCY_l2@j|h?oVCY#Fe4I%ft_S?i+TCvU;{^yGe}sA9dS4ttPz+_VjH$O zqAA&*7qa9u!uJ}7XS>TfO~4717`0MvaaN!C9Q?3W;LrUkl^^2( zLZfYV+UD$;pZlA3BNW96|7;FSbgJh%1cD zLw#6gO`a8>ORd82TDLvr!aVRb@a@BvqB*SRlbI59mZcEe*L7swR`r>O!^t@x>8f7; zWc7Nl?%ABKqIm93-^!oie1h$zK_>-4MZbENg$Nryot!`ioq=wt+^@8$L`1*|g)UlW zUJ|Gm6FB5*Ti;53iP-qr+{vf3KBCk^Rc#%Y;hw*AQu(C!{KXWd zUiHX0A*|q3O{#da4~cVqDYy% zmFd0x`MJ*$JJ!l%ZybFK{Y}f^w>r$tnTV$M!fBvVufp6_?la6-)xb<+no&B&AFMOJ zGNS`+NIbEjJzgOIvk!ix*>mO!Eb>Y&2x8`mh+?7QG#$MB8h!~n)1=^mTck=elQ^G5 zTnCd=2gc=ta#Gn-%S}H7`D`js$y^V_@3N3V*r?<0MMtU^%_hwTUC4TFnio4z9c5%R zm-N=``V)yrkP3WD)|Uj>suh&32fgauS4qT@ks~zH8xa3cgUV7c)P61tRl#-%-Vnuv zY%?BFgqKRak0J4ORB_7z-KyI~k5p-@gBUf+Xp zE?i^?IDVbS)}sHstPsQD@iOQ5ANT+O-SGGXQ?vk#eb^w03)L}Q0hto;um6Ek z40X=QVEfD}1(HYFNiowDFn)?6-Dp(zeKZzy4hO8XvE^tPZbzb=Fjs#-;X7{%GrdRD zXVPfSg;tO6DJ>_APeI({oe0hli|!nnUy{Ql1+yLqrh#H_Rd=`+eCEgMD8`kXa*Bne z80Ha-b!BahgFA7xwIb{18xN@dDZ^dRWq1oT-{OR)klG@MgTjZjj>f9`2pehSoYv~v zetQm69&Cm~2@v@9&bTv z+miHAWz6gYwreX&A)j3_T3W7Exma-D;N;zb6^D?nULosW=JLqK4p?o20 zRx#yzcJx9upp^Nplpn@4!|DGahvOZqWg+bPd=cxw)iFFdF0LNGj>-3%8Xu2|cmm)(`he(dqFU7XExEGYW3|O1*ORp7^JHZ{pT?%-stH#`$ zd`1t^&;19xwL*GGjPin_u#yJppd=hKdK3}R{G_^vaa?VM%#m%Xz4<~JT0l!Mo@HVO zC}!oXx_H^2)F}0fWb5xa2C#W-=fdCDikB(T>&VzFVs6%%(I(KL*CBQScDR)Ri{yi| zlVuTG_w+=x=tyJvGQ^RlpLWl ziDP&UheqEALt&_@^lM<5U}-77FYCwbjMY1;Fc2jf3$m$EoW8{@Qi^sB~MqFC>?Jrz!fD467~;7<5T3l!jDchm+ML zUn?$8AGE6wC^q#)wkt4G>{~c;vV772k{hVDcpvdQPKl9oXhZ!e!tFC31HLqg2PFJS zsCeAWc&E;YE6dj{tUg(CRDw)Az&yWFK}<0KC7{ zI+^XRPm-x0(aZWu{i$D~A6ZYf&Jf7SEZS*)aiG-dd!~Nz;bc(u;=>b+F2bXF-?L*H zh+C`e(|UnTOJFY~m%%cBGx4$$C=YDVh!*sI{2d|BWKAr{&swmAD`y#iWzMeGbn*jf z;%Im@^{k8Tt$q395l|hwmcH`)i}`g4r}uvhKaRucROgpz{wa*2ZtZMCA+q5Kq-(vPWl5VA!c;4;(q$6$a zbpN{(uoP@It&E6=jqb_&>WQ2tb>$@cla)`}n-6~4VfqGO6YBm>5_`t9|NDC|fT|Y2 zx)x{6e>AvRN!0-K)fn0#v^`KE;y7hvYTV*oUr~D}2%pWx6qI(%$F2)KL7@0~^0fV9 z0FK@nm~!hMr8cal9y2_xRTxwMrvY4nZgcIQ_B0-n&AjjY1WCX>`QWZ5cg@7z*B{^q z9p)&8Z_WlYDL0nJu=KT^IT|Lb-F$<=# z2-J9yURTHr0bkRQJ=2q0=b&;B;p8Rn$lgEr;lYsfeIP_sPh%pXMq4TvUlDQWxgK@4 zdE1}loL<`N`uFQ}3?lZ|d*V1Bo4x?pIpTr!YCol!x5cm}y3o=!Uv{+;;jg4GX4awq z7*=hz#|skJhw^C!;Y%alLht6BwzvI|#0GE15S_xegyu41toxE;+HAJpu&PB%zi_9O z^_@~EP9_+*6_e`N@Hx+8lWRmQDe7Ua5+QRrXNLTv_?Q)dV^cWXGNcgdENMs6aEF;cg!yjc{(;Ok}? za2eo1S9Z#+gPQR!-W43@tH{Q)`HOOkcHJZfdehbMG9sn?LCFoNSl+f>cEs_ZJnJqM0)&RXwZE7c)S!Gx-C}y$Nv1BJC=8NJoP7t|Y`n(b zKh&_4(dQVJpA!&dpy#wc#vd_HqG9Nfw?3C-Aiw@4Q`qjBWBtOl)6<;`&8a!7dw_71 z@wO6i9g&(P+fLx;k%O1 zFLw+?y*h1-hi@fvcswb#hF!-uulc{9p!1i3Jh2fD!Q8PN9=Mj=KLF)=hp)Y+&+sXZ zdC>p;eF}fA(r{#5(MZuR)#Dz{2s;Nly)KT6GEMPpU$`17KKM>Jb`IfPlKp?NoDAdU>-b+^ zJ}VkeX_{F)_eC4m`;p4wVow-$@o}3R^Hi>55Ah*Ti{tiHuyOz&;Ccls6L;5 zNaB*08E8#dzb8>-)ImG(YF`guA>J(v{_byCosmx+DQgvuD51sWU)O32`S#C=7-Z7! z#Pp@A+l{}EGf~&k~<+qxpco#^Jjaubs=g z&1@tQeZFdZ<=zm9V;A81`guMyaf;9@=19?h6)wPKVFSj~k!FBc;P$@(Z9{eKla5-G zsUs^~pf=Jue>!$ofW9FoJLd-fi~aRD(Vw2}Q+YkTi6ZtB<_bH25qqxdG1nC(O7i@g z?{wu8H8eM%P}3XF6-*>^h?*MuR-i%Sz57L3%yW}5Mc5A8w6a6R3!SDP{8bbLFI zKeEoG*-IGY6#sW+E!4e+=wb2uTlIB;M}Zafei!F;ziYntvEu=J(Z-(X{ck)u#BMOT+gw%nG;n$L75_ycR=()Rdy^ppRXY>BTqzILz07b77Kmflz_>ycotRabyWoRYFY-scV7x~1J8Dj5r+nxiQv3j={ad_DLDdK8B$7zf(^91 zR#baW0u!0yUZVNUy7kIlb%ms(Wp(jv4(nBd+`ehQ3!l|B@-_g{M1CCr?oDu7Z-|Iq z>N-Gd?P%(&PH4D)Hvy!LnQwMwpU{ZJG*ig>HAzW?Dv?ks3xr>hr@%w&CC@ zfrZLmg975$hY?Movn@2jr3L8wl+sSICw6;WPct0(@1#72=FD9{Ar#SX-K;1ma~`M3 zJypNy%V(--GqsGTp2UA+p>d-{jbc_^+JYMElf|J`U#8#rWS%$wlOPhaFcth?(Hw_^fWo%%h#`=TNJ=)o+=1`azMj1MR6=~J%v92#8-WJ zvp12zln`T65UMQ}+#%aK{lPYNtVz5k36+gF9CugcKBW+ydniV9K0s?cSNkvr_29t3Z;Xn)1F z+W!`jF5%V}0LvW&FMLl@2YFk_tWu7g} z^pzM8U!rxLp9K|ED2UA|=^csm z3=fGvhDi2WXZl!ecOK2$_8(~5Wn4_%vGXbcI6wd3%S1wh5h!OLVD5G)sNKdy;y{~n z){cRvKiK7%G|aMz3k7pI7-q>BES0Wc-bus}NPqdoN~u`rCV7R_-M}`1sA0&A55xn7x&?C!dwt#t}T&D&DG3fb*JDTs(DsLliC^~{hzHnJ8(bE+Sq;>T8 z)nbfFNgn`HK0crVe8$;F9Zb|zKPT%v$4tdbY`AE(DeAD?dLvhAXCi?KVCvv z2V)zE-qm9KCQVof_ehU7b@Le>p@7g>Q6(5X0Q>1KCV6_e{q4Hztdj8@Hyr(&VJA zC7jTIYp^K4yhY@n1qM5bj;~1A`*G+F=n;6jnwFV1_i-O@^BLLY?o9CXS-J z0fZw%xaV$0f(e%Mtpb^!ue!n&J!{G-gT^0T9L_~~{t9KMW#DD$Ss4Ls8%n(ksqp05 zldZ*$62mI5Ey?G}Zc{fD`}2|_3?7q|E29W?&he4IA?)u$x2w@5Fk?L+KRMon+n}1I zUFKC(PxgQJKcSAm29w(eE=E3F3p&$ZUjV+~eo&4w`S~$i!c$n(i2;DR>iV=i(4<_< zAyWF`@n_2RTx)jFt_%Ugb6TFur{iwA8&$i$b<;tq7O}q#;&1Bl4)G(?(Ee0xfKS2! zo*SK->EaOkeuY71%A|aIr7$xkT*Iiw-j-LlemN(+uXogsFXkJG~BCxB$P zbpE+Nsv?haxP4;=#!RegHVL|ij)0wcu6TsbOPHcgt&j0>oJMk^mxAcXSDP%N#-Aqg znb9}lm4j@}<`C!u8cd{ntE2LAE-LgNElTtzatLW6EOn8utTYcFicUCdL;m(R&4 z;v`2!BNH z86=J66-h0uj^FEdzQe$6)-pB!S}R?GLehPmKS$C#m;I#VerZbRfY4q6==Ukb?HK)z zert^>ElpEt)&G>~?A`rR7x_4$;3Z~=eDV0~%I1bHO2b1wZUd%`j~awoGw)o^hfu)1 zMm(_WoF)5J&;tTfdG1rsM>NQvRa^$!dsI!S?CQdg&J1miJvAqJ$#3wfC^z{K`$O8%)E3ej>S)~Q^9F!$ni6|tN$wref1$2JxCr0 zstrM>zgy%=`N^4@{yu=F$O4lki!ZK)ZM=Wjn9}Y2);WNE4fMep&-T9y*bFhovJbQc zV>vG7X`G7MPn3T>?(?yJQP9KxWU*oGy-!yBZZr?V(ZVcH& zW-!m5)_3v=tR%2Ptmlqg4hwHs9&s34l1boN4qjLY_Pr+;q#^T4T1w@~_uKkosp7St zs(q{am>fJ1cP*o{O@;%R{9+)E9L-o9p~oOzS?5<>fcc1&%$@8T`o0%u9wS=&>ysBx zj1DKuPXGE)x4H$B16$i?!SGF>nVq_)fxCek371#mDB%-|w%gsy+F)ge!QT%5j+nP%$e0efqg-tuPMexVipP@WoS zTo5p;U^aj$HXP>Ot^iBVq|(09hgU$)XijrBU?hhltIpCr4OguByaCREi)?Mb9P?v@ zuYd;QcqJ~re-Y#h4M}rg7e2O@hnS%_oJ}7#-(C3T9i&hE%11VV^6uDwX8~qDa4hBD z#({PTM6MZv`muIYR%8@}Yld&v z!nh%R2|*!!V{`y`0}?o5yU&eSdf~U9lw{1xOl3pCe1Adwrf6EOx<% zoQ14d8B`J6=*9LyY5ILG&*&%FjQ_7Cn9a)B(Cf$tB&Z8Os;Duo-zzLrz*QfmV?~P@ zgz+0HasI06b~y-)hrWOB&n9%+Qgy?^ihc{)oHdK?iM1q#S^j9261X`w&@6rsCl#lxEL-^32bT@eG_dANEoZr@RJ-&(R8?bukr9SxUk}3t>s!rkN0M^%BjH9g zQb$9@ms6g9Z+k>~k{(8Z<`sl0+`7fmSuu?nO(>Uo{Y1x=WRn2Kt|%@lwUf4Ob6{{7 z6uJ)fMhr6mlucupL3-v(IUMM~@Mta=L-l52{QGS|q|Vh7R4!wtOP!V*JKP-)tcVbi zjS_XR_i!FX4;A}cqNi;ms_6YPr(xwpa$Se1q}5n%u!!2>do7U-i4imsoM9&XS`XR8 zVC>@zQFpkpvV2|&#lAqR23*4p-A>vpdKQSuTg}uPg^@&VVJv&kj|3$Qt=8Mb#pZM? z3XQtGBz1NXZJ%;+8=#87W_6vbosSPmw6IvBU&ROXLS~5XLUf|Cqtz^L!_Y#{hOL91P&YRa z-RwDaayohdMP$x`q%*9vtp9Wz=#)#LFtzaM`zwWd$~fUpa|i*OA~7cy1@k|vV?60F zcg2E}NV$(Mv1rgEYdyl_COdO_TGD&gpiJe%bpGOxfy05Wk*}ut9AP0xTY3<^tAr+&c$ISDuKtC-|zf10CfMv^(!qumx0{gW8As)Liu(-zaXwF)y}>X36gf(8E!S0xmnt-cU{`o zbhB>zcvr`?NSAf@_jgJPu5IFLm;FRsZvR$nLh28Q!_C>kjUSB%Hj8=q?6@l-`XPz=Ej`#Nl-1ZJ1MUMoo-rj%1dSnSj~{N75*~lj8lSHWn-bD| z5z`FYe)K8K0 zO3G>k4nC-;OzJLI4wDW&3GclDBnsnl`g7YKsoMiH7eJCgufWJ^@^3OYCnu*N7(zRE z3ZSiIhPzSLZrzRLI1aOlM_8aY8w@5LPb{r0FE3u#Y0ZE-n>voqOol$UJ2%ULGSub! zTYUix_+y9`YttNwH;TzmCx?LS;OoKa2QYs%-un3g&_>(&kn!EyTvMM-jJ2o!qbo3) z!wtU+ns_?!>so+*ZnHXr){>trVtZa3^eWdZ$8kbk8uRq|cwml3#+Wk*=rK3|%DgE| zq%zG>5ildIDvOZ>U=dQnPrtqu0bR)EDJwFw`>v{SQS{Q(=&{T1POZPyWS_)ca`ty; z;tfnEE5Pt&LRs_vH@M^+NFUjG61I3#S7!*h9zJnJZuMcI%;KzlmIFk*j-7QipA6?G znjYT;meMP`)j_5ng#Y}RYTUuvI8hT6x)k>=1l;tW*-);eDC8wm6LH-20A<1nC@1dm z0WVxESk#N3H#STr`7$WP%nzX`AZn}oU%F&3{>EjQYCZIY*Vqh0p#DkNCK2!W{h4sR zoTe8Ce`dC#el%>I^H7keT!Cwz`CECRQ31;{x)!hIj@_3GWbW^_knB+ATt{a6y(hb=EZ zrg>sijvLnk*iof{B!RhC=E*)!?b5x--s|LI)BBx1(=RiXj@j%^WMQBAx}mJi&z9Qi zIM7RBU{W!6veCK})4Q^mq!!nBS61NIw?jfP=MnNp{nu}C;M(Y2*uH$hNrc?bCPeFG zwcYrrj1ZTVYnm5X%VLTh|Fd5TaX0VXv`YQzk9&A2y6+adkmQ^G=|0bn;~hd8kq-@^ zjXW&PD4!#BzajL8FE%m~q9-g*?Jse?L-l)_o>3P3?u4Ds4WN0J=?D4=4fl$47wj`r zsVfkg&3I-mpIHwbw-i@i$MBs$Ywq0*H4v@Z3Fvmal z6L4T3WG%PXe7c*q?VI?P8V!m;Bc+x)&y15ph zPv;2+I_V4%trzE#Ef?o6KpET_bH4su@cHskl!W_w_d!(bd&vZjT6wHq`a|A(%(j_Pvlx<&y}LP0u|kVZPByHQF&1qCFeJEW9U zkS+m{E+tgDLzHfQ64IR_AuY{Wx6k)J-+Rs&=dZoT*qi%z$91i1t-0o$OaJohO|&sD zTl1kDk!|MFU6fREc|62cdApz+p?~wyBv7^y-4b zyk7=%QZjz2)Bc2=A$C~OPNPz-zeB*qG97SqWdsPQu?-;H36cLEG81)c+i}n^%~#TT zh*f;+`=u+LhuaH{+R4xejs&(W-lZ$oq#@du07N=;p^+{*Ret+H9)V&wg})N5H;IPS z*&TK0xj+9sgd<=x+`^9%f4#ppvH%Ul_Km7tYf>ZVTQJx5jp#z;;@u}-tQ9Z-``7d7a|z7sczC=&^}F~`l9{O+Y7+Y2BG_VcZPegcpB!3knRy-V zbq)+%)f-*#9Ri@BH-dN3ktQJqD9I}zeHZ}%&g}fvYA|Q$LR*~Mpq51L%_+B7e0tmV z;m@C+b&`w1({w&Pz=254FPjqMTKVVaXL5w*O#i#*?l)j}mzZ^v4Ps@l+R_FI6q8A> zay%|g$)FC6J6Wx2eyR6#)&%W7>Lio;X2}QAB?o+?$Y)>MzA=0(YVpNCH){kA$tMyn zeUAnmT8Fl){_rSJs|nJ|eA~g5^3x_nzaVxrnL3&2_T?yFE1G)Ie|QeXH*LezX`1VDsJmlJweO2AmrxWRry%EZcLYqk*=xW6>q7ICammrcJ<^kHq`_>o^)$BO+Ss;7B= z2^eM)gBAUQLBvUK-ws@t)}T#V&bzB#kFz^)t6@yyVDwq!cut@EcTNoTFE6WXU9t`9 z&c_SLBO;(+7mQgeLrG4^mxw+8Wi;@hdi!nv8f=62d!(Tj%dhz>0_l?p?G+myvyLVn zIJ0S<112>WAvDdi`aC@20``dFYy!JZALjM(N;{k5NTu&g@b!I9K}gVwcKSit^Ui11 zQMValjA1Yx&5T z%L!Y!Wb?IrbKy6)HHC$fju-}B<0ecHagbZ2IK8PsKb#NaRs@EN)6a)GbI|&e1%Tr| z@cKyrJ>oa-;aO!ptt+)1<3q?^&~)QRN3+M}1gxVr#u@j7?cpb*xu(F`kLHv2$Fr5L z6U*eXu6V$S8q%GL`}M{C_z7Iyxp{>(WXJN6rsKr(%Z>@IlXKgCwT*hpWau7}UhU4( z-Xig1U^g@NA?@X6jUjxo-52M^8hi6C$&PS|o@Az(^#}5|urX6~|H^Ffn`RsMZVZRD zxhr--SIS;p%D4tvZQ;V^4MGaL(1W#ox`K0?WP?tVCM{H}+R@nSXej|6y8WBHh|nu% z->a>n8n!hT=O=2uNM9r*+z^;l8F;5h3giZHHxi@T5-cY;p4PwjWwAgALwHPG*&iwD z-L%oyF@JD+LdIRrFL$ofxL0>K`h49Ct$Uk$iQ?D(cPA>1Rd%O<^ywdv z0ewU7SmiI%=WP!WGN!R3q6eSLHC{Qx0ZV=B<=^?>$;0q8ZSyx#VfW$vh~$)qR^E8I3Q&R7zm23e96?9Q(-C&mbAy@6U47mC+zw zHJ#&oUofd{0n|N6g(I#muI@B8v*nn%=W`iQ7OWy^wF&(_h>PoRc&6P%MH*s-T-%QS ze!u&TsG5Er{9jRh}YdQ5Q2kxty@w2(dY1R2J`zRIlYtKV&U;@sCs~D_Z;Y~ zQX%sxj0z$9YSoY$%(kz5eMy`giV(`-k`2R3lC=iDsfFYGcL)CBRDpwNq{qR&JKtxIXmc(EfU8zFk; z=rSpkz{3-UBWy^|-DS(kYA3C&x3TBU1Zyu2s@yLs`wG9Qk6gXM@@UgSTdh zNoktp4T+Z(&(q)?jCqDlZuBFUb{YBH^5?9Jy=XE^+0${k=C)tsuwutXBkzgfyFRS% z5&7K6E>82p)MD(~(0Bn=9{&-foow?Sgtwb_U`+395 z&*cm93P#yU#CsZ`PnszlMM31_ zhGid`P|i=Oa#LbMqpd$%DS|@Ec`^7*@zZS#E}~*~NrO9$gN6S-l18fP_r~YPq2E|4 zKE*FL4nBN$GBKq6M2PgeI@%P*x}3P`h$714KFUZn#Q7w4yZO3Q29)-*c&=>pvsf5z zt+}+eu1#0!&P5GSI1{lo*X{Q9{;az&tZ<^)9~6ZoK52_O_RQ3TK!`A~5gU{fOEvH# zJBckZX~lIRncrEGiDVl6t0Qox@xAbH?;Fbjl4=a@uRqRxLJ1(~-<9$AmyHDco}S-9 zy?e-sChkBD!SB)4hxb$vgSscgB2M^<0- z>896zg&+0p^~SMbV00<6IO#4A{z8fCS6BB$ap)e1s%D^fXxE1VI%kPdBSt*{>4JKB zN!aolMxaQf5Od>rQf}!FeQ=~ZaPhmR@GyKD*SF%|ot`StMGu2Ls76kNkc%_FsV5qi zmx=ZV^F=s^6fr2ste*WON0%Z74DG-o7x*xv-+ZNx_tz}G#aM?*$6+Fz9wO0OgRvi$myt?2Gtb4D zH@m^onx&X#T`XcprF0cIxf>dae>bOMq^sA=fHf&We_Ur`ulxQRD73FjE)^h-OK~Gku}t;Vi7%6h)5eWwJ4ZV*H_aYaggDlh@L0 zvj+a{W&51iGSEsM_ME7og<@Dzq|Z@P?n}L+SZQw6S9DQOLa+4RNb@Tk1%Zr#==E)B(vMO6GjPIff(8 z_I+?c!9P*J<`FS9P=t2wK3oq)$tPkVTZ&uru(dx*+kS5yJA(sFQK9A~eFyniOH9dU zx!tlFeY8jDP@A~wC|zR4IK-32N@&;oOFzqwu`TKy=#bR;=&hC9)6!nfGdvB6yUDNK zKE<5G&j#{N_Tw^h-AY?Zb)e1p3Y4%*ZiJHth#~>&DZ%LBBy@T`FM)L|1%^?9YYZND zndUYNy5yaB^zSZxVwWs2zpv_v?L4-~+s)~BKMkRq`=SM2fH`-b!m4YqI>vsengf+S zZzHbs!XBna^_F__tNC{-ot1@1^pdDr=6-yv_Gd5N|51a>Gqm>D%kmgUo>%-<(HhXV zs^xqHF7u3v8k^P0d4Edo!LiJ?SPmEeOsBbtXI)3%c+=+2Jx0t^`tO5fB4fkx|IVz= z7n)PP#L9L3v9C@Y&?GLiZ(*h+sel(sXY%DVCeAv)x;~ z&7N@S*WB|zzqvOtpQ0BNes@fM=Lk_%)T*7C^P$QOjBRmmrbKmy5_Y1u=5w8zO4y7O zwWUblq)Pm_bskP4-pzk0Cun3~ z?PrJPXBos04BlG3lg4L;>$W~>?Bfi0>m7$_DqwndK7S7#c|53md8?GhIT1nn!>goh zrd1&YFUpUA4lh`xS8HGm)Kdh`WV#m@JnEZkH&Nrlg0vRJ_kPG#pE|dZUl`0+6s8>f zwDd;P%GIgf4xXtBp!@$U_stSmiKF8fKP@ZvSy>kSBdz$P^o3hyuPAa=T=Z7FrZEeVcX|) zTS95zXJtX{!QCf%gtXM@{mm-svR%%|rM z^x{LILVT~x8z^{!9m0lNR5ktLDj-+_XHex4156>8tDgF!&_HYu%R%Df4_wmFqP|qI zo0&?+YMGA!62oPB)=68o+7GSob<LQ+@1mI2z(tg8bJhy z(#%#&bc0e%s4@ajWfAAa@1V_p4aoA;;Q&aV{z;7)OKDnJ0!^T3?WG7;ybfoRm#8%1 zjh%E`OT%o%dbUdxqR;a}p6(K3k4w*~`m!daaslO#H|sA5bK26O8Iar^&{3vem(}xsI^mCZ0$V*%i|Hv`V;69eLZS_$vjCdm3?GZJ79=L{N_qxtx+ zRie=?dj>)c^+aMKw1+y?8b#QVgJLDf^X zGn|rt3s%*$rU0B}*H9Tnf!n6H{f_t>6pefSQEVdaF4cj(Un*k}jJYY;*X2D)N3 zfDk3nV;MZi&rRby=vP1gh9Eu4Kw=;VF00UW463kK`_OjFKQGd&!grtc%w1i9r6A~C zMtzczXByfGgctuXuG)_Le0~=hGK4GhELO?Gc(2~9``xlXBax=MziOsxOiz%Yb211k zz-Pcho{th}&m-cxP{E@C-H`%0GMnEl0WK@#Rl@}!p%JDhs7!k0F449e5=v{sx6$7;r4qVH?(GA$I9$ybC7^6P zs5x#I=-!f84<70mnAtAra!{qyM_<21@&-`PkDk&;fQCeDeRNQ5h7>mFZZ`Ij272Ew zk)sk=OIW~0eoNRA{-o$h#qYRu`2dtv^(cObJ)g>n#7bbpVt2xI(3AubF73qzra#)E z`vve<>t%9%f*4lcS%0jaW)dk8;}^u9juHAm2wx@+M4&U?_iarYk{z$EE6oM21zx+| zz@m|sa$GDwr{3TDfMk;$JL*lY!3c2pbB^x3@gBXahf`5?@^bOH1dA_sY&myXzjl^* ziG`Gxj{_FZfas>9wn-i5ANlVh7^2>-BJXm4cNQu;4kUpXZctuc4yd%$8PU&L#KbjH ztZ*Ux^WMffrM!G+5SwBuXV&!Ir9nxG#vCJy01*jIN$5z&%8PnuDnFa%RiIV46UcOu zHF}+V=}UJU4z7&iOrmTfUOrt(--EF7EOArX!AIcuVn@G8GDs)3v)K6>mSo~kIPjba z^r2~IU%Q$!bWC!X*!T8BSK@ zuC9b$M^pVfr@uK-bzPNuUJiHSO_=jRw_Ert7YQd$*o*FpUZ8C&Cj-I~s^;wCpbt~^ z)4X-FMayLw$40ya*1U}1>M+Ga27+r&=)BBXA!1?j*?#@i8P({aH+bI`%e@M%ZkZxr zc-3a`VA4r00+*22so7xl!n3<3*8Epu;bzU0w)Y<$RP&|2p#stUH0zVkwYlmg!m(h+ zPE7k;k7!Azj)i7{UQ@EIr;@*{Vf)VKNB2~v=h&I9=qvAv{(6jEFuz7dpBDPZY>9<1 zJ4RUcJ{^b1q&o;CMf$^d6gUVMy6yP{-Tq8ZfqZn;&4_RYo3%;$%{9t-w=da_Z)@d6 z<~YZ&ByenQnJ2puqy0K$ zQXwW2r#NR z$*X9GW(l&(4@q{Kg^Sattf!@K<%A?C-6#IsXTpW6G>?Z9#L>0~8YdzcC8-xdP%0F({e1XiAABX5ZOIJfy>JFxGSsLA}Vd>{B`<+tL z>jIkDgDZTu-0#IDv{MEK=T>vb63{gfmXpXn2=>h(i0vt438MKsqk z9239%s8Q3P_*eJOP`>92ZVz#UGOw4~-p96}n-lqk zrA0g7_WZ{k6aPFVWu?phlz}G2+I`0I0g_@%*ll?R@*GafByIk7>#rE{aNIL}-1M;R z$*znNM5I$#kIxct1QwGu*$88HZc3dEyGK{jod@7K@L->G()bRU)9rpbpu99bD~@)) z5`obWg=;JlQ1L@@P96Ku(q|GnE`9DiDS%Skb1T_utU3$}#0#uxoo8ipWPE`8@=)Yz zQJ%fzqlSfux~GzLdnd;@zcMUD&9UbrCqxLB+2ZAZIdseL_Q^fZlsR^yd&YQ8iCTuW zBJvz!sbg511a-l^KXnYJs4*C6lBNcLOy2iX;l~JHSb z0>}upUKv_P#gu4!yw+0nXR;%1dOKy2X?GMUwD}@%JpY&-xXK#s;nCHfRi)Uym^i+z zW>`?3!mV~0h1Mz)_{%PnVm-mBqDx^B6H{lJ+oE=u+-D2X46bB5xflF2WQG|E&%{ry zdw^kZzeag0$Wp3mlH<_d@2H=*Q;74-i59v-H@AcQm8{AA%u0AUfue;3WlUXHGe6eS zN5A$fEsC>vau^z*rQ19wdLHc>@a^uIk~ojA@bmT=)4|}fMU^x1k7zv!5sY2;w0@y? z3bzwE>qUqZ-FO}L$77*zgyNZ+ZaQe#Tr3Oc;#A{=x`vH$sdDHFRWsQ9HrS~5I8a~q zNOABgnDx98Wc^fZg@)v|B@GKbNsE5ARxOK7SFe#el%xf~eSCOE1HM+xs1WLp5WMD| z$JNSXJXWu!j0v+;meWzrSMgJ-n+aTpYmm!Irqnjf^HYW0G!OaXadqs^)>py`<9E3y zR@8f{K^H1)U9?_XG70JN#J@ow;;e%cq%SQB2LJ zpVry+rR8WAYD51d*wlF4D>TZfv$4#2DF{0*9sX}2C=PHvQ+Gy_qD&EC5+k! zk2LuTmZLQDLyjUxs-EU=JUl_%gf-e#Srb`(k5_Z~gZ1o17p}&Jkr7Z%8j)E)F1L)S zbHW$9I>yu?l=kg>zi9JH)wqtu%1f@!Z&6{c*jXvbQ47I$X+M5jD-Jz~X`aI@hFjDw_L-;GXyc)fMX@(d`5qni*i*Mv8)A2ozC4e72#>+v{}EO?$k!<)_9q` znyP5>tsAITYYp7&H5Qn~-CC=ckKvFOD6|B)rGCPH7LAzOm%0zHsQI01 zO7%2w*$W%b2h-0Nf5!?{yL}UHW}+Ky47?7F$mbEMlYMQi)T03s+PUhkf+J?vFZdVU zRYDz76DC{jlNcsa>fTioDq_uDYklLS<93e%;a(J{zI3Cm@Q2nV5{?@bS4Vi7L!E*D zQ&&kLk-Wr-pyN$dAS6a`_P0|Cr(?ab`*11iQ09D%@tD6Wir0Q_@QT` zt&{l@=aA7qy1b$8RVQMhirO4$@!pggV}lyI?OYunq^!PiNZSZ|}l zCMc*`m)Uc~6PkZlg{dF!1@=`1^j#&ibaFS>-@LWH+K)&w|8g{4nd@aD34ZdMAp<(5 zvu^6ft!AVgPl_XY2tZ-@p)6u8A^X+rD?*`GfRj*D8A{YGJXguudhFamonBJ+IO;(6 zIgfeEF(jqmLM1qgEy<0q(9U}n)&6GtEAI2rz&>$-6Me(!5;X?2iVyk{PQ^@m_ZMq^ zZ+-MLtRXCcO`+Z~0vt$5?;8ej34ABGKJpz})pxEq_1XFMQ2OZn!k4(p>J>J+;?iny znNBU3^gL?=s`F0owk$oin)NrHt@=y+H@wXW>-A51n_>iOjJMuDi*$0Wv5F34^_hbH z{gju{bpI8n@x8{ZBn9@3Ba-Dy7gBWBBBS4~V~~@T>)+L|d=Tfvcr133`iy@xX;W>HSwBK83ovsm-`~^>qYJ(^892f#@`nT13M5HYKCQ%g! zE4A>Ch%-puUjMRvgEsSY`sJURDcu(AeQ4@z%Cj3D1Gma?;}%i2%@@Mw>x-I)2o=8o z$OxbJy6t(q8QiQ5kHWVU@)#J$jMMK}y(O*~P4A{@3&)l&*V4V^c*)T15Lav|n5^ac zt9ih4-7we=voq{JsR6$*hMNCR zGnL8uX~^2lI$v2wCs#eXiOl786L}FArrOg0Y6mj$`k!;Xm@aQcI=mLI$dYwaeJ~|# zn^Te(2~#Aq^?Fwlv0a++k#m`O*I!7heyRqDN2#kESB}KDL`)j!JFZGRBjOy;Egs?O zUr7rL(7K2}4`aA1@j=b&7&!#qF3Wa2yX$-qztLk`n3#F!V{V?DJk~*<+CR`!$y;_= zL2_m^Dqb~i22vWzfU!qz%t z%XceTgM`GwV7JE+mp0jpiWS5-7x9&2o1JAFUQpLj3@(N#YVmfq!j%(U{ zdwAO?HT~WfayBHt9~1iN+rWiy` zz%7vd>>|RmXX4f|E2Axwbqn?LPoJE{PSy*RETl>)x6DIpfcVsaIlwIG-M*%WN+|m+ z5H9`p`V0ClT6+8Go?(Hqie6jem}-K>U<@tB2PXEVuoFQd%JW*9aX5}!EkK|y^E85B z>gAuW2Etc;%WT@oIq6CKmG$q`S$S=j5w~wrmlUqIn1}5sV-Arm#Yme9k!|1gxYm?l zV1UUj_PplUOoyNmDr!ll@faurPKwbp<5c354(b66*{qQ89pyEPX~cKhc<WMhVyrQ%Gr(8lMKCA zpntTPg0L@`QC?`fncuueS0PUOB3cgraci1ou6Q~t%QSUzUVB#95842_rrRdHCQLfi zEq08PlzN5Rp!l!XKsUL8p2nH?E$XV3(d~x^q|>)}eWs0W?GU!qi*%CZ2Zrvfw-Rrd zUR|}-?6U|grd+1XG;9gj6kS;t?RvxQl6QN* zJbcIc$JBf+dCbmyxTi-L$2}*pR}iX}Yi8wviFlu<@zRaW|0Iw3m>=XPPHBH#kR0w& z=P!%DM16DlwX?VZpJoGyjjqj+VB6zbb&pn5 zSOdw^K?V*&7jg0sy@o@==PmW66E^}I<@X6N*V%~J)#wi5r_5|SesW($YoTirhRP{B zis$1k#_S&yO%}x!l@cwl7~DL#*!n55E+U(BbWI5oBC466x+U9HI>}zzMxiJ!iR}lk zfEKULJcwWPK9&j@MY#-}m(ft3Cf5T-w$b)NflMKi`C(%jic2P8O&Gu0ZaO?S=4|0o zO39`^gAAvdBS6cry>r1Tg1ll zJxl>2E^iH9%C>JYWb%x5n^4JUc#x)}z}u^L>U}TAhMsYIc2s&J| zXO5-3biIY%@WcKRmpN8E_mhVD>?Rw_7VbLZF)8jhye59BW8)i(zlH7tCQ(O?pN3%Zmg#x~l)#UT|Y>t!94TI)Ktuy#Go;@X(^h6AX}@{Dbok8Z*UmP_Ht8zr@Hr<%xWN@o3C z8m7^4RW`+uOtvuV1qT&QUDE~C9#pa%S+7N9z!|kuDL4=+JAs8)1dYC z?@SbRIzXZzWJ(SWz~qd!IN^jbfxb!8K-D_t+H*jE+>f>1 z?%n)L&;LxQUQ$!CLbR=ZV7}8>i%)a9dQfQlYqtY>jc!)Gc%QU5p}oQ{vP{zy%{l2i z7DB@Ajg;5ni4iqYX3K+byx5F$87mDxu>{4E$0p3(5)vgW%|{jqmIgmHKTzi_yV+yw z&JPL&u{oTdc`9)ZTGTcT8vE!!+O{6b9miGMV1I3U`N+pc{NVGRKGd#9Z{9!RmFd8> z5Y4)j9#G`y_+Hp`{^vva3<3bw|Bwv}$=jMl{?#Vkf5>L|OQOnHM?vLcy$&I|A@ZX~JDOj4u;E?alYe{Fhu|0O z%M7MVICE?P#a^qL>1>N>464iQsAIZ^v8>lSZi&l|y_ibPHr1UL28es%oa%)R!?*>?b$C_Wmx)L>%LPVN4@ zYP~N_Vq&v?w=e5>KyH<8R;=i)ETi3m+S{4n=HUciE%Bt6R67o4$2vY&I~=+iwBUC` zLqM3J2mG)pybrr8s4AeA$G1!GYsFxCtvpn~=L+h#y6N!tlvy*4#9>$hKcuc}ucS$* zX(t)~#k)q@z~B(1=$u>v-lY?k6iO=nB=v#3(L;>DNykJIAJJh5aW?1N6g=3-E&OlT{0GpncM=}C1jaKX$qdIAP?Gs6%<^Y3bR z-VC*EzdwQm5eqr@ph!FY$e#(`W{qz&yj59T-F?IZyCOw}VBx@enTNhH) zd*MN59RwxuN3x_3J_#bm;+DwW!8)R-#(^fydq#ZL_W$%~qCSWoUXQ$CWt zO0*O|%QQwI3ZgeYkG#w*Dq=DBO60K`K(HN&v%70W=d2n&22dEaC1bNST@$R2Vr;oS> zXwL5Bu>R%Y)_gc$8{)V=T5?e2d)8f8zjLn+EWe(wS!9`qTw%cIWFVm4OR+JZiWL%% z$V>7&)l1Zt0i5)kz@vkBKY|AzZiB?X42tG)=(}%nQ4AEMlddXKa$@A)A_&>JZ0?ql zC=nZ~nQIeWmo)Z(e}dDXS-r)Qw1Mc|&6`UZ7P{DMgJ4oK4r1nATVQZkPJey5HjL_e zp8bW)zlAusv`Ce|46E1n#k^8V1A3Z~`N{jo7khXH3}pVg5DD5t%e*3hT|DGA?P&p- z^3+O`#ol5O9F{@B>!`mCy#C4x%l@W5(KQaCch!7fABa=nAzlsG7@7y9 z;UYhJt|X7i$mlXN%-`B_=YN)ankIZh`=%L72COxur|v;SOXwom=sC}!GtnpUS0C;o z`3wN8;evELqkJWhKF>43CkcN-ZD(%te#~KEL$N>r1HI+Y!d$SP@c2=}HYg{ff}U{{ zz_R3^cy0LsXp2B2;6sA%Rbx8#?pO^;$^bh)S-7XKnATy=%KFyk)V=E#<_Bgk`X8)! z;5@Er?KylCbE`;KNVFOGR2Q&VIeba%dZE|p@a=Sod!uG0&BgB5Fe*#)w+!q!3LN<{ z9cv*^zr?ejJ=%O^{S2&5Yg@&A0y zd()+6rVj4*7o_7j7rvp#h*!y2&afz?BY4*EvSRk6qo?-arZBP1kvP9z(TlmgxY93- z!>Aj&48->j;uYPbacU7i$_?PsyEWmMMhZX>z-Y79!j#Xrg|Q#H^XiF~8Qh*O4yzZ_ zqZB+FRV;~{ZQMh-AYr!@nsj`NY6c0nti%&QHL{MniUA<7@GiQfPj@4c8aPbf(@TxdZ>k1H$E?IV_97 zQrMc|@6~FcPhh+Et6vcK`AP3w*DQLXe}V(G^iy~To`m_YusY5 zt=sJrvjZ;-0g2UN6p@_HF8n-%%_K~E0xk%IeqPz@?C2sem=)1`V!)nd`H_D^%LE&~ z2#3z^|K2fuG%H3|!&_E&YaX}&6NrwV%2sO*0s$iXD{vn-rM{i+sEEx6GuT2&)H$&W zkg;jY>puW@dafywPJ8XD{Y60>@sL{3CQSZk<~ib-uL5Q+>u~Ot0a`#Jey%lyHY~}?3C~G z!MA3T`}{^B!x)pKVSvY{M6%#@TFHFR*k}-~k7W?G+j|5Rs2sQc{Bvv$Sbxvp;mS`W zE)eK+T}kRVX5zM@cVrFCue_r9c&6J-hEc8Rcspq3e8A6aER=53|!-1iaD;Av=4{V3XZz}$_%RIJ=>1* zEk|m@um0cP0K2++`x>=q`JVd2>+o5o;lyga8P66fJ%%*t#$raMGUYv2Q?~)se++C5 zYf({<`Ap#@!x3rUi$Z8!NSs9QLFD9I8!gF30t@s`d_|1!9XCzb*YzU)y9n+)h59>Y zLHiP99;wj^XuD!QZ+OgePX|=EU1bH>*83RNt^Uwn90rvkV>zmT?b#YBb?yPDO1u6; zxBbp%kvxj>hOk8X$kJB+*lN;5fIM-=%Vh#!xzu1<9wWc~KQlA<;#<;d6IDgsQRiR^ zSYrX)D}4)Ex1AZOlQC3EH28#-3&121PRw66AeKGo6`C0ePZwWNj?HjSMXyUa z`V4Sd#^6-7ziD4>yqfxn2K*H2;dSH+u{|X5N9JO4i|y2kJvfLT5PMB+>Gqy*A?{hE zNZX6OUvqBh*|P7ih_zua8=n2v8E)`?N#5YEtHi^QMwl183zl-@;JarBemKrqrw+a+ z$}xZ=n6TbznCt}tT!vGjgYf#AVEyP!Z08vR?q7M=cLIZF# zoVe+Z{R=kM;D$YeFn8TLANfHezTm4=>Sg_`Q*a>ByZeE^9yZ&s0nHHj1F>Wg{wNO| zMFAwrQ*r@wVYHR`xhXjI!6*^BTkExkPOYTFV98z8jEX#i zR40Moiz%H|NgjN1TR~1GGGQ94W($Hu_26bEy_OZCrItQ^pTpwaK8#n6H*u+qn+ji= z%6apiFu9c_k)splCBu1O{cm`gv?LP0qwY8RqGUAWg=LdugK@V}ajwIU+s`3=sg;X@ z5AsdobgOxD1>3cW{~foXJ?yTJpHZ8_K}@%?eQ_{(o5p zuK!^f&@|a7qa!SXdvQWu4WBL(vcf3U9U|$6vGfF3a3;6N@a70Y-oqSC(|arBHujHN zT>5uY&*!_dUyUl*7D7UQ0Hn3r*BLEFzQBel#ti1MK#l+Cq^Zhp&peSi~& zwNb%S1PBoR%iS=I@pxxVRTIINi0y!&xo=5!kx0Y&ZnCo4gh#BnyWGv>(xQN6!Z8?V zO8^63ckW9JcDOkezwHhdmVaF5C{5g`KE3tfU3th*d4y2@uHX+RARlBz#yp1mM<;~) z_e&wnxGlC(9^0MOgqK~>ZJunm%*9XNYC@zkE^iIRa*N2#t@n&wm)s&%RfOQqFc*EP z_pWtNq9@7vxHyN!Ch_XrEsa*$868;eMw5Dy&v$oBM{CI*ud)frYozg^{_mjlPm;yo z0rkiPU|o+&9H9)`e1n4X7|Ysw+x%`EIZ9n4R9X<7dT+=Ji4ow-yLi(K@*7I@WIj0K zo{gk*zp*Bl)fGO39jb9f{8f`$EGu_PVj6*e?R3)voCwHKqT!GxdZ+iF*Jh&zY}~1c zBvZqD1E}ED-M4ll#Wt~r&t>u8598kd4wJTrb)S{^pLt-ARGkUuze}vM+5^_<^D5_I zE`85$px;R5F_xv*y&K2U6mdhI0xABkNp;{t!WfNIj~&xEs7m0T4npWQcBtQ0R9`?e zH*!6Z6pdF)9K-C_^nVIj=>-Ici4j{R!W50EPclsYeoZn8xhF-0 zKerX!yPgM0;ke>dd{FZ{4j;)?&pVGZFh*8c@fpL~0J4xS?R-O4S>7#eU_6?kW;G}I zia?>48k>g^(b5D~YIq98A|`4TWoHj_wg)BHjb!O*>HD{>1r^Vxoa1 z>{8d*jwR`OXH6kP83T|SMqvqLs69Org6J(0Eb{`2qvOq>s1AsCMf?sLrTHt` zx$@e|z0B!w;T`WUgpXMNtwZ5|>yXs8i%^Nm*!y^|1ULsvRF|u~K_@%9j7m2g&;)OP z08ew%G*aJ+YDBQSgMiIATyaAe9lg?61PA|*ux=F_QAKbWTHd|i3zQ`S2B5`F5a&op?wPFUrp+JeunD}LormCyhE z#YL1!L@^1hEESoDcA?VZ$nw12Uj3wB234*6iMnnfQq1=EP`(?q@<;Aw$NjHArB-ty zWPO7iE^=xp$iF`z6IgD1ecHsE4&;nsQ1e57t88ukeCHr3m~r)e6z%`s?I_j9Hv`D%XFHk^5B4|6 z8g-8Uw?+dJOQ9yNK6G~LU1M`X*4h8O9U?&mffPO@itzbA5~9D42^*LL<4HbE3@bP3 z-@XLL8t54rQ z$XCVy?Y&c#k9wMTO&=LwVmR1gw>FNLP_tc`Mj+e)Q;5HoZe}*8%4b*-t-$OF5 z`=2f$q<&Xef9l!`)%*Pa{jC5=l&N7cvi)^OI(T6F8)3ut=P6V!r-PAdiFT!(%$49X z{GxsA2eP3!pw+02%2m%|$-kn5Onv_N;d7-z20Btkpm{h+eR`edMEvljD{v@iNjt>N98E-zpC>;nMt&dAsa@Sd2cB~@c($PN5{+IO!dEibaeV* zCQOVxH#hGM)g=U}M9)eh@FybtA%i0F z=gfH=vI8$XQ5a_(QfoKR%Gc;gt+7k~?*|n=E36!lgJ3+kU=vZk1FHvXk^kRmbz87i zhhVvOzszbdhwJYVgcO^G4aUnA|5^2SafKqX>)}qjyY;??NYlY;(84Yl82KrK8sL9) z-4k2<{BVIQU{dVmXK&TR!&N1;(hf(EfknEtAS714^+^pt4 zIkkZbSKRG`^?l5Fx{R%O58<|fQ2ZBduQVfClrz9igiA5y3r9saJf978sf*$Bk4PT> z%1$yYD(e%4VH?5=e_MlhkbEA|S4@Djs$GP@@D@#PQ{dH&;%59ZsMq^IgED`nuTSdz zpWk&FD{vc7kRH<+SaOJ{Ed3YP&moiQpyX>M1Z`R(0u>!VNmwLn3U+~lFu!i$`wI9; ztb+%_r~E-M4`Z0Fr_elyT5dGSrqGJk_q3z}=I>1quMd1f02mmJ=pjXHp>{uL}{o>jDrsPV+bFD4#%#S+IK#r8*fLYHC;rMuc=CBJf|lK{^C# zv!Ej_0p$dOnHX@#Lf$YEqL0aI^7Vb{4iuR`f-RI5FJ>;z(ur3PSpb4f93O0~%}*kA zzP_8hWiQlzC^i{12#a&Y))-e{BBAJSIL zyH+<2G*bVt+lJsgKjqxbDQMCXoOHN@3@|CIS-w$aGztL01`JoK1l`p@1vtW-jQ#QX zEB4pr_{G`&NGcC^`5~?qk}Kd=Xzc$t;3A;e|AeoZQB2{Mn&+!pEY-X5CV?=w#b*`Y z^DRp%wVkXAwu(%!HaNaiXemN^X{YG}7p+S@MkY#1yk@odn>^*-;=Up4(HG^j2RYzJ zl}2NBO}+nQMJ%+wXHPMIgHdxK>|cMP`-k_WBO$%;OQF+tCc1T9AM)# zD7UUM1(CW@q!}UsO1fbfMQ{SP=u?HYMXVPKv=^4(4FY!Z-ow=N2Ee}3&q=VW9BVU+ zj$uWuJUnX{K=R#>FYh0)B>saE4G<`iPWw0Gcf2&DT31Wt`}p);Yv>L#NtC*hkt>A) zP?pHUY2zSjeVV;q!HFKOhC>r~KjIiQRS$@KBkD)6eLjrgn5^ zS@Jc^J_87TG$8S1`yn5KM%L&42aOz3G5YR@Zl9af9P1iyq9Ecl9Yp0E97gR66vxR% zkf>PHMw6B1|6mz=%o>mzom2uIWL$FB+@^dl&bTI#{*S;Md$3a)-Ehj~%kyGvElC%@ zb-tCtR_r}XnK~rnd_p3A$?HF-cjJ}c=!85jLK}}l0{%bsMMu&L?BZl^XhV&G8B$!? zBe_5(46lNU;jXa*--{DNckDQdzsSsorMnK}<@r_LJl;5x%Ie~J7B=Aj=jxGvjUH-Z zmO{V{&G|Q(<`58)zhIq-PH?NaOYx?iX+W}kV9#w+87r`To`_**KDmb`$KHkcpX@+8 zF6MFl6cFCK9^9Pf9yWKTG_NxK`i}5kO_!4*Sd) zfdp``bmjL9b+9BBWVQ>{qtNAO7nMXVE6(qm z;hf`{XCBY<_xFC^`@P@y-oE$u(zSTNl6V*B)N4rjVWhPgy$2k4>ZM^hnNizrmzv4q z@_U3%y~@Tlrc6OVilQnfq0PX0>AylMzaNCKw|-pGcQqMI&L``zpOgQF!b!`crM|3N1$P2Oq1JE66lB0-zCXq{2(QM6%s zL9+_ISr-~Rq4zfTLK!O#)k}8jsN~vA89AOyV!O`Xp1Y*~2c)h~V?W}ev4`vDD9k^t z|JZ~eee)-2&J4@gF&#C(i2LT_1mo&oo0*|6ACjBu4WBNSUT~otFNj2})B2ZO-na1Z z#ub$(AAS=!SI2>!otf9XfCH<;OlN(7>(w2tvp+dkl^-J#iK6W@Lo}1vImCU+*_g(tfiLol#@I(S?g|1V-Vk&kr@3sZeKYR`Hf)& zgDVuJCf?fA)GRBd1y=#u5KlENKiGO^sd3=2sTgo*<44ZR5FO2^WxUI`79)%DD&Hb{ zSeZ^Q`F@tLS9x0Yi(gcp6pKWCYkxv`rQPO=Bl1ifDs=i&Wu3(b+|$46k<{3tOCQuA zQ;pfGHxS`dES>L`w`AUAjYz~5mskL2zeTj_zxI95))|RyW3^`~W(wF0o9lNXzYWTn z7($DsKi&GfZv1tIlr0x?t}j{(*!)H<;uJSohDq__M&tgB$|ATK3B=8aNO5vd=c9c3 z>wJ^o2f=}n=qJ_p?9v=B)I<&?s8>u)^jl3o+PdWs4-l{Qo_9^?&A>G9H3s9d>j|!3N9)ssd|rNID+O*y>|F$lSnR|M8+|YN; zUviYogu{GdGgHuVCM>YF-9?UGugs60M_R5#qL+>IIv@|Z3l9fc73PI+X3`Y}#9SJd ztoGbvB*Y|rkL4H`6M$KnVJpF8ipu@q1xiIc5i5>U1Oi{knQ0OMgwJ*P{YOxqeXPlmS(C zDjrxOpOFcAWUiPP0Odr?jZL)e>%`BoV=T+0z~M^w)%D@@93@VMzxaio)p(05O14*a zqV8eA>edLM0YUiGbkU@6O=Ws7HuSS!AsMbW(bV80{lR-OlX?MLSManCsi~SWB%^Dh zEQ`40B_=gb2Yn%{Dgh-I^AwIo0cxLqp9FcO=|h3qS__gXj_`Y7IsnX@K2Hl)|g~80I0IjcKb>g@DsfZ9E8Y_7oExFdQtzQWAsfZDP zf9xY{2_L;QMW)TBIav3!!F~)Mho8T`t91o$6D~e2_@y|A46@LWvK#Gd56MOe^kGx6 zg;I>Wtcf6-_8rsk@H^j_*Qe~dR^V`@dw6)9yt7BZIpV9`o<>$(l?{mc zX8aMa0eE26Q>bCfymet+{|@~07|dQID*Jw$Qpjus>^3p!aN&mc1v)rQ9K`1LAw9cd zzV@ved3$(o$|wM1qxnP)A{LUNr{$f-O@P+nx+0QVB{Z$y z5Y<3ZQ7;yD49+}Z{tP_uUYRW>6Jx?(6JmVYMkZ7vXCia5<=@HS_s|bdegLGFs>q-5 zGhkV*ndfP`np!|I)eCG(=FSoghoj<^b}1mZ8ov}vx*28iJ$n~}DUPM~j@=@QaQWgX zfBF8>RJHS zouRdw5866W_N)}L$7ep;q;GK1^9#2K((&xaodQ*d!3>_07UHzO-S20JWe?e0GTV+3 zMpO?`%=;BuHq6W}iFCt0V*<@{#||L`%Pg+_4AmdV+-g$;b~=#NI-BX`MK2}zm_8td^GBXOHAu}sP*@R@BNM&SZ&mxJ;ij+N~>``Xf zdpz&2>i@g{&+}ZD`@XnuzT-3A@7MaA2Rd46R20k<1OxNu<(0Rhn@0Rdqv*&+BZ z@)dXD2na9)>R3g653{)>(pdVQwbiyGC={*Yg8-Z=8qdGclyU>lkE7OzPCU2FhocVQ zc^sb2^9gro@#0Gq`6I;uKO&7o_Z{b#&06e&nKIv2=2qt38!o$4d!gsu&z)cM&#q*( z%C4K!u*-O0?0FQ3{`;lKy>KdQDW&9REf1R8e}B7XKgMTw3y&s`COG&~c#2Npr-^xx znedBip~!dl{080UE4Nkc)csI6_^Ra~qMzJrAAiSVafC3;C874E2_1$I`EufipF^|` zw@^3|T`(vAy_vQC=f0zAerYm0KN8tv{@mf<-xA>jDisaKgMR$I?ZJNx;vfuLT1dEk zca8`y;atblf&F{&;%NBw`=|RE$S@>#0ZL7IWe*Z*;%j6>x zN=96nZBMMl_t&?0)gSSkwWuZ7-G3}(#N)fS(^+ILZ_}RQ?=#(H9}c{n`ta}D{O>)S4*S;ylAdpSF8jh|viYUcsL}%# z@xm_&;@WrDe-#9?rOiX6K;)06k&dSm3)oo}T! zlYJ%E^idXK`{ew$vM!OwR~v$DZtV73C41xi#TmW> z`nyUM-XQzn$+OJlLN1FQ)izQZb{st73)kt+r8-iMrk%~@7bfKnE>rECo=czjM7NUU zyzk~*%!;H~=*rUm9L{uZ;>)vhf&&*bFXWh?e{wQu$)wEo@r6VQUB**p`TN?55+(U2 zX$ZPb57|VP62yS5tF@p>t|c?6A_zEna-BhBxzp9p(??` zpI@GF!|35BKR@|=)ZB}6du=|3S=6>El$JZ=_Rm(l_B%Q~CR6A<6 ze@ScJ@tn!Y+7W%S{_hR@=->^%=VXm*9k(#pqUh8;uKhgUMgYCp*G`@qp>ZPnt;^Dc z`lo2lz_inXuP~(VEb0PWMrsL}M69D7&?>p-TN5QMXS#Frsyr5Pu31`1oz)9>Iy2Oh z!ed|K|Lk?Ybl3~hckf!Wl5c2@{I1!L6Cd(W7Rtoj|6na-RM@T082SrEPG>DA*?D@yDfMrTY_m^I*9qE|@VK5@P-VWPjrf_D{(Rk9zd^1Q*o=moQA*O{tJ z`s}Qw29M~(Bko<^+dsZ=UU*da@1b{LYtLJKp~cnO5gEQH@{lclz=D}^&gFAyis3zf zbBx+d>0eDnvp~t}BU{|&L8FKaZ><~-m}Y(ScQSFcq);s76vl;lEW}|?RFc2){kKOL z;b*HB(gIFG-jRtF1})lu_1A9|ZeC#3%>K8|{H=0Aa3!?KmYG4RSVfOEgWVG}AE3tSeu@O%F zqU0MIP%x_>jeGyS*@zb0{8ys5vF5+GhL(a>f;%W%pPi=o_vA1ke$p`AmA8ki-9nF8 zY-xmQpC0^I6#3!TD|m3V{-`${UH&mjG5?-a6)rATZ~2vHLd6yxpUc5i zEXRv#_TJuOJSxmfNJoZ@-c`{-SJ-B?6hwK31&>}@ezrAIi(-EqD{xU`?EQm+sd(=` zg^}jRLN6VM$o7>@cf@4%ALw*n{NUMapeG zH(TOA{Y>^{zZ==X4G;en8QJ;njs>0LwJQu(tqxGlGpsw(kVR+3jLW-loH0cVFbtsNAZnX`M1tMK|!rSWqP@)Gio<*CPK&piQfSqKpF)l|H>w_a#k<8uWz8aE)ClJ4J5)8Rcv^_H2_eB=@dFnF(ihV(-D zmGJZHN!tN{(g}ThbM3q1Bi#I*?8ic?~RY-KR354xb6reI;sTd)%0e(&)YC4 zv5BHr42{e;=gWtA+-ACGCTQe$Qm(DTO3*8|P^`;rfU@ujo)AF^NZabK_O6_0OE&*- zE@)eg!<(n#_RqM(OlO;*K4s{nDlL6A_f?0!c>^9@aBlo##5URcsgI&q$*uXWtf+4; zzlUV!$CK_1;Q{>R%C^LdcAfcNYBQeysic-U%!Ly3k`gv^4Q%)Ah-kKGq;t^aTg_y! zXK5#&Oc1;E2x?pIF%EHu#CX!tgyWjb)5A42iSoW?`@6q=_f67A*`!>fWB9al(ege! zZ|3@6sl9cZU5h3NnNmf0_Z44##IKvCW*TW@8oHY(dAm7)gmR+ntJ%YP`+n?~}}mTO<{&bjyk5Sq)6FHFx) zUk#|*n#t=dydo{>Iu*^M8Q=X=;N~hra>t@7i~uE!fC*X56}%tP^$NJAt9&q0oW>zE z947%&SH(i%)YETGmN(y8oz1f0$ula7>wjgHHe^f$B28%dcmgAWO8kz(o4xbqfHP=?jWw4znI4X(DTA}uXd0rn34O}rdbjXsUqX7Quw5!t&d9!*2g$38)$7Rb&Bm^z!* zhSPi|eWy@0=<}0PMFH+R8;e_R6CUOjE-K=oT$AI;zI(bIkxf$7QT(^#PF!YZ=6>xQ z5kedumFTh&zQUCLfv1_4R_#|si8_6bfUwEySU}2ZnUlx76%@F8P(4~Q7q3Q(M@3|y zS3E2b?CKKguq<#%G4UvLyWgb@4JQrFOIF7lS^;rgfLba}9`*#rJm?f*9#(Rxk(OL7 zm(=AAEpKpkflg{bqMWyJULmIbux?~iiS9EJ0Uo#6-k$8!L3X+NIU<^<`tPCG>+Z#p zB1VO#XDyrQ2X|~r`#x$um+>r>oP2(6PePaa9{SesOUs8_0WF#7su7)<=>^7Rovu&e z^KTvhGuw;CHdSa7BCb=VRhDdUT(X?7h3 z=KeaW^c-u~1`@;_bJ1!upJ1d{N5>mEe7or)g^f$wr{wIhTk72!C!5$B->St9;OrE; zSy{A!Je}~}+sumP*HxUP*=4UN}4hRu0B~yro>Sd;qWt9LJ zDa@J7P+vzqL977bO5*-$Op?Rs`Rw|D;8lN9mkBEtsGWxI#qfYMQ&~ir%M!R=+Ec(v^XvlD>5hzTg&gR#7wj# z7;#)$@be39Nm?Mm-xgU3A(I(8S<5&P%d+}?_p(5K^%#xp(0S-)4A8#wa4$LN{t6Px zRYLYJv0J3Mg%~OlJHGaL7!a4$8H0!sW!HOR83idocfVM@(rMxK+1ZdO>9z`}!UMRV zg83ReeZKU}92tVqE;?AQGFW-4+95y)NFcPF0fBRh((pa&c|XUKtkR-!$3D&o#aZ0o zgKgyli*)@6#*q&oi#PP=RN2PF;r8d|ch!g-kY*nDEG_A($^l4m$jdE~_$*|69RdBu z{~rKC_l5_DaY9}4=5@-Kq$7`3_yP}BA%&oH{$@4_0g(rPsSMC>T?cMccTJ*dI>0AI z6LS?&Jw|h*&1QdS${uJuiFp8z{ZQ99YsHVIzQe*rOMgBY!uy#3aHzD*wC02Bx5M|O zRNe}Qe(W4Vog)R%6)H@y;%DV~8z1dAjtmCVMWp0!jz`Jg`1173*<0U4td6M&od7Y% zK~m2hkn#vIp=z_Y-$xT%tpftEj`)=B1MdF}vd2mrP`k;IxsT$wBH`y?=e!k=QGVl z{^maH2$P#(?!a&RG*Z-Kjx!6Gzq%QLl%0L=pRH`5)|^zSD3}NwmgIOFAWvEfOGY6G z1|QZ6B6RsD(CZs*uGj1f559Ga0c3K1b*4wucn?$^TMbI%#z;2H?rdHEb79;hu?o|j zFD&8L2-#u+51v;aR+H%wo)HD2@BxyWdYEd?_xr>9+rE#4O~lw%U{2zo^?w3vH94J? zOdZb2%4>}(&a&vuzZ@XWdmz!EFTf@0GTKViF^Ay0wNq6329i!AekU%!oJg&UWS14p zZqm!Vz_cAPLI#6^d@YzGccnn?3bpHw9?~Tr3mSCBGMu{wz$o6xEJcCv+>_-$%!QC{ zAs&Y4Ym)~t6h0Gj`~H_I%Z;ET@u-B+pPBy%&3y936SGIdKHDuTvQ_PDC{QTUr-l03 zKOgj@It0dF*kVlpDrbRmvKYhLcrH9`FY@uYy+EF_!n=2vMeM}J$95uwa-rj} z6I>WVHM$X1ZM#fFK8r6r3ra_hVL`lTQdrZ6hevtg4gS#~q!8d;3V|_TpycfHk$Bd7 zjQhDFbaiVuRgDH@p|7mVX7BEPs{CEea#^dRQ?c)vB}!*)0e+5gtl2f}%+fvu`q}Gz zNB0Ba(-07Qc^J<&C*UR6XYx;>UPnV6Zw4Z_J3Dpe&hWc?M#nYddA=52x#%1z1_cXO zM>;C*Fd&g!E$1|%Fb$Y=XjWU^uz!5b=QPd3qo;bqq?(BHkxz5sLz)h;vgk(Xqz*Zk# zI3r+~e+d&jaSU0GBmA%&Bjc%=m5KHdZu8raNKe*%u8$M+s+C_KF=i2=B0oGqQ8Z3l zM7DkJ2(s<9b@CgO>yxDKh_yW&}A(zHbjju1m zcrh`Jv}$!#(np!~f!hHq3?fycwZs(c)O|>q!X!d@4e7G)GWrVjKFT}S{ntRwB(+ig zCC(QbkHdBkg_w0@s2_=l-(TCv#`$2y-`x&3Pyx7P=simpX@U$U> z=E>_@-=$nXS~DU9x=$G@Cwt-CW7G~;?aJfEab-~|dn~p>8S6UOx3@7)>_1@SiiiZibVJ6vRfCYtcd4g+seudy=RYBvLHDd{$&pF-;-gX|45 zK_ICqV?Re=fQIO@0d%kroO7SM_+N1-9 zWKP(hXI?u?&~@UXw;e()^CIqUPCha4x^CZ1(|D<7Z!51_xc1|q(tV1{vOL=dQdA?9 z?vQasSP<^;!To1800Pnxz_KPo-L#DRd4 zEKX4S&w4K1`1Isda+;+3+|wJ)v7bIYwPr!Cbx8x}LoabI9F@CT(30&H3$Lqrlj9~7Krf9|Gk-=^v;pUPGhD0Y1^uf~yfzr3+ zl;pXkNkq+i8tje!H_qv>?psqCKRPfjrP59&1Lf1tGZbE9VRctYQP~9J0I@5UKA)CDmQX(&99EW5COkU@I5F!40lt4+BIyNnD#XOr>CKMX4tTYp5Ox__%w4vMof%9 zE%@$iZ@~)~6S?*ASQg@Ud#Cre*MC0&J4l*@a2^&v#@`Q-uq9lmaYQ9Y$V&Oc2D$DP zT7$>KqY;Zcu9axrMnE{lz^C1tYjEB!yJo3bi11j)9w*VeJuWV;mD)5=K>L5Qsv(ym z3_(`+)cf}Qh2!X%X`!0kwS>$(h47vt^Wn0X!7Oc#{*vn<`KA?}WpY?6j}#Tp&vC-M zcYX~7tZ;}O>_j6dexp*$D+#Coti@>k{kw@$od#!bemTY?w8QZlWDSG?PY+g>?0#mMVdzuVX`*@;Q$=g44R5WT1N}wUUKnd3v2@l$X$e&Tg{cb38H?Kx) zdqgxM2q!7`f4c*;2x+xvp~9FnIxGG3H}eHi3UBVaYw+Cj2NT~LZI**eAqrh!KvLkN z+HrD}F@3)%hjQ@Y4iOr04hrIuQ2u@JTP6nY43|+pPT!)M*sAp2eht!uzQf?#N&Z73 z2htr8sI)Te?k$fy=}{D1$r6>Rl_Uri{%b@uh_J8nhZWp| zTq$JqMQpoqVC@}gN@Y7|h*8-Djr;~EJTG0R^%^&{l@G?`7&0b`N!K4KoInjct?n$f zxwvZE94kE&bR|hb!&&PR+IcroMf*o#wxZZce2~C63Ac z*)l1xdub@#sWphA_|Q1ri8fQrBmrzaS18Sc$%O%T9l=S>0KclU#&l6k+%o9+pr9XIoJLt!Xhz?Fa#~O=b*RG|S76L>c4FcY0=mMtFN#Lw?@$Xl8E@uGD zgjcdfUTOH=Z`nb#X_gjS-k^o=0b~$_*D8!Z$$Xxu6{7>jRSpORuD^!{nGS7Qd;ll% zMtu-D^CaT=(Z~k<5{4;$asEl1**n458r^?cfV=8BdM}iY^2;8ysUtAXnhImQaQS`C zVz*gCSsZD_*KFbW-rf{NVuag(mDo{mvd{JoM#Zzxw4!uwLa3(QQC~)pWn1eBS4k4*D-A^jXr)+wj&m9R zNcBwe_RaQNi#>iV3F7A;(D90p?yY_`UoT_%{JY_Jzh;7%`f#;3i6eH-0qS`>C#Y){ zw%M?LK7)8z#uX+$4>%-1TxcjU=q-5AOj54d@Ngd!Qayx`4jse;JuW)YP)dcUnzr^+ z7|sZ|#N$%yj{0&fR;RDgoXR&~vn)aVkgSd0e@0LOf~|4E=iL_ZVO*kwvwgYa$8aXW zm-p7@Y;5yE6Pj<_)_eF;D_($Wom%zvF%E&fLD_t1hM{ko*&_^RGJm_-* zpmXRShKKF&Ogd{BRs~XSKrSBu9D|8G*bUZ^zT7 zuQ*k;5A?EAaO(=#Umw-GG))dB+75Q28TU99kA=Lyg5wHS@B_fuiSLdzWy6vQs($S% zu^jabc!S2QOAWb}a zE!lhR*$6=vFqG*-z3>P6f(E%fqPE@9u(3YDIC(k;nwB25(%paarzRW)XhlVX!h-3f`eNCHxu2 zQ+pBf#GvohfoPn7U^~w>>w8UfWEuR855mxz5D{?}Zi72CgYN}h=fvf+jhp_gOEhvD z3bW42RcVhj^>rg%SvEkBo2f-4Sd0c|zHSnV;l{#sg2B`gy!+pelHMD0! z_|+oC(-xt5mWA05M5WKsE(RAj5)-Ahpf@}!Ekr_H!b6tu9%Qu*#*cW^^*4G8ub6O9 z>t$&b1iS$>EjWjtaaozt1sCyw>cUlduI1uMc(S*j80*<~N9yVp*hqCa>#3yEmfv}A zuXTK*G{H

JXV@Ru*e(4+y=(abQ$G`qSq*&tuTjLb#&}H$1^8DwG?m0$KYlb#&b5 z`?Zle;)w9TbDd%*&8o6*j@dmII05`M4$-~0$l046hN=WWkLgXcQ;y}3biszv3#8=; zgz9mJ`G5z5RP)RPgm4kDLusOAT|@bYcb$#cP7t-rmnH7 z^lQ#++Ma#jQoAES}A%uO`yoUR3NCV}EJrm{nYv8hK6ng>QMO!!WHzzX643gqo|MRKz?3r!9N%D>B;*|86yYr9d`_hs(R2=b zBCVbOBDt`m~fQbft*OZ zbW{j6c8S(st($#-sKmMV(`+~r5_yA|f0aL@dSm>)qN3~l1}w#Yj$9k(4m!pz_|mHF zmuR5Hke>)m#%+b}Au!CJ`)t|N5g$tHKzH>8JqVU@A+y7_E_!Fgq;T@Fjvb{MW2+a1 zB_1F%OGtcZ>MJO~!ca^MO2TpHmby62ppNEO3~h)r2MOz8Ww^hfr;pLlpgagRzI16;uPsu%oEmX76B>5Q;Hl0AII9eF(F`T zPDO*J+Ck*BqJz`{xp{-oyacPL*C8her8|w#mMVL~OHO!}x!&G%HJX-|;9ANLH?cjm z29Y^TIl{WjKk*@vB6Qdh+Ginss=p*%x)$Bi6L zn~DXm7kEaNWiRS%V(_gqo9s#N-Y~A$lgjiJA8h#V$cFbmWaEpYE54h~r$-w7(odfp zdv5TXjx%br3slWJd6cEwH7K1zT3R1_oaF)5phKtf{}eHz|2DAxSL`&+9@G|uyMb;0RX~iL2EgqGRGgo{|Af}Wyz-p^iS;qOBX|n_V zNHn0uhsaQl{^Jc{mmYjrkcRNz_!fXaVjlD(loJ6}r%W*uO~Ty3x8hV>m%J$W?N;DBTj0-^cIjPZ}=_KRT^x9>|~I63gm$y7RRM}mZobiSA*P630J zw0(NL;pC5}2P6UqO0*py7xmh&D@fMsJZwPj^W&186_>*6X%_(UFi)xpJG`zvLKei| z$a>u%ek+h(H>!ZiTzL$1?=z@y zDADJG-`wZFz0mc+s-J%>IB6)Z^v$7CWfM9bk{xjYh!On*_w(RE{)8^Z%6QNiB3+dZ z0OAHsz1h0T02ApkLirpwq4}Kk-g1pt;=@JPZV~pS^P=V$|I3$7lsriS^cAhZk1&M$ zmosQM)oL%&k4h|!eR!A`Ev%m%ZRcG8F+Cpb=cXSc*=CJ7>OVNa;{Yj`L}aU7)Zanf z2WHUvSb>O|$0DOBMd|~>hL(S{P?vTls#~=tb}-LCdL;%7hlI)pMkJ5@2Gdw`&L82^ z>MmxXVus(1n>;eqnm2@9v#N|ONLHWqLJ#5oEW~_T1Eh+>^2oRM&v15*uXEL;y6 zgc#F5yT2#`eO^t>VSqo`XDcT%f?w8F<&^20g;eWJ@R1N`@3OPuJpHXKua=$FB53w& zfe9Z3P``FbvWPq6@22jIBEb{bz;vvPLHoYTE#i7$r#zGKd>L0GbopgQWQA25yW$qn zM{hvDqQ+q_AADS=;y<>N-#<&3->;+1((T(cU@w7SKhb2P$qObd%BH()1~0eq`bhC4!6` z6!hW+C{d%O&erH3IkYZsNuOD zLw;8SN+Zt!MmH)jevw9za8Z7H?wt3VTAL(bl=>y+5C_gdVi=%msI0(CAa?j!pB;~g z*z|7hAQ5)df;iUrwS7Nowf_~8bZZ1BXqDf6;yZ_wD{lKrh06;h7K{XY=2^tvt@fHM zb!p411CxINIc_Oxd&PZ-jKl%$B+K>27+6D4uH;)W0C}8z-H)8C=M?P}&aJ22!(1{7 zx(iHPn-TW8R`vt-*2vE2BUs}EZ$L}myE)1orCMDPI zALfuVCJu%#JfqZR)N z81lRF8l#qw*0?XruA|e%O2m=%lzIN)sp@E;O?Cy#>1t6k4Rt_*+++rZORlR$yysqu zETQI*%QycHp`~vR#pyy5p9mRgsrbT+x!t~MFJ@ZlF5IboF@_=FXn)0CapZzUusGfS z)!5NQeV52aH$U+uPX%kdYAIZKT(iGta$h2C6$yPqhDh#OCzRn;5#5Yrg+$J`k(#NY zdMs(I0*D;f(M{iNsP>+j)}xz0gv{qx#ZT8G0%?mLXvHu}REe{^?Mv=*Hd@Dw_z|-{ zqQyCZX4yt%-zicWiqM7hARc>VS1*_v2$Hk!G7sSt^0`-ULcNSXejB1Nn*RQ6CH;O) zZ=EChHz~?y=jbPKsm9e_6(gw+|BD&y`TV*cK<%^LQh_jsJxALY0`x%W@%@eQs7K%_ z`Qy|wm@7t#UqN&$e#pNDZ|3J^-yIu_3HA_sFUZv;!AhoUyAh0&SgPUdJZIU=15*|U zIi}B0lnWCysC22F!xMi)FH{Esp6k9F$g2jWKOx6&3|SJHAc;y-eiiBE9)7=Mkr5-ta{7=;g}2+%V*AQ$QBAy$-Hn@9{urZVB;cNsBMmQ- zyZxo*`3j%iw?8gO=V(IWe7QlEMavM|y#(#L!<9C}Q{jSgdm3jHYdtgmwik!zkabhP zSlT}Nci;Q4^e+5aUv`n%l0EA&8eWHM{4{Vz~X zH>CAsoai#IwES9Oz^=21Sf|i1fXY?=nQ9GEFVOiAAOH!73U!^TaFmk;bu$|^(^+hk zXZfrCXL5d09gygI1<{6Xt)qSABq*m)TI%n0oa)(gfusVg&*@E+2R&mG>|sCJKbcu~ z4(&!jDZt0C4H+^lTW{~K+TQTKdy(y~X9p>xFmmv=@ zA_3Y?WC9)Y1eQpGfHb7XB9K`uw|{VsZW7G%C(w08H);TkpxXy9R(c4w4AA9oTNYwFO3GdRTz5Q;Mt3%=SEN%qGDRQ0MeSLgWsa} z{75OVo9UUy-}GcT6|>Ycb4db{24Kw?ahHZeZYMS31CkzeuDG}EP>`|=sICcMxhMk( z7jqh20F$@r8z)6XO8eQG6a&$1GOoqTFK^%hFXZZkgnn2Jl~Rr#GfoUM&eazXUf=v? zSCpfJWJ(RP$I&U&Y*JGR=>W`fsBjVSU@u}`Ol&NUpS5nsFwAGGQ~O8FLP{-P4h}QP zyD-b%^0x>XUSOhnoeX(d7^?g<$cFU4yxvG%H$g8m}{&UBG- zc%`h5)2!*b0#f2$h5&na8_CxDKq2-MwZzHyWD-XR-632n*tuvy%QmY}B}o~ws>J~@ zDlFgv3LJ)+h(*l4J(h5Xs11O%9g|{;f=D>CXmgodP~Ir$I%#LXW)}3Ky|PCCDs+ph zo|2YA@fqZEkd&Ni(4sr303Ui1=dBpK2^kZ-c?cSaSb5kmlA+$GQj`ZP4dbw?B_)yCHL@0vUvwOF<^ zR!2rgVPhmM`55%vnLtvmjlO}y=3qby;e6|eNtxMs(7I|50Auayk9~~D+Z`R=cor?^ zy`^?3WOge9iBR=|QCj|7;GU}_A@*zZ4}bYO+r7*pjNSul4h?ZGrB6TPh|s+?J4@#? z$kb-YJNYrxvcn#x5X`+OhZ&23*HR`IG<=2BIjEZ zg}`yIAZZ4M9&|ZCR`(445R6(_6TCwai?B1>ulu1YJj|f)wgzuB)HIR}wX9hX9z>Undr;TF&o#Ng@j|{MYt^1^e6W;JsD!sTc=T}0{mA22i(Lle;4NoF%Py+;S!Z@i0IP$HlPn}Lkovc>uc64Q+y2AZ@RT!N}6-RBUIr%8#+`Fg1q1^!iX7ELZ_$pUrcSmPs%!av-(pP*{ zjj$5$RN$qkGHw+5hdkgn*O36i?|}{%4)T<1C_S0t%O^q8kXyU<`_|1)BjD8OZn29N zb*e;$bPI0U4deC-x12MKt`ZCVfmXHY2s;u)0;ueKVLBSrx_jW(*~nUo(G|!0K@l;y z;v~h_EBs#|_9j5#LK>M{BSeF8b@#Ga7EDKvMYxg>N@FE^h1Xz>fGa7tC8?)Z5aTmv2QuA+ME)%LaOW%gF^yUbW9}BRE<}XW$zin z5dfP_r(pgN?Y%fztnkgmUm@RqpijArvob}xgrG&UIY2LRKT@n5duOU$F)9bid=38i zb5hWW7toxMR)V+cxb+P5uIa1Hp@$lInwHR+Ej`N&oGc8v@u(EW!(<64R{sub-Y`N||rjV^;!{Swq_&$D{ zH$Z@87fAUwI4L6UMOTIq8-h=hN1D)`XQWM|=ef7}bG6T9OH43EXJ8F6;5xx#^f25qM)aa(`uKp*S5fr(ZYjzP(?O93}aqoLmj zG@S<%dD?kGLiUUliv`U`;M7@tZt)!KE>1Xd`+YbcSD~SfNkPq_C$ky5LE3lFqMo|o zg5=7?A(?ci%bujY7rH+V()wu(lR}31(U4+NV~>gGT5w8xPWxE(#?fm>X)QyY5+8-= z)9fr`Xixvk0mE`s|u21)sq^r(fH31l7&^@lm zEydz8(Znfnjn$DL{Ca&sB6IQkoUmd_NBTY-rX6`Ji9QONPac#X|EQS=8G+psbz~Cv z`>EKi_{`*yC*{tps~|{nr+0qb#cBEzsM50~s0rz3V<1)8MfDIYbcT%2m^K!obudA_ z_;Zus43fa5;*?DM$PlOjw(ke5(M3(ZA;lpIYCO~eCyL&!#rGR`Czq?nC9IjVc0qV=GvWYme=~k8Q^E_AHx(dc|Zc!dRnV-g{Ybz3d-)cO6j^(uo;V)0uZe>0280a807}RNRJaxW1(IvypVw`6w z5uZnO2E07xr6$gNTFJ(BPQQ-mD5}M9&DZWk>86-ixX>c>Jzfjtqy)-}3v8g<`*!#A0$^FrU$$vM$5U z@_LrhXJh1jeIjDVuM(xPUstgrX4hD#^@|@3F4{|LWI=`7IW3mU4`~zap^J?bkONU& zhQ%C#f93#l7uoJiaz_+UEH=D-GE5iG>6xVU9#UhM(lgeHAn{f}VP?WhsSqN+*;Evn zV<@#%1JuZegN90V8^n&%~yP3pngZmdNkSU(x32^ER-g{^llH59Nc~HBic^?rx86Qw>T#tW^}C~%Y;R&{|uWL9EZ8+P#oF7e^6@-~HAYCu^cG0*4mf z(z?v`l^B0(FQDQ*8pR=B$Y9fltDQ#WL$=O74>@)G#`$_yr4|y&#$l|Xy#@w9CvH?F z*vtZ@zr&TGTjb7lWmz{F{S*OeSO)Ha7%P(cZiB$1YkAj|qdkfzIxOkV5;6JZSNj4JAbzrM; z**V-o+!Ii_njvE8?NVo}I}t-$C18&Xmy`7|IZ6r3C;ywo+EEl#dSQqec11)NRd3}s z6h!n_0VlyhHM4{Oq1?7b8UIp0dgxdjD&NePAC#YD-x)=V|I97*@pky^3{X)X<{Ota z*BQzUl>!H_+uML@F#!iU*FMu^7H!;*65m#@)zGCr6N%ddk6^bn3^}M2tzdn`Ld(;- ztw^Zq9%$%gXNhg^RF*-7(*)}Pycxt4HW^GA-E}c|k{xm*4BYs9izV0qjfU3i8Puo%xr~o%dD{Xae*5(lQvyd#CdfbU;UXZ!36MY}m@#K{GNeJ?N zb$txS^ZKKnr`7am=-#Mso;y83eyL(lhr$^=5jeLH6#Y|`^Z@?+`#=~oT8VpW`Kocq zSuZ#nyEtRUcV*y}m3DvSouU&aufK@B%BNR(&Gac!InM9#EKNe3bEqMN@|%{@t{=!t zBmi}U_+Lg{BL9*kZG>2ms=SBFUxQwJ9%LC2wcb`Z0p;H5c5}GziCdLXhkw@Cx(`O)u*_$c8haO9&-d=jS7HPK2`!46`5=sWyW)e-fx~ z&pEP#q<$?GDw0F~WM2mL)yUypI3!^3>gXx66Nt3|k&FH@P-Ao0GvoO4+#jf}bLcm{sHv0oNGp z+@6Sd2OGnJ8tFT~vX9k3c-O=wda)n1-oU65Gc4v3j}$?J74LDNEYT#kcp${hwfy_DzFYH4i6Md29f z5XH#PddLnAPRQMY52U~lCX=zsc<8j9L_W(xJ7HO%*}J|qL5AFGYdS0bhkw&c6La14 zi2)YJDo*5>4n+A~0@Q0AiLxbPWjtMs9H?Jw#QyFpQI^;KUhcS_z$XnQ0z+7iR&b)p zic6<26^}?8U@@**qb2eX<&gLBLi|3v>3p+y#-Gb};nQ-KU=h9TrD6Vcm?%4csa5*7 zM8h$Tf?|!>>w$brC+20WKdT+}JSg382n|XTa-dF;MEn(8_Jn}hI+`%nslg7q(!KLg z#;=?wB`rqIVF5##EaZ>QiXKy`5V%m)6hpw}52OZ45P+^qMrvx?x*wGQ=cHOm^RW1FR?4e^ zh|7GCqMGjfuxe&pwucm9QcS9rBnx>E1)-N*eS!0K>hE(ta1>LKLGT1LB{e;Tr-^D6ZQ21l?FMFrnYtUF~3X3ZIR|hRKBrqQx=xZJW;P%l_*uIHn%Or z#3Lx7#{J7@IQv~1iUl*D{kyD61>1^psW#c=2{TBA^_Q{3jy4lF$N27m`{$C+0UvoW z*Bx~Ks^KnOA)M*a&%dO*z~~==Qo8SFkW5p)Kql@xAAdAsJ5%GahH<$A+c=tj(!GNw z)d!y$!2Oc*24KrM<2KA;h-w^f+J;slM24Kv|7-;=-9Xqc)~*nPnxMtwxgeSPaAc1j zJ1r7~(mB6nzW>Nc<0o=)(+NnJ9$_s$)>Xj(m+qwTujUj3<@p5AtaoAL|KsItCVd#ZGBxd5qs}vP-L*PLXO+Wz=DrqNV8x<#ZIAbY{;hqMo>uR|Js{VStZKT1@2>XeJgVKFMUYwK5$*hFNNa zG^yv4OS{F-z{TxxeSCLyMjf!e>2zS157}Efu86f~ehjlTR!o>%llAv0<-XUksMkI~ zld>;R~UgG*jsr$nW3Zag?=mQp6eqT-2}s=N^n z?7)0)Z`-lXpTpr6IJmLM2hzONfUMHsOfVeJtGEayL<%`O0UsmKU7@;V^Nayzo09sT zM09mMEK6){SX)SfN8j`*u`goJfS@#I;8m`AKqnExVR7by%|}i zvp1qa`bCZ#tXARQIFD;Ue2DJo}h7`+kh;ab4$ip67$^ zf*062U7K=BE%u#l!H+4NbNoa+gc#J^#3^hY+XEKBidT@G2pK&dp*KT=vK9k+o0t(N zY@N(V@!AYLRdjsx)%WJuU;h){S8<3QNgS(pHbb}xugIr9HUt!;{P zsjJ!e6#4{GH`sxjCv{0^xZ3y7&Gg5CMT>s_G%R# zVn%)mBkMUOV00C%igZX`_hajzAZ_0C8n^_W9)VvbFHoK5_Q!t)OloskVAqHlx;!u; z-;9EFJeh|T;#*=A@ZD@NT3q=4WCAqh0H4#@^~_3~j-yW?+NE3|U4<0bAU<~L4@B}o z$6lsSt}h8N9>6BiE#?54YnUI@k~brQcd0Q$?<#@O<>)QfQ{ubTobebER7OAAKt-($ z5*6yH=&O}l?Vmd7DK}KStrp407e>Q>y zQ(5D|N+;j&kN*Y6P2=xJj{9QfbO*4dZn&lKTgt^)1su$Tzy1&$lN4db`UNNW;_Riv zX=Uf24#iN~AcY%fPt+mXEi>0y`W}$<%Xcq&{dHGz!2hD^_PkFV8ElIwIgpj)CToOtE5vLLB z66pfXZ}3^T(3wn-H1f;15>*R`7Cgxd$n-eN*Bgjyxl?rWM4*4AL z=^yu6;M6@6_@Q`r5<=`3yw^@Z$~bx;r?tVYkTel~{%PyRuM-Vl8qggVE?Sqn737?J~je8bb>pK4|W#WxDsD@+*C@_PPAuo!t*xqj8 zO1+B7lz6p^!!Km^ovHMxY3Fdq^x)w8Th!la<+-j1k)v+}0iUM>I2$)#lM#Q9GH>!4 zcpCl+F!nGU>7yl$f3xiEGH6~l^!e+f>`_&%jsE142Hpe~c?1O*R-PVwLU#ZWXgM$c zfj#gsl9D|?WWca18VTxZqCMxoCWqQ5QXrp_wq&3tl#G0e%l3qQB}?EDaE%50TJ0&U z6D5ZmpPEN>Xt_ByH={b~n5PKDvzl|a!YxF;SMK=kJvyBsje{~_xrYg6kq|3j=!@~m zjkSNXIR2~t#hOB$xKN>h_C}UbNyAGxTjTvU(2KbksAR4_;a?J zaGo{ozZIh!j7aglQG@?OU8>b4FOkFJ=y-=o$oSAN9?Fwja5iyu%HO*Vg`wT+b`VyW zu99aub}+U(ra}hr2{Kj&AD}A;#s=ABjgOCK#`5o%^B5L4j1$#H0dbSmC9^NTUf?ft3y2%lWNf2)0AN(p~|@kiJb;Z;OKmgJuNSpa9Mr6_3L+V zHhzT=br*08jn|5!6X+f&3QgwZISZh*zn&w11C znhUKbT8s~YM%gXMt4swESb17BpPc{rn;5%^VZE1MtgsinnG^O|;iARZ@AU(l(~2Xp z`|I<2Bou~__Wd*%ujbU`J1pyw`g!77#c=EgNh|RAf4mL3;W*e;pko<5VH03DDj*_G z^cen#w39$jB6m{y5|o~e?9;z}h$~NFHz04hGpJ$qmmjMOk9C39{$wAXi7}qd^e^Z^u%0 z2>wA$I{n(6@(-|3S60W{zS{3=Cx|;MT9qsU9$igF{(0#4mp|#>^8aRhh9b4k_a{WL z&x3e1M7Bp!6QN&Lr`GuRe*1WA44fTB$CLoybI=_Z@a17-Xd5f1#?#vD38lDwGHuaFl>LQKW9Ppa95r!7;hOpL__^b@2=MuTB>(UbQ>f6NW+^ktSYq zhROW@8wkXYmO7sx!WHus2m`Cwj_-tBLC^=4JCG_2-(!99Y_dC$M=m8)Ck7jQ&NP5e6w0zXM0w9;)usvt zzec|cfsN;beGo=*A|O2ok2-<|lLhhH9f1IA{O$w}((8{XihLpvkF*M=uCr~omjiXi zbIATz&UL0v2GoiV^w;P-zkNwb)*(JKE1H$a29G|8RiTL*ooXAkFkeD+&E zSWH?3AD-|CVRz_{raw+aRms(4^fWVWt`W9pDN!uXQcU-oMQ|W<19N$d zGy-t}(e@i-z{%U}4nPnLbPP8=-US&4RMpB z2<AoB*NL2ww2+EY7S?u8f=(DtD3%h|y(z#0Ts`Jw&vO@+6YDv8W1 z^&G;Yp$WK9ML^BVqio$+$CDZyQ}s!4$5yiDLdt7LWO2B&a&wGyt#D(d6QM^QO?oRq z50D&5uU#N}f#4Id%>lkH)v+TK_#&?YN6gKLJsY`yo@@yN69+^}3@XV2?+vWCXE)iV z3(h@@r5s$NxGnLvcMG6#$%m1!DPzj4{`;Kl5xP0W9^mfcOL0a-Ngc111yit(q$<3D zFozgNA*jY#;}a@zS~rCfD$Nkkv@}J^6n`w5PO|w_@nAHL+QpqPKvL&+=7%>JYAoUA zB>*(E!6{*++9OW_m=Vv$t@M3gU;0Kg^y**CZfIK;l49vZFlWf+5mS$qRVBK+A<|CW zR|3P?om`N5V_Mb$$f(s{J3ArS6fJlg`$F-v_cztj>wj8z6B{0Q9&y?P*o#e&l$WGkI98c7kM-|JgHOpap9(@A>-J_s?-H&eyhGY5eo(X#wS;;qeW*{7_kQb1S@sGj*E zMT$hW@Ivos5>&9BAKUn#?5D%kX6=V9*EFc6DHq0=dwAoIZ67IT7O%z<`r33w?NbFRq3p-_Ts|_U^=Pt`bKVdo-h%uk5&NPfF5G| z{Jo&|B}(q4OmT}^0t0c0bDja8R6PX)x#i|;qWvbYBpDxSMN zQX$IcmN_8PN(8hmxfCM%P`(j`_jGV>iWeDvC{~aKNRhG~uGlEZ(eQ{?+c5_{FgEyi zb`fUtIMo~(faM$mC7+oII7U-elFALZW?eZ2Uf?!#Oxjmi`b;(N+5UJVTVnQ&aE$}( zZ;mh3Sfa}6Z%j-_U!o0J;M@O>SW&6B7`FkYs{BDM+3MycHhZM;l&$pI8Bt2#5(K66 zK9$Z;qsQ>}JhvX|Lfv+KC)$~le4~R?b zf3+Dm+&?cnGIff3uF=Gm%b!dt1DVdG`r%i{zv|+5x0}GR;}%F1=E3y}!0u9IU>alr zhlpfi{d;wA2cw|P)7s4@!p<@PK@rOqQxeEye}vs zcAdr8Pu6~ZgL3(McDi?h-iTU%%#1~^VXgeRpW_WE@M zz%0m=&+rzQiw@#p4TpFKih#{iX-hQqv&``oh55iU-Gu0lkO!Sc56}Gxd`qCJ-mQ zf?44BzZBR7s7SA>X}!G*CvPe6*pek5SNhXik!q=6DB?HcC39QjpbeHjrA4s3=Q!YHYUyKz$1qoJOL;#3? z-a}CF`C$T0R)poV4J&l@3jnxU#}*uC&ze!pS+4sE;MKY-d(K+6I8!@mMw+-d54@W8 zUjQDnPW$IANrYp@nr-mWGp?-Vr?v>{`$9u2AwiqD2>PL)c;sbbcwJ0!%n8iT zw|?NdfD|PO66TM}xXti^t04oRPvJzY_W*{vN_zt()=0gIo!G_H0a~aJBt%}M&GHyv z)7)U0EVgV`R`d!CdY}|aPvQp_c0Ws)*Gh1iyT0Ej%YgVRXiZxbnH;EZ4lwdM8~L7m z_|Uxr=Adl%6MhS5fkvbur+CSq`H72-mxsi;3u?q@&RDQx(1I-@m;HHR>hHYkQEcu} zN3);~4uzrfL zn8KLWf;g4x#6qPCg*U>BFDR zk7N*&V*lmkglCT_W zFEILU21%(knq>(G;SF}U_ef^nr`+F|$g`eS7o7ABJkJm-K|4rDBj*d9xS@EGTcjJL z6UqvWb=W<2Uj4VpCvE1OFC6i|sI(ftzwdr`&tvfP3VJFXkUqX!FjZn|@6C!)q*J zNeIQEeK~GF2rYyXXX)yV+~$Lvo;jY_n5mDHEo3Pq;z=>aob)(7lWvIR9=055@1rbD z3;B=CgYgbzdI_}q%+83b(Ph{0S3+ybiB}VyCaU#x1TSb2Gk*dKHQ)X(fn1_CKlYKCleUA@Cob z4!@GJgV^;Qx>KSFzmQP`6}N9(hXbx9hdGuhhl!`;a}gwySACe(m|w=tMK|C&@qoLM z{zo|!)hsPGfjtwad~`%byaxFTP9>k{MaA1$H<@f{Pi4GVJW#phy^(HNflQsT0&U;` zGJuw5*+-XSlIqA)IKv~JEXnUuvpfR${;Ei0t4@m&M<-d#lfTm;9pVg3aV{aau@=Fc zWtKQC>ncc9{P- zXW6Z+)I>`zkzh$bQ-JNZ9KVmvpzN_txuEDk5NsXMW_|4vmw-j=p+bQ2F)*UKyn=2wAhGwQcjmu@g^WkT(*IB)f$*9 zVvj5$R&3HeXBzR`qYs-xh%RHqTB>>*$a1xd@yy#pl3bmgeE+3TB_plBrPF~Z^rVrX z@ycAYOM){2PLh0+0wmh29TEkhGZqyiF_`yD9p2X za2-kG&o>Wg(!R))=(o8MC2b@9kU2JJBR-HDKXhzSSo6#$>b^Kiq_wh4$|dNIfyUM- z0$(mt{+)2Z%Paci@vF9YY%RPyn%A2PI7oqj$t()92WTyz!qaD%E+@R*`Otc$A!eyc znCwB6^ZgF-#XJ77nHhxl``o+IBUVz49%ZmWsZ%GqGip^6K!YTqt7TTaYCy{}*Yq)nD?X^hkO*InG8((k;3L8VjfbTv?Op1a(CJvMjb^lFC>{hgEirs{Gn*XI5^v^&Twd@(9az;lL#qF-t5abv)ZA?Tbqufd$-2mU_%@4 zCWw84c+^Y0maLu|O)hg@SQD+{%l3<1V}~lTY-q%p@AxkX;@0r*gP0@ zRf`bG{id&5w=FSWjCl3-C3hwaY?yU100wV{-nl&JcXcHp#t)*0yiPU*H=g@kcG=7{ z@i6K!R8E;f?RQB&%_dZWiNLBXvxxsCJa;d}PFKpZ-VE3N8*B3Fmi@8q2gT>{G#jX$ z`kH1az3c+{M4nvA{{hZ_<$MPuQs9;0YquAS?{w~VIS5wtLHqHJ&h?1SaQ48N?pII| ztv_=((GI$oB27|6>Il+eAAUTI3!#jFV14GNiw%<3+`L0k+d#cP6X9m!Naut-2;{GT z`O8k}Eg=_vU3}keK|QFU`}bYm4&BnTLemUZUr)uSG;cj3UHgwA2iv9bcy}nU0q?)} zb_aU$M}ODT2>bPQ&e{X!$p~>+8+;xoz+Hb2teT#lODq zZIyHTiYX*ie=1A1puORi_aWN#8re#{)Z<^ZJG=X}zsoWf{knHjzdb4%_&E6}!_WVC zXKXR`_hH)96dHSLTF5B$W#(Rpr&q(~->!)Z?uXaP>VanSV5}eSd=1SzS>gjA zm^jz0SEh_%QTUJRV8>r8NsE41?Qh=UL1`;jce+?2h%2 zm3>v}I0oFeR7uPnDJo*nX`jhw_zA5mAzq~_gOJm(PbS?lnLxShjfD4=qy>-$y0)`8 zkxn|v1?9Y#!uIKr{PQqg$FRicJm0n9woXz6U9T)Tb<-p@;kU~E1P1Bd>u;3kTlo2S zc9SPI3TwAjtI8PGxB6r=$X0=yW8TAHR%}OYiOaa{4fK=s*!H6GZ@lilY;Pnz-^e%o zE@@}zPC5~{_`W|svcfO5#VyVPAo&3P;m1>@tfZRI&qoCQm=5jX8@R6BE&)$3`6u1y z<>1pipepI{Xsrm2V{au0=-n6pT{~A;Vv(dORAXs!UH`LUh$B_wx5}wnkv?|u3+bMt z_JzFSMB6M;=FsC>Yr~m~73%o;i>_98r%&~ zpV=qx*Dgz{7ne+ba}4#x7V7pLY}ppN6j46xOZ&3*#Tf;~d86|E8J>_rTT!>BgFV}r ze~KNi9~<@SI!p1cNVlEmPwcE~?0spAZHzaZJR+Xp`nSe9offITCgUkils=gHGj-Vi zy=wjAUZlXz^A|=3*e!$?=?KXdo+euCP_RtaZf@#@^HDriX=9p;r$>7m*|4zjS*rPC z5_bAJ2x*4|DlG;1%FA1a?@DYcL=}kxv-GoIT9RxbULk86cd*2iD0}0r%Y18~ft}c4 zdHen^`PuYUw?C>|bUdYBD;&D3>g)C#?mZ>mXV)AYFo?LY<^2LrJeb{UlSSF<(~bql ztaewEtuKQN<~IQsnwCLBqIg!jsdfo*I$W$b^&DxPL9=|oJj4mm05{%^+8qt*(|Oyh zNTv^wcXvqt*8e@0ezf*^G2Icm9}l%=)Jh`I7IaL5f(oLU0>gKb(bt;zhhJXCdN!Qn zV}=n}>~`YVoP*V|F115}|7#&|$xKg6!*mUi$$xWab5k}>m}Xz6m1=UGV`Yw4IW zu?Mr1=ReAyI-S~@97ZjF3{EPd!K55E?=RH-?ap?ob2cdGp(Z=I#jJoMy%ntBKsYO5 zt`&RV{#p4Oi!L{!8TrgHxDe)nb;{P)9`6wX0=^29TUkb13(-v`WNo-|67CWYr0E51 zy2PsF+OC_2h$u2Wd6jZ{nR1VyJxYX#E#?JX{bepWsYqrh3DtYc~5fc>}Zht z{kpwWQqA9F*B*0YD&p}6hrZA3@Y#Qp-o%LLUZij3kK-hk!`{+SmSu`hC!5!#?o-Tt zh7U0{oH9tckjolvseU7&e#;)XSaf`6><;ZARAf;ySNLArT_*8oamD=Z@L(fP0Wuev zrd|RbcDx`YxzcSVV{4qqa&<|{ZcSB_y5ebXr3-dMB6nkFF zvv(9}SW>YEYxD2sg~$+viu-*V4A>y6v#S0$P2_7bw#u6itNZDXfI5p!mO!Ro46nGf zcfLs5)=Bhge2D7?^#nIciN-d|-&N8*&DD?m%YATZzHXG1WL#)#mU-!2qt~U{p%X&9rGfpXzAET>xkr(rr zl(sQ@&?1wbM2^Kfl@{wOP9P^K32!DQ%Xag@n>dQDkV3KJ;vh8jUI#6EPtuaFvs5#+ zm}YbBQpX;98nzkb%WJF->QkVf|IN_1*$=8=a-d4VQDG2z2};*+ujZ(8tMK2j zk&D0H;3|uMM0z5vY(B#lQtQ-m^RTpLqdJfFRz!fmB4b29Yx?04DMlm&-=lc`e25Td z770jS`rb#O&L2nIdErCxRLd}~EpRw<6MJ_$grvHh$}I_}TNWdoB57j0$x#qwa?cF+ zAiyRo^}3=eRy*F%RX&m0xbyic=3n*06E0ya;xsVzANLst7kDjAm1=d6@&H{OSJod9-xAVvC{ND?xUAOq|#gMwPHeGqX0b-7EuFzkMJt zm~xw>_{wtYCVUi7 z+}UL6YgT8NxdAA4QgF}%jKvl{RghvVn;d#ZqZ7pvx95{r_G~B4rlAw%8>Q$gAWQaG z)%4R+@X4^%;@54>Y$hu2xWRJI>twnNoRvM9mGevX?cqD zZ$_!siN5$|hcgy&ws7Y#WQV-<7uy>LODBr+wy9+2ZJcb=DYWSFFE@(dcPkLIgNdoh zsEVevMeE;42OBDu+}d5HxC2Q#(h1o{#X=l^s;q;%(M$%m&x)i)(xvXE*IZq7^7vSp zQSi4;C{&N{X*e~X(75dX`YIkJhUB^K~xABh_;+!_Vbjkf4be=5J0{Q9--12vTA;$&~13Sn?t! zrzx)1f1_NGxDZj-t3N;<;l{2>cK8eTnKEyj&vfwaDC1^JVE% z)h%T|?hs|~qvUaJ`*=}}=5DZQt~>Lx{M(E(_l1GyJNep@K^Fcfvi7bV&yr;3PNS80 zlQXBey5YrJ){iX<>$E9)iWy(mmTzNsL^CW}b>Z^q)yizmHvASa7ZWsRDoOK6_{bL> z_LRcOf&pc&8Par%?~j9-*`|EzuU*O}nRdzync*R=t!ss*(TBc?J;IWOo-db|Oi(P$ zotG#hOjv$l9(nO^awYD@G4(x9Cj4sKr>2!IdWrv4OTsk=+6cEZc2g$95*%erNr)bo zR|m>_F?o?le|Gxh^BQYZ7e@d=#_3l!#kFv*?ov^4XW@4yICcH?w#!P|B2Bzm`1k7=5tIAe2aq-%d)}7Tyssn|1 z!O5IA>~ZoN;Ko{lSUi-xPTlpWi?Hug&A+#_|2Ri8!G{NK0>v?yl$zN1lE1Dd6C6ue7WbHNGW(S3_9OFuT0V68hPT) zdw5N)`mlQ8Xf!Xw8g+E^t?P%!H|~JQ{Ac)6bnbE z5f!iD`;f0@S>;SlTxu+{Q<&zjd>nqEO;h=2L1>#`hRoC@&jGy)Io}pJ4+wn38t0iR^%Uv#E;J^Dz@Rbzmb|EG0~+$ zp=a=-!-kRKwo16=VygwYo>b+Zk(zyZr%LL3(l_}n ziK4=HUPsqr^p#-Z5*o^S2{TkeDxMRT#3)OAV}Jh0Yy%nP&)%yiU$4aV4;WQ+~6gqmBlUam(q zap;?-e&!9bJ2PA6?%lnWTcA3&xBAmuC}q?>J8k^+QWu*xS&#|Jw@t55Xm6EK^gkiqT*mdOF$=mF85D-85S36zZqVaBn)X}1yUa+BB1o4E+QaW%A2I*o z7J5mz|HR@idi}`gviUL(3lnxL2a}eW?}vDYlDKe_EU`Fk>x8=J+U*9*o}-0Ra>w8N z$q{3@F`C%4^orH&LPj#upiju(bKNn-+NU~--gcl|TM?Qv;@d+xTKLDfe#CcF!rlN8%fw=1d6Sz={4ga52pQ1;et<@@BwbyXj(2XN6x!YuQlg z9aPka*<$fr{HB#>RQre6;L_`|56v8xQlpfPk7D6wB0K7JXIT0)(%i$GJ`w2dVu0l z;0Jziv-dvd??IU8*_g!rXr&MD(tDX^;f^2M9m-i-bzrxzy|Rui0a;HC(UbC3+Y?{T zXa%Ngs8r4==A<%E_%Hq)pCj4q%6BbRz^p- zjKB(&ep%SX2|C`cFQhA0ZKY6pFzRn8QR6Gz(8Rto2A+8cIi9OK*F7PwArzm&L}LN` zgl>81r>UZzE6HR^)JY-z@(P!UQQ!>Bij{yBk>gO$LAB1yXqilWCw?%9tUx>+O&?8o zL~ht!78iy~mSEVEiD>wlVkQhS6yDgO5^x`WV;%e-E`a`xgnRaOKs9Zno>5}(Zb{3S zkV-!J+)ON<(8`dei!6&$vSAA#tT+l!Jk&V`ajEkQbs@6%p`^g4V{{sGuBrgFD0`OD z^2(DwgT96PdqTe0Kyxd>CAvHeO~#3@yL+s@zr;e&5rNWJRUlhWEr>*%aH>A!sS2RM zwR+?lz9;-Gj(750t6?blPP)R4m!S=B!x5}* zmDc$C;^M2{9E-l_r?7kE!#kn-UpI(lrVjE}FR?%qG5vJ?en1(Rq6{9P9Nv_pl#P}L z3sO=#uE?6**>9Cf=`)B@FZWzuX3O|TGskY8^`lE{zdNYNl=_!0h;=UPd7^^z=@OXh<{uo5E0ha~Nm$K9pjJl-84NOMF8ZZ3MCj~TZW;PXep zxxQHTn&+1psHvQ)#yAve{Mo0#$XwjF=MMlGwK5DmKgX?KgL{ zHl{)yq5{#BDS^nR>8?jM9HD@WVNW8l8MqF>_eRp_p3XZM_%|k~s|My=Nmyb<=m;54 zY}KS69i%~<`1yt@-RP5~1|JGZ1w{zBm*KnK&4f1) z{F@QVpO-UrfHqVLd@Tc^JX7gt6+`F9K=c7=9Os|6Sz;`jxY(-ZTsXUzHU90Ocf*6o zt!Thcs_6cxH@a6goG_3MZ=sGVEv#d?3lSRrTP2UsXg_=h&Qx}wO09x{9IPmShsPQc!4UNC>Q%0YEo8Dm81+XUUr%yZ z_6UuT!4E=#-^ysx{|{TZ=- zCTRl}aD`5$HU3Y|-BE`>oi8s#A$gMt!<2~(!vf8}LFl+lMv6tId>K}1mbdPy$_1?V zMVN1iJ}OZ!rSHZdOY=$$@C{U6D-9qAiY8h)Aw1aQiz4*dWsiXmG0_8!MkVpB77FJf z(?{REx9`^-wsqj$2ZXS@S~d*mP%WorK#ZAeKhSOYwbLU#Ww$mS=Yl*_;)870{13;EZVl_So-f_&wk3(NgE=JtTeJ3jXI`yq z4ZPt2we9f=+U9~_lLP>D2Ph_IbtY2zqgmxLfMjX%+4zcASI# zUjF6JWX-lOC?Cs_laImwz>*{X@6k~GDBvo_-V*hTkAQ7abGm#xpcIKGK$dU;e(1yX zJ}FNij{|*GhZgSPcsxP_!r~Z4=}CHFAY9pw!5zx>KtL&jACJXK9oY>f0k>}qUfLqX zr8+mS-7RJ}7kFkHbl^W&37!r$ML5|R{yz8qT81Fx5_i99h!_Rt_5K@#C?TM#wC^Dt zx$E>?0>-H4Dl&ORSAG_)=wYKdO2O9hb3R^a^m;qp+@Mkbv?eA>koa9jB7XQ6(EHWU zT~Pkyg9FZnLZG{;+gG-ISds?&86Ja=cb8^HM|^j_n~u>b(QbZOcxjvuiml4;LSyp_ zjCxd9WauQ=$WOi*6hzB3!VV%-r0xuYQ<^e=^@b|yHCoUr)eZ7mBL#)9Xeqp~8$>K^ z=wBe`AK4$aCP}ITK@mS|I$2Hxa-YB`*u<>Gif=r~JONkPa|%m-(u`(r&{0yl3?BhWt|Cz2ylOzale_N03@o~Itx zH+I$=2|$VTx*uqz`>b%){Ce$Z5_%R&p-~Kq(!m2Z7T?>e7ht!Hz}9y(iIBm-USYcv z+j~p<;OyV&TlQTjuM3}JU>=U;M`XXdB(^{;F#;mzbvQoV>g8qbmwsjEt&W+l-5+%{ z9mOMaLWz_8a>_h`Zj`A~*3V{WDYZH?l*3vSCyNREaIMAv17|kx=eeu~Qa?nHJ`vh4 zlMhCUzLHuD!%Cy7AS^R&p}$T)Tms)VhFlK>L~8f|KO&sL(8p zr8j(Vl%Jl2ML6%HbYVFosgej2z%x?vsy<8lq8 zs-^3dW8L7ZzZ5EBBB0 zEsrEdZpcT>kdU$aur!hNQEKaU1bpZRqx`y~6>j0>Ex0lD%NMvOeq#~s(I*JDhL+HN zD$Be5F!k$m%Fvbizo++E3Dm3c7cD%f36^>0jctDgP%N`|#%yLYl^cB^o9V%*IiX`o zUP@)HTV&eH;A=gLBPNim20IwuqBLc1@Qg_N!J!=+NIbg07AWCPaM~c0IgNJcGtJ2u zp$bl+#&JR)-HsJ==Er|2cZX#8CJ&a%ZpMykLb!Cg2QC?BXduR$x&Pr)bf}ulGB-1h z{!Tnw;DZlLGI`t)hMCLrNqFZ=qKC)r-o>neUEw1N*{_Yrbw;M9D8GTG>sfWW*M=K< z$Y$qrC=#e|+o zq;gYQ`_=o;+g}Sq}S8I3kPrqGTrg63Rj2_g9c-TFnYzjm-LUud!21s$fH$w>?v~L zG34z|3CCBo81DV;(KgqD6coM;LiEsc19gTAb5<_&`UuBI6*|jzQ*8vL471`G1*2O z5qM4YNKY*u8kchjdY%d%~mU;qt6!|J}XBj|&j5-6^CQ{dRj>WK{n% z7&dR(-mpT<#npHo2~)Yi@dnTJ(2taZaM=DKXv|#mSj^-|4Vi_2{5TP#ltiCt zzf0O`jZ&sWpS;k1n@U3aWY~%fIWo_YW_lTYm_Rmi6$8-Vx(&MgSsxoxKf5x(_L0pQ zyFa^L$|G~-_Q*wN$j;bC&*D9}*x%3wM5PrKay)4O!}bD5hdiO~vsu^ae*_hwUkCCN za>2lX_~+z5Fg?#_2ftz(G6G#r&hk~Ybl^pU-2{4i3r)poK`R9wwnC10?`l_VM>pi9 zPWN(Obj9bJg{ebj=G5e1jv~17W;YSLG3?h z&S4gsxu514Vha2U>RE9}cK7z@z&%)8C-_UUsnUYVP5I@!=LI#JzAZn(m&aG{Ek<3 zda^a_LX?R((qYDDP4kJ{;vvPKUAj2}9cT&>NE-{la&-{A!}2eVtd|A+Jre@CyQbC$ zbwg-iXQ$sz(@e{5qqv3SB7+qxhU554ZH$jdI{nEyG~PgJ)7G4%XH=S~y>^sbc^rK| zz;ETSzjv5ANDd8?{V%R01j0bF1*W!hL2}7ZtXrXo4c@F}kj0OHDmbrzdcQEkzNY@* z-_z(#A?-#(E-aU`Ohd+r1<7#2IX8yvA0Z+nh96(QXn<>W9Vpn3jnL_FR z{mgExHv}wkVKAJ%#gI~yf;pfM>0=q~#9H(Pda-^GecD1rn!Eq07=Z!2_ScJUOJ<09 z6QT5|f-YdcUdo;MS3rnD;(L)Oo*%8Nt4lL`@~+V(R_>zbeBhm3z+YD+^v&)g3e8SRrOMPzAOFy zx;bGt5ZJ*$>N1;G3fDE}3Tn+9#l6L+>4Smx7 zom?G9yxbsNq%!)?AAo>VUYmvIT7D9baqO1ZQF9!tEZBoGY;@xyUU-!;Fb9DzPqF-T?oe||&}@|K-~RTTm_>k3UW z)KR4L$VL>aQ!ZtLyN=6|qZZCVEo))N8rVetGngvdG)-;npf~*pC^Nt{1>8H z(i=b??gU0eZK@Y;CXT{V6S7<$Gtcmm^01Od4+NDWA^Bco;fp947Z}h*o2<($ak|)$ zz~9LH6|wUgxsq&Rz6Z2D3t=q(J0>rZ1a*y2XSa}=j1sBIkQizWYZL|r*AcGXa{zLG zCD=jt4^PM;E9SKIKXaI6>HG27f3ZLZEmV{oL{NoB!*a!4Cf>`cHxux%w!azG*TUm& zG9>8qmB#V{^F|GYOH5}644x7wuBsFXeIXL26z~3L zQ{21EF6P{YKzFaiHGgp=!4CeqSKz}A8JpcuVOk7>MBoZ{h_!T9ip?5{BmZISY1LaYnYHD$ z9ztZB>~D52-q&Tz&B1e3;a|NXimru=X?er$flx%$ND{i=F0&kmOK=}fIp{8oKnG%4 zF6lfs@}H@he;9i;LWT+%m1a34Ig>yy$*y6k=&8z0|4O-E`Aqt3IC9m5ZFp}7?8}iG zg^fT`8*|aL?|-8RZ~f9A7EOBCZ2iMX6bYh(qB!b_asqmHZRF?KLA&%QK(2=GjDWH5 zpvJFs@DN-@Y*U;ola0;d_3pf)eEo8kI|~VeDuf1mWd~5GMEB@ki;^ioO58#802K7E zWuQ7YjTOl&)x8GwVD*IGpNYy=D&t{Dqje5fLG?Ixt!murq>fbU3>+?pk-84Kavd06 z!zr-q>KxnYAEhE_lRn%Q`g<-)uh&5N7}@7MH3J&eNag#s^*m4VRqT6A z#b8V_28b7E4(_d{or0B6Dfn<*qL2camo-x0-b;hEPILGxEMgnXWk!TX?ouFF(z8a7G`@j9vu5%wv-bTKV>%+POw|Rq=v|s1fbwnEC-~{XO@4`AF%+yga-??>{CO& zE81s00cSruY?bB#k7?9Mtp}XL6At<@qyXL?2i=#QlV?lz!7aWo{ih6C6Clk>5wwk^ zSVOzIO*|8sMX!!k6+z%_$3F(JwQJ#vIp8Q8VK_l&2%`^}eD}d-whWxj#9X_*BZiNu zfYtzqGQD|??roF+Y|x7_cnE$3@DpAM-W5GATC5fxt;MLQs8k`+7{wN9mQEFO`;pT3 z-_s?>B4?3O6yFUc7LRzZH||db1W?ByI*b_IOPQkxvG5yL26!x~r-=UW$DDu~#R<8ogW29mTkj9#&%;@nR(IL18Xg<`tuJzB+zaVFRtZR)JN8_*5y(4xf6@dldi%Y2$S~QiE_3fM73sezMl^IzqXp!QwJdjv7=Lj z3<}8+gEit+SV%3g^X0qABeSySuZ9tPlS#mf-&WJopnMEQwpHLG(mPt|?1yaU(uBN_{Vc|ZBSh6i8{0l)LtyRXl`-jNGBASU?yizCCg)DMd8jYZ5H+|)5m zGFN}|znRq}?#ekbes}Tpt8m)LqF9vS#Orm7Vrc2z4$sd`KG(a3bfAC-n^Hg0+v}13MFpl1ByD@} z|4G|ANG-w=5p-5r7@vSUyO^MpSbS}o#Vrho^U1`&>?8h@gnj)fp+_C0iXD~U$p3#1 z#VanT;VfH78eG0SAXsM2kCpR(^uM`Aumh{^cdd={LEBu06?BK0JEphpM`rHhQGqp8Wzz*t4ewR%wt#>|Z5Bih~W3au3HO z!T9GDtwz+T<9d;9k=ryErgW2YBGYr-s>EnHkiOFP%YR4WUG?uM!}p;%w0X&!;R70w z4b2c&uYAl~n~mms$=MV9W3g^kM`=z6s0Yw;ME}3Gt~{#AD~n4SF<}ZrAZi&FlMq4_ z3Tg)$D4_uo42uZ?QBYZ`Al9NH5>(0-s!RffMnED8f)oV70jelL#Ij8W#6bsS5wLUNU`|kVR^1Hv^efQcJn_|pJ;8y6dI-V23QPc47ZRaJ1 zuHoB_5cfxhbvJEIMoAdyGP?B=m`XM1uwSXSZeXjga4 z`_|(FhBaELox6HY$R4zVbzE(%fxZPAbQUMt{Gc}?bs1wtnS+|<4pLt>iHG%j`>k?% zI2tG&dEOR-W?U39l9Bejx^I)<(mH;5{!bKmNWWSSMGET$zRGYELb<~sy8+a9_m<39 zn!J@HEj^OF3zmw`nl{WWqFnpi9<-iv)s!?O)C_F*p?dI;?rAKl#(Mw&jmLM@*GBEP z`7|sBjuEDv%OGy#r~b41fgfQyPTW%Fv&A-ht>uN!k52WR*hvt{^|>mDc6aXCRKnv8 zOpNuN=_GU}v(O=?;w*;}47CFfrX_vW!DzOJWf{b_l@hUsXF7m4K6u*2|ZI=C{efohqC#^rr#NC15~M zIN{L_v7w=apSTCatYK^?%@JnNp2~!4Qy-h_mO=BrIoK(e_wXtgv&k z=r)0jK}32U#2~u|Nu>Oc6oeO>oxXDd?ZOhX;wRMv`o#7Awb4k1v>9dP+9$x7-*k@}%7Dgt`MSdO3QmB}^31Cp$uRE#V03lwjeNG5&BkH8DCn`1LX7P7`9q)k*v#?Wl*ef=9o5VeB zYa$}(T%Ff(`#NB^8?aZ{^e5l52VDKRBr-tr7+Oo#^V@jP9b7`NfkieKu+ILX$&4)& zX)4Q6f5Y*cYh_O@N*t^%x!2_Bjf~Cz9Al02D)r@s!Jw@21NE7)GvS=VJ6PVsjcig7 z2vBXZ^W&)i_7UE%#!I(3f|~oqbi+qx_kMh}$=@9F-n77|Rr$SU_6-p1a!ytKanZYSuR~HKjqMLZkxkshvsb9d$foq>D zFc}AQGHb`silbaye4X$fjw}ouxz%n}*Ye<`zoqJ#b+t6InIX)`e2QSo#Z8EQ+mP7+ z0d?U(N58%H_d_d&xtT?PEcH0q@W{cE+{lz&>&@=fZ@!v=%uy1QQ+2N=ueV#>Fgzk} ze7&k$4|(kbu8|0@PwRt*XX8~y4Z82g5^1ZV886&6zNcag1b!vMb(| zflMHSq8h_8=+>X)=@3QJ$oqG=HF7Y=;ka$8^2zJ?FxW(@L72*F{x}b!BT&Ez788~y z%+8Bw!beXxuIVikN@jKscQqmthWLp&%A)F7pt9bRx{0cu5EDg^dZrK3n!JKkwN_;T zdLx{FTve$0I-doR#Nn_m*1$>LO$&n?E+PR}%<{)UrPYT|Zb0_g@X_4k=?lL-zpHT1 zTFUTd66j>ar|3o-@aI6`>PPYpz|6lEzRc_t4y{^+Iq$}H+JWLwdOUL(|Gr@J%>ivg z+o*(cH1^<(n8y&H2bB|&hpHdgLd`>z|EV6*&&{{!^X2*0;4ClKKZzbVv+sNMDY{>G zDeh4^c&k0>L4!ZgzB~pHdj5xmcg=m;>1{4V1;+~&?Or^({O(J7i<41Qz`L-I#V+wi zkN_eCg6IBKScM-1y6#otM>Z?| zr9W9>u*>Ci`4vuq7qs$6t@0RXmHz_^R@RP{&rsU*dpECfO2nDlXI8<{9 diff --git a/tools/aws_benchmarking/server/logs/master.log b/tools/aws_benchmarking/server/logs/master.log new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tools/aws_benchmarking/server/pserver.sh.template b/tools/aws_benchmarking/server/pserver.sh.template index fe2360ed20637..5e46a4246f1a3 100644 --- a/tools/aws_benchmarking/server/pserver.sh.template +++ b/tools/aws_benchmarking/server/pserver.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run -i -p {PSERVER_PORT}:{PSERVER_PORT} -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINING_ROLE=PSERVER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file +nvidia-docker run -i -p {PSERVER_PORT}:{PSERVER_PORT} -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINING_ROLE=PSERVER" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} \ No newline at end of file diff --git a/tools/aws_benchmarking/server/trainer.sh.template b/tools/aws_benchmarking/server/trainer.sh.template index 89f405811e768..56405a8e31d0c 100644 --- a/tools/aws_benchmarking/server/trainer.sh.template +++ b/tools/aws_benchmarking/server/trainer.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} \ No newline at end of file +nvidia-docker run -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} \ No newline at end of file From 45d87ade441b786c8d6cac20a6234816fe5a2019 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Thu, 12 Apr 2018 20:55:01 -0700 Subject: [PATCH 10/15] minor tweaks --- .../client/cluster_launcher.py | 21 ++++++++-- .../aws_benchmarking/server/cluster_master.py | 41 ++++++++++++------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/tools/aws_benchmarking/client/cluster_launcher.py b/tools/aws_benchmarking/client/cluster_launcher.py index bbabd98246599..3a6cc57b3ab3c 100644 --- a/tools/aws_benchmarking/client/cluster_launcher.py +++ b/tools/aws_benchmarking/client/cluster_launcher.py @@ -49,8 +49,8 @@ parser.add_argument( '--pserver_instance_type', type=str, - default="p2.8xlarge", - help="your pserver instance type, p2.8xlarge by default") + default="c5.2xlarge", + help="your pserver instance type, c5.2xlarge by default") parser.add_argument( '--trainer_instance_type', type=str, @@ -68,6 +68,10 @@ default="ami-da2c1cbf", help="ami id for system image, default one has nvidia-docker ready, \ use ami-1ae93962 for us-east-2") + +parser.add_argument( + '--pserver_command', type=str, default="", help="pserver start command") + parser.add_argument( '--trainer_image_id', type=str, @@ -75,6 +79,9 @@ help="ami id for system image, default one has nvidia-docker ready, \ use ami-1ae93962 for us-west-2") +parser.add_argument( + '--trainer_command', type=str, default="", help="trainer start command") + parser.add_argument( '--availability_zone', type=str, @@ -104,6 +111,12 @@ parser.add_argument( '--master_server_public_ip', type=str, help="master server public ip") +parser.add_argument( + '--master_docker_image', + type=str, + default="putcn/paddle_aws_master:latest", + help="master docker image id") + args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') @@ -322,14 +335,16 @@ def create(): # set arguments and start docker kick_off_cmd = "docker run -d -v /home/ubuntu/.aws:/root/.aws/" kick_off_cmd += " -v /home/ubuntu/" + args.key_name + ".pem:/root/" + args.key_name + ".pem" + kick_off_cmd += " -v /home/ubuntu/logs/:/root/logs/" kick_off_cmd += " -p " + str(args.master_server_port) + ":" + str( args.master_server_port) - kick_off_cmd += " putcn/paddle_aws_master" + kick_off_cmd += " " + args.master_docker_image args_to_pass = copy.copy(args) args_to_pass.action = "serve" del args_to_pass.pem_path del args_to_pass.security_group_ids + del args_to_pass.master_docker_image del args_to_pass.master_server_public_ip for arg, value in sorted(vars(args_to_pass).iteritems()): kick_off_cmd += ' --%s %s' % (arg, value) diff --git a/tools/aws_benchmarking/server/cluster_master.py b/tools/aws_benchmarking/server/cluster_master.py index 38d09dc869401..5e63b5a8b4c0b 100644 --- a/tools/aws_benchmarking/server/cluster_master.py +++ b/tools/aws_benchmarking/server/cluster_master.py @@ -53,8 +53,8 @@ parser.add_argument( '--pserver_instance_type', type=str, - default="p2.8xlarge", - help="your pserver instance type, p2.8xlarge by default") + default="c5.2xlarge", + help="your pserver instance type, c5.2xlarge by default") parser.add_argument( '--trainer_instance_type', type=str, @@ -97,12 +97,18 @@ default=os.path.join(os.path.dirname(__file__), "pserver.sh.template"), help="pserver bash file path") +parser.add_argument( + '--pserver_command', type=str, default="", help="pserver start command") + parser.add_argument( '--trainer_bash_file', type=str, default=os.path.join(os.path.dirname(__file__), "trainer.sh.template"), help="trainer bash file path") +parser.add_argument( + '--trainer_command', type=str, default="", help="trainer start command") + parser.add_argument( '--action', type=str, default="serve", help="create|cleanup|serve") @@ -124,8 +130,12 @@ ec2client = boto3.client('ec2') +args.log_path = os.path.join(os.path.dirname(__file__), "logs/") + logging.basicConfig( - filename='master.log', level=logging.INFO, format='%(asctime)s %(message)s') + filename=args.log_path + 'master.log', + level=logging.INFO, + format='%(asctime)s %(message)s') log_files = ["master.log"] @@ -304,7 +314,7 @@ def create_pservers(): def log_to_file(source, filename): if not filename in log_files: log_files.append(filename) - with open(filename, "a") as log_file: + with open(args.log_path + filename, "a") as log_file: for line in iter(source.readline, ""): log_file.write(line) @@ -335,6 +345,8 @@ def create_and_start_trainer(trainer_index): DOCKER_IMAGE=args.docker_image, TRAINER_INDEX=str(trainer_index), TASK_NAME=args.task_name, + TRAINER_COUNT=args.trainer_count, + COMMAND=args.trainer_command, MASTER_ENDPOINT=args.master_server_ip + ":" + str(args.master_server_port)) logging.info(cmd) @@ -446,6 +458,9 @@ def kickoff_pserver(host, pserver_endpoints_str): DOCKER_IMAGE=args.docker_image, PSERVER_PORT=args.pserver_port, TASK_NAME=args.task_name, + COMMAND=args.pserver_command, + TRAINER_COUNT=args.trainer_count, + SERVER_ENDPOINT=host + ":" + str(args.pserver_port), MASTER_ENDPOINT=args.master_server_ip + ":" + str(args.master_server_port)) logging.info(cmd) @@ -553,14 +568,17 @@ def do_GET(self): if request_path == "/status" or request_path == "/master_logs": self._set_headers() logging.info("Received request to return status") - with open("master.log", "r") as logfile: + with open(args.log_path + "master.log", "r") as logfile: self.wfile.write(logfile.read().strip()) elif request_path == "/list_logs": self._set_headers() self.wfile.write("\n".join(log_files)) elif "/log/" in request_path: - log_file_path = request_path.replace("/log/") - with open(log_file_path, "r") as logfile: + self._set_headers() + log_file_path = request_path.replace("/log/", "") + logging.info("requesting log file path is" + args.log_path + + log_file_path) + with open(args.log_path + log_file_path, "r") as logfile: self.wfile.write(logfile.read().strip()) else: self.do_404() @@ -631,11 +649,4 @@ def print_arguments(): create_cluster() server_thread.join() elif args.action == "test": - init_args() - if not args.subnet_id: - logging.info("creating subnet for this task") - args.subnet_id = create_subnet() - logging.info("subnet %s created" % (args.subnet_id)) - create_trainers( - kickoff_cmd=script_to_str(args.trainer_bash_file), - pserver_endpoints_str="11.22.33.44:5476") + start_server(args) From f3a55f2192b4094e81b3a65b15e3ee00250dd339 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Fri, 13 Apr 2018 11:20:45 -0700 Subject: [PATCH 11/15] add no clean option --- .../client/cluster_launcher.py | 19 ++++++++++++++++++- .../aws_benchmarking/server/cluster_master.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tools/aws_benchmarking/client/cluster_launcher.py b/tools/aws_benchmarking/client/cluster_launcher.py index 3a6cc57b3ab3c..594378ff8fc07 100644 --- a/tools/aws_benchmarking/client/cluster_launcher.py +++ b/tools/aws_benchmarking/client/cluster_launcher.py @@ -26,6 +26,16 @@ from scp import SCPClient import requests + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '--key_name', type=str, default="", help="required, key pair name") @@ -117,6 +127,12 @@ default="putcn/paddle_aws_master:latest", help="master docker image id") +parser.add_argument( + '--no_clean_up', + type=str2bool, + default=False, + help="whether to clean up after training") + args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') @@ -347,7 +363,8 @@ def create(): del args_to_pass.master_docker_image del args_to_pass.master_server_public_ip for arg, value in sorted(vars(args_to_pass).iteritems()): - kick_off_cmd += ' --%s %s' % (arg, value) + if value: + kick_off_cmd += ' --%s %s' % (arg, value) logging.info(kick_off_cmd) stdin, stdout, stderr = ssh_client.exec_command(command=kick_off_cmd) diff --git a/tools/aws_benchmarking/server/cluster_master.py b/tools/aws_benchmarking/server/cluster_master.py index 5e63b5a8b4c0b..798228b35ae7a 100644 --- a/tools/aws_benchmarking/server/cluster_master.py +++ b/tools/aws_benchmarking/server/cluster_master.py @@ -27,8 +27,17 @@ from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + # You must have aws_access_key_id, aws_secret_access_key, region set in # ~/.aws/credentials and ~/.aws/config +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -126,6 +135,12 @@ parser.add_argument( '--master_server_ip', type=str, default="", help="master server private ip") +parser.add_argument( + '--no_clean_up', + type=str2bool, + default=False, + help="whether to clean up after training") + args = parser.parse_args() ec2client = boto3.client('ec2') @@ -414,6 +429,9 @@ def create_and_start_trainer(trainer_index): def cleanup(task_name): + if args.no_clean_up: + logging.info("no clean up option set, going to leave the setup running") + return #shutdown all ec2 instances print("going to clean up " + task_name + " instances") instances_response = ec2client.describe_instances(Filters=[{ From b8577266c584e2a785d45a2756c3f4f6efc05a99 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Fri, 13 Apr 2018 15:31:41 -0700 Subject: [PATCH 12/15] change pserver to use regular docker and some other tweaks --- tools/aws_benchmarking/server/cluster_master.py | 5 ++++- tools/aws_benchmarking/server/pserver.sh.template | 2 +- tools/aws_benchmarking/server/trainer.sh.template | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/aws_benchmarking/server/cluster_master.py b/tools/aws_benchmarking/server/cluster_master.py index 798228b35ae7a..21f85a5fc43e9 100644 --- a/tools/aws_benchmarking/server/cluster_master.py +++ b/tools/aws_benchmarking/server/cluster_master.py @@ -478,6 +478,9 @@ def kickoff_pserver(host, pserver_endpoints_str): TASK_NAME=args.task_name, COMMAND=args.pserver_command, TRAINER_COUNT=args.trainer_count, + TRAINER_INDEX=0, + # there is no way to use 0.0.0.0:port to start pserver + # has to docker --network="host" with host ip to make this work SERVER_ENDPOINT=host + ":" + str(args.pserver_port), MASTER_ENDPOINT=args.master_server_ip + ":" + str(args.master_server_port)) @@ -588,7 +591,7 @@ def do_GET(self): logging.info("Received request to return status") with open(args.log_path + "master.log", "r") as logfile: self.wfile.write(logfile.read().strip()) - elif request_path == "/list_logs": + elif request_path == "/list_logs" or request_path == "/logs": self._set_headers() self.wfile.write("\n".join(log_files)) elif "/log/" in request_path: diff --git a/tools/aws_benchmarking/server/pserver.sh.template b/tools/aws_benchmarking/server/pserver.sh.template index 5e46a4246f1a3..e648ecaac18eb 100644 --- a/tools/aws_benchmarking/server/pserver.sh.template +++ b/tools/aws_benchmarking/server/pserver.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run -i -p {PSERVER_PORT}:{PSERVER_PORT} -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINING_ROLE=PSERVER" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} \ No newline at end of file +docker run --network="host" -i -p {PSERVER_PORT}:{PSERVER_PORT} -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=PSERVER" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device CPU \ No newline at end of file diff --git a/tools/aws_benchmarking/server/trainer.sh.template b/tools/aws_benchmarking/server/trainer.sh.template index 56405a8e31d0c..4ece636a087ad 100644 --- a/tools/aws_benchmarking/server/trainer.sh.template +++ b/tools/aws_benchmarking/server/trainer.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} \ No newline at end of file +nvidia-docker run -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device GPU \ No newline at end of file From 946dc16ef615708962b3192c29341732b7f72f73 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Fri, 13 Apr 2018 16:18:35 -0700 Subject: [PATCH 13/15] update docker command template --- tools/aws_benchmarking/server/pserver.sh.template | 2 +- tools/aws_benchmarking/server/trainer.sh.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/aws_benchmarking/server/pserver.sh.template b/tools/aws_benchmarking/server/pserver.sh.template index e648ecaac18eb..18f9b800da7b3 100644 --- a/tools/aws_benchmarking/server/pserver.sh.template +++ b/tools/aws_benchmarking/server/pserver.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -docker run --network="host" -i -p {PSERVER_PORT}:{PSERVER_PORT} -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=PSERVER" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device CPU \ No newline at end of file +docker run --network="host" -i -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=PSERVER" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device GPU \ No newline at end of file diff --git a/tools/aws_benchmarking/server/trainer.sh.template b/tools/aws_benchmarking/server/trainer.sh.template index 4ece636a087ad..301ad26213b53 100644 --- a/tools/aws_benchmarking/server/trainer.sh.template +++ b/tools/aws_benchmarking/server/trainer.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device GPU \ No newline at end of file +nvidia-docker run --network="host" -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device GPU \ No newline at end of file From edb199b58c2476358bf5c7f75c38c317d2d80ab1 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Mon, 16 Apr 2018 16:27:15 -0700 Subject: [PATCH 14/15] adding more env vars --- tools/aws_benchmarking/server/pserver.sh.template | 2 +- tools/aws_benchmarking/server/trainer.sh.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/aws_benchmarking/server/pserver.sh.template b/tools/aws_benchmarking/server/pserver.sh.template index 18f9b800da7b3..2612856d1e627 100644 --- a/tools/aws_benchmarking/server/pserver.sh.template +++ b/tools/aws_benchmarking/server/pserver.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -docker run --network="host" -i -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=PSERVER" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device GPU \ No newline at end of file +docker run --network="host" -i -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=PSERVER" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device CPU \ No newline at end of file diff --git a/tools/aws_benchmarking/server/trainer.sh.template b/tools/aws_benchmarking/server/trainer.sh.template index 301ad26213b53..a4b2876b08cdf 100644 --- a/tools/aws_benchmarking/server/trainer.sh.template +++ b/tools/aws_benchmarking/server/trainer.sh.template @@ -1,2 +1,2 @@ #!/bin/bash -nvidia-docker run --network="host" -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device GPU \ No newline at end of file +nvidia-docker run --network="host" -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "PADDLE_INIT_TRAINER_ID={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} --device GPU \ No newline at end of file From 1e7c69fda42e0d5f8212ba815e9dba0bdf11d3c1 Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Mon, 16 Apr 2018 16:48:20 -0700 Subject: [PATCH 15/15] doc update --- tools/aws_benchmarking/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/aws_benchmarking/README.md b/tools/aws_benchmarking/README.md index dfa2a5f478400..837fcbb8512bc 100644 --- a/tools/aws_benchmarking/README.md +++ b/tools/aws_benchmarking/README.md @@ -54,6 +54,7 @@ Training nodes will run your `ENTRYPOINT` script with the following environment - `TRAINERS`: trainer count - `SERVER_ENDPOINT`: current server end point if the node role is a pserver - `TRAINER_INDEX`: an integer to identify the index of current trainer if the node role is a trainer. + - `PADDLE_INIT_TRAINER_ID`: same as above Now we have a working distributed training script which takes advantage of node environment variables and docker file to generate the training image. Run the following command: @@ -81,8 +82,7 @@ putcn/paddle_aws_client \ --action create \ --key_name \ --security_group_id \ ---pserver_image_id \ ---trainer_image_id \ +--docker_image myreponame/paddle_benchmark \ --pserver_count 2 \ --trainer_count 2 ``` @@ -146,7 +146,7 @@ When the training is finished, pservers and trainers will be terminated. All the Master exposes 4 major services: - GET `/status`: return master log - - GET `/list_logs`: return list of log file names + - GET `/logs`: return list of log file names - GET `/log/`: return a particular log by log file name - POST `/cleanup`: teardown the whole setup