From 9477790c718234ecc513639c33771426e3a2a0d4 Mon Sep 17 00:00:00 2001 From: Roope Astala Date: Tue, 18 Sep 2018 16:04:57 -0400 Subject: [PATCH] Notebook update Notebook update --- .../04.train-on-remote-vm.ipynb | 19 + automl/00.configuration.ipynb | 24 - automl/01.auto-ml-classification.ipynb | 5 +- automl/02.auto-ml-regression.ipynb | 7 +- automl/03.auto-ml-remote-execution.ipynb | 69 +- automl/03b.auto-ml-remote-batchai.ipynb | 121 +--- ...uto-ml-remote-batchai-compute-target.ipynb | 527 --------------- ...emote-execution-text-data-blob-store.ipynb | 62 +- ...ing-data-Blacklist-Early-Termination.ipynb | 5 +- ....auto-ml-sparse-data-custom-cv-split.ipynb | 7 +- .../07.auto-ml-exploring-previous-runs.ipynb | 5 +- ...ote-execution-with-text-file-on-DSVM.ipynb | 71 +- ...to-ml-classification-with-deployment.ipynb | 43 +- automl/10.auto-ml-multi-output-example.ipynb | 12 +- automl/11.auto-ml-sample-weight.ipynb | 1 - ...l-retrieve-the-training-sdk-versions.ipynb | 3 +- .../14a.auto-ml-classification-ensemble.ipynb | 412 ++++++++++++ automl/14b.auto-ml-regression-ensemble.ipynb | 437 +++++++++++++ automl/README.md | 53 +- automl/automl_env.yml | 3 +- pipeline/06.pipeline-batch-scoring.ipynb | 29 +- .../project-brainwave-custom-weights.ipynb | 617 ++++++++++++++++++ .../project-brainwave-quickstart.ipynb | 309 +++++++++ .../project-brainwave-transfer-learning.ipynb | 567 ++++++++++++++++ project-brainwave/snowleopardgaze.jpg | Bin 0 -> 62821 bytes tutorials/03.auto-train-models.ipynb | 6 +- 26 files changed, 2482 insertions(+), 932 deletions(-) delete mode 100644 automl/03c.auto-ml-remote-batchai-compute-target.ipynb create mode 100644 automl/14a.auto-ml-classification-ensemble.ipynb create mode 100644 automl/14b.auto-ml-regression-ensemble.ipynb create mode 100644 project-brainwave/project-brainwave-custom-weights.ipynb create mode 100644 project-brainwave/project-brainwave-quickstart.ipynb create mode 100644 project-brainwave/project-brainwave-transfer-learning.ipynb create mode 100644 project-brainwave/snowleopardgaze.jpg diff --git a/01.getting-started/04.train-on-remote-vm/04.train-on-remote-vm.ipynb b/01.getting-started/04.train-on-remote-vm/04.train-on-remote-vm.ipynb index 59ba86fde..9713264cd 100644 --- a/01.getting-started/04.train-on-remote-vm/04.train-on-remote-vm.ipynb +++ b/01.getting-started/04.train-on-remote-vm/04.train-on-remote-vm.ipynb @@ -182,6 +182,25 @@ " dsvm_compute.wait_for_completion(show_output = True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Attach an existing Linux DSVM as a compute target\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + " '''\n", + " from azureml.core.compute import RemoteCompute \n", + " dsvm_compute = RemoteCompute.attach(ws,name=\"attach-from-sdk6\",username=,address=,ssh_port=22,password=)\n", + "'''" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/automl/00.configuration.ipynb b/automl/00.configuration.ipynb index 752001b18..f056c79f1 100644 --- a/automl/00.configuration.ipynb +++ b/automl/00.configuration.ipynb @@ -231,23 +231,6 @@ "print('Sample projects will be created in {}.'.format(sample_projects_folder))" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Install additional packages for demo notebooks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install pandas_ml\n", - "!pip install seaborn" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -255,13 +238,6 @@ "## Success!\n", "Great, you are ready to move on to the rest of the sample notebooks." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/automl/01.auto-ml-classification.ipynb b/automl/01.auto-ml-classification.ipynb index 3529c3ba0..3984534a4 100644 --- a/automl/01.auto-ml-classification.ipynb +++ b/automl/01.auto-ml-classification.ipynb @@ -50,7 +50,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -268,9 +267,7 @@ " metricslist[int(properties['iteration'])] = metrics\n", "\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" + "rundata" ] }, { diff --git a/automl/02.auto-ml-regression.ipynb b/automl/02.auto-ml-regression.ipynb index f8814de63..92b86c5fc 100644 --- a/automl/02.auto-ml-regression.ipynb +++ b/automl/02.auto-ml-regression.ipynb @@ -50,7 +50,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -248,12 +247,8 @@ " metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)} \n", " metricslist[int(properties['iteration'])] = metrics\n", " \n", - "import pandas as pd\n", - "import seaborn as sns\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" + "rundata" ] }, { diff --git a/automl/03.auto-ml-remote-execution.ipynb b/automl/03.auto-ml-remote-execution.ipynb index 626bd57df..2130b049b 100644 --- a/automl/03.auto-ml-remote-execution.ipynb +++ b/automl/03.auto-ml-remote-execution.ipynb @@ -58,7 +58,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -143,50 +142,6 @@ " dsvm_compute.wait_for_completion(show_output = True)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a RunConfiguration with DSVM name\n", - "Run the below code to tell the runconfiguration the name of your dsvm." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import RunConfiguration\n", - "\n", - "run_config = RunConfiguration()\n", - "run_config.target = dsvm_compute" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Change index to use master packages\n", - "If you want to use master rather than preview run the below code. Once Public preview is launched we would not need this cell." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import CondaDependencies\n", - "\n", - "cd = CondaDependencies()\n", - "\n", - "cd.remove_pip_option(pip_option=\"--index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1\")\n", - "cd.set_pip_index_url(index_url=\"--extra-index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/master/588E708E0DF342C4A80BD954289657CF\")\n", - "\n", - "run_config.environment.python.conda_dependencies = cd" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -263,20 +218,27 @@ "\n", "automl_config = AutoMLConfig(task = 'classification',\n", " debug_log = 'automl_errors.log',\n", - " path=project_folder,\n", - " run_configuration = run_config,\n", + " path=project_folder, \n", + " compute_target = dsvm_compute,\n", " data_script = project_folder + \"./get_data.py\",\n", " **automl_settings\n", " )\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the first run on a new DSVM may take a several minutes to preparing the environment." + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "remote_run = experiment.submit(automl_config, show_output=True)" + "remote_run = experiment.submit(automl_config, show_output=False)" ] }, { @@ -352,9 +314,7 @@ " metricslist[int(properties['iteration'])] = metrics\n", "\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" + "rundata" ] }, { @@ -504,6 +464,13 @@ " plt.imshow(images[index], cmap=plt.cm.gray_r, interpolation='nearest')\n", " plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/automl/03b.auto-ml-remote-batchai.ipynb b/automl/03b.auto-ml-remote-batchai.ipynb index 0104efbf2..8802127e7 100644 --- a/automl/03b.auto-ml-remote-batchai.ipynb +++ b/automl/03b.auto-ml-remote-batchai.ipynb @@ -58,7 +58,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -134,7 +133,7 @@ "from azureml.core.compute import ComputeTarget\n", "\n", "# choose a name for your cluster\n", - "batchai_cluster_name = ws.name + \"cpu21\"\n", + "batchai_cluster_name = ws.name + \"cpu\"\n", "\n", "found = False\n", "# see if this compute target already exists in the workspace\n", @@ -148,11 +147,11 @@ " \n", "if not found:\n", " print('creating a new compute target...')\n", - " provisioning_config = BatchAiCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # D2 is 2 cores\n", + " provisioning_config = BatchAiCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # for GPU, use \"STANDARD_NC6\"\n", " #vm_priority = 'lowpriority', # optional\n", - " autoscale_enabled = False,\n", - " cluster_min_nodes = 2, \n", - " cluster_max_nodes = 2)\n", + " autoscale_enabled = True,\n", + " cluster_min_nodes = 1, \n", + " cluster_max_nodes = 4)\n", "\n", " # create the cluster\n", " compute_target = ComputeTarget.create(ws,batchai_cluster_name, provisioning_config)\n", @@ -161,73 +160,7 @@ " # if no min node count is provided it will use the scale settings for the cluster\n", " compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n", " \n", - " # For a more detailed view of current BatchAI cluster status, use the 'status' property \n", - " print(compute_target.status.serialize())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a RunConfiguration with Batch AI name\n", - "Run the below code to tell the runconfiguration the name of your dsvm." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import RunConfiguration\n", - "\n", - "run_config = RunConfiguration()\n", - "run_config.target = compute_target.name\n", - "run_config.environment.docker.enabled = True\n", - "run_config.prepare_environment = True\n", - "run_config.batchai.node_count = 2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Change index to use master packages\n", - "If you want to use master rather than preview run the below code. Once Public preview is launched we would not need this cell." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import CondaDependencies\n", - "\n", - "cd = CondaDependencies()\n", - "\n", - "cd.remove_pip_option(pip_option=\"--index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1\")\n", - "cd.set_pip_index_url(index_url=\"--extra-index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/master/588E708E0DF342C4A80BD954289657CF\")\n", - "\n", - "run_config.environment.python.conda_dependencies = cd\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_config.environment.docker.base_image" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_config.environment.docker.base_image = 'ninhu/amlbase:test-utf8'" + " # For a more detailed view of current BatchAI cluster status, use the 'status' property " ] }, { @@ -307,39 +240,12 @@ "automl_config = AutoMLConfig(task = 'classification',\n", " debug_log = 'automl_errors.log',\n", " path=project_folder,\n", - " run_configuration = run_config,\n", + " compute_target = compute_target,\n", " data_script = project_folder + \"./get_data.py\",\n", " **automl_settings\n", " )\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_config.environment.docker.base_image" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_config.environment.docker.base_image_registry" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_config.environment.docker.enabled = True" - ] - }, { "cell_type": "code", "execution_count": null, @@ -431,9 +337,7 @@ " metricslist[int(properties['iteration'])] = metrics\n", "\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" + "rundata" ] }, { @@ -452,7 +356,7 @@ "outputs": [], "source": [ "# Cancel the ongoing experiment and stop scheduling new iterations\n", - "remote_run.cancel()\n", + "# remote_run.cancel()\n", "\n", "# Cancel iteration 1 and move onto iteration 2\n", "# remote_run.cancel_iteration(1)" @@ -583,6 +487,13 @@ " plt.imshow(images[index], cmap=plt.cm.gray_r, interpolation='nearest')\n", " plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/automl/03c.auto-ml-remote-batchai-compute-target.ipynb b/automl/03c.auto-ml-remote-batchai-compute-target.ipynb deleted file mode 100644 index 2823db00d..000000000 --- a/automl/03c.auto-ml-remote-batchai-compute-target.ipynb +++ /dev/null @@ -1,527 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Copyright (c) Microsoft Corporation. All rights reserved.\n", - "\n", - "Licensed under the MIT License." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# AutoML 03: Remote Execution using Batch AI\n", - "\n", - "In this example we use the scikit learn's [diabetes dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) to showcase how you can use the AutoML Classifier for a simple classification problem.\n", - "\n", - "Make sure you have executed the [setup](setup.ipynb) before running this notebook.\n", - "\n", - "In this notebook you would see\n", - "1. Creating an Experiment using an existing Workspace\n", - "2. Attaching an existing Batch AI compute to a workspace\n", - "3. Instantiating AutoMLConfig \n", - "4. Training the Model using the Batch AI\n", - "5. Exploring the results\n", - "6. Testing the fitted model\n", - "\n", - "In addition this notebook showcases the following features\n", - "- **Parallel** Executions for iterations\n", - "- Asyncronous tracking of progress\n", - "- **Cancelling** individual iterations or the entire run\n", - "- Retrieving models for any iteration or logged metric\n", - "- specify automl settings as **kwargs**\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create Experiment\n", - "\n", - "As part of the setup you have already created a workspace. For AutoML you would need to create a Experiment. An Experiment is a named object in a Workspace, which is used to run experiments." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import azureml.core\n", - "import pandas as pd\n", - "from azureml.core.workspace import Workspace\n", - "from azureml.train.automl.run import AutoMLRun\n", - "import time\n", - "import logging\n", - "from sklearn import datasets\n", - "import seaborn as sns\n", - "from matplotlib import pyplot as plt\n", - "from matplotlib.pyplot import imshow\n", - "import random\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ws = Workspace.from_config()\n", - "\n", - "# choose a name for the run history container in the workspace\n", - "experiment_name = 'automl-remote-batchai'\n", - "# project folder\n", - "project_folder = './sample_projects/automl-remote-batchai'\n", - "\n", - "import os\n", - "from azureml.core.experiment import Experiment\n", - "\n", - "experiment=Experiment(ws, experiment_name)\n", - "\n", - "output = {}\n", - "output['SDK version'] = azureml.core.VERSION\n", - "output['Subscription ID'] = ws.subscription_id\n", - "output['Workspace Name'] = ws.name\n", - "output['Resource Group'] = ws.resource_group\n", - "output['Location'] = ws.location\n", - "output['Project Directory'] = project_folder\n", - "output['Experiment Name'] = experiment.name\n", - "pd.set_option('display.max_colwidth', -1)\n", - "pd.DataFrame(data = output, index = ['']).T" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create Batch AI Cluster\n", - "The cluster is created as Machine Learning Compute and will appear under your workspace.\n", - "\n", - "Note: The cluster creation can take over 10 minutes, be patient." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.compute import BatchAiCompute\n", - "from azureml.core.compute import ComputeTarget\n", - "\n", - "# choose a name for your cluster\n", - "batchai_cluster_name = ws.name + \"cpu21\"\n", - "\n", - "found = False\n", - "# see if this compute target already exists in the workspace\n", - "for ct in ws.compute_targets():\n", - " print(ct.name, ct.type)\n", - " if (ct.name == batchai_cluster_name and ct.type == 'BatchAI'):\n", - " found = True\n", - " print('found compute target. just use it.')\n", - " compute_target = ct\n", - " break\n", - " \n", - "if not found:\n", - " print('creating a new compute target...')\n", - " provisioning_config = BatchAiCompute.provisioning_configuration(vm_size = \"STANDARD_D2_V2\", # D2 is 2 cores\n", - " #vm_priority = 'lowpriority', # optional\n", - " autoscale_enabled = False,\n", - " cluster_min_nodes = 2, \n", - " cluster_max_nodes = 2)\n", - "\n", - " # create the cluster\n", - " compute_target = ComputeTarget.create(ws,batchai_cluster_name, provisioning_config)\n", - " \n", - " # can poll for a minimum number of nodes and for a specific timeout. \n", - " # if no min node count is provided it will use the scale settings for the cluster\n", - " compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)\n", - " \n", - " # For a more detailed view of current BatchAI cluster status, use the 'status' property \n", - " print(compute_target.status.serialize())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Change index to use master packages\n", - "If you want to use master rather than preview run the below code. Once Public preview is launched we would not need this cell." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create Get Data File\n", - "For remote executions you should author a get_data.py file containing a get_data() function. This file should be in the root directory of the project. You can encapsulate code to read data either from a blob storage or local disk in this file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "if not os.path.exists(project_folder):\n", - " os.makedirs(project_folder)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile $project_folder/get_data.py\n", - "\n", - "from sklearn import datasets\n", - "from scipy import sparse\n", - "import numpy as np\n", - "\n", - "def get_data():\n", - " \n", - " digits = datasets.load_digits()\n", - " X_digits = digits.data\n", - " y_digits = digits.target\n", - "\n", - " return { \"X\" : X_digits, \"y\" : y_digits }" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Instantiate AutoML \n", - "\n", - "You can specify automl_settings as **kwargs** as well. Also note that you can use the get_data() symantic for local excutions too. \n", - "\n", - "Note: For Remote DSVM and Batch AI you cannot pass Numpy arrays directly to the fit method.\n", - "\n", - "|Property|Description|\n", - "|-|-|\n", - "|**primary_metric**|This is the metric that you want to optimize.
Classification supports the following primary metrics
accuracy
AUC_weighted
balanced_accuracy
average_precision_score_weighted
precision_score_weighted|\n", - "|**max_time_sec**|Time limit in seconds for each iterations|\n", - "|**iterations**|Number of iterations. In each iteration Auto ML Classifier trains the data with a specific pipeline|\n", - "|**n_cross_validations**|Number of cross validation splits|\n", - "|**concurrent_iterations**|Max number of iterations that would be executed in parallel. This should be less than the number of cores on the DSVM." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.train.automl import AutoMLConfig\n", - "import time\n", - "import logging\n", - "\n", - "automl_settings = {\n", - " \"name\": \"AutoML_Demo_Experiment_{0}\".format(time.time()),\n", - " \"max_time_sec\": 120,\n", - " \"iterations\": 20,\n", - " \"n_cross_validations\": 5,\n", - " \"primary_metric\": 'AUC_weighted',\n", - " \"preprocess\": False,\n", - " \"concurrent_iterations\": 5,\n", - " \"verbosity\": logging.INFO\n", - "}\n", - "\n", - "automl_config = AutoMLConfig(task = 'classification',\n", - " debug_log = 'automl_errors.log',\n", - " path=project_folder,\n", - " compute_target = compute_target,\n", - " data_script = project_folder + \"./get_data.py\",\n", - " **automl_settings\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "remote_run = experiment.submit(automl_config, show_output=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Exploring the Results\n", - "\n", - "#### Loading executed runs\n", - "In case you need to load a previously executed run given a run id please enable the below cell" - ] - }, - { - "cell_type": "raw", - "metadata": {}, - "source": [ - "remote_run = AutoMLRun(experiment=experiment, run_id='AutoML_5db13491-c92a-4f1d-b622-8ab8d973a058')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Widget for monitoring runs\n", - "\n", - "The widget will sit on \"loading\" until the first iteration completed, then you will see an auto-updating graph and table show up. It refreshed once per minute, so you should see the graph update as child runs complete.\n", - "\n", - "You can click on a pipeline to see run properties and output logs. Logs are also available on the DSVM under /tmp/azureml_run/{iterationid}/azureml-logs\n", - "\n", - "NOTE: The widget displays a link at the bottom. This links to a web-ui to explore the individual run details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "remote_run" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.train.widgets import RunDetails\n", - "RunDetails(remote_run).show() " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# wait till the run finishes\n", - "remote_run.wait_for_completion(show_output = True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "#### Retrieve All Child Runs\n", - "You can also use sdk methods to fetch all the child runs and see individual metrics that we log. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "children = list(remote_run.get_children())\n", - "metricslist = {}\n", - "for run in children:\n", - " properties = run.get_properties()\n", - " metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)} \n", - " metricslist[int(properties['iteration'])] = metrics\n", - " \n", - "import pandas as pd\n", - "import seaborn as sns\n", - "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Canceling runs\n", - "\n", - "You can cancel ongoing remote runs using the *cancel()* and *cancel_iteration()* functions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Cancel the ongoing experiment and stop scheduling new iterations\n", - "remote_run.cancel()\n", - "\n", - "# Cancel iteration 1 and move onto iteration 2\n", - "# remote_run.cancel_iteration(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Retrieve the Best Model\n", - "\n", - "Below we select the best pipeline from our iterations. The *get_output* method on automl_classifier returns the best run and the fitted model for the last *fit* invocation. There are overloads on *get_output* that allow you to retrieve the best run and fitted model for *any* logged metric or a particular *iteration*." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "best_run, fitted_model = remote_run.get_output()\n", - "print(best_run)\n", - "print(fitted_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Best Model based on any other metric\n", - "Show the run/model which has the smallest `log_loss` value." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "lookup_metric = \"log_loss\"\n", - "best_run, fitted_model = remote_run.get_output(metric = lookup_metric)\n", - "print(best_run)\n", - "print(fitted_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Best Model based on any iteration\n", - "Show the run and model from the 3rd iteration." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "iteration = 3\n", - "third_run, third_model = remote_run.get_output(iteration=iteration)\n", - "print(third_run)\n", - "print(third_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Register fitted model for deployment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "description = 'AutoML Model'\n", - "tags = None\n", - "remote_run.register_model(description=description, tags=tags)\n", - "remote_run.model_id # Use this id to deploy the model as a web service in Azure" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Testing the Fitted Model \n", - "\n", - "#### Load Test Data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn import datasets\n", - "\n", - "digits = datasets.load_digits()\n", - "X_digits = digits.data[:10, :]\n", - "y_digits = digits.target[:10]\n", - "images = digits.images[:10]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Testing our best pipeline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Randomly select digits and test\n", - "from matplotlib import pyplot as plt\n", - "from matplotlib.pyplot import imshow\n", - "import random\n", - "import numpy as np\n", - "\n", - "for index in np.random.choice(len(y_digits), 2):\n", - " print(index)\n", - " predicted = fitted_model.predict(X_digits[index:index + 1])[0]\n", - " label = y_digits[index]\n", - " title = \"Label value = %d Predicted value = %d \" % ( label,predicted)\n", - " fig = plt.figure(1, figsize=(3,3))\n", - " ax1 = fig.add_axes((0,0,.8,.8))\n", - " ax1.set_title(title)\n", - " plt.imshow(images[index], cmap=plt.cm.gray_r, interpolation='nearest')\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python [default]", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/automl/04.auto-ml-remote-execution-text-data-blob-store.ipynb b/automl/04.auto-ml-remote-execution-text-data-blob-store.ipynb index df3a5fd20..17f5f236e 100644 --- a/automl/04.auto-ml-remote-execution-text-data-blob-store.ipynb +++ b/automl/04.auto-ml-remote-execution-text-data-blob-store.ipynb @@ -59,7 +59,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -78,9 +77,9 @@ "ws = Workspace.from_config()\n", "\n", "# choose a name for the run history container in the workspace\n", - "experiment_name = 'automl-remote-dsvm-blobstore-3'\n", + "experiment_name = 'automl-remote-dsvm-blobstore'\n", "# project folder\n", - "project_folder = './sample_projects/automl-remote-dsvm-blobstore-3'\n", + "project_folder = './sample_projects/automl-remote-dsvm-blobstore'\n", "\n", "experiment = Experiment(ws, experiment_name)\n", "\n", @@ -144,50 +143,6 @@ "dsvm_compute = RemoteCompute.attach(workspace=ws, name=dsvm_name, address=dsvm_ip_addr, username=dsvm_username, password=dsvm_password, ssh_port=22)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a RunConfiguration with DSVM name\n", - "Run the below code to tell the runconfiguration the name of your dsvm." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import RunConfiguration\n", - "\n", - "run_config = RunConfiguration()\n", - "run_config.target = dsvm_compute" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Change index to use master packages\n", - "If you want to use master rather than preview run the below code. Once Public preview is launched we would not need this cell." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import CondaDependencies\n", - "\n", - "cd = CondaDependencies()\n", - "\n", - "cd.remove_pip_option(pip_option=\"--index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1\")\n", - "cd.set_pip_index_url(index_url=\"--extra-index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/master/588E708E0DF342C4A80BD954289657CF\")\n", - "\n", - "run_config.environment.python.conda_dependencies = cd" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -297,7 +252,7 @@ "\n", "automl_config = AutoMLConfig(task = 'classification',\n", " path=project_folder,\n", - " run_configuration = run_config,\n", + " compute_target = dsvm_compute,\n", " data_script = project_folder + \"./get_data.py\",\n", " **automl_settings\n", " )\n" @@ -378,9 +333,7 @@ " metricslist[int(properties['iteration'])] = metrics\n", "\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" + "rundata" ] }, { @@ -519,6 +472,13 @@ "\n", "cm.plot()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/automl/05.auto-ml-missing-data-Blacklist-Early-Termination.ipynb b/automl/05.auto-ml-missing-data-Blacklist-Early-Termination.ipynb index 62c9f2e4a..417e7bd28 100644 --- a/automl/05.auto-ml-missing-data-Blacklist-Early-Termination.ipynb +++ b/automl/05.auto-ml-missing-data-Blacklist-Early-Termination.ipynb @@ -56,7 +56,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -267,9 +266,7 @@ " metricslist[int(properties['iteration'])] = metrics\n", "\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" + "rundata" ] }, { diff --git a/automl/06.auto-ml-sparse-data-custom-cv-split.ipynb b/automl/06.auto-ml-sparse-data-custom-cv-split.ipynb index e1b864482..2e1733c1a 100644 --- a/automl/06.auto-ml-sparse-data-custom-cv-split.ipynb +++ b/automl/06.auto-ml-sparse-data-custom-cv-split.ipynb @@ -54,7 +54,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -264,12 +263,8 @@ " metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)} \n", " metricslist[int(properties['iteration'])] = metrics\n", " \n", - "import pandas as pd\n", - "import seaborn as sns\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s" + "rundata" ] }, { diff --git a/automl/07.auto-ml-exploring-previous-runs.ipynb b/automl/07.auto-ml-exploring-previous-runs.ipynb index ee6a5bb15..52758166b 100644 --- a/automl/07.auto-ml-exploring-previous-runs.ipynb +++ b/automl/07.auto-ml-exploring-previous-runs.ipynb @@ -48,7 +48,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -196,10 +195,8 @@ " metricslist[int(properties['iteration'])] = metrics\n", "\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap=True)\n", - "s = rundata.style.background_gradient(cmap=cm)\n", "display(HTML('

Metrics

'))\n", - "display(s)\n" + "display(rundata)\n" ] }, { diff --git a/automl/08.auto-ml-remote-execution-with-text-file-on-DSVM.ipynb b/automl/08.auto-ml-remote-execution-with-text-file-on-DSVM.ipynb index 7d00aba48..1d447e071 100644 --- a/automl/08.auto-ml-remote-execution-with-text-file-on-DSVM.ipynb +++ b/automl/08.auto-ml-remote-execution-with-text-file-on-DSVM.ipynb @@ -48,7 +48,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -114,15 +113,6 @@ "**Note**: By default SSH runs on port 22 and you don't need to specify it. But if for security reasons you can switch to a different port (such as 5022), you can append the port number to the address. [Read more](https://render.githubusercontent.com/documentation/sdk/ssh-issue.md) on this." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cws.compute_targets" - ] - }, { "cell_type": "code", "execution_count": null, @@ -142,58 +132,13 @@ " dsvm_compute.wait_for_completion(show_output = True)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a RunConfiguration with DSVM name\n", - "Run the below code to tell the runconfiguration the name of your dsvm." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import RunConfiguration\n", - "\n", - "run_config = RunConfiguration()\n", - "run_config.target = dsvm_compute" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Change index to use master packages\n", - "If you want to use master rather than preview run the below code. Once Public preview is launched we would not need this cell." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.runconfig import CondaDependencies\n", - "\n", - "cd = CondaDependencies()\n", - "\n", - "cd.remove_pip_option(pip_option=\"--index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1\")\n", - "cd.set_pip_index_url(index_url=\"--extra-index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/master/588E708E0DF342C4A80BD954289657CF\")\n", - "\n", - "run_config.environment.python.conda_dependencies = cd" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Copy data file to the DSVM\n", "Download the data file.\n", - "Copy the data file to the DSVM under the folder:\n", - "\t\t~/.azureml/share/{workspacename}/{projectname}" + "Copy the data file to the DSVM under the folder: /tmp/data" ] }, { @@ -285,7 +230,6 @@ "outputs": [], "source": [ "automl_settings = {\n", - " \"name\": \"AutoML_Demo_Experiment_{0}\".format(time.time()),\n", " \"max_time_sec\": 12000,\n", " \"iterations\": 10,\n", " \"n_cross_validations\": 5,\n", @@ -297,7 +241,7 @@ "automl_config = AutoMLConfig(task = 'classification',\n", " debug_log = 'automl_errors.log',\n", " path=project_folder,\n", - " run_configuration = run_config,\n", + " compute_target = dsvm_compute,\n", " data_script = project_folder + \"./get_data.py\",\n", " **automl_settings\n", " )" @@ -378,9 +322,7 @@ " metricslist[int(properties['iteration'])] = metrics\n", "\n", "rundata = pd.DataFrame(metricslist).sort_index(1)\n", - "cm = sns.light_palette(\"lightgreen\", as_cmap = True)\n", - "s = rundata.style.background_gradient(cmap = cm)\n", - "s\n" + "rundata" ] }, { @@ -515,6 +457,13 @@ "\n", "cm.plot()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/automl/09.auto-ml-classification-with-deployment.ipynb b/automl/09.auto-ml-classification-with-deployment.ipynb index a80233479..9de5cca7c 100644 --- a/automl/09.auto-ml-classification-with-deployment.ipynb +++ b/automl/09.auto-ml-classification-with-deployment.ipynb @@ -53,7 +53,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -231,7 +230,7 @@ "\n", "def init():\n", " global model\n", - " model_path = Model.get_model_path(model_name = 'AutoML3d05a78138') # this name is model.id of model that we want to deploy\n", + " model_path = Model.get_model_path(model_name = 'AutoMLbcfe9c23e8') # this name is model.id of model that we want to deploy\n", " # deserialize the model file back into a sklearn model\n", " model = joblib.load(model_path)\n", "\n", @@ -315,9 +314,9 @@ " - --index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1\n", " - --extra-index-url https://pypi.python.org/simple\n", " - azureml-requirements\n", - " - azureml-train-automl==0.1.13\n", - " - azureml-sdk==0.1.13\n", - " - azureml-core==0.1.13" + " - azureml-train-automl==0.1.50\n", + " - azureml-sdk==0.1.50\n", + " - azureml-core==0.1.50" ] }, { @@ -333,20 +332,19 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.core.image import ContainerImage\n", + "from azureml.core.image import Image, ContainerImage\n", "\n", - "image_config = ContainerImage.image_configuration(execution_script = \"score.py\",\n", - " runtime = \"python\",\n", - " conda_file = \"myenv.yml\",\n", - " description = \"Image for automl classification sample\",\n", - " tags = [\"AutoML\",\"classification\", \"version_1\"]\n", - " )\n", + "image_config = ContainerImage.image_configuration(runtime= \"python\",\n", + " execution_script=\"score.py\",\n", + " conda_file=\"myenv.yml\",\n", + " tags = {'area': \"digits\", 'type': \"automl_classification\"},\n", + " description = \"Image for automl classification sample\")\n", "\n", - "image = ContainerImage.create(name = \"automlsampleimage2\",\n", - " # this is the model object\n", - " models = [model],\n", - " image_config = image_config,\n", - " workspace = ws)\n", + "image = Image.create(name = \"automlsampleimage\",\n", + " # this is the model object \n", + " models = [model],\n", + " image_config = image_config, \n", + " workspace = ws)\n", "\n", "image.wait_for_creation(show_output = True)" ] @@ -368,7 +366,7 @@ "\n", "aciconfig = AciWebservice.deploy_configuration(cpu_cores = 1, \n", " memory_gb = 1, \n", - " tags = ['automl','classification'], \n", + " tags = {'area': \"digits\", 'type': \"automl_classification\"}, \n", " description = 'sample service for Automl Classification')" ] }, @@ -380,7 +378,7 @@ "source": [ "from azureml.core.webservice import Webservice\n", "\n", - "aci_service_name = 'automl-sample-3'\n", + "aci_service_name = 'automl-sample-01'\n", "print(aci_service_name)\n", "aci_service = Webservice.deploy_from_image(deployment_config = aciconfig,\n", " image = image,\n", @@ -454,6 +452,13 @@ " plt.imshow(images[index], cmap=plt.cm.gray_r, interpolation='nearest')\n", " plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/automl/10.auto-ml-multi-output-example.ipynb b/automl/10.auto-ml-multi-output-example.ipynb index 6e050ee21..e8dc247d2 100644 --- a/automl/10.auto-ml-multi-output-example.ipynb +++ b/automl/10.auto-ml-multi-output-example.ipynb @@ -37,7 +37,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -133,9 +132,9 @@ "ws = Workspace.from_config()\n", "\n", "# choose a name for experiment\n", - "experiment_name = 'automl-local-classification'\n", + "experiment_name = 'automl-local-multi-output'\n", "# project folder\n", - "project_folder = './sample_projects/automl-local-classification'\n", + "project_folder = './sample_projects/automl-local-multi-output'\n", "\n", "experiment=Experiment(ws, experiment_name)\n", "\n", @@ -260,6 +259,13 @@ "source": [ "print(Y_predict)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/automl/11.auto-ml-sample-weight.ipynb b/automl/11.auto-ml-sample-weight.ipynb index 1206a049a..53fad5b37 100644 --- a/automl/11.auto-ml-sample-weight.ipynb +++ b/automl/11.auto-ml-sample-weight.ipynb @@ -49,7 +49,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", diff --git a/automl/12.auto-ml-retrieve-the-training-sdk-versions.ipynb b/automl/12.auto-ml-retrieve-the-training-sdk-versions.ipynb index 359c75c51..2dee41be5 100644 --- a/automl/12.auto-ml-retrieve-the-training-sdk-versions.ipynb +++ b/automl/12.auto-ml-retrieve-the-training-sdk-versions.ipynb @@ -30,7 +30,6 @@ "from matplotlib.pyplot import imshow\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns\n", "from sklearn import datasets\n", "\n", "import azureml.core\n", @@ -233,7 +232,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.6" } }, "nbformat": 4, diff --git a/automl/14a.auto-ml-classification-ensemble.ipynb b/automl/14a.auto-ml-classification-ensemble.ipynb new file mode 100644 index 000000000..87d633dff --- /dev/null +++ b/automl/14a.auto-ml-classification-ensemble.ipynb @@ -0,0 +1,412 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# AutoML 01: Classification with ensembling on local compute\n", + "\n", + "In this example we use the scikit learn's [digit dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_digits.html) to showcase how you can use the AutoML Classifier for a simple classification problem.\n", + "\n", + "Make sure you have executed the [00.configuration](00.configuration.ipynb) before running this notebook.\n", + "\n", + "In this notebook you would see\n", + "1. Creating an Experiment in an existing Workspace\n", + "2. Instantiating AutoMLConfig\n", + "3. Training the Model using local compute\n", + "4. Exploring the results\n", + "5. Testing the fitted model\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Experiment\n", + "\n", + "As part of the setup you have already created a Workspace. For AutoML you would need to create an Experiment. An Experiment is a named object in a Workspace, which is used to run experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "import os\n", + "import random\n", + "\n", + "from matplotlib import pyplot as plt\n", + "from matplotlib.pyplot import imshow\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import datasets\n", + "\n", + "import azureml.core\n", + "from azureml.core.experiment import Experiment\n", + "from azureml.core.workspace import Workspace\n", + "from azureml.train.automl import AutoMLConfig\n", + "from azureml.train.automl.run import AutoMLRun" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ws = Workspace.from_config()\n", + "\n", + "# choose a name for experiment\n", + "experiment_name = 'automl-local-classification'\n", + "# project folder\n", + "project_folder = './sample_projects/automl-local-classification'\n", + "\n", + "experiment=Experiment(ws, experiment_name)\n", + "\n", + "output = {}\n", + "output['SDK version'] = azureml.core.VERSION\n", + "output['Subscription ID'] = ws.subscription_id\n", + "output['Workspace Name'] = ws.name\n", + "output['Resource Group'] = ws.resource_group\n", + "output['Location'] = ws.location\n", + "output['Project Directory'] = project_folder\n", + "output['Experiment Name'] = experiment.name\n", + "pd.set_option('display.max_colwidth', -1)\n", + "pd.DataFrame(data = output, index = ['']).T" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Diagnostics\n", + "\n", + "Opt-in diagnostics for better experience, quality, and security of future releases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.telemetry import set_diagnostics_collection\n", + "set_diagnostics_collection(send_diagnostics=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Digits Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn import datasets\n", + "\n", + "digits = datasets.load_digits()\n", + "\n", + "X_digits = digits.data[100:,:]\n", + "y_digits = digits.target[100:]\n", + "X_valid = digits.data[0:100]\n", + "y_valid = digits.target[0:100]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiate Auto ML Config\n", + "\n", + "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "|**task**|classification or regression|\n", + "|**primary_metric**|This is the metric that you want to optimize.
Classification supports the following primary metrics
accuracy
AUC_weighted
balanced_accuracy
average_precision_score_weighted
precision_score_weighted|\n", + "|**max_time_sec**|Time limit in seconds for each iterations|\n", + "|**iterations**|Number of iterations. In each iteration Auto ML trains the data with a specific pipeline|\n", + "|**n_cross_validations**|Number of cross validation splits|\n", + "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n", + "|**y**|(sparse) array-like, shape = [n_samples, ], [n_samples, n_classes]
Multi-class targets. An indicator matrix turns on multilabel classification. This should be an array of integers. |\n", + "|**X_valid**|(sparse) array-like, shape = [n_samples, n_features]|\n", + "|**y_valid**|(sparse) array-like, shape = [n_samples, ], [n_samples, n_classes]
Multi-class targets. An indicator matrix turns on multilabel classification. This should be an array of integers. |\n", + "|**enable_ensembling**|Flag to enable an ensembling iteration after all the other iterations complete|\n", + "|**ensemble_iterations**|Number of iterations during which we choose a fitted model to be part of the final ensemble|" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "automl_config = AutoMLConfig(task = 'classification',\n", + " debug_log = 'automl_errors.log',\n", + " primary_metric = 'AUC_weighted',\n", + " max_time_sec = 12000,\n", + " iterations = 10,\n", + " verbosity = logging.INFO,\n", + " X = X_digits, \n", + " y = y_digits,\n", + " X_valid = X_valid,\n", + " y_valid = y_valid,\n", + " enable_ensembling = True,\n", + " ensemble_iterations = 5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training the Model\n", + "\n", + "You can call the submit method on the experiment object and pass the run configuration. For Local runs the execution is synchronous. Depending on the data and number of iterations this can run for while.\n", + "You will see the currently running iterations printing to the console." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run = experiment.submit(automl_config, show_output=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Optionally, you can continue an interrupted local run by calling continue_experiment without the iterations parameter, or run more iterations to a completed run by specifying the iterations parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run = local_run.continue_experiment(X = X_digits, \n", + " y = y_digits, \n", + " show_output = True,\n", + " iterations = 5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exploring the results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Widget for monitoring runs\n", + "\n", + "The widget will sit on \"loading\" until the first iteration completed, then you will see an auto-updating graph and table show up. It refreshed once per minute, so you should see the graph update as child runs complete.\n", + "\n", + "NOTE: The widget displays a link at the bottom. This links to a web-ui to explore the individual run details." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.train.widgets import RunDetails\n", + "RunDetails(local_run).show() " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "#### Retrieve All Child Runs\n", + "You can also use sdk methods to fetch all the child runs and see individual metrics that we log. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "children = list(local_run.get_children())\n", + "metricslist = {}\n", + "for run in children:\n", + " properties = run.get_properties()\n", + " metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)} \n", + " metricslist[int(properties['iteration'])] = metrics\n", + "\n", + "rundata = pd.DataFrame(metricslist).sort_index(1)\n", + "rundata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Retrieve the Best Model\n", + "\n", + "Below we select the best pipeline from our iterations. The *get_output* method on automl_classifier returns the best run and the fitted model for the last *fit* invocation. There are overloads on *get_output* that allow you to retrieve the best run and fitted model for *any* logged metric or a particular *iteration*." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_run, fitted_model = local_run.get_output()\n", + "print(best_run)\n", + "print(fitted_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Best Model based on any other metric\n", + "Give me the run and the model that has the smallest `log_loss`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lookup_metric = \"log_loss\"\n", + "best_run, fitted_model = local_run.get_output(metric = lookup_metric)\n", + "print(best_run)\n", + "print(fitted_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Best Model based on any iteration\n", + "Give me the run and the model from the 3rd iteration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "iteration = 3\n", + "best_run, fitted_model = local_run.get_output(iteration = iteration)\n", + "print(best_run)\n", + "print(fitted_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Testing the Fitted Model \n", + "\n", + "#### Load Test Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "digits = datasets.load_digits()\n", + "X_digits = digits.data[:10, :]\n", + "y_digits = digits.target[:10]\n", + "images = digits.images[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Testing our best pipeline\n", + "We will try to predict 2 digits and see how our model works." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Randomly select digits and test\n", + "for index in np.random.choice(len(y_digits), 2):\n", + " print(index)\n", + " predicted = fitted_model.predict(X_digits[index:index + 1])[0]\n", + " label = y_digits[index]\n", + " title = \"Label value = %d Predicted value = %d \" % ( label,predicted)\n", + " fig = plt.figure(1, figsize=(3,3))\n", + " ax1 = fig.add_axes((0,0,.8,.8))\n", + " ax1.set_title(title)\n", + " plt.imshow(images[index], cmap=plt.cm.gray_r, interpolation='nearest')\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [default]", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/automl/14b.auto-ml-regression-ensemble.ipynb b/automl/14b.auto-ml-regression-ensemble.ipynb new file mode 100644 index 000000000..921c2a4d5 --- /dev/null +++ b/automl/14b.auto-ml-regression-ensemble.ipynb @@ -0,0 +1,437 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# AutoML 02: Regression with ensembling on local compute\n", + "\n", + "In this example we use the scikit learn's [diabetes dataset](http://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) to showcase how you can use the AutoML for a simple regression problem.\n", + "\n", + "Make sure you have executed the [00.configuration](00.configuration.ipynb) before running this notebook.\n", + "\n", + "In this notebook you would see\n", + "1. Creating an Experiment using an existing Workspace\n", + "2. Instantiating AutoMLConfig\n", + "3. Training the Model using local compute\n", + "4. Exploring the results\n", + "5. Testing the fitted model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Experiment\n", + "\n", + "As part of the setup you have already created a Workspace. For AutoML you would need to create an Experiment. An Experiment is a named object in a Workspace, which is used to run experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "import os\n", + "import random\n", + "\n", + "from matplotlib import pyplot as plt\n", + "from matplotlib.pyplot import imshow\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn import datasets\n", + "\n", + "import azureml.core\n", + "from azureml.core.experiment import Experiment\n", + "from azureml.core.workspace import Workspace\n", + "from azureml.train.automl import AutoMLConfig\n", + "from azureml.train.automl.run import AutoMLRun" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ws = Workspace.from_config()\n", + "\n", + "# choose a name for the experiment\n", + "experiment_name = 'automl-local-regression'\n", + "# project folder\n", + "project_folder = './sample_projects/automl-local-regression'\n", + "\n", + "experiment = Experiment(ws, experiment_name)\n", + "\n", + "output = {}\n", + "output['SDK version'] = azureml.core.VERSION\n", + "output['Subscription ID'] = ws.subscription_id\n", + "output['Workspace Name'] = ws.name\n", + "output['Resource Group'] = ws.resource_group\n", + "output['Location'] = ws.location\n", + "output['Project Directory'] = project_folder\n", + "output['Experiment Name'] = experiment.name\n", + "pd.set_option('display.max_colwidth', -1)\n", + "pd.DataFrame(data = output, index = ['']).T" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Diagnostics\n", + "\n", + "Opt-in diagnostics for better experience, quality, and security of future releases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.telemetry import set_diagnostics_collection\n", + "set_diagnostics_collection(send_diagnostics=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Read Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load diabetes dataset, a well-known built-in small dataset that comes with scikit-learn\n", + "from sklearn.datasets import load_diabetes\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "X, y = load_diabetes(return_X_y = True)\n", + "\n", + "columns = ['age', 'gender', 'bmi', 'bp', 's1', 's2', 's3', 's4', 's5', 's6']\n", + "\n", + "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instantiate Auto ML Config\n", + "\n", + "Instantiate a AutoMLConfig object. This defines the settings and data used to run the experiment.\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "|**task**|classification or regression|\n", + "|**primary_metric**|This is the metric that you want to optimize.
Regression supports the following primary metrics
spearman_correlation
normalized_root_mean_squared_error
r2_score
normalized_mean_absolute_error
normalized_root_mean_squared_log_error|\n", + "|**max_time_sec**|Time limit in seconds for each iterations|\n", + "|**iterations**|Number of iterations. In each iteration Auto ML Classifier trains the data with a specific pipeline|\n", + "|**n_cross_validations**|Number of cross validation splits|\n", + "|**X**|(sparse) array-like, shape = [n_samples, n_features]|\n", + "|**y**|(sparse) array-like, shape = [n_samples, ], [n_samples, n_classes]
Multi-class targets. An indicator matrix turns on multilabel classification. This should be an array of integers. |\n", + "|**enable_ensembling**|Flag to enable an ensembling iteration after all the other iterations complete|\n", + "|**ensemble_iterations**|Number of iterations during which we choose a fitted model to be part of the final ensemble|" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "automl_config = AutoMLConfig(task='regression',\n", + " max_time_sec = 600,\n", + " iterations = 10,\n", + " primary_metric = 'spearman_correlation', \n", + " debug_log = 'automl.log',\n", + " verbosity = logging.INFO,\n", + " X = x_train, \n", + " y = y_train,\n", + " X_valid = x_test,\n", + " y_valid = y_test,\n", + " enable_ensembling = True,\n", + " ensemble_iterations = 5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training the Model\n", + "\n", + "You can call the submit method on the experiment object and pass the run configuration. For Local runs the execution is synchronous. Depending on the data and number of iterations this can run for while.\n", + "You will see the currently running iterations printing to the console." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run = experiment.submit(automl_config, show_output=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "local_run" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exploring the results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Widget for monitoring runs\n", + "\n", + "The widget will sit on \"loading\" until the first iteration completed, then you will see an auto-updating graph and table show up. It refreshed once per minute, so you should see the graph update as child runs complete.\n", + "\n", + "NOTE: The widget displays a link at the bottom. This links to a web-ui to explore the individual run details." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.train.widgets import RunDetails\n", + "RunDetails(local_run).show() " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "#### Retrieve All Child Runs\n", + "You can also use sdk methods to fetch all the child runs and see individual metrics that we log. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "children = list(local_run.get_children())\n", + "metricslist = {}\n", + "for run in children:\n", + " properties = run.get_properties()\n", + " metrics = {k: v for k, v in run.get_metrics().items() if isinstance(v, float)} \n", + " metricslist[int(properties['iteration'])] = metrics\n", + " \n", + "rundata = pd.DataFrame(metricslist).sort_index(1)\n", + "rundata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Retrieve the Best Model\n", + "\n", + "Below we select the best pipeline from our iterations. The *get_output* method on automl_classifier returns the best run and the fitted model for the last *fit* invocation. There are overloads on *get_output* that allow you to retrieve the best run and fitted model for *any* logged metric or a particular *iteration*." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_run, fitted_model = local_run.get_output()\n", + "print(best_run)\n", + "print(fitted_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Best Model based on any other metric\n", + "Show the run and model that has the smallest `root_mean_squared_error` (which turned out to be the same as the one with largest `spearman_correlation` value):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lookup_metric = \"root_mean_squared_error\"\n", + "best_run, fitted_model = local_run.get_output(metric=lookup_metric)\n", + "print(best_run)\n", + "print(fitted_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Best Model based on any iteration\n", + "Simply show the run and model from the 3rd iteration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "iteration = 3\n", + "third_run, third_model = local_run.get_output(iteration = iteration)\n", + "print(third_run)\n", + "print(third_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Register fitted model for deployment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "description = 'AutoML Model'\n", + "tags = None\n", + "local_run.register_model(description = description, tags = tags)\n", + "print(local_run.model_id) # Use this id to deploy the model as a web service in Azure" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Testing the Fitted Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Predict on training and test set, and calculate residual values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred_train = fitted_model.predict(x_train)\n", + "y_residual_train = y_train - y_pred_train\n", + "\n", + "y_pred_test = fitted_model.predict(x_test)\n", + "y_residual_test = y_test - y_pred_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn import datasets\n", + "from sklearn.metrics import mean_squared_error, r2_score\n", + "\n", + "# set up a multi-plot chart\n", + "f, (a0, a1) = plt.subplots(1, 2, gridspec_kw = {'width_ratios':[1, 1], 'wspace':0, 'hspace': 0})\n", + "f.suptitle('Regression Residual Values', fontsize = 18)\n", + "f.set_figheight(6)\n", + "f.set_figwidth(16)\n", + "\n", + "# plot residual values of training set\n", + "a0.axis([0, 360, -200, 200])\n", + "a0.plot(y_residual_train, 'bo', alpha = 0.5)\n", + "a0.plot([-10,360],[0,0], 'r-', lw = 3)\n", + "a0.text(16,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_train, y_pred_train))), fontsize = 12)\n", + "a0.text(16,140,'Variance = {0:.2f}'.format(r2_score(y_train, y_pred_train)), fontsize = 12)\n", + "a0.set_xlabel('Training samples', fontsize = 12)\n", + "a0.set_ylabel('Residual Values', fontsize = 12)\n", + "# plot histogram\n", + "a0.hist(y_residual_train, orientation = 'horizontal', color = 'b', bins = 10, histtype = 'step');\n", + "a0.hist(y_residual_train, orientation = 'horizontal', color = 'b', alpha = 0.2, bins = 10);\n", + "\n", + "# plot residual values of test set\n", + "a1.axis([0, 90, -200, 200])\n", + "a1.plot(y_residual_test, 'bo', alpha = 0.5)\n", + "a1.plot([-10,360],[0,0], 'r-', lw = 3)\n", + "a1.text(5,170,'RMSE = {0:.2f}'.format(np.sqrt(mean_squared_error(y_test, y_pred_test))), fontsize = 12)\n", + "a1.text(5,140,'Variance = {0:.2f}'.format(r2_score(y_test, y_pred_test)), fontsize = 12)\n", + "a1.set_xlabel('Test samples', fontsize = 12)\n", + "a1.set_yticklabels([])\n", + "# plot histogram\n", + "a1.hist(y_residual_test, orientation = 'horizontal', color = 'b', bins = 10, histtype = 'step');\n", + "a1.hist(y_residual_test, orientation = 'horizontal', color = 'b', alpha = 0.2, bins = 10);\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [default]", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/automl/README.md b/automl/README.md index c2743277c..7ad8581c2 100644 --- a/automl/README.md +++ b/automl/README.md @@ -1,12 +1,10 @@ # Table of Contents 1. [Auto ML Introduction](#introduction) -2. [Prerequisites](#prerequisites) -3. [Running samples in Azure Notebooks](#azurenotebooks) -4. [Running samples in a Local Conda environment](#localconda) -5. [Auto ML SDK Sample Notebooks](#samples) -6. [Documentation](#documentation) -7. [Running using python command](#pythoncommand) -8. [Troubleshooting](#troubleshooting) +2. [Running samples in a Local Conda environment](#localconda) +3. [Auto ML SDK Sample Notebooks](#samples) +4. [Documentation](#documentation) +5. [Running using python command](#pythoncommand) +6. [Troubleshooting](#troubleshooting) # Auto ML Introduction AutoML builds high quality Machine Learning model for you by automating model selection and hyper parameter selection for you. Bring a labelled dataset that you want to build a model for, AutoML will give you a high quality machine learning model that you can use for predictions. @@ -15,37 +13,6 @@ If you are new to Data Science, AutoML will help you get jumpstarted by simplify If you are an experienced data scientist, AutoML will help increase your productivity by intelligently performing the model selection, hyper parameter selection for your training and generates high quality models much quicker than manually specifying several combinations of the parameters and running training jobs. AutoML provides visibility and access to all the training jobs and the performance characteristics of the models and help you further tune the pipeline if you desire. -# Prerequisites -### 1. Whitelist your subscription -The first thing you need is to get your subscription whitelisted. Please email your Azure Subscription Id (NOT your alias) to viennawhitelist@service.microsoft.com. Expect to receive response within 2 business days. - -### 2. Enable Your Subscription to access EUAP (optional) -Some SDK functionalities might initially be only available in the Azure Canary Region (eastus2euap, aka EUAP). To gain access to this region, please submit a request here: https://aka.ms/canaryintwhitelist. - -Note it appears that only subscriptions belonging to Microsoft tenant are approved. MSDN-based personal Azure subscriptions appeared to be not allowed. - -# Running samples in Azure Notebooks - -The simplest way to get started with using Auto ML and trying out the sample notebooks is with [Azure Notebooks](https://notebooks.azure.com/). - -### 1. Sign up with Azure Notebooks -- Browse to https://notebooks.azure.com and login using your [Microsoft account](https://account.microsoft.com/account). If you are a Microsoft employee you can use your @microsoft account. - -### 2. Create a Library -- Create a new library. This will host the sample notebooks. **Important:** Mark the library private. The default is public. - -### 3. Upload the samples to the Library -- [Download the samples](https://github.com/Azure/ViennaDocs/blob/master/PrivatePreview/notebooks/downloads/auto-ml-scenarios.zip) as zip and extract the contents to a local directory -- Click on **+New** link to Add items to the library and choose to upload **From Computer**. Upload all the files from the zip to the library. - -### 4. Running setup.ipynb -- Before running any samples you would need to run the configuration notebook. Click on 00.configuration.ipynb notebook -- If asked set the Kernel to Python 3.6 -- Execute the cells in the notebook to install the SDK and create a workspace. (*instructions in notebook*) - -### 5. Running Samples -- Follow the instructions in the individual notebooks to explore various features in AutoML - # Running samples in a Local Conda environment It is best if you create a new conda environment locally to try this SDK, so it doesn't mess up with your existing Python environment. @@ -55,18 +22,18 @@ It is best if you create a new conda environment locally to try this SDK, so it There's no need to install mini-conda specifically. ### 2. Dowloading the sample notebooks -- [Download the samples](https://github.com/Azure/ViennaDocs/blob/master/PrivatePreview/notebooks/downloads/auto-ml-scenarios.zip) as zip and extract the contents to a local directory +- Download the sample notebooks from [GitHub](https://github.com/Azure/MachineLearningNotebooks) as zip and extract the contents to a local directory. The AutoML sample notebooks are in the "automl" folder. ### 3. Setup a new conda environment The automl_setup script creates a new conda environment, installs the necessary packages, configures the widget and starts jupyter notebook. It takes the conda environment name as an optional parameter. The default conda environment name is azure_automl. The exact command depends on the operating system. It can take about 30 minutes to execute. ## Windows -Start a conda command windows, cd to the folder where the sample notebooks were extracted and then run: automl_setup +Start a conda command windows, cd to the "automl" folder where the sample notebooks were extracted and then run: automl_setup ## Mac Install "Command line developer tools" if it is not already installed (you can use the command: xcode-select --install). -Start a Terminal windows, cd to the folder where the sample notebooks were extracted and then run: bash automl_setup_mac.sh +Start a Terminal windows, cd to the "automl" folder where the sample notebooks were extracted and then run: bash automl_setup_mac.sh ## Linux -cd to the folder where the sample notebooks were extracted and then run: automl_setup_linux.sh +cd to the "automl" folder where the sample notebooks were extracted and then run: automl_setup_linux.sh ### 4. Running configuration.ipynb - Before running any samples you would need to run the configuration notebook. Click on 00.configuration.ipynb notebook @@ -178,7 +145,7 @@ cd to the folder where the sample notebooks were extracted and then run: automl_ |**preprocess**|*True/False*
Setting this to *True* enables preprocessing
on the input to handle *missing data*, and perform some common *feature extraction*
*Note: If input data is Sparse you cannot use preprocess=True*|False| |**max_cores_per_iteration**| Indicates how many cores on the compute target would be used to train a single pipeline.
You can set it to *-1* to use all cores|1| |**exit_score**|*double* value indicating the target for *primary_metric*.
Once the target is surpassed the run terminates|None| -|**blacklist_algos**|*Array* of *strings* indicating pipelines to ignore for Auto ML.

Allowed values for **Classification**
logistic regression
SGD classifier
MultinomialNB
BernoulliNB
SVM
LinearSVM
kNN
DT
RF
extra trees
gradient boosting
lgbm_classifier

Allowed values for **Regression**
Elastic net
Gradient boosting regressor
DT regressor
kNN regressor
Lasso lars
SGD regressor
RF regressor
extra trees regressor
lightGBM regressor|None| +|**blacklist_algos**|*Array* of *strings* indicating pipelines to ignore for Auto ML.

Allowed values for **Classification**
LogisticRegression
SGDClassifierWrapper
NBWrapper
BernoulliNB
SVCWrapper
LinearSVMWrapper
KNeighborsClassifier
DecisionTreeClassifier
RandomForestClassifier
ExtraTreesClassifier
gradient boosting
LightGBMClassifier

Allowed values for **Regression**
ElasticNet
GradientBoostingRegressor
DecisionTreeRegressor
KNeighborsRegressor
LassoLars
SGDRegressor
RandomForestRegressor
ExtraTreesRegressor|None| ## Cross validation split options ### K-Folds Cross Validation diff --git a/automl/automl_env.yml b/automl/automl_env.yml index 0d8af0695..ecb53bcea 100644 --- a/automl/automl_env.yml +++ b/automl/automl_env.yml @@ -5,7 +5,6 @@ dependencies: - python=3.6 - nb_conda - matplotlib -- seaborn - numpy>=1.11.0,<1.16.0 - scipy>=0.19.0,<0.20.0 - scikit-learn>=0.18.0,<=0.19.1 @@ -13,7 +12,7 @@ dependencies: - pip: # Required packages for AzureML execution, history, and data preparation. - - --index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Candidate/604C89A437BA41BD942B4F46D9A3591D + - --index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1 - --extra-index-url https://pypi.python.org/simple - azureml-sdk[automl] - azureml-train-widgets diff --git a/pipeline/06.pipeline-batch-scoring.ipynb b/pipeline/06.pipeline-batch-scoring.ipynb index dae220339..bf53e4d95 100644 --- a/pipeline/06.pipeline-batch-scoring.ipynb +++ b/pipeline/06.pipeline-batch-scoring.ipynb @@ -410,14 +410,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Create template and rerun the pipeline using a REST call" + "# Publish a pipeline and rerun using a REST call" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Create template" + "## Create a published pipeline" ] }, { @@ -426,9 +426,10 @@ "metadata": {}, "outputs": [], "source": [ - "template = pipeline_run.create_template(name=\"batch score\", description=\"scores images kept in container sampledata\",\n", - " version=\"1.0\")\n", - "template_id = template.template_id" + "published_pipeline = pipeline_run.publish_pipeline(\n", + " name=\"batch score\", description=\"scores images kept in container sampledata\", version=\"1.0\")\n", + "\n", + "published_id = published_pipeline.id" ] }, { @@ -471,18 +472,18 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.pipeline.core import Template\n", + "from azureml.pipeline.core import PublishedPipeline\n", "\n", - "rest_endpoint = Template.get_template_endpoint(template_id, ws)\n", - "response = requests.post(rest_endpoint, headers=aad_token, json={})\n", - "run_id = response.json()[\"Id\"]" + "rest_endpoint = PublishedPipeline.get_endpoint(published_id, ws)\n", + "#response = requests.post(rest_endpoint, headers=aad_token, json={})\n", + "#run_id = response.json()[\"Id\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Monitor the template run" + "## Monitor the new run" ] }, { @@ -491,10 +492,10 @@ "metadata": {}, "outputs": [], "source": [ - "from azureml.pipeline.core.run import PipelineRun\n", - "template_run = PipelineRun(ws.experiments()[\"batch_scoring\"], run_id)\n", + "#from azureml.pipeline.core.run import PipelineRun\n", + "#published_pipeline_run = PipelineRun(ws.experiments()[\"batch_scoring\"], run_id)\n", "\n", - "RunDetails(template_run).show()" + "#RunDetails(published_pipeline_run).show()" ] }, { @@ -521,7 +522,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.6.5" } }, "nbformat": 4, diff --git a/project-brainwave/project-brainwave-custom-weights.ipynb b/project-brainwave/project-brainwave-custom-weights.ipynb new file mode 100644 index 000000000..8e7129d3a --- /dev/null +++ b/project-brainwave/project-brainwave-custom-weights.ipynb @@ -0,0 +1,617 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Model Development with Custom Weights" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example shows how to retrain a model with custom weights and fine-tune the model with quantization, then deploy the model running on FPGA. Only Windows is supported. We use TensorFlow and Keras to build our model. We are going to use transfer learning, with ResNet50 as a featurizer. We don't use the last layer of ResNet50 in this case and instead add our own classification layer using Keras.\n", + "\n", + "The custom wegiths are trained with ImageNet on ResNet50. We will use the Kaggle Cats and Dogs dataset to retrain and fine-tune the model. The dataset can be downloaded [here](https://www.microsoft.com/en-us/download/details.aspx?id=54765). Download the zip and extract to a directory named 'catsanddogs' under your user directory (\"~/catsanddogs\"). \n", + "\n", + "Please set up your environment as described in the [quick start](project-brainwave-quickstart.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import tensorflow as tf\n", + "import numpy as np\n", + "from keras import backend as K" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Environment\n", + "After you train your model in float32, you'll write the weights to a place on disk. We also need a location to store the models that get downloaded." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "custom_weights_dir = os.path.expanduser(\"~/custom-weights\")\n", + "saved_model_dir = os.path.expanduser(\"~/models\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare Data\n", + "Load the files we are going to use for training and testing. By default this notebook uses only a very small subset of the Cats and Dogs dataset. That makes it run relatively quickly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "import imghdr\n", + "datadir = os.path.expanduser(\"~/catsanddogs\")\n", + "\n", + "cat_files = glob.glob(os.path.join(datadir, 'PetImages', 'Cat', '*.jpg'))\n", + "dog_files = glob.glob(os.path.join(datadir, 'PetImages', 'Dog', '*.jpg'))\n", + "\n", + "# Limit the data set to make the notebook execute quickly.\n", + "cat_files = cat_files[:64]\n", + "dog_files = dog_files[:64]\n", + "\n", + "# The data set has a few images that are not jpeg. Remove them.\n", + "cat_files = [f for f in cat_files if imghdr.what(f) == 'jpeg']\n", + "dog_files = [f for f in dog_files if imghdr.what(f) == 'jpeg']\n", + "\n", + "if(not len(cat_files) or not len(dog_files)):\n", + " print(\"Please download the Kaggle Cats and Dogs dataset form https://www.microsoft.com/en-us/download/details.aspx?id=54765 and extract the zip to \" + datadir) \n", + " raise ValueError(\"Data not found\")\n", + "else:\n", + " print(cat_files[0])\n", + " print(dog_files[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Construct a numpy array as labels\n", + "image_paths = cat_files + dog_files\n", + "total_files = len(cat_files) + len(dog_files)\n", + "labels = np.zeros(total_files)\n", + "labels[len(cat_files):] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Split images data as training data and test data\n", + "from sklearn.model_selection import train_test_split\n", + "onehot_labels = np.array([[0,1] if i else [1,0] for i in labels])\n", + "img_train, img_test, label_train, label_test = train_test_split(image_paths, onehot_labels, random_state=42, shuffle=True)\n", + "\n", + "print(len(img_train), len(img_test), label_train.shape, label_test.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Construct Model\n", + "We use ResNet50 for the featuirzer and build our own classifier using Keras layers. We train the featurizer and the classifier as one model. The weights trained on ImageNet are used as the starting point for the retraining of our featurizer. The weights are loaded from tensorflow chkeckpoint files." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Before passing image dataset to the ResNet50 featurizer, we need to preprocess the input file to get it into the form expected by ResNet50. ResNet50 expects float tensors representing the images in BGR, channel last order. We've provided a default implementation of the preprocessing that you can use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import azureml.contrib.brainwave.models.utils as utils\n", + "\n", + "def preprocess_images():\n", + " # Convert images to 3D tensors [width,height,channel] - channels are in BGR order.\n", + " in_images = tf.placeholder(tf.string)\n", + " image_tensors = utils.preprocess_array(in_images)\n", + " return in_images, image_tensors" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We use Keras layer APIs to construct the classifier. Because we're using the tensorflow backend, we can train this classifier in one session with our Resnet50 model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def construct_classifier(in_tensor):\n", + " from keras.layers import Dropout, Dense, Flatten\n", + " K.set_session(tf.get_default_session())\n", + " \n", + " FC_SIZE = 1024\n", + " NUM_CLASSES = 2\n", + "\n", + " x = Dropout(0.2, input_shape=(1, 1, 2048,))(in_tensor)\n", + " x = Dense(FC_SIZE, activation='relu', input_dim=(1, 1, 2048,))(x)\n", + " x = Flatten()(x)\n", + " preds = Dense(NUM_CLASSES, activation='softmax', input_dim=FC_SIZE, name='classifier_output')(x)\n", + " return preds" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now every component of the model is defined, we can construct the model. Constructing the model with the project brainwave models is two steps - first we import the graph definition, then we restore the weights of the model into a tensorflow session. Because the quantized graph defintion and the float32 graph defintion share the same node names in the graph definitions, we can initally train the weights in float32, and then reload them with the quantized operations (which take longer) to fine-tune the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def construct_model(quantized, starting_weights_directory = None):\n", + " from azureml.contrib.brainwave.models import Resnet50, QuantizedResnet50\n", + " \n", + " # Convert images to 3D tensors [width,height,channel]\n", + " in_images, image_tensors = preprocess_images()\n", + "\n", + " # Construct featurizer using quantized or unquantized ResNet50 model\n", + " if not quantized:\n", + " featurizer = Resnet50(saved_model_dir)\n", + " else:\n", + " featurizer = QuantizedResnet50(saved_model_dir, custom_weights_directory = starting_weights_directory)\n", + "\n", + "\n", + " features = featurizer.import_graph_def(input_tensor=image_tensors)\n", + " # Construct classifier\n", + " preds = construct_classifier(features)\n", + " \n", + " # Initialize weights\n", + " sess = tf.get_default_session()\n", + " tf.global_variables_initializer().run()\n", + "\n", + " featurizer.restore_weights(sess)\n", + "\n", + " return in_images, image_tensors, features, preds, featurizer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train Model\n", + "First we train the model with custom weights but without quantization. Training is done with native float precision (32-bit floats). We load the traing data set and batch the training with 10 epochs. When the performance reaches desired level or starts decredation, we stop the training iteration and save the weights as tensorflow checkpoint files. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def read_files(files):\n", + " \"\"\" Read files to array\"\"\"\n", + " contents = []\n", + " for path in files:\n", + " with open(path, 'rb') as f:\n", + " contents.append(f.read())\n", + " return contents" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def train_model(preds, in_images, img_train, label_train, is_retrain = False, train_epoch = 10):\n", + " \"\"\" training model \"\"\"\n", + " from keras.objectives import binary_crossentropy\n", + " from tqdm import tqdm\n", + " \n", + " learning_rate = 0.001 if is_retrain else 0.01\n", + " \n", + " # Specify the loss function\n", + " in_labels = tf.placeholder(tf.float32, shape=(None, 2)) \n", + " cross_entropy = tf.reduce_mean(binary_crossentropy(in_labels, preds))\n", + " optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)\n", + "\n", + " def chunks(a, b, n):\n", + " \"\"\"Yield successive n-sized chunks from a and b.\"\"\"\n", + " if (len(a) != len(b)):\n", + " print(\"a and b are not equal in chunks(a,b,n)\")\n", + " raise ValueError(\"Parameter error\")\n", + "\n", + " for i in range(0, len(a), n):\n", + " yield a[i:i + n], b[i:i + n]\n", + "\n", + " chunk_size = 16\n", + " chunk_num = len(label_train) / chunk_size\n", + "\n", + " sess = tf.get_default_session()\n", + " for epoch in range(train_epoch):\n", + " avg_loss = 0\n", + " for img_chunk, label_chunk in tqdm(chunks(img_train, label_train, chunk_size)):\n", + " contents = read_files(img_chunk)\n", + " _, loss = sess.run([optimizer, cross_entropy],\n", + " feed_dict={in_images: contents,\n", + " in_labels: label_chunk,\n", + " K.learning_phase(): 1})\n", + " avg_loss += loss / chunk_num\n", + " print(\"Epoch:\", (epoch + 1), \"loss = \", \"{:.3f}\".format(avg_loss))\n", + " \n", + " # Reach desired performance\n", + " if (avg_loss < 0.001):\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def test_model(preds, in_images, img_test, label_test):\n", + " \"\"\"Test the model\"\"\"\n", + " from keras.metrics import categorical_accuracy\n", + "\n", + " in_labels = tf.placeholder(tf.float32, shape=(None, 2))\n", + " accuracy = tf.reduce_mean(categorical_accuracy(in_labels, preds))\n", + " contents = read_files(img_test)\n", + "\n", + " accuracy = accuracy.eval(feed_dict={in_images: contents,\n", + " in_labels: label_test,\n", + " K.learning_phase(): 0})\n", + " return accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Launch the training\n", + "tf.reset_default_graph()\n", + "sess = tf.Session(graph=tf.get_default_graph())\n", + "\n", + "with sess.as_default():\n", + " in_images, image_tensors, features, preds, featurizer = construct_model(quantized=False)\n", + " train_model(preds, in_images, img_train, label_train, is_retrain=False, train_epoch=10) \n", + " accuracy = test_model(preds, in_images, img_test, label_test) \n", + " print(\"Accuracy:\", accuracy)\n", + " featurizer.save_weights(custom_weights_dir + \"/rn50\", tf.get_default_session())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test Model\n", + "After training, we evaluate the trained model's accuracy on test dataset with quantization. So that we know the model's performance if it is deployed on the FPGA." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tf.reset_default_graph()\n", + "sess = tf.Session(graph=tf.get_default_graph())\n", + "\n", + "with sess.as_default():\n", + " print(\"Testing trained model with quantization\")\n", + " in_images, image_tensors, features, preds, quantized_featurizer = construct_model(quantized=True, starting_weights_directory=custom_weights_dir)\n", + " accuracy = test_model(preds, in_images, img_test, label_test) \n", + " print(\"Accuracy:\", accuracy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fine-Tune Model\n", + "Sometimes, the model's accuracy can drop significantly after quantization. In those cases, we need to retrain the model enabled with quantization to get better model accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if (accuracy < 0.93):\n", + " with sess.as_default():\n", + " print(\"Fine-tuning model with quantization\")\n", + " train_model(preds, in_images, img_train, label_train, is_retrain=True, train_epoch=10)\n", + " accuracy = test_model(preds, in_images, img_test, label_test) \n", + " print(\"Accuracy:\", accuracy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Service Definition\n", + "Like in the QuickStart notebook our service definition pipeline consists of three stages. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.contrib.brainwave.pipeline import ModelDefinition, TensorflowStage, BrainWaveStage\n", + "\n", + "model_def_path = os.path.join(save_path, 'model_def.zip')\n", + "\n", + "model_def = ModelDefinition()\n", + "model_def.pipeline.append(TensorflowStage(sess, in_images, image_tensors))\n", + "model_def.pipeline.append(BrainWaveStage(sess, quantized_featurizer))\n", + "model_def.pipeline.append(TensorflowStage(sess, features, preds))\n", + "model_def.save(model_def_path)\n", + "print(model_def_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy\n", + "Go to our [GitHub repo](https://aka.ms/aml-real-time-ai) \"docs\" folder to learn how to create a Model Management Account and find the required information below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core import Workspace\n", + "\n", + "ws = Workspace.from_config()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first time the code below runs it will create a new service running your model. If you want to change the model you can make changes above in this notebook and save a new service definition. Then this code will update the running service in place to run the new model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.model import Model\n", + "from azureml.core.image import Image\n", + "from azureml.core.webservice import Webservice\n", + "from azureml.contrib.brainwave import BrainwaveWebservice, BrainwaveImage\n", + "\n", + "model_name = \"catsanddogs-resnet50-model\"\n", + "image_name = \"catsanddogs-resnet50-image\"\n", + "service_name = \"modelbuild-service\"\n", + "\n", + "registered_model = Model.register(ws, service_def_path, model_name)\n", + "\n", + "image_config = BrainwaveImage.image_configuration()\n", + "deployment_config = BrainwaveWebservice.deploy_configuration()\n", + " \n", + "try:\n", + " service = Webservice(ws, service_name)\n", + " service.delete()\n", + " service = Webservice.deploy_from_model(ws, service_name, [registered_model], image_config, deployment_config)\n", + "except WebserviceException:\n", + " service = Webservice.deploy_from_model(ws, service_name, [registered_model], image_config, deployment_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The service is now running in Azure and ready to serve requests. We can check the address and port." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(service.ipAddress + ':' + str(service.port))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Client\n", + "There is a simple test client at amlrealtimeai.PredictionClient which can be used for testing. We'll use this client to score an image with our new service." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.contrib.brainwave.client import PredictionClient\n", + "client = PredictionClient(service.ipAddress, service.port)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can adapt the client [code](../../pythonlib/amlrealtimeai/client.py) to meet your needs. There is also an example C# [client](../../sample-clients/csharp).\n", + "\n", + "The service provides an API that is compatible with TensorFlow Serving. There are instructions to download a sample client [here](https://www.tensorflow.org/serving/setup)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Request\n", + "Let's see how our service does on a few images. It may get a few wrong." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Specify an image to classify\n", + "print('CATS')\n", + "for image_file in cat_files[:8]:\n", + " results = client.score_image(image_file)\n", + " result = 'CORRECT ' if results[0] > results[1] else 'WRONG '\n", + " print(result + str(results))\n", + "print('DOGS')\n", + "for image_file in dog_files[:8]:\n", + " results = client.score_image(image_file)\n", + " result = 'CORRECT ' if results[1] > results[0] else 'WRONG '\n", + " print(result + str(results))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup\n", + "Run the cell below to delete your service." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "service.delete()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Appendix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "License for plot_confusion_matrix:\n", + "\n", + "New BSD License\n", + "\n", + "Copyright (c) 2007–2018 The scikit-learn developers.\n", + "All rights reserved.\n", + "\n", + "\n", + "Redistribution and use in source and binary forms, with or without\n", + "modification, are permitted provided that the following conditions are met:\n", + "\n", + " a. Redistributions of source code must retain the above copyright notice,\n", + " this list of conditions and the following disclaimer.\n", + " b. Redistributions in binary form must reproduce the above copyright\n", + " notice, this list of conditions and the following disclaimer in the\n", + " documentation and/or other materials provided with the distribution.\n", + " c. Neither the name of the Scikit-learn Developers nor the names of\n", + " its contributors may be used to endorse or promote products\n", + " derived from this software without specific prior written\n", + " permission. \n", + "\n", + "\n", + "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n", + "AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n", + "IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n", + "ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR\n", + "ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n", + "DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n", + "SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n", + "CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n", + "LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n", + "OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH\n", + "DAMAGE.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/project-brainwave/project-brainwave-quickstart.ipynb b/project-brainwave/project-brainwave-quickstart.ipynb new file mode 100644 index 000000000..ba71f37ba --- /dev/null +++ b/project-brainwave/project-brainwave-quickstart.ipynb @@ -0,0 +1,309 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Azure ML Hardware Accelerated Models Quickstart" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This tutorial will show you how to deploy an image recognition service based on the ResNet 50 classifier in just a few minutes using the Azure Machine Learning Accelerated AI service. Get more help from our [documentation](https://aka.ms/aml-real-time-ai) or [forum](https://aka.ms/aml-forum).\n", + "\n", + "We will use an accelerated ResNet50 featurizer running on an FPGA. This functionality is powered by Project Brainwave, which handles translating deep neural networks (DNN) into an FPGA program.\n", + "\n", + "## Request Quota\n", + "**IMPORTANT:** You must [request quota](https://aka.ms/aml-real-time-ai-request) and be approved before you can successfully run this notebook. Notebook 00 will show you how to create a workspace which you can use to request quota." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tensorflow as tf" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Image preprocessing\n", + "We'd like our service to accept JPEG images as input. However the input to ResNet50 is a tensor. So we need code that decodes JPEG images and does the preprocessing required by ResNet50. The Accelerated AI service can execute TensorFlow graphs as part of the service and we'll use that ability to do the image preprocessing. This code defines a TensorFlow graph that preprocesses an array of JPEG images (as strings) and produces a tensor that is ready to be featurized by ResNet50." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Input images as a two-dimensional tensor containing an arbitrary number of images represented a strings\n", + "import azureml.contrib.brainwave.models.utils as utils\n", + "in_images = tf.placeholder(tf.string)\n", + "image_tensors = utils.preprocess_array(in_images)\n", + "print(image_tensors.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Featurizer\n", + "We use ResNet50 as a featurizer. In this step we initialize the model. This downloads a TensorFlow checkpoint of the quantized ResNet50." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.contrib.brainwave.models import QuantizedResnet50, Resnet50\n", + "model_path = os.path.expanduser('~/models')\n", + "model = QuantizedResnet50(model_path, is_frozen = True)\n", + "feature_tensor = model.import_graph_def(image_tensors)\n", + "print(model.version)\n", + "print(feature_tensor.name)\n", + "print(feature_tensor.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Classifier\n", + "The model we downloaded includes a classifier which takes the output of the ResNet50 and identifies an image. This classifier is trained on the ImageNet dataset. We are going to use this classifier for our service. The next [notebook](project-brainwave-trainsfer-learning.ipynb) shows how to train a classifier for a different data set. The input to the classifier is a tensor matching the output of our ResNet50 featurizer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "classifier_input, classifier_output = Resnet50.get_default_classifier(feature_tensor, model_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Service Definition\n", + "Now that we've definied the image preprocessing, featurizer, and classifier that we will execute on our service we can create a service definition. The service definition is a set of files generated from the model that allow us to deploy to the FPGA service. The service definition consists of a pipeline. The pipeline is a series of stages that are executed in order. We support TensorFlow stages, Keras stages, and BrainWave stages. The stages will be executed in order on the service, with the output of each stage input into the subsequent stage.\n", + "\n", + "To create a TensorFlow stage we specify a session containing the graph (in this case we are using the default graph) and the input and output tensors to this stage. We use this information to save the graph so that we can execute it on the service." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.contrib.brainwave.pipeline import ModelDefinition, TensorflowStage, BrainWaveStage\n", + "\n", + "save_path = os.path.expanduser('~/models/save')\n", + "model_def_path = os.path.join(save_path, 'service_def.zip')\n", + "\n", + "model_def = ModelDefinition()\n", + "with tf.Session() as sess:\n", + " model_def.pipeline.append(TensorflowStage(sess, in_images, image_tensors))\n", + " model_def.pipeline.append(BrainWaveStage(sess, model))\n", + " model_def.pipeline.append(TensorflowStage(sess, classifier_input, classifier_output))\n", + " model_def.save(model_def_path)\n", + " print(model_def_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy\n", + "Time to create a service from the service definition. You need a Workspace in the **East US 2** location. In the previous notebooks, you've created this Workspace. The code below will load that Workspace from a configuration file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core import Workspace\n", + "\n", + "ws = Workspace.from_config()\n", + "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Upload the model to the workspace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.model import Model\n", + "model_name = \"resnet-50-rtai\"\n", + "registered_model = Model.register(ws, model_def_path, model_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a service from the model that we registered. If this is a new service then we create it. If you already have a service with this name then the existing service will be updated to use this model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.webservice import Webservice\n", + "from azureml.exceptions import WebserviceException\n", + "from azureml.contrib.brainwave import BrainwaveWebservice, BrainwaveImage\n", + "service_name = \"imagenet-infer\"\n", + "service = None\n", + "try:\n", + " service = Webservice(ws, service_name)\n", + "except WebserviceException:\n", + " image_config = BrainwaveImage.image_configuration()\n", + " deployment_config = BrainwaveWebservice.deploy_configuration()\n", + " service = Webservice.deploy_from_model(ws, service_name, [registered_model], image_config, deployment_config)\n", + " service.wait_for_deployment(true)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Client\n", + "The service supports gRPC and the TensorFlow Serving \"predict\" API. We provide a client that can call the service to get predictions on aka.ms/rtai. You can also invoke the service like any other web service." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To understand the results we need a mapping to the human readable imagenet classes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "classes_entries = requests.get(\"https://raw.githubusercontent.com/Lasagne/Recipes/master/examples/resnet50/imagenet_classes.txt\").text.splitlines()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now send an image to the service and get the predictions. Let's see if it can identify a snow leopard.\n", + "![title](snowleopardgaze.jpg)\n", + "Snow leopard in a zoo. Photo by Peter Bolliger.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = service.run('snowleopardgaze.jpg')\n", + "# map results [class_id] => [confidence]\n", + "results = enumerate(results)\n", + "# sort results by confidence\n", + "sorted_results = sorted(results, key=lambda x: x[1], reverse=True)\n", + "# print top 5 results\n", + "for top in sorted_results[:5]:\n", + " print(classes_entries[top[0]], 'confidence:', top[1])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup\n", + "Run the cell below to delete your service." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "service.delete()\n", + " \n", + "registered_model.delete()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Congratulations! You've just created a service that does predictions using an FPGA. The next [notebook](project-brainwave-trainsfer-learning.ipynb) shows how to customize the service using transfer learning to classify different types of images." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/project-brainwave/project-brainwave-transfer-learning.ipynb b/project-brainwave/project-brainwave-transfer-learning.ipynb new file mode 100644 index 000000000..6005b5ec0 --- /dev/null +++ b/project-brainwave/project-brainwave-transfer-learning.ipynb @@ -0,0 +1,567 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Model Development" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example shows how to build, train, evaluate and deploy a model running on FPGA. Only Windows is supported. We use TensorFlow and Keras to build our model. We are going to use transfer learning, with ResNet152 as a featurizer. We don't use the last layer of ResNet152 in this case and instead add and train our own classification layer.\n", + "\n", + "We will use the Kaggle Cats and Dogs dataset to train the classifier. The dataset can be downloaded [here](https://www.microsoft.com/en-us/download/details.aspx?id=54765). Download the zip and extract to a directory named 'catsanddogs' under your user directory (\"~/catsanddogs\").\n", + "\n", + "Please set up your environment as described in the [quick start](project-brainwave-quickstart.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import tensorflow as tf\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model Construction\n", + "Load the files we are going to use for training and testing. By default this notebook uses only a very small subset of the Cats and Dogs dataset. That makes it run quickly, but doesn't create a very accurate classifier. You can improve the classifier by using more of the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "import imghdr\n", + "datadir = os.path.expanduser(\"~/catsanddogs\")\n", + "\n", + "cat_files = glob.glob(os.path.join(datadir, 'PetImages', 'Cat', '*.jpg'))\n", + "dog_files = glob.glob(os.path.join(datadir, 'PetImages', 'Dog', '*.jpg'))\n", + "\n", + "# Limit the data set to make the notebook execute quickly.\n", + "cat_files = cat_files[:64]\n", + "dog_files = dog_files[:64]\n", + "\n", + "# The data set has a few images that are not jpeg. Remove them.\n", + "cat_files = [f for f in cat_files if imghdr.what(f) == 'jpeg']\n", + "dog_files = [f for f in dog_files if imghdr.what(f) == 'jpeg']\n", + "\n", + "if(not len(cat_files) or not len(dog_files)):\n", + " print(\"Please download the Kaggle Cats and Dogs dataset form https://www.microsoft.com/en-us/download/details.aspx?id=54765 and extract the zip to \" + datadir) \n", + " raise ValueError(\"Data not found\")\n", + "else:\n", + " print(cat_files[0])\n", + " print(dog_files[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# constructing a numpy array as labels\n", + "image_paths = cat_files + dog_files\n", + "total_files = len(cat_files) + len(dog_files)\n", + "labels = np.zeros(total_files)\n", + "labels[len(cat_files):] = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to preprocess the input file to get it into the form expected by ResNet152. We've provided a default implementation of the preprocessing that you can use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Input images as a two-dimensional tensor containing an arbitrary number of images represented a strings\n", + "import azureml.contrib.brainwave.models.utils as utils\n", + "in_images = tf.placeholder(tf.string)\n", + "image_tensors = utils.preprocess_array(in_images)\n", + "print(image_tensors.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, if you would like to customize the preprocessing, you can write your own preprocessor using TensorFlow operations.\n", + "\n", + "The input to the classifier we are training is the set of features produced by ResNet50. To train the classifier we need to \n", + "featurize the images using ResNet50. You can also run the featurizer locally on CPU or GPU. We import the featurizer as frozen, so that we are only training the classifier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.contrib.brainwave.models import QuantizedResnet152\n", + "model_path = os.path.expanduser('~/models')\n", + "bwmodel = QuantizedResnet152(model_path, is_frozen = True)\n", + "print(bwmodel.version)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Calling import_graph_def on the featurizer will create a service that runs the featurizer on FPGA." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "features = bwmodel.import_graph_def(input_tensor=image_tensors)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-compute features\n", + "Load the data set and compute the features. These can be precomputed because they don't change during training. This can take a while to run on CPU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "\n", + "def chunks(l, n):\n", + " \"\"\"Yield successive n-sized chunks from l.\"\"\"\n", + " for i in range(0, len(l), n):\n", + " yield l[i:i + n]\n", + "\n", + "def read_files(files):\n", + " contents = []\n", + " for path in files:\n", + " with open(path, 'rb') as f:\n", + " contents.append(f.read())\n", + " return contents\n", + " \n", + "feature_list = []\n", + "with tf.Session() as sess:\n", + " for chunk in tqdm(chunks(image_paths, 5)):\n", + " contents = read_files(chunk)\n", + " result = sess.run([features], feed_dict={in_images: contents})\n", + " feature_list.extend(result[0])\n", + "\n", + "feature_results = np.array(feature_list)\n", + "print(feature_results.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Add and Train the classifier\n", + "We use Keras to define and train a simple classifier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from keras.models import Sequential\n", + "from keras.layers import Dropout, Dense, Flatten\n", + "from keras import optimizers\n", + "\n", + "FC_SIZE = 1024\n", + "NUM_CLASSES = 2\n", + "\n", + "model = Sequential()\n", + "model.add(Dropout(0.2, input_shape=(1, 1, 2048,)))\n", + "model.add(Dense(FC_SIZE, activation='relu', input_dim=(1, 1, 2048,)))\n", + "model.add(Flatten())\n", + "model.add(Dense(NUM_CLASSES, activation='sigmoid', input_dim=FC_SIZE))\n", + "\n", + "model.compile(optimizer=optimizers.SGD(lr=1e-4,momentum=0.9), loss='binary_crossentropy', metrics=['accuracy'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Prepare the train and test data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "onehot_labels = np.array([[0,1] if i else [1,0] for i in labels])\n", + "X_train, X_test, y_train, y_test = train_test_split(feature_results, onehot_labels, random_state=42, shuffle=True)\n", + "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Train the classifier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model.fit(X_train, y_train, epochs=16, batch_size=32)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test the Classifier\n", + "Let's test the classifier and see how well it does. Since we only trained on a few images, we are not expecting to win a Kaggle competition, but it will likely get most of the images correct. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from numpy import argmax\n", + "\n", + "y_probs = model.predict(X_test)\n", + "y_prob_max = np.argmax(y_probs, 1)\n", + "y_test_max = np.argmax(y_test, 1)\n", + "print(y_prob_max)\n", + "print(y_test_max)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.metrics import confusion_matrix, roc_auc_score, accuracy_score, precision_score, recall_score, f1_score\n", + "import itertools\n", + "import matplotlib\n", + "from matplotlib import pyplot as plt\n", + "\n", + "# compute a bunch of classification metrics \n", + "def classification_metrics(y_true, y_pred, y_prob):\n", + " cm_dict = {}\n", + " cm_dict['Accuracy'] = accuracy_score(y_true, y_pred)\n", + " cm_dict['Precision'] = precision_score(y_true, y_pred)\n", + " cm_dict['Recall'] = recall_score(y_true, y_pred)\n", + " cm_dict['F1'] = f1_score(y_true, y_pred) \n", + " cm_dict['AUC'] = roc_auc_score(y_true, y_prob[:,0])\n", + " cm_dict['Confusion Matrix'] = confusion_matrix(y_true, y_pred).tolist()\n", + " return cm_dict\n", + "\n", + "def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n", + " \"\"\"Plots a confusion matrix.\n", + " Source: http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n", + " New BSD License - see appendix\n", + " \"\"\"\n", + " cm_max = cm.max()\n", + " cm_min = cm.min()\n", + " if cm_min > 0: cm_min = 0\n", + " if normalize:\n", + " cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", + " cm_max = 1\n", + " plt.imshow(cm, interpolation='nearest', cmap=cmap)\n", + " plt.title(title)\n", + " plt.colorbar()\n", + " tick_marks = np.arange(len(classes))\n", + " plt.xticks(tick_marks, classes, rotation=45)\n", + " plt.yticks(tick_marks, classes)\n", + " thresh = cm_max / 2.\n", + " plt.clim(cm_min, cm_max)\n", + "\n", + " for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n", + " plt.text(j, i,\n", + " round(cm[i, j], 3), # round to 3 decimals if they are float\n", + " horizontalalignment=\"center\",\n", + " color=\"white\" if cm[i, j] > thresh else \"black\")\n", + " plt.ylabel('True label')\n", + " plt.xlabel('Predicted label')\n", + " plt.show()\n", + " \n", + "cm_dict = classification_metrics(y_test_max, y_prob_max, y_probs)\n", + "for m in cm_dict:\n", + " print(m, cm_dict[m])\n", + "cm = np.asarray(cm_dict['Confusion Matrix'])\n", + "plot_confusion_matrix(cm, ['fail','pass'], normalize=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Service Definition\n", + "Like in the QuickStart notebook our service definition pipeline consists of three stages. Because the preprocessing and featurizing stage don't contain any variables, we can use a default session.\n", + "Here we use the Keras classifier as the final stage." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.contrib.brainwave.pipeline import ModelDefinition, TensorflowStage, BrainWaveStage, KerasStage\n", + "\n", + "model_def = ModelDefinition()\n", + "model_def.pipeline.append(TensorflowStage(tf.Session(), in_images, image_tensors))\n", + "model_def.pipeline.append(BrainWaveStage(tf.Session(), bwmodel))\n", + "model_def.pipeline.append(KerasStage(model))\n", + "\n", + "model_def_path = os.path.join(datadir, 'save', 'model_def')\n", + "model_def.save(model_def_path)\n", + "print(model_def_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.model import Model\n", + "from azureml.core import Workspace\n", + "\n", + "ws = Workspace.from_config()\n", + "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')\n", + "model_name = \"catsanddogs-model\"\n", + "service_name = \"modelbuild-service\"\n", + "\n", + "registered_model = Model.register(ws, model_def_path, model_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first time the code below runs it will create a new service running your model. If you want to change the model you can make changes above in this notebook and save a new service definition. Then this code will update the running service in place to run the new model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core.webservice import Webservice\n", + "from azureml.exceptions import WebserviceException\n", + "from azureml.contrib.brainwave import BrainwaveWebservice, BrainwaveImage\n", + "try:\n", + " service = Webservice(ws, service_name)\n", + "except WebserviceException:\n", + " image_config = BrainwaveImage.image_configuration()\n", + " deployment_config = BrainwaveWebservice.deploy_configuration()\n", + " service = Webservice.deploy_from_model(ws, service_name, [registered_model], image_config, deployment_config)\n", + " service.wait_for_deployment(true)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The service is now running in Azure and ready to serve requests. We can check the address and port." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(service.ipAddress + ':' + str(service.port))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Client\n", + "There is a simple test client at amlrealtimeai.PredictionClient which can be used for testing. We'll use this client to score an image with our new service." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.contrib.brainwave.client import PredictionClient\n", + "client = PredictionClient(service.ipAddress, service.port)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can adapt the client [code](../../pythonlib/amlrealtimeai/client.py) to meet your needs. There is also an example C# [client](../../sample-clients/csharp).\n", + "\n", + "The service provides an API that is compatible with TensorFlow Serving. There are instructions to download a sample client [here](https://www.tensorflow.org/serving/setup)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Request\n", + "Let's see how our service does on a few images. It may get a few wrong." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Specify an image to classify\n", + "print('CATS')\n", + "for image_file in cat_files[:8]:\n", + " results = client.score_image(image_file)\n", + " result = 'CORRECT ' if results[0] > results[1] else 'WRONG '\n", + " print(result + str(results))\n", + "print('DOGS')\n", + "for image_file in dog_files[:8]:\n", + " results = client.score_image(image_file)\n", + " result = 'CORRECT ' if results[1] > results[0] else 'WRONG '\n", + " print(result + str(results))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup\n", + "Run the cell below to delete your service. In the [next notebook](project-brainwave-custom-weights.ipynb) you will learn how to retrain all the weights of one of the models" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "service.delete()\n", + " \n", + "registered_model.delete()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Appendix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "License for plot_confusion_matrix:\n", + "\n", + "New BSD License\n", + "\n", + "Copyright (c) 2007–2018 The scikit-learn developers.\n", + "All rights reserved.\n", + "\n", + "\n", + "Redistribution and use in source and binary forms, with or without\n", + "modification, are permitted provided that the following conditions are met:\n", + "\n", + " a. Redistributions of source code must retain the above copyright notice,\n", + " this list of conditions and the following disclaimer.\n", + " b. Redistributions in binary form must reproduce the above copyright\n", + " notice, this list of conditions and the following disclaimer in the\n", + " documentation and/or other materials provided with the distribution.\n", + " c. Neither the name of the Scikit-learn Developers nor the names of\n", + " its contributors may be used to endorse or promote products\n", + " derived from this software without specific prior written\n", + " permission. \n", + "\n", + "\n", + "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\n", + "AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n", + "IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n", + "ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR\n", + "ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n", + "DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n", + "SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\n", + "CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n", + "LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n", + "OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH\n", + "DAMAGE.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/project-brainwave/snowleopardgaze.jpg b/project-brainwave/snowleopardgaze.jpg new file mode 100644 index 0000000000000000000000000000000000000000..80450160bef250f5680d8d2046e0ebafe7cb0ea9 GIT binary patch literal 62821 zcmb4oWl$VVxHSY%aCcqY-QC^Y9Ts;dg1gHCi`(Mv?oMzg!C{dEP0(E4uj>18f8H}y zGgZ?)T~E!^{q*T`{;mJ}34^I5rzi&l3kw4S`+mXv+k%mXL4bpYhlfLWzabzXAR?h6 zBfSIT(NYS1GsQFSZbILSg>$du>VG2NZ;c`fc-Dr|BnwauyFA2 zQ6j&$6278sZNi|!!NPpNgu{G)@vC|>G_Nx;sP7=)SLO0ySBT=sxFI&a zw*PlWDk>}a_%IKTC92VAd`_8EOdL%3fm+grGQ%RUVwpZ?XCa~_>EN&j)nvXF71d6j z+FWU>%w)#sy4svD&WDj@6lvPDGC=!Bkh1$3Zo962=CG8GrZTa_h|qL0Z4NN?cvBf1f-nQiqe$8~Rhr6zDVh}4FL4m~ zM&OlXHmcjYl?^+99TBJ0jopJ2?i@qTg=!k<>kY`WIfYG$rpeD9j{MX~O9oF*)sl>w z%Eg4M2tSg(BtlJKNCYMg55I`AQesizQt+M;=0eQqDRt`EEo!6<2vd>rRgxtcnI);R zNj3;qn@HYcbXGMR<8#RLW?YOWR*)LelLyGCNqI?{ZS(07QSV}6ZjX{y73t+r$%QGp zVZcMp_-T3%0`5-v{mmL#bx8Xrsnw`J&+=@bL`&$PD4Xz9+^9d| zp?J2w+35wNRV}~=x>6AO+!sE7`h-a{Zv9w!4&a@446ANSv?9#mFug}w$6RWcC2~ zt_+?)$0F4lt%(zA1a0a!qTZz8C}D1VtgK1Z$&o`#*XoKfB(~j(wU?Hsu2YY?&o?GM zu{~`TJf}&yXyC3fi^_xk$^YfYZd}h;8ofN413m!I(Gk%TZJ?)pyIM?TRG<+{55n+;rAEE-%5;|$WCb1BeE{%! zG_8cynCS$kM+RGIB$Wt2=ZpNctdUkar(@*@p9Nw=vLLlp4x?r8M!EtoF@Paf3)>U3 z5iglWPhaIV0KgQ3LOE=q;BeY_vQC~h`1^eB7lAgUs6y|DnHLFmbk!M4x(mJuvf?&A zAMDOIsh^zT%_872=ep(neWV}2J$rdPvbyVPP;0?YK*k*d03^?-)TDlM zmx*T~eorEHOg?rF2(C(Ml5?2Nhic?jOYQ*R$f~4>;VLtevJC^?=Nyt`@(j_%`}w>p zFP-?z9qGEg1IE(sR+(><;w~v*dbD{@ck<+S76+^p7Pb4r=0b?BcSGUm-6uh#oJcT( zJgaol^vPX^Pub^BB?%HBtNqnAJ?O&h8mAA};rXk;!dnmLZfJ8VS#ju@ROqoV2 z=#=j~hmm8&;LTJO({3gKj=-&vaF1{lvB#4ev7;wD4W(U*a(^9C)S?Pd1j8B~5E8wbDvLa|TO#t;*2_H; z;^S=+HM-_yGUX-eAuvDlMhQzLPxAtopn?#=U&SYn)p*8MrXoOW|D~Fx!5{bT@JfeR zt#DlkI}+VRU@Bo+Kyr3695V}(9KuctAUqXYl~JH@o0J+?;Ti(Lb>KlE6554Lh~AW= zb|myl7==C2?b9xt7ks;&d7cyX!|XRxqOVfTZi0r8G9DD?Qm8a} zk`YQJ?50MT2tKX5|6OAhYYceaEx~<0`81f)xrz`HUpYPYB=lQ|1lC@>V;n67+zXkz zP(}2yyC4==ur)t{xZZ9>8P|g<&VGVdzQuf%N!Aql#`A#>A>qTwurSq<0BAAR`y=2> zOeD5t41-E)cPz>vJWDtTr?fSFF4D@vU_F5Z%gSxU{l@i{>vo5o#6HT+v~}8h=dXw! zanns#^qk=zM?-LH0fhAt{zwiW+#L_jp!1fQtGoCa6FuNTB=S%u&K|gtJWSL+s!=d~OR@`{V+*F!Upx zoLZ#XSG6zJGP>-XV>kJYNxV~kNM%nili1Y_0awySPu!O(5?HhZKJXXNa1bGAT-Bt3 zG?Uz|NOPab-t;`+hc0XM&zal%^_i%Gv-~9RxmQhdL_5gNCu_3#d%#t9#>sc3G2iBD z&}TuOl4da>a(;Pb|AVev!#3{@Kd)xjdvaT72<=^#nSz3xB9kS2hJ-Trn1a-P<5&H- zPhvdn^L4`ftsA$hM_0!~YugzBE=2@zvUX8gS97Nl@&&N83u0eSje9q9!Ceuq$mBgFHKCGPp#qrTLu%L|oXY`oCr*yN zPee+@_az}yl(C>MPy`jca0;@XYeB!f;pL_CpEcbWq2nO^`I~6DQMYy4y(c-7x3ibg zbvSKJFA+mRu)}!6*s$Gkv)az_?9hqb%y6cs7AB|NuICW{bhl{`(WIevI(MGqNj`Vd z#dK`BwMMV=G_VHkYJ0-edzee-`y6A!a3BS>2fnVjxNBC$t;_7m!lf69X~3BX_Mr8J zRz2(Jd4L+34Tp%!A9q*s-1_Oe{CsS{1Gf;_=#h6gl_jAR^~Q385(Y6XF-tnWBM#<_ zf;}!~1tCbgoGPs-d;!-Nre0l+hJ>WS;b-7tE!Z2|%Grr4o9ef>NP$pmitieEiOuQ$ z((Kaea^bW@#9in5IiG)Mg;48|UyrVjVU`;p#jqX{9jIi|732?Uj}`FCC<7HDFxxWq z+T|Au)GI3Io^@`XvUq~U_}h!L)+*){%r&_%lZzl`PJ&|MD~-*;nRAPpd)Q z+vd_e&I>t3gSW=(q(U=R0h{?G%xm}CZq}!JwqPfgk+0t2Yy)4z8wVYnzs-0mQ>IdD z0d8F0P6H*%s*&d>Ayp*P5LE#nrp{mSLXP|d&k{GcQ)I`hm_)? z@2ZeQg6G|Qw4HJ*yq4F;&jrtgmW+0p9KTMdkuA;+pZ#ua<->MUN2Vbi2D&RR_Pf+E%|^9ZD8RHrLSWfKsu6TW)3qtD6Rs*2d>F8b#CYzGQlb`YotDX3g< zW5cT!lC6C7d#}(+eWCduu6@B6@(VML4=qn8#k<$_a`qKoVxag*hUAVRi!d`cYIEJj zpS`Z^_DM;(xtmUpB>MH+*QV2=l1-lc$$L5EqAGyzW7S5a>-{lOF0xCu%ut;!nJAq^`eu4j)C^2QOweYk)j_R_yix!if(I5=T%k@R;<+Ym1AayBMSG{ zJeq2-2{U~-$0;Xyn*W#(xU3Zz1GxS6L5roysg@{|hU+%TV5+o|0ax7zHsQ&OnWw$= z#2*|*(3WPKUNcp@p@d661kB&B^Z9gByv764NxQ{cGH161hv~9;Gp9|UvSC+37+i*8mH!voH1j-t(b`al zKx?Top~VN`zL4*C<1%+AyBMT`sK8YK&2)96B8#16dYW#(Twoa_mRo*aeeHUOLgmkc zgZZ6@BGvn*OC*dsGEf?RaR5ldy|3i<3WAGo;YN{*9L6n6*91glSkEBP7n%o5f{EbuwaKf5I}tC~YxiVKBLM z>91)xjN0Q)*Sa~m(zTF^r0UZ>DmX|Hl`xYcz><<^y-XpQAyw|`A!V3J8Nj9Fn5X!i zbBcHVnxN8ihqUWa;D=+$Nq$&@Xk@6UlU@TE_k_b!;y;***+bosaT&#qEB~@X)sCPB$ePwC~j?Qo23R*?Q<`IJ)#HMFO**Q9zo$j_*!uRm1CRu0&vF`@8v zNCm=OY40beh?=Uwgp@&8vnPkgQRwUu1~w^y~VDUzcu) zI*{ZHtE=Ep4dfz~Rwc0`TgVyrvGG|@RjuOp8^Ot*!vKW?wo)S&r6(cPd*VHz6-#qTYe%!fq8}v14@Qw`Blsc+d(U92}lXr|85ejH&{F zh*a9hIC~J~eGV!BdvuYk1)S1?p1pAVWTcc10m98orA748oV19Xa!A(Ts7kHHj@ThO zcPCS=+m;|?L7&6Y#lhP^4INQD_#dM~e(l`(ZxubFI0e5X(N!mFtUq!9mkfKtmbdx0@W&yn^6*J=d z2(>z{+^25jv|;XsNv;%F^fYWtLNF7Zf;T`&Nhg&CWls)wIGL2>D4M zFV9Y&6BixGaM`2GM<z$5%L-_J*+yS7*B;Hq|)~_^*wva+W9RPex>aRZ{@-AzcUt!(g=i;}; z>Uu344O7V7c>(DVDl%}CO6pP^l!}f<2Bvx9Q@4~&q<03LbhALJ9u6nGl%%l0)ueVA z3ZQOsOf9v=7ZL$LRQXayjfDCo%EKKbsx@+Uy8jbTu%~x_p+wMxO`%y1I?#yAwK~~5 zYhV}@s5_-|{-|o><;w!}`VAe;&1JpGV!0Sc;MHfTIWjsinnCp(x8Ot|T~Jp`o*Fts z{-$smBqrbRYw9j#vZqT)XJo_`I6$ve?j1k33|i*$ZWdrhI|xEc`!YsNAY~IiH>4O2 zLf@B8!#op+HlZdU13>7pm2}cRmUUI@7`Szc=mK{!N`zVzRVS%(F(7MX0P4Xnb z9d^+ct}HD)COMfQ8=p?n-ZIr)QK*v|M;QHL-&Yc|LlLaq6s5>yZ;A5JD+7s6$hgP# zy2bmXS{=9QY9hIWfrnjs%(n-fQ8xR{?9VT|^wm-#y_&^1as0W+&*JTnbqlAHby0C9 zc@u%0fR63YL8Cy!{7!2vx$9bg^kNgjco>I^CXY!7j(|?$I0B1Z>6t>R)&^AIwXf&d z6#cV(y>KbK5CIBz;>7_c$0V)I1(!nQ3S2;g0Y8na%6UX{bPNK=6?`J#tIR-m5spwd zfV-4lf!-dAdkUPVa;8o7k%$vDpF#{>l=iGswDe9Gl-2FoZdF=sD|HlB&jgZ_emp4* zxXTkeA!r?Lmm}JI>)PMB&jr;C>(%+s7*}{+Ik`g#11JK)J zE7*WcQ{_~$l1>8RU(713qkc3mc;C9*jeV7*p7_XR$UK!wDcR9+#i)padWJrUo))t) zFKwG>(0_B6=db5z)p19}H=J-|Z@Hh-q*?jS;>zQ5P zWoUhtY6yN>mL$r}b(cu#c9g5x*Ok9(5No4`-(=ox{(Z;!ZsI(@_N#C`d>Io1x_PfxIMpWmlB%SHZdW$)*@1(0OIVt2#4%g>yv1)RK`wo-kk)*Bz_Z@KQTWrWcY_YEiiC?4_;q+@Gcd%-()?{=Flbi>;YoA#{UXizp%3jWdNmTdnw4E6)sCRhfZp(biel;X=KII8R#amme*=9nkJ1i9wJ1wrHiJB@iMj!mmb4;%1 zgmL^?bmj+me)XljK|>?k0rm-MQOSuai}9v|2;ofUvzZc7XPY zPpjEOS-lxz-_8muPOIngrRwa<5x7OExBPt`I4)524~Bhlg{va^x}(y#%*J!%!91rH zViSLm4!c*3Ox<7>jXg7qn!?aRp%z+X?j#Xpp{x_{Kk zI&g*Oc=nj-Y=P2|rdprTbe5s?OC)4j3X?X?jXo$~y!S!LDd;tt;wx&yjrVLP*;o-_b98#_ z?h}i|%h}C32^lvFA#LI5vd)RCGVF;^8!|H$*xH&FKJt1Jc0{Z_zc>1;yp`$~gn&cY z`LW#Y)5NRSWZ6NLs(Jps&@M*%T=kxhz*#=4sD3kv%nF;0GO4;FWn+<_XIry!&F0GF zX0z3+an4M5te&4VAbfw1$~_H}Z9HNh*ekE*5-p~g?8DDSBH+^&eWesf=8DC6?tW*D!YQ-Q?_#CIl2y=` z^o7i`7p|{b zGPpD1bEQn(dw(evg3;n4q_ZP5fRhMw=Tf9jQ3Rl%IH6xCU9sWBpo8`R2f~86<#WPK zWVDL(8QG~`9j13qX@RyDIShoXABH4iP>~5#8Oz;wxsb`3@#XZX*hTGYJEYxNP3)Nq zo3|l0-FCgnlQFibf|zO8rKeT|WZ+1(j75!i7T#qu}>W>4^Z4ML9)gKqPLRbpov-;N6a8 z$AtOt;R7rj9Q^O)vdAhVMKUWUW5)_y{7f`yARnjV1&&6hX;;l$=uurr?w)@n0mYNBFrSm zsD=k3u-+(AMD~1xF!@&QWoL)DU?hI-_OkhixXv`{y-0~GhVMTZ{GS|>SJXiA;m+_N zb(f%~e=w1f%&usQ?LVwha&z;F94KFojoCYj>gheJ+i_;rLT2BVR3ex80*Z>?qQ9~K z6#W|3Eq%d(GUkyNlF2sa%XfsT*~qf4y*IjEq`4YUs#UXe^^V)>cs zW2z4xR06D_Bv$!aWwGKalJI0RVtYtl$a(feV+ zAJR~zz3RY?7Ln<-%O1x+LV?bE%RF(F;;P|mhf`A}gr5UPclk^S20l|I=0;{`Kw&So z|HF@?ZA3VJ{$H4(G=Evur0GB9mg8r>ct*T&Rc-%u&E@`Ld%<8-Ay$XjInn zJDW@eB0YU`zG)dz6flLDo16Pm_&#f%|M0)#a#{S#P_x!k8@D5a^DTKo8-lHx__cv( z$#pj6mEAuWgm^62XVaIpKeE5qDi~jdGxdI1hR+WE-8I@w@-3XW$#P}Q{aTzYuI}aK z6}oUK9zWA1&8BNYo{s8qJ(j|YE4e=AIK>!tO+MV1xS3y(?(+OJoqo1~R&|w~<>C7Y z*dBRn`ZpkBL+OXHnshVaESxMyC|zdV(LIsBkHACQ^@A01L+JXi&W_JSnE_-{wei+N z+fv9MoH{e5HhDL&3dJ*WOx{>pB1gMD&|gdEnlnRG3V!C^GwS#jBJk;@WM=T9(gE9}oa0uk8?(yvU-9uFef`x8Uj{5*=meMB*3G@>thW9f@C{tq?h;Usxt6lCWnw`ON(`!kw+>r< z4P<$AGJmLvc09!AjEM=gwrd(493w&43K3?YA7<2wW>ZJqY&UduS5AU}(S>!p@ALS2gG0AH)^}lVtLkzl! z`*;d%f8yE-4H)E^&fq=wAO5;-X_}R3Aj&<`?J$k7TlVEFOM=+$hA6ZVnKq*TgHbzP zJj7nz&<4h;dQt68M(35QKJY&Bn0|5k2XisnG#C>Rf0>e^vFJq9m9Vnx3{l8dn>RE6 z2eSah)9`3awz~hgA)p`OM(rE91(s8-aO^198$XLai~0wnF*CyH9JqP$MQiOJ487_^ zdOy{AjEiwnk|>!Paoo>?Y;l>HA#Z){_^D226WSAXy_sOYvlUv!q0bjJ_hl#H`Uj)_ z`S*^r^0g!=dj_Fuu+p=3ss0f5SEZJaoZ;{+Gtz12_nuozE{a$yjj{1V&VMjZ>}}7t zoH$5)&$`?xCgFW!E-K|6Yd&EZa2yD(WfR`pVxKnx;gT*xye?g1ciRY->x6CCnSMJT zgQiqULw-pDKbJ#y3QAS%qo2PF{)2&^2BLEPeh*-?WXU|f>;b8v2Kcj%>CIKMPBv+o zGyNZo=QwRs>cT&mvut0eejv}0=-F4*j3uEmr_V}fXFc*&!@g8J$F}-hY;9E@jq=O) zEdT5$-I3tQ9Z5>2fA11N#Bh6}o4dkMZvXwDpdC zVxfe_gJoO0|F3g5NMJCcg_kuU3nIH$DEt6)y4mp5;}+_IIB(?#7UqU}Dl+E=6j+WIoGUOfw%aZgG<{tCH4qN3!u5cLJrM zb=6iT8wyMWCFNXr+Wq?YMfY*nmz?oUvy%RCc0V038b>CP^iA|klU1yn8VzcOCT|m5 z2gJnp4z&o)qejk>4{Wg*J2)uPxM>+payyl84W`!8gABNGRU`dmD4djuZFeB|^ch52x$Ss`9KKwK>rlcsR#FwS6)<^$ z>nLmEKN!>)ThDEG+q>)RKj{LG5+m-P3ACZgal|KWV@+exu}!OV)`|4XhO_Xp{98)l zV4qsuGyPlDD;R+%JWnH+^5@uGlmo%urOGr_GPVL$Up|bjpj3u8)IQAcy1smn)?ADM z?@vpu4MX21ZTn((mbyHV+E*>z(Z&}OG0|EsF`=;2o109a^&!;SydwcNtP!Nxxa_q5 z+t=``T_X1d#+b^FhG(Qx!fCeZp zDaLJ**DDk=WJ@O%1$h6(NR8Y#+DURZZ-*t*)G)9g0}S*cKdlJ4Wy8~UIo{Hz*7-(} z*A1dYIqzmF(+Ss>3S6}Vf4W*8Z){i1$HysFpq#5tZ4X1xtkB2b*3Z|n|?hxjV@4{h6Nsr zsg&;w*vXIUP`HOR#0dpltB>z5Qv)%qjt!znYJi%{RRN|G?>1cBp9+Tbkmx!4VBTDo#HZIXxn#X1CYkj(+I45wab?)r;z#mq(8Ck%QvP=1dILuB zr0S%du(hEff3=Z2m&aujG$#E*_KmNilF?LGJ4~B~qn;>)SF(Hni5>DewlBF0Liv|7 zS9>MSf;)7-JF*dtS`^OE+ux4iufEdO=;c8r*?{m@_yNbn;Ls$222LD;N@K~acwZzuCF!d&KCiD2EUaw7rD z5}l0>AW?_o)g6ZjlK!X+({FGl&B!6@O#19_VrQD|pGHx=++6;&cBEGlXZ5Y6 zncsBFA{rJJ&v_m95Lu$v}4*C5M!^aYx)#Lk~fZvMYV?T?(S3)M7Za`N$Uf*fHR zkx$Ngzr_{Qvv(=!EDuHbEKX{x`Yg!4T!_}D7>H|{vsu~Lm#M!V=|<_5n`Y=bnW?J6 z!L7ge@oXtuDnL_`4`bJ#oYy6yubm6H7yrSqspe<+ z2h_}_q=!*YaTg7jhw15R*IDaYEo3Tde*ZNM6k;VDx{_*4Sl(!a@QyjYNm|?rEGM`G zm-Pz$=_iuqjicujA)G)3mUj%=6>t1Uule~|T+mB=Bc(6To8wiIFdSWnFl^=3zXj8e zpQ23Z1@C)rHPM?8-g}N>qaHn1Xuq)?F=RbgkYQk*X~2k%+F& znA97wN1Q;+q-3eeWr62+3HPQ>N+qe|fjHs~qj8Z>i2~Qu8SU`+2}V&{!!FxaYQ4BG z==z#{xU26GW_$w@-15|4f(f5?jWV5b7h1V=GkfBgoScK?j(}2^7NM*f&qifC{f=E% z^7wPIups8HC)NIaw9d@3s_FVAXOql2t6d`VsSVp0Uwp+iHVKw_j5AB`%}sQ$sr7S+ z?9Hi8_I)qSCq`EM>$_nimhtKDb?+4H{$ov36^T%_!&a?8jCKO`iCI8kL$<+aN0^!l z5!ZH%PxrT3Qsd&j5v&4j8hUMdW{)3P4H6b#f(TA3+G9h-rAqn46^BinJubQ$t;(29 z5HT#dtBTNa_tg0WKNaJiS6EkfC1UYOj|b$GUBSf{y42BofBD3ZX}%0x(J@TLZp@q~ z0*`TdW&iC1eK5(bs94{vg*l5H*Be>yPR-$hA=Li_<+*iJJkPE<*4aurwAO+3LFD7W z$dD(5oEe)j*g_ROml+#~FZpJeMtwZCdBgDFt;g(bfL@k@uC9a&ia4N%8M}xw{A7oT z7z^&K`07@!IzU?APgP&+qE21mOwGNun9C(2>jH|Rs5cV#8)&O&4PD`b?UBAtKmDrn z*9ly0)&X`V$w4O|TWzqb;qP&+MR=u0`1J_2tQ4K&lS_j`XI*HHqfxo_HIOH@$%y(& z;D~X!#?W9NwG|G%XlbD=dg7JtK{4S);O475u0x}-MgZ}$?Z(~kM+cdDh+cVDBbKUx zceTfN1LH}TPpsQeKqUDPCFWQHszQxS@vW#i(a70zrd0Omoi zOGbybK0}_S3SiCO|jh)SYb@~{JZJ@5Ora3+H&-ygx z5OqUEJw#*!<&JfF#uR^<;8Di7AXC#!|NC*4KZipYjtL}916CBgWN~Xw4oec0?`j0* z>D~_B0~g|GTnR^iCjY=I)q2njbWL9uw}Ck=WlZ0T`&D9vDZY)Juwd9YI8Gq$CI-kM z6tQ&1#W@<5-f${5v?5!}IHnO$EN{ydiyZjWuw!8_Dv5v&8pZ zh{*m$_;=cz8p9$dQp?+;z@lzU*Xwof<$wZacJ?j$1@W^+oH_H z!C!jqXJ0dmvmJU2bRf_$PC71-4K3=KEU3Gb7pMW3=ELkV{~6$1nyrr-v3F2u;80Hi z!3Ks}^&c0^?^ASoLh$K)=;GY3?ru22ZZ$6TpD2tz?-z{qXJ))meBI~v#|-Arh8LrO zS=?6h;85c8KX?z;!TJHN*99^^ns3)lK7@XL8~L50o#XF67;nBr(MFfA`z=U!NZno` zUFVd}JrJ)?LH$J(Vim-`himVYop z%iFlZ{7O}uC&|4kPni@5Mu=YvBRLZna`11pC4$LEi^+yWC@j9AREl}T>t6c=rCX1W zU8lv%Xy}A71Ze+7!yJ8;qSviy^|QY5J9{!=p>{oMy}qElgg_wmuDzWCJ|ZyC2!Kt*()#(5^qQg|}ei1c^_3y%HFpM~v%VAIZtnaCiwM&HLg3v>Hk(ctqQK;li(SYOOtj>fn$f$sp3gx=8h-hMDSHv> zAjG@z@XMaaQk_7%GZfHPF0Jjb;uhZQSwe8@wo&&uWmH;2)v72!_!5oD0(aNcC5Jbw zFFU4f#)>z%U6uW}rL!4r(MeW5YYjtF@waiZVUSq$w9-ZSVL-~%c}Pe*R?Uor9lAcR z-+q;Ju>{RuGR@J1(}3y=5aTgiCAnAlILCuFR&hWg4WQDgISAgMH-~Y-IM`a8CgJ+y zc%7~Fg$-40=A}xIuSep3DCm1$xaCv`8OhT}KV3$xw~jxCS|Z(e8BJ}p+*t`7`3adS zJ=i30Wd4UR)ywJ2|6nwyDV~!^m}ZWgPPnzBz!uF!aSR|Ng*kLNO@=y^o$V>^_Lj1oIyr9>_ajW$x2}8zB+BdT0F%vmb>C|# zxzz~^ZZ8cMoyZR6Xx^P5!k|*K@r|W7k2hhEPf(_H$jTDL2+Bf{2UaiGa+}SKcaE7+ zWA%Y3)fheEF|oEvvuh5Z-dT~L2{O?6wg_v=-wf-Bq_B#n+k5bSOuD|H=2F8%7N@#WS-Wa@Svx3eUc95>o47_^DKYBx zz1M-)+(*#xJ7B*;ZRSREkzP{JAC{3FYirIfgF;Lg6(&EZ`eG8I9>EO+QCNg^UeT+k z2V*$qyrQXwUMKtv+xncu)vuNDY~ff>GXX&MDHDSiiVrrmDx8{R778gpwBgzunyJav z9G#kot&8td8b~g+$qRB7mT>?lS+7f8ndcG75B!6vP3Eh#d&U@FX5K$INyct3;Huja zw%)&eINEab<~x!`}umac4Eh_xSylI;RNNTc0@E*iiuD}S(+%6f;(=hO0TApBy0xyhKteF zcS-IFhf3%9?>T)B7T0 zqMo@T!Q!|nFAA~wcITkwNBSkCZj5{ysVDeA#e+HQHgaReS80va2hArfOs-QFsy)CQfG5cNl* z#>S}Lm21UL|MHZrNpPZA=ii#g2CsNzYMmppz&L%z4o4S*N4(Ov!=isM9aK(%ak)u` zT)RaX$)NR2-s}#2-pmaj91`J-Fj>!6BW5Dq4I_t{ypQ$fWo^~xn@h`=C!dx2_06`e zLY9~-emI|@$niji(VKEB`id~KiP^JsDeYTJptXRF)B(NC@o~NM?r!4PYJ)EV_eS+{IinC3M%#eD4@JX4i$~B}-MRZ6 zH8EY^xw6RBezoDxIwhjngjH3{plq~RZs(%Dy6*FtO6EUatp2sfmA1)}mku(YkgRMY z2{MfQ!kRf(0TdaDJ2_mx>4RGdm71<3?>nitb1!=CF3PfBrP>+oBO0Xk%^kGb{p)52 zEX1>)QO`|<#-M`!dR8A_cyj=CgV$ohpDRU_$Wo_!`9d5p4!$eRB&vL5Qtkb%pIAaS zhI3Zi!wylE>T<2G{Ao*b7spoRV`NAH$6&@SHq{j>aF@ahlLOX=F{msh^U@#lAo71` znHF=xuKa;7VfC!uFvd9}X2;~xpr!r0Ayl6t;IT8~%WsA2+F2~z0lRh_FuG6)EyXT7 zr_ktbEm%6&;;1aOuUcQo)5#wd(G~6y!%sx_$4Y;tW&FrQ)l86K1$!I%KGt#3nD${> zCuA|XR(QKn+nAdQ94V?et5$|K$+9QCkzpEBAbU%|Z`B3F-&=NRD*X=E8(WvPc6zk7 z?~HkeRCnA zZRD8#<^rEkTZ3g_nkNIg{G?;Ic*>@y>ws~{F<1QgkttKW9ITCbtp6I7A!uOetq!Wi z=^Juo2N~Q#A`(44ory`7VN^1?*$yRR+l3t>jd?^V1bk@PKRE?DC}%OFWmi2E@c3B`>zB8Igzv#;{G%M-?Z?Rnw~C%eN%F0XX#91?FBC z`MS+?yn|vHNtlczRPR91V8)ie_xx^<6GqT6AI1(;ujI;!w@kg)1G2{z0DtyYWCxc> zkfF(oGgc#7kcs? zRlZuDKO0&Cch%mvI7}|Fej5!uKvNctA!v-ZCRDe59Lop>RLWpVJ~n@@@T?AcQrE2G zYIQBA&56@)5u*=7zI;oE>iLio!Y^ANga)8}E@2iE((R!j{%-dIDWG=yeY|MG5KBE8 z)~-EJA(WTHHrkbqPwn3{5O?m`UXFC9XRkj`-e_f0#Q^^b8hP@KV{gVGT&;p3Fg>fI zq+&M5i)?}df0q3Aam|qPJk+_{{bs#0rt9EvT58hof%%}&FqUX3dTuUo&YXdw5lcx? zpsh`VLqSa*v*fS6$iWn6py`Uv+f)aR(IoUA3c+Vb%}C(& zzenx&b7ed?F)f@u_Qh#hxJNQ3AnkWWxK880n`o&%Bu6OJLdqI`9J-xvEi=1L;Y zMM(!*w}(~NM#lH}bc7L`Vr%-DS~3<5sF}Y0?XQNG`e1xf#6e%0L|>zBfPMTP+5ZWi zPFPO#+epn6Fr%(miB^go>eNp1MasYUz^g7-`-PJF!G%-GUPll-NY93Z0pXfpCADY_I(bhT&FuJ#`@xyYZ zzRb&YG49zf;Ay6%>jjRd2F;)GvSwAn)@?y#t)+5_Os*^+@Yn)6-VnpNV|*E5(jWYK zJFnAvq!qjGui_8Q{ZEK-`iwA6mVWBI3XX~xu}mMPdvy+(kC@ieW|9 ze*ApGq()sYO}o-HwRakV?G|hE{{UKr`pr#7_`z;DHGdS9m?Fi4^vIg*_15#KbS1Ri z((@>GRsx^pSlYjdXar+bZHC0Ysnhvh*!bwGwWyjAkxYDg%zBEq22iFC$UXgY0=F;H zar$c6)v#7#b*k*BbzYoq%Jn{;ns9P5u&Bf6)^%*Re2hHH3^~=~rdEC?Gh_7}8}dDe zz|Y3aW()S}6>0MsJakexRa{76FdxhPVoB;lvrd_a0AGYMuOTAA`scA(ZYDjGFQC@! z_!$%%f!+s9H!e0cR4vrkYUAQnL{ZrG)Z-lk4@P=qO&`WR_w?`XU1I)>E5W}LKl@#{ zokjYi^vhGK<@&!%i%xU&{Em;7;@+Lqy$i8(pCbHR+i%0b&Rp)B)2zkn-tMc>i?~)o z<{>4jz|6LEj695WRn*>~G5%C*u>613KmP!*OrsX-O&vI`m589kMwKq2#exVwRla*` zu>j2eo|m<0!8?h#J>)`@UC+3Fjvf_Q)_xwR(<;kcUW;C<#K)|GHQR_X#n4|fj00ZL z?cB}kSeTPC%X4#Zv8_g{$m8MGSMsqUh&!+9m>`F-Gkvw#!T9w!!n1@aI2o;X5PsPc zQ(njT3c_=nn}^(H=k*3@qO2Y!x+p;nKnj;eGHwd>6h!%0JCE4h+|<{zU!PM`P^{q@ zVEwX0Y}PC{?k^{YaxCn>($gSnU5lv9y=WVR>UD;^9G2w<9kMKNIPKjDwiFt30A z!~iA{00II60s{a70s{d600000009CK5FipTAu&J$K~PZ>GD2Yg+5iXv0|5a)0V4kZ zLtnsIXCliUEKIynMHEp(T@709udNGrr4^V<^h>tK^{tj35k+bK$fAxT{uMozXslk5 zl?q9eOXA=-e_tYHmzylSz2PE%3^O?OBD*ga*!-^Ay&>yISfaC5aeoc=Kx?w}Ub-mY zk64nY_9jcdD%tCyJkZ_0miO?=@IL4&LwNDiCrj>9q511b;wa(ai2Y@hAulK7>PWAV zcswzx-A2mF9*Jo46Y_i3v{{sXRLa#Z=a>`sqV{_nbdF!joe$1ZyuHxN*?JrEs&sNg z^O)vcUKrv1cBN$tD55J2VkF4JlJH4Ty<)KabiYLrRhji!XCyf4k;LF|9B~y{Y*<2& z@k1<(=vY~04;7+UiqdPTy3Z{lu!M@m3oKcF1oJIx46HcWz~V|cO2n&9SeI4a(N-l# z8#t6bYYR%Oyw_Zj4O$|{>)`xjWL_mIALOx2=7r?3V#+9y7nh3B^fV<}$X^V7VG{hc z=x+5-`_G{~Nejlck4GVNzJ{!E?vJ90Ki5TyN3tg+hOt_+iRfkS%hpKrg(@z|cE(gc z$=Au~zbpNUSF2@<;g)Xx`2J7tN-K5OKj71?tmEG=;ULP(60yGd{{RUWOj)wZee(NM zM!amjiI;?NE5N;s{ph+S)xc}vS#i}YC5bkf88J%6hRDm#Yo=XnuYx)_oEgO8dJu=; zXYqE+=a5%B2*K|edvHt*CHF$28V~#`Ucz+F9z-qc~j%-f!lcXkI zv(Y;rk$C83jr7{p&di}?@W#)1FAop6bKHJz>dY|~Syt+Guf5mHN^I9hx92zWMjnVQE zo%)nfTl0l^I)0B+>oAtx?$JeUF@KBQZnSkj+YbuMJB!Z(l6kLdsEE5;jy`lZ=M+{i zJQQ z;PQyeM6iYHC0UcK-i6Lv5n7)lRV^}BcClK_`OJ9QQfQA*wYc34*&!4%#_*_Ei2O5= z^NTExJ2&LAPtIWcZFyuD*o~M4mge zboQehxF+;YUqWLTn2-28c@W-Xp~mxlv!o%#zeGT3EC!z`W;vFD3Vo(#j$kKA583ItrRlY(;LJnpMhxZ8_L*D zhVM)nE(9(4^NA4X-y%tAlz$4b_%FDU#9b2du(IAcI|eQ8N-d#FdS#H0-osW)rg@1k zCuF}cq%!oQxNgKXrQ)))79^~3v8IT<*}}!r^XF}(^HxiuzJ~`Mb67~B;T4sJwLHTO z30J=^lI3W$DID#sP~-S>jp1mcx`gj~AH?d$o#;b!a&TDpApO^_YS7jwNbM6t$JXRM z*DE6Y>U<3qhQ&6E@;@kEN5*Sr8(rjIqTOaP(RWWMz9!2$WjU1#FTl4;FOvIMA&R&K9Ir*T%+$hl?*nUG2&ha#>{rP5*K&c{*M0ub&tsMHjiO1c7RGXPPH6dyv7&wRyh|8<_PAGWT;8v z!ha*5WUT^auz)J&WkH{jQ^bDDoTj;$a}KGbxB|Cc3fIy{xRnJDVQ*$54mz)vsDY4G zrLuTNp>#jGAE;LaY0+o`8_w>J9!bup+MeQ1lyF~a0gX|D;q+Us2l5rcN2-qzRV_E$ zZ#XMyHPw$Bbp2GG&VTM!=9~E+csmEEj~l*9r&nsDh(*m38?QwG zxwz+rBxtj$^AN6cT@r@z1EM0H?^0Fu-fF-02)?sz9Ak?6HhmMTTfM9fEBJb}_eg}t zP&@AXywG*cvxrg*^j~ThciDkgc88jWmjnPGK!npDQohW?_;gU|y|{P0N*=zW-N`ze zfCg3LL>}DF?xes)tl%<;IBcV1409@9A3jJ4COIzgwM%JpM%vDMxj3!jTR0Y~ZaK;? zXZdKkwHzgPeW9e=mbqNRaGa}w!ghB;wV`VZ%&_pIf_UaV;HdTDnDX&kqElIEatBUo z%4SL?rd)XNL;j^-!i8?o=;)?mnj3(-^+MKe9RlqPfw+Y*S{^f%_CT4*LG^so3f9yN zsC>Oul|)n@y1kJHhXStqj?8jGr8eq@>@2J`>r%2>Q=Rnpq3cRBCuew0RCq#F*osxMV6;Q^-K+$)}V%x$qk`cO@WF0xH7z+zW6dhrz z=A%u5+Jo+tK$&W`I4nS3cCG57Khm%fd}S~{%yNC(3p-=h;d4@@*(g2Q6vk7A%DiRH z$vn}fb>LB?>A^yaW=bxp7ZiD}*;s%Zd;`sPn%D2VZv2(?4XeG8Y+OqIE|8N@d61+; z&Z?hmp@{Pa{p24*#7|ZjI5bLj<)3c$%T{(9=mi&Mx|g0Pwv~8 z!UCfS%JK;2oN6-R!03P>?dqFRu9d7JJ3D}R;+xr;`|8%m`=iN;}s3W4D4%kZ(lo|vLeeXpfLG+UM0Ii}fQtFocmR)qB@Iu{J zJ5r1%KBwx7v~SH?4(kzQ+7wc4uI8OYi>?|j`k20xwdke_r1xN4iUQDild2AO>1h2G ztah3nNVVs2$8+>mJD^-0kkmhZNvtLYr9dF9`i9O(v8~7(r8K)xhcyMFeo2&RV&kd} zB%e*Mv&&Q!ub`JGV{PiC?URgP}gNf2SCvlv&=$s%-^up4$x#PjWT1V zsu6rcqk^CT$#S%8Bub4{u3#{(4<=R6F`(vAb!WRLg2h5tcUS@e)m?Cfxf;Mm?m0vp zs%M1yh(6CPSWtd8qE*f+)~PBQC20tqR_kGRkMgKkC>CZ;R_>QHcSTd?Qw0Xnr|x}Dt1qwb;E`=|pb(N}!M2&nQsa~stL+!r$TZ+;(FHT3%siWYYlj zQ=;>EauEj*xG6i%C@^TTvPhH^Cq@cy4biYZqliB>yH7+7pb&LbyC5HDPbhTRK9KlA z9@QaQWoN{5y}>#4B$YBy)E$32Fz+B-f3n8GbOFc-L89e4eHIL;aL7>T9oWt^ zLCqo&yFeuxjF2?jI|-E!%C!9ya#G`Jm5V`l?y5fAXsXVo(egk_kBNY^+nwiQ9?Jmk4>%%^JtC~RwT+$J1q2TJO zS8JT+$B|?f%Xp_#sM1(f?X}J5zN$Lx?#3QQ5C=F=@4Dr>VDmwhe%Gl`yYo@n`Blvb zk|2{lnEJdGtc8;c1bTTX)R~~w6Tv_;vP6xg+64eYZb}~Nl+&%ePckS2fDfJ0%I(1zscog;2~iCffItI!dX&=sjojP-=@| zJ*+npARy+)cJRk{A?rdOjv6qosgj@#kXL(5e9$eBh2AM7BM!nL+rbA!5wg;7hRHz?R6^WwSJXQ+ z8z2%?lus3+fFLVCostYKHXM+Qx>dOiLS;-!ld6NqU6PCpywn+M;~=i#%{05rk}?7|SD{!80F3uQ4Q(jQw4>cbuIi;eJe9QogTYI5 z7Px_C3;&keGnuFB4yVO{Ih@R-Qo<~G})W-}W zSnlw0Q3NYvcp!9Y4-L30vK$r_;Q;3Sc0}xvu*#8;n^5pl)IiV`oq@d2M`Ub;2f>}0 zU*;8b$^zK;#;6JwTMBMig@U1tEZlP!z@9VID zB}K9G;)IJ~OsSKy3N2W|1Cc@wU9fOgwMpKTb*uu!YaK6(I}{VAs}2 z)f>u@mCa#9gdJ0LpSf1kY!8Vn5)*_&p!xu&K$5QFjo5 zE9c^bN)9|iA=+?6O5GB?#YN^6+9e0Qx!I_Xd@hMR5iVP*0YU))sjUw@2LbWn%H~Ul zBzvOzPW_d&hCdKmF8YwSoF>SXk|Al0ofMTMWeYefOyNf_5eU-grBee4DMhZ53L=>{ zm2d^iyCN5vz;ayF?`fy6(PP$OLX|E>@f-)DxXWMof^*$ZzoI_#b^54ya|+z33O2!~ z{TAzI&&y;J|(wDZ7r%>kRpkFSJqTHm>;K&6{uvD{nBRJb?1xzC1PtiNP z1RvFRj&wJIYlC5+{-`sa!&aUi6|OtmF4w>s3@!+_RX+DHPt{KU0P3w^$FANA>UkWI z2&_pfcr8Fd#Qju|hf@lh+ri4_@RlI+68qE+SM#7I23mS~ft6iUBk4ukc zhqvmBsPAJu*ARNHYkA7nmI>9*72+QbCinz1b=l(td~sf}{<9C=v1O zbd7~VgW%mnWE4RIJVKXj(3SRjsi+jV5~Ym*E1;42MZ@EoPB{fi!W&~|szhwvD*nH*3XBdW#GnAEF?T zRvr2)jKal0R46jAXYzoe=MeB@LYBN#6QtxV{{XslP$W6O6SLC$(p~gSW}FarRSi<@ zQ0^K_g8u-ReDVW=fFK}@w>-ix?DGKzJeWEnNSQ(Qmb9EGE;3Yy6(R@GJ=TEWIDASK#BP$(r4Q9%rCj-yWg1#3bFi&?)o_iP zY=<@i5jvo9vHt*-W3`WK%YZ3?ggK7YZn{Fso_@o9#8~n5S4P{K;yQC(+l{W|Ry0xY zKqq?f9d=rTg~e#2Dp>@hckHbv9aO0WV^8lcdn_|MT_ zsuF?X)FyJQgS>-P*tFH|MniT0WT;(-R^zhIx zFe(+I2Ex!nCYwlV7TpTkt;g=W+G601IRJ%wAzuC1Ks9BcorbL_`6zbbxr=OyK{F6ZE9om;GpV(ZnA;m zb?|qLEv6dW;Ycb9pHMvn@G28yfLQQnXf64K9fv!Ty1)=53wWbd_g)Sg!55czU}_2s zkOrSMy5M~oB|5HP=$hNhVH$M?NAK`wLX4~|r&Tz+O_KPIs#}rrM&)sFapa*8?G&}p zIV^AVLbU@Q%7Z&YOI^+|QF{Od0_I>WW58J9AnK}784WulWP+gkE|NJZYSBU*RPgXm z6$DBqJXcf&(AW@z$&QNd@t}OxzJdD-j|2-~J-QxhMRFe;u!*ff;0`<%6fAhKg(Ie} z`6%JCPD->m0=(B09DJ%ct8JdBx~t2h%6TGM(xjOjmh=XJ}xHp}?*S2^o+f>uCM+3PCc0SK2k zM=nYfc%IZrN&e-p&2bJ%vN@OxEa6qL#{?|)SD*CdTK1N>a615TmZ@VL?~w3t*_l(S;HPD=wpJ7rT22mr zI4u%N+62f}xVgPHNLeBkl=R|I$>^f! za)LMi0IF$rr5RQYIjEWVB>5`-AfHM2b4(v-J@pnp<`uug5AKbfow?_g0}FnOPtg*} zL4T1?{?%{r^sWB@N;cCJo__OH)SlX>h|zC$YM<2~vwk`*N>6|ZTv5u71$T8{rxhX7 zwtg9NaODTY%C?DP_2nv7@sffgf)BV059)#193T*_FiI_Mp$nP~qf(~Po=8~3^;!ZO z`=>nlT5!6Bj3CzV_-ri0+}VGIcF(osA{|YpL>w*N(~tD8w1+c;#)u=b6qi~$pwUnn z!XfQ~nAp+=t9RpaDi4L!Y6)w#Ponu2pVdf5W!@8jhz(ni6pSM{>bDC@4YdhT@hOq! zvpy_rgR%mjYli?rE&$SZ^6B~@*ADTn^0;gzOeh=`Hyv#VFT@aY4EjnW9s3N9i&Y!~ zD9;ZZjgys=Dt(oKvCb{A?)$+Em(!0lZEFwR3o(TX;?h`k3+&E4 z9d{oqO}{{`W*f3oo!8XcQK7@Cb#BYLK=N}|y`&wPSXuDEC|sfuEz{dAa0DRwf8|vD zwux9-&Txa9GJwLjkD+t%Wn*)Hzk$vA`n#$P?rLcnP(d4WvkDeAYNZ=mO7a!uD=k?b zo>uFD`5z3pfTg4n@SuP~eL5#}aOKl)MF@iXNA*m7!+R<~9)^J)V^2S>*g+3jywFi~ zC)?3xkv}C4k>QGMB}>{uwMC(57K=rqhJ`26)a_ljwN-O;r9p-POBVTH2kNi$LfBDy?dj2V|vM18F*>{a5gvp1*V$1M5^8f&T!H?5lr>_L{+z90O?6ae+VX zsI#|Jr%$w0`;E$DYgPfzh#I3pk7;A-f_HviDxIzUo$U^J;IqkLb6JT;S1y_kDPM*QPNu?Ljb%k5z1B%KIser9xKvub+qhn*om5)hVw0HfYfT0Xy zo5kE=c7xfUR@V%Krcm zpvn}9tY^Ku&fmZnVyb%$6H1WX?EU5`9t87F>`!e!;wx0TvxE1lGJpEDR~2YeH&nP+ zE|3QZZ~LmQr`f2}?Cof4Dt+QO{{A;r?2so`f3kB-+!U^B6kVK2R@I|!b(!>4CG)yI zofZ}vKtb;_2OA?_VajWS0tyZGn$%u`_15`lvvZy zcw{X6Q{L4Veir1awOY)*)ACojp=tSo0_X7s8bpV`Q^+HauQhOJlCk-YPOG26eOiVZ z!6iDiLgRPrgy}z#KmWu4Di8nz009I70R#dA0RaI3000015g`yUK~W%KaWH|Ap|Qb0 z(eUB%Q2*Kh2mt{A0Y4#60XD-}27b9CBY0BzU#glANK6TlH3<-e@*#hQhpX@_*q7zx5$_qpwnoN2FJ3SAzEUJz(PlF|$0A@?If4?+~BD4Bc_K7;V3DA+qkWP3vhUNoiA_^$-KpMY%` zf()a2kAVED%HAw3q6SK6G9N7(6>cT+Buu3cPr{VG4HFTwJs|9ikTau0)0$^QAdv~8 zeHW`4bVY7I0s90SXean>YQ|EHqmhQzx@g)VaiIKZagC!I*sZ|{xR&I0QO*k;(Hw>H zF1Hwj(R?XJF0e{$Jz$!SV0s}2yj^;TLVb2I>^+Bgph%P~3H8>5(7Z;*$~{k`5K)s* zQ;#I5Mj;rfxMFOIWMdFecNgU51dJG|mR5+73F7ojr4!W%BZQOT4gxE?61-6sv zF-FmA#;?a?r(ksqLe}X@QvEja6gPYp_QD`vt@ue?_T1<@tV1)UsrzIAJ@Pr{W zno%8`11d1GL@l*sIV+6Vz9^V%*W4cIa4u649!6QG!Ty9TwiDSPnK!lk5VSDx$2)rP z$jpp3icZ;;U+8VHAet3hRhAs=dl}mqFO6l!z`lwKVt5>Ao5Q4fh4t6Yl%>^#%2Jo; z#PmVnhtxV@g*(yjhhb=a-1lpZ$mNiM;X$fG$kIjdu3?Fkw*+KV&CF*^Xwh(Y@J8Y~({^Y!- zG?vD|$}Fxjqm1AO8Rl2|%F~+|5#CXYgW%;i{JG@sXQvG<%qcmL5zPvazvF z%QJAuZzdU@x5IoD@K-M8LpiL%DNIz+7W}4l2Nk|cb~VG;+Rxb=NIyvvlk;$>2_!hX zCXj?72tp8q(F?*5gdqv_X+|!vgdot9uLX>S0iUyx)rJ)}6=*`c5HN-2wRj0+ktcvz zE=F))WN1tIV^o1#@czc31~}9<2smIhB^ydA5D6Yq?2XgflXIo>YJ3Vvagf>;YkZ|d zV%sFUazsyZTmJyEeV$I=B)cn2O;^^CGq=lR5k&cia^T#Lcfq^P(q!hYaZb_gI*S=E zZHbS7O*Moe2tp8qC((o_lG{pBl%+3<5QHX_YzQfL6;S-9gle_B8X19N#knRBh4i5b ztvA6VbRT%wwx6mH;n#DRB}28)oF&0R3BsD=(UmnG=(aHw*w+HPaP2vf;R+nNt{fZ^ zzUYWfBToX83ASI6kmM7rVVr?9Ss6o)=L77mnL8_U;Qme<1Aiav!g?qDkk>Okox#p? zh3s)P8yvs?044nhLTH#W-4PWH#v>{x4#n>1(RHx~!|7p@FmALhXOhvc0vlhGb&+ju z_#av5oRSK8Qi2NXP6`ldkB8vdnE+tp1rt8uh))BJy%{o5n2Ta4J&4}!C2#Cv+NOK_ zJ;j>ahmOYyK2g3!+CPXyiS7)bj;&jmOtR$L!v%pZvQ}I3vKOXvfGT3&20G$+68``R z=%xCP{0F2V1erDzH!O!nQ4HKgmXKvfsPqZm!XoOvwT5+=^kl}p_0bktaIVG;L&A8c zJ*0CaB2w)6bJ&4=V;?RPxBC})-@G%V`!dW~lDJ>&bk-^Jox+?67yFQD1TEkG555`R zcavfzd5U`*9&B35Ms*VGs}M`0hD!EFy0benn{lM8wKIB#?HSMR?YzIg^i&;Y8PMT! z{2zt#r722CT6>7p5d2?-{{RjUf?f@fi_K;5Y?lkN6r7E$!D5^qvO+}B3+Pdz+8}?i;o-8p`~N zb`)0u>m%!TO;V}`jvWe^&Q7&q$;8uZlQN1MBb(l{V4u+*k8*(UX&9H!Q z8>R$w_ouUwd%nbIhATDK{*9SkL9@OG2^n)^0)d`j-E&Ca%6lt3wsF@#KK+?Hz)krS z>{}KS=$J{!y#k7tCd;xCB3L&@Z^Wz%w;dT|p}opu-cBd6r((X0PX)sYFiMtN^JCh4 zgiyLMOb)@=LX%{E=-J4(5Jcocct03HA`|>&G4lkL#tNcT5|_?s^hDC;_(SI-u!khQ zltCIvE0dCao|Q<~2|SMLh0`$I2IND+ZauBjgmE@r1~9jmO8EW&uBOgNA%Vxk8Y!D> ztcp&bLm|${eVc<6d%x(u%Lly3+4E*UW?IgZO<^v-V>y|Jak3K>tmBf@UK3_LLemG` z8+hniN{PWg>O8$i(TG~UV4NwXEaiEPqux6L`4{IwJtm4x40}12K9puYsj=|ZLdkEU zo(y|=()}nx4N>a`AT!iFRX2di^Yk8wrCaJMa=YPDt>Luy-O(X%g13`|4bv3aIulvw zxMV7j#w6#IP^~g|!LwOygD_L=@J2y<+>qAl0Yl*Bg%ZVKc*hd@fK{>$mHMW|j8P7ZXoF};Zy z8y-u+y$#!mi&fEhr_qEUf>@l0WQ-I8gKahz?u4e|W$<)HO|OFmUpI0$a}RM?eVH*! zoMQq>KW`=^a%zw_`-)lnQ3{w;dybmm!#m(GaJ;0b*e1z?aSC7MoEHNv(abCSY4PwO z7PmHy&s-t1j{#%;!CM=Rgmx2omPdv@?npT-=aaC}h)lv}$B(hF)n&en_YIu=gk+9O z5m7r3&K^zyrc1M6xD#?rSpJ8z$qoQzw2hR^q2R_@7Kj=$vATJNP8#=-Nqe*rn&;)uP60;s=io`g&cq+O4Tf%suYGQQK35W?$obQjF~f<;3f z1|r-ScAA=R(9$QUiU_DB;0PTNt~$uCE0!q-3;6$ zGUVW4OSx6J*^bM@+-}et=orSl4K#!)$r3FbZ}K3+`6}>))_Jr+tx*$bHt5@h5G-+b zD&cd2yz$yb;r-B}-IB7kmD#8NDeAVU!x1V9!$HmViytIgprT+R3{! zh0pA^N4GSX-;rC0cWZVzx7o~{5y>4=Msr9d!{LsL%e z2f7;J900q}cZ%qjV)O)wsn{#@P-ui9JWWl|Uvn&Pn3k;eH;+G(W|S+@5PBDj@k&UL zs3GhNC%F{ZKMCnbX*c1FD^4lENsMF1!$n=9$`$m5vIfK;Qsm%{c!g1I$=FlmcwwL3 zfG~R|W*81HWP4w6NchR&`}q{QY#5kNIabfyVu@VxG)7e(K1DLspJOc=u;2ayze!(% zeGK9!7ySvqoGK3TbCPsox62eh{v4-gCrSI5Q|S$ZgjaJOxXIGtJB*^~A@Wk_rPg4{ z0;?npiRLesgn|=9Vd-=qAK)@%w1k*ef*px87V43L^fbnV(3+YdVGd8^{{V0m*#;!L zz}gUhXJc|An{q|ndo~7#29;5_5I;vTyKay}FLzEkaCWz{@->&j?h(R!nQunKT(}u;X_VfP2vNPU)I)i~;!lv3 za*Dqv*vNJ)O6Cj$nA0Y4kGp15w93SH>}aV*iDWsq+&{pghQ^^a(An6R|GQ|_ZN%# zikQ|}OrID%gXWH&J2(;IFl7WdTc5~*z2I0(CixAN3ML}vm@N` zh$5yhITL4UkW+oVn*hdwUWhKb8G~`0rP$SxCA29e!2bXsTwWd(-*gvx8C#td;+`x(K!9Ar(x-3MZm;pN&WDrWjd@0U?EMkT9k`QuG)r-LSH3UZO z3#%U!eaIP>3}N|B5mgxZ8w^nm7bXPX0_~L>scdd(L4Yvax}~T5l+@w+DO^4>G-AOy zgT)$>i;;9CW|Nt|!{ktKjkt=<84Amy(Y*V_eDn{f&)y!0C{`W9}!wFeHE0+%5$O?+Y+4eRRsu)m_;PO>!l<51L ztxbvE-%N8WymlCwDtg0Rh@VG4(>a;K!WyndU2*-A53Fo+2_dmBHk8Coe5x-vW7mW?9j%W;(Yl#jdyaDhX}xg74-T7<+gi#|JnsWftsp zwdjA?r9GgH@&5p1hVs}&V)-y(3>v1H7-J&x1oVfa#DnnDThYLrxxp|*rkj5g?8r%N z*5wI=f6MYFW|=z05Nu?g#Pb9u?Cy5c+XleZTnWEvA|vvn*;j-`Z2REIH=x@(-Hhyg2Gb*& zK)5Hd1Hwt}==h1YcC?_ySKu~y7+L!z)dNgCdMssNxJ+q#nQWYiE&@Ch=z32@e3=?i z5?u(C;=}019AWfGad64T%8H1ztY@Z0R=-4%kcGodXS?KmcI;z~avp&+H-V@zGYwIb z8WPY=UBUt&I8>@3tspUk$}g)6AA%_2J(aR(h%LUQ9KKRB_$VdPAZLx zzXB0Tf;kC~K#vO+bT;Min`Jq{I3RB&&~Cb!z|PCOMlyeJMUEe-0e~*$z^d15K&@cl zi$ff9Jf}wT7V}?_zDAhXFh5y>8b6H}9%8Zo0MY~EL*$!o1Y@RwoeD#dfJyv~4T5iV zL|RjASEf(V5ZPBAazrk-_%{YH`#CS!41`|MM9wv~qS5HuSfglmH8s46syE=YwsI$b zPkXVZ!1xWN^JCHzRw0F$vV0;l?_@`__!4X` z{{Xlzng+v|h5rC}CSvZ&8AEFJHvl(FY|%xFrVI(5;%tih6fQz(HGs_UW+w%H3KpboL~SbWIei@GyZs;k!#@ zco&;)$WI5>3>&6w53pu3HPckUhHZh2%ObAZb$KofLQk_8b`AdkfVessx{qnV+(v&O zh1l4^`ru*t22iXe{{UkC4y7QA_CbyMO%uS_-Z3DCu&Z#;IAt5cV@3mLNL3V06WFfA zv)Q1TOZ2%oCX^uyR1KqLuSWXc*p6R;I*9jkVI?fBdN5W+yik!o8#ES@xU7G%9!NAj z!QVG53O8#|l)I|`0E5uuHX@gx*d&?w45EYkQeY)UcG!l&KH{Q>Lp8bnN35w%Na%7U z&vnUmp!7-q0I_dtyY_C(x%4&_u~o?SItDE?HB9Hyg7AX}R)Z z-N{jCKXhe}!Wky{^kW}mp*eK#F-4=y8Ag0XA{{X{@7HZn!5mqV-*Ln%wf!&QXYy@cZc@shDdTnZfc zW{_e-^g@~PzbCX>bfb8nA5|#OgwYE^Xggt>I6O441?`$HL3BI`_L~0w~+DZ;(!0A|@6(aV|^a{ev&Mf8Z(i9Y@>^ zgnXFf%sr642s>|+G7bz;SmccO9g8&?voa)m)O;AzRSMV05pCW08pKJ-*37Jl-3&YF zHi+sSj{+Nn-;hR&`!9Y=cMsD{~I7JR!8((141T z_hd(EZuYe!67d9xLTb7+IpEPZKk1QAWV{!l__M103l#4SQE5rkw;bWUDfd3IM*JEfca%wRV3l_vA#v)1286$ zG#{X`FHPX8EXIoAyUW?5=#E>60^lWq3|m4OgV2VwqYy;0Y!{Q&55X5mGBXF% z$tG9a+#dOx<8MarE!)z1AiNjTYv@;o!nfZi<&VhQf`0;HmT3DAWQ6N~8mB^fDitS| zSrEJFK7`IHkfmx4i5Tpr!8ejpKH-v@nYG!|BNG=fyQ5+^4StM}Ltsn|rdAT|@-?=8 zWrOweNsj;D~>`>isSYv^$5HPtK^A$BIsF({ziYY3bg9WAF@&nv@>eu1$m3 z->PcS&Q3H?*wJXU2rFP;4HA7L@VY^`t5y>8en?&qZ=jm3ZkNmE>Vn}J?PQ_rvzJ0{ z!kafSd>Eplz|V=0c}a+EztC_=J&hQb=?>*4>p~mF*!ASJ_!lhA&R{&@9Y$0-@S8A& zDl=yXC`{!UG(LnNP)H&gCWgSn(it_JTkFJa3tNcmN2Eb&{(@^PEydcC=}!bN8T1&k zQiRXem$~EFxnHy>2O@-4C(h>n zz1N{)Dw~rY&~U+*V&RdVyN@TZppBY65M3M4wI)g39mXYsJ^08m7t!5 zq85wA>RTp6CqE>XC2S|`Vl-#u$VY;Z(J*-!FOj$3eHQplo37o#uALy73~hn7vBFRI zM)OSFaD1?Ii1iOnmK~2s#Aw(sa~xY6Jj3x9{D5odUnC)FzM~lv?0ZBOic*xmOGYj4 zk(6mSD{-WUy}g1);4@*JmX0}BO~1uGI-JQlF7+3|2x2`bRgG#pU-B6$`_(}-`p-e@ z(y3O)J_Blh0@Hc0VtoqwO`Q~_{{R66{s!!1Y(;Vs=r$%&h@yTCeZpz%#;$z{@;!{n zycuxItQT@9w;#0NW^qA=l^>b{#X^ zD1rznk?|aeVLe5mh8WgFO%hB<#wYY&o8v~!Mev?CR6#HB%%wic_!07ZbhwfxiRiUl zvSCIupOFsYoI)S78I`sS4?@cY8?m~72E<3oWlDXdC1MEReaXuRq8qJ~br=~JLs^D3 z3emy)I7~>2H>l7=7>23oT2aRFs7$;$ak_y_pTkxlz{dLZVpu zVG2@b7-WV=cqQ&*ye-=Go8Md+qh^xA-bhlO2B{TBgT1e5K77194ag z3Dn>v5VcPjFz{mic<~|{tmo7ga;0CS3wbbE3dUV<`eJO5aqtxPPXW!v5Ae}(rx=-p zj2lwpYisM#Ma9q-i@j=L-WlIVtE%)Rmbke1TJm+>Od5Y1SU0*RK48pem0F{VFp#q3SJa_Bdq_~ulMvD^m zp1=YxHbBrYC-jV z5A#YeGJbFShVE-?^|1|TuJCtQy<~Ka!6i7;%I+S`2v48(5iPwgOIXi= zYbRMne}cB1@~&b=2TfB>s9@fbTDgSX&z;oag27xwS4V=P+e3_)V<@VkEm56b@2%T6 z_39IWA76!l`IaCTL^>_!KN>SFm^|K9=0m~cBn!MnoZ30prE|I&RlzF3w4xnz6_Pf6 z-TK8_exlgjUr3Yt8QMH7?25l)_tuv=wjtL(<3kGvI=%<3&=KezxJyAhSmIZ#(@6`m zHq{xjRo6n^qq+7ASSm}Clxd?_-@O}G0^_-b`Sas%J?g={u^wH-^*I6u&bI5IF@W&BB5ZqNs_seshYtXc?bfUev zi`R(~7l9Yem&9+SDf9oxG}x$Hgs41Ut8;pi(Te?}h8OzUC|MK~yM!8{SD5l-F;_U##4C&$&M_#u2BUrXeu1o!K+z z7Yfyx$rf~KD84#yzV)~Cv;ehybT9c?85}30pMv$3vS1!FgmQD+woa0{c)0km@%H@C zTFBxBu$vtx+=_YP8<)KzVjK2O?I{If{nLb>V#vEV=pQq|4OB{ABjJO;?ABy_zz?J= z*#>Nj&}uN%*)yA*`Hz4Q+CCDQq=witu)?_4xT7id@GA+CQw8zYV43@mXRUs(Hd@C+ zzTI+;RhjO(p$Wy$8vo{6|C=}vgMy95u^OF(iu~i@j4_QOfg;s?BbvYqPfIh$UA0~q5PO{+PPFNIlxuFqx>t=8`hnWX@3+W0 z@taXGW+G{!iKSnxSje=Zn!N}%EtK%WH|-1_8S@@SUDiWE%5vsGRc+z|a*BgZgmOwD zu1#+n$YW=JZSvOc;tOWv*Q5^0WOjdew@?*N_KMfKdofwj&rU~=*PHTtY=J4P1|7qA zqH&g(9t*aanm45BxW|HQQ0QV+0hXOWm$_~!8{cQ8zl222Y*Z;PuY&2YUR+wrq>{TP z)k&{7QE}~O&uMNKE;w#_+ZbrGFtP=xnZ<^(u?n1Yueg!h&}d|Q5e+RxSKE|FSzFs6 z1Z{~@(`xeG8)bAg#$1giWApofo3jT z618u!K+?1EUw7-Y?397AA6~FmS~!qZuVcmGqRksWlPE4l{M&?f6}-4YB~0gL21c_TSKXL_~xxa zi{|_8r*wbV%Z#k=+iT?8S?H2Q|HPfWTwn2u`*SGG_Oty5IQ$%I6N-aD;1)woBc$fETZe@xPr{0Cw)xShT74>@grwBL<9tuOeuxk>>l0SchdHZ_+1Tsn&*m$ zd=Z}?iB{G^tZmoqqw^^q9#V`*mLHS&CO2uRruraGcMYv2Ykopanj)jCdqVHw%uoJ`R7d32j0bt#(=4Iu$7@-F~@@zi^ zoozUHalENupC|J=vj7q!rXNK`%0vDax8=T<^YF`s%n@k8Q*xv1nbF-V0UY;8xiwHmKhv_p^2`P&d{ z$ku85CgFt%UV>Z8{1){CPO;b)`eKKJblFOqUdBQE%2mQF5O3xsG@CEWhk3zw^vHRR_*AZau;%A6jvJ6uq8D7kWRRU@&xS! zi`8avxFRN>ne9>0c6G=Wx367;(&&BflHQ+nS$-Z>9GH@Zvp0cdJwj#vBYkSTtcYEq zalqCRM$aaDptP#xZ=vkN9);ynZ2$ux(nm3=s1n`E5I1P>oZ{)#?pSH9_G5059coABR{@%%$w zPs3jgc$?%JTTQ#EH$S+qE(f02*oobQhw#m|<$KvH~@E@uytS^P)H2d|q5^=1U zUfv7nE=hImq`&)zW$CtOyuDGkX0aOvs>z}akByW4Ny{%M*xCiNlFRD<2X zZEyjxSc2R9h)8*$AT_ZF#*D+fbzp-SJ%PyOmPzfMTGo4Ne2J z)S~tSRqBPBz^d5Ne*m33W*AOKf1yA)CQ>J-kSfkBWBc6_#ci-4-|ogfm?O=hv$Hzy zh`(8uWKotPPGZ3v#l&W7(IMZmqB=X!{G4~tx=J^PZ8{jiUPW^aWUhWoQJopzTg*nAY4Ge)^P$x&hlO+yWQ;mH!*dmC=WW`FK z&TEzSO!~V2T#rR1xi2g`8&?x5$e*DTbRmaY{2^4jb7DAoVz?jU5^MNaeDe5RHI;Nx z50O6>D2RjeZNUcnV*gNO4GgOALQj%}@>16oDTsg&#; z4FrPGsq>rtw5l=x0Egw#kA$db1E4dplkLtPc#{Lwp%mX^EHK`F)2X{>D*k&XCY~{v z^sV-JXWkJ8Ur3LUoBtz57o!SmAi?~Tvw8Ij*3Iaw)ANeFB*^A|)UWem`TToqM{jCI z73s1NG5N=%p)_YDp87nAYs&pemu&7`_CFf`?l>)W=lh z^CilnKUkwCIV@`!)ItfFj#3L&e$qhg0Bo?jk$hqg>ZDzV6Du@ETa-P1U#LLP%Q2P22A#yPf0L{4lY>kbb) zH{o!IOBjb(O+V+I2IxctjBdM_TEnSY8OWmKrJWA`?U4IL9pFiNk#m}BKU{6-zKTrg zOI?9;b{qdb=VW8ab`KfXcAQ!|t7uNg(h6pKAK@fH4ensy!U*ps*1@%P(PB=karcPj z0S7-lCR=fNYMnReu93YnxAh|_{ggNA33E_Qm|ow4kxVvqib<9||9>UUXA*v`7^}}o z#Z0#69ks*{r37~0KC}@=MOvq@tk*j=6ika=A1SqzqKb0YkLJT*_{2#1TlPX0kWhd;43UrP80)?>RVgK+KJO-7P&3W0M zpEz)&mdr!P53iUjQo)94d}-C<(c1IZVnXrlzUxA7sebMa6%aLvCo+2<(EdoMbk;li zT=hNI)kL|Lktm0~GnzjYRfb0B8_e7hS51+;7-W!Z&enUbaYOW(g}mXbz-qNr2~Oj@ z7R0TFrp-Q}vTA(Ha2w@xB*3arK(aVl`BqskfvOnx6urM3G81e;5bRs2?58kA}%hZsAx%H(2u>Hr(sa>hJ`BqpEBfQ z87MP=k+&Sg_y-iJ5I;D>5`teUtYkRi0h=O1f?+MfOt|U-dKsxco|!%vl(}XqmDM!> zk_^@2K~kyM+V7ZTw3w~?S~js4d;i!Z(?2XzM@~~feFEOTPnLN%v(nnY!a?yu(@_HV z!*?JB5MtDEgAjXng3tMuL?chzTCe>{g){K>(@#vFqXR*!xV<_1#m`)0a;N>k5P2?-is9zA+=b;sOsv8x};IMfxY7h9^t44-cXsPxR(>SD5JJsDXAqUWrl_{ z5-}ge<>2YZ5hjH;aEwC;M#07sA=HObphSj8C8MXYR-Lpc`q<+uL9)TRwwlmg)QhPI zpwDe%JL9vE4Q@v$ll{V3@D9v)UY6{gg0>g*u6~;%reDJsHxc&c#}VuMwsjg2tb)Ou zYw66EEOdPc5hq7t>H-EysAbIS!UYKh;QL{Arkf$Ug;mW18>6D!B~_KB4nPu}`U2}$ zbo9MCC1oJtb}tSSYdmZvHZAvJn`=5KPBf-wczI>oJNtay5O6*M2G_(ZG^LXlL_3Zl zuURR1RR%%9z(JN-C4`-v7={T2nK{EW1@-U=VcX#2m@{d+5pu9zRkjcTz@l9D($9mi z5R;7qL1s=#(My*zf07BKKv^;|I0u@f4g?!YP;Q)GuCa=~1d4YQcz}vQqd9+QZlE_; zSjb}a!!O2WOG;)OalZU`^dhue9{_LJK!!phegV&svH5;7E~RK&;!Dug;#2 z!-Gr<#xA+6wg)+95J~EhiMJmJerpN|SkK(Z|J*BPRQ`sy)jdem$~C0SN07=}ZnsNV zvGL*#)%~lK&su+Bo3%ShX#`E9GD14dTwxL8A=~9Yz#VBQl8G4F-l-FK0hJujkSkG) zuVloNs6qSw0_|5gvG1()K6U_|loVQxb4)V{H83c>;r@*C_L+>#By%LA&J>#YEU+W% zUV6(9;PD-nUBV~2}a>h##1V zHMJn2`M^q<-I+v8pThp(PAWPE&U_uq4nj$yNF%u?-eK|sBVj`;fO(}2>%5&i2o+FY zf7dAo3rxufG1t_`+KXX|dzsGWreB~QL951vPGI*M4B9oce81CQb@hTOa7gg+uh?OV7HhV>wSJJ@1Z2RYFlm=n0>ZsbaPc&4`Zg%D83g-S>o)fU@=JOPQ zlV>-hPl@-iYu_0Oh(w`PR1K;aE2nXh7!}PyNi)eC0& zq%g$LODA)c)Q*((5M5;&Cq zNAOzQYorKWqo502W)`HqL<8x>23!ng?p>E*4Hh{GHKO8x0C`LNEk%EZM}P59G*mQD zfFWsR7j1$eV4MLRTP9Q@48>0y%P&I!qV=YK7HKgO{ZE?pKYiB!eg9|ZMg(CLWKa}< z+4v|#{hv1L`Db~^G~aif2>Bq*C&}xM)6vbmO@v>lsb-+fxl z5N4<_A*ZS;pA;nU}r6D|MJQD zQHQ^{8kKe4G0t=lNxDBO!q$vmm zWHgPX6m@N_EL^u-St&!lMJQ`}Rh2`I^5&{hrXoBKtGle*lubpokoi$Bfkb8`eOo z>E#TYKQ@~`pGQ~kfN!ajdcIkS7t?d%X}{LgtbX))U1_a8``~8<(CZBBkxcST3~Ayg zUGmLAR)oI0;FA1!nt$kdbMcs~x}O(e-SWuuOYY%r{u9_sDimv~7~52j4#j^mq141% z>dyIdJ4X8j}UWryKvr>1vZt*4gdvIgQA+!i64k);?o^@8s>pZ9bz5 zvz}o$ompLwAk43Wkm+q5t=)FbP1hhlin|zaUV%a@NJ2+@j!rvI>_#ede*kp)@4UpV z$(|7x?PTUm*V=azjojN9YH4ZDofF zED&ZiPL%FwF-M6uhf1&5AzsY*Q z+S+?lZl0bUB-?{3JN;~Mizi&~Z{ii&Ce@io(?^fj%hQK^!j^mdGct7bD9=#f{(bHp zev)A8tMobKoZ7=((CvAN(|F1Lz4>`m z-hY@aQQul?r}ttbRzvJ2CCelMng67uFJJv=n!sMYus@db3^E>fX3;O{3L|*3m@U<2*N9Atm9=~u4Tw2xt03V-Zn0`0u1*gfhyxr)F zkuDCL6EmLM$_Sk1jh;JcjogF?ww4NiF1`C5+%WTr6dBa$fwajq@_zDc? ze;%n4;$-(y?0$LuQ_e=cHp}yA!AYy;y^~lABz@armAAeAZN;beHTg$Ab(PU$-WJlg zea0u8o@-O*d6s2D^ZosMgba*fZ6OSFr%A`EUd+=&w9KF)Evi+)Wv>RLwgtKk!sizq zxw;cIv|}{`CAQ|pbbDwx&2K~U0_|q(&ZEUHEWQ+6}x3)I)~R z&{N9;&*ND_X}l;O=!7PjBscXw|NpfgfoC^S1)rh+Yd`<9^o6XwAyS;Xc=$&{PwBKO z59*zl@J3185wO#wWt8irfP&^r?s(Lxy>=cTe-IAkAbDncWI|di>i1Z41_BiZeq)!h zUE(z7Q}U^APh0kmfIcu(hV|@6xlX(6rxAOK@1fR@?EI5n2a`v4_`3QTS*5c9g)m&auhqty;CZdL8TuZ_AuqJg)g^$& z;}4-^(6rR9?dsNLB%XX``lmmu3cn}WHOa;Ltqz+lJUZ64VlJk6{-N9PSmkUCSfF-3 z^(H+Y8#!F_AQh}p+zbx^S97n`eD>P9TpJHIB0*K~?$eV1~R7$?)kx?$^D?0beLVhVD z&>?=_>)t;&t$*-p!*T9!-mEvEU(aAK8;Y?mZqlXhe~Cx8FZBACI!|eZf_syA>macI zq*g;BY<_nG5QLbL63%6IbdStAiD@Enn9FW`Z1z`{O|f2bH0y_){1ZonWM)~@yiK@S zA204zDxu_>ZU#23QvTYr;va7f_}cGS>UGaenLx?!@6oRnEYrgOrjl!tmeG5SR(PI4 zwz3q8?l3xDV)u;oqzADHVa@Mjn$wyh zEErexbXW7c&%m8ub{|w2UcGPBiz>UDkm~bLsjEf(+0N7Brp?J@>zYr9ycXWW;onj2 zSARX>s2!4?*1OTanfAE$k$sLRz?}uj(BrMLcRggEXcu>!^t`!s_69@Eymo5e`(u|d z*lPUwwAUpXh4s@yCc!5nopQYpZ-RKHgSKG5UQ`0uuD)C1(f4a@8i9(o3!>Xo^OuS~ zqqp#DE(T=ovXs8I0Jl!x+pmu84BxCWKpup9$b)Wc zRy4TpX64_YjlDV!V!zYEs=XAAZ>zYH><>7&vtV9yeaFJKj6FZtcy za$<$tG}ynhaAphX%7jbs6uz&;S4I|cWC}Sy{0ESr`%FS0&YpdX2)#!gDkkdAf4UF4 z5GI*Bz_<4P4*>f1QJs*dOz}SedB4GnsgJ`eoKoYFW~U=^bb8_9zv?a~R79I=WiSM+ z+qw_ALUE>|U>Uz4?6A7cur9V6q7;LHQMSv$RSpyHRzlxyKy)fW8*!J85I8 z{Emchxiyu9lD*Jhe$pX7-S%Vx8@6T%KaxAY%_X7Cvl?;~*hU6Q17f!CBgP}{1(z`0 zZQO>QSfiIun%|U!j5m3!$j6?pMt%J{b!rbJNqK(q02B7jC-2{U{1*vzz9-t(!Q89a*&=`vCl&C2wv->D}%ZGKF}`sA(Xz_3tQ-fLSWyS!y$ zTNyeI5lOx_#}v)>e>MXSdz-)Qu&46y+&SLB)|Nrqd8fp@i;zJwZ;t%GWHq+Sg!{H`o3Wn~D_J=U1^};N$t3U5~OE z>(IzU?d$R!gdSZe$8iAP6UXL4SlBl6)z-Mgrub<(fv;0Oa(la+HS%KqSbYd}@%X>E z1P;hE+V=l_|F69~8i458(fYXs`9v}N&!=KhU>kM*f4v9|@Z5`}1v)rqhRWu`w-tb0 zS@|HQ`|eOILR#M}`r!(8a(>9fZ+@V|d&x9p9fT2`(o{L-GEh<@3qBQroq%}&84)Lh zAZ=#St0|}NCS+~TB|#Dict<#eH5Jvb#iLtCeGnD3Y^`c5DSu(ncY^VjAVxcuOoi!u zibziuO@nGW#-u`PHH}pRLpo-hq~d&R{R9xidL)|WSsk(?)BJir7Q(?W8X>UT<3c;+ z?j0{-G7B-soI+t$3WKK`(FzRiLb=&A3NHVGtc7W`(!s1y(H0?{H3SU6;{IPguGevnF&?#tFbQ6qwjP6vJm}cL(lj*%^J(g>$NaT9fU6OroQNCYU5mk&y=kq5 ze3*vqV{T){`o!GQGLK8`8K8!(u;ioEC$g)_1>j)sb1Ee|sjjA%LU{t+gk-^RK752y zgPp#AzCAvXr@b;cNPrdzwQ!IiGEqturUy$jrm0z>Ye=%>w~O*1Tx06aRU{+l7I@;% zMUHuLr`kNpcdUq5G7Zph3~Iu)cJ68`8w3(xB|xnS&}ob`Ex}#y1_l%KFPAlZhG4}K zxJuImgycMd81B?@1ZZa56XJlip3 zN_;$>QKL+E$`nJ)61)5?_9dwTDVbHQtP=J}Vi1iHtByYs*P8B*fItv9bh)mC=wQWr zdZ>5@52CG3=KbP%p|#+YDndy5(ZHaX0zp1g0oO1EV|){`0v}4h;axO%EFn08oDe@W zzrz4y;zI4fB_%d#0*eh(g~sg*mkdikgRPTb40wm(Q{s95U{$MW!J^XMIiHeffC^tU z4Gf@|pV$Y?zIG$1W54%2xlNC9>5=MGw zWzgM6F{aw@#1OX~^L&i6nPGZD*BF>U>0H#CP!iHq4Iy=5#92bkHrPao!zfAZd_EbL zGnUWX#SUYko)G9cW!{611*iWa)N9t>5A1|Ck}mm#=}1GNZ=DsJV(H0>;OqJv3m+O% zh~Ewg+DiTp@h1c7C^>3l_X+D-JO?1UQAcV@C-D^V_^dV{XcZ9+5_VojhG^p3{cZqzAmY}-3fESZ5-b{Lz~ zskp$M;y^OZk7jDw8m`OSh4h6?*T~pS{lK#jq6V@bA5<~W z#(WN&lC`tCk)P48>m+0!)}j9t59N7-yOerIZRg3MZ*cwWb4F^9LLX@o z3_2n%(D0A(=J%aw1lAE(!oeET@e&%dKv--N-pLSVtdVamu-kWkXt$20Wq%~f4@!X! zVnJH(s}q+;TAH>Kp!4nMFp}COrRWN_52Ed(Y-!HITE=d^e)SS8}b2P~GP8u%~ytn4cQqRm-N07dR3r z6((nto0xq{2j%tgO=ZyaU`G10->0}LHFBtIc_NA z-KKo#C!T`z)+)U~>-*U{C^hx|Z=@mY<;IlGfn&Y>IW#^Ixy-pk;{GGII z8jhE+6vFglD7l!n+`Oq81V(IqbbIJ%c zP7s=yNejY;KI$hlQ&_09c-9->8Iaj;=p5t*k0Y&#y0I-k2HwCZ3ox|Ld7bTwtIb}ArV6W#HQC>uV+2gf zT6dZ0O4%^=^5!?J!t3;{QqG}DgGpUAf5H^;JfF`0nuLwHrSN+{xf)N}Q-bw>&kWoC z%5?mTXy$b{z9c8wc^OqP0>J$9A){~MMv`z{B8@iZf>iL6jy zsL;LoZfvSR5*A+SG9S$LHA9RJ56`k+jmNk{8@!=T$(vE}lT_iTD-Ep=Av=>18!fJ# z*e01{Y@;R^+#vALf#b|{anhHcYi6iZ+ZI(XWJ%!TTwP(KCIHl`#(qD^p@7Bk9s6J^ z*7?&*7cSy@hf%y|be*Ns5oAI5G_#eent}T{s7sHFOYnH}dE%2gj7#iH6Ugwi?w7cw zvd8F2{duo5Rp@xQ;9YraJzrzN?D}wQ{IRrsGn_FxQd-;h^T(Nb96pn3$VVRgp>R1H z_x5m$TRcFL_F!nla$V?;&TosDW^{HaR<&C%a*Dq=A#aIcA0tA!QG^cw(pd@cXx#|m zb{?EdMg|NOu%3P}$YKTy4fQJvruL}=9iX|5GdY`y7)OX27RGe%uR(cVwud2PgS)YH z#9fj`Y+|D|Lm2NE7!9`1#R>F1KG{SYcQ%>czS-Z?_#^lj)yedYc}*ye(#&y@%YWv^ z(m4EIB5tuhP%ed0Gkn50pRh3I+tfPUWB(>?f)q|0(NFDK&zVD29}j5+YA z-kfZ^^=gf`ACrRAr)NadixdiFU^tID;}CPN|2#SBZ03q4&vc zfHy3SyW=7N9CeM)5m;i63hQ<-$UFnJVu(ah`QzQ{xOkIrF9o-*nsynsZtyvXmDRG+vV(2XkXB34}pwPh5{{ma0OTj&R$0r;Uut%NwMjsTKL|40FbdAr5-qk51;_ z;ywk663kHtw>u1sGEeyRZ<+`lOU5>Z?$OV@e7Z53AF-My52LK!Y`*2s<8L?q{A<1I zX9p$!!^rt_v#S5!edJjX^${f_P`A4vDx3to^itpsz>U4zjUYer6d<0nP6kuTo&ZNd$r=PMw|fRIPxPGDSYdZ zbL5BnJJG7{v1!##?Y_<%(jUEKH@hfSS}kBek%M zJ16(m)-6?2>M-sP-?C*1%x}L#F7e`*k%k}7k?`Ah%7-ac!>ecGkK@&w-T5nCdkMCp z#q?9-skwi+)%(sMXvxPW;@B56T`iNoyR<40eza`~TZCYsgwO!7(dV(ftTvlC#st1V z7Mn##>L$|qgK+WYHuVAKR8v{CI^=HxAHR3(m{yYh4sn%wBz$lvS*+q!0H-Ol_j&UWH!Il~id z>I2(iK0R@~=gWcSL{A;cA;f%zR$9$QD$hsspN{I6ZkrKW^D|7(R-X87^3ErL2+f}N zRh=BkQD@lbg))EKbp8m`vCj>(1@X^>4xnu#Q8N5x z_mY|h-OqKS2DMfdDEM2M+Yi+)`UmGKs|j=Q!GX0=KELWmv9Wr`CL}=zXCyVnHBCdz zh|d{Sc}e}MZ_ysWWR$d)3Iwcb((g&$SrQL|q2 zhBBYJ^+W8e^+u|^rTxhV0Gs>p4}}k5+U6{6U$D!SxFvtjG)#Vz#K$>Vhv@g#aQ6D| zN7iE%X*n*8d`LXgbcQ8Q6Qy{UGC^Q zVL@^%ar9fX|>(? zxUjJ3!j`=6nAF;B9S}N2?NGh?^SK5}S`>6J9ZXwX4t88B5QV{)Sd>n*TA^jhXQBuR zQ&aJVf;mU7>`J&>fB*68K)a2k%8+~?5N=For^-9+)I+H%bX?OD)#-1DFG4TBRKl>i z<7$*;wF-Ei6Ht*Y{i%dPU9~?I&SJVqQhr*=G;cewpmPGwNxaAN+t_s(;$IrI5TO2m zo9rE!Y^O7@WK7VvVi~zu>S4klWuY3C=VjB#Vp2&0s&w}^L*OOqOflFc{}`{Vi6J`` z3b7_TRLoP)F=W^L_66xITKFkfDk&5;ZI?j%%FQR{UTkcYxv}DVAF`n2agT9k{ZBpD zn3Iw1@g<*5`v0=2#gTi+%tUID54|N=J7X1~y3BYi^nWMw_Oxr8Iw=gl=yAr%UiL}N zlsX!k_ezUWs6Vz74CMRtivs##%sIeZ_CR8VK5CYtapkn_l_s#`Hct51#j$hoW2l`4 z9BEO(tN8I%bsT6}K;Dit3{GSUkm@0XX;hFBH55eD{Sm`aHc1St;%)AEH8ASSu0*6w zPi{vN&GpIHIhB_iL^8XGLk6g%JA28w3*UASK&KdkEPpd1{j^J+q?wjjJpQBJTQhpH zfHGFCzMKe$wvvR8=QT-W6sVEv0rQRTHW2*06@3h<)S$NQ+QYm2K4B_XIMkW<@qODJ zihK~(aH?%3e#LEHW6{rIE7c3SgXW&7;+eX=k9s(YZw?Ag#yzGu)qjcNixIXjkG$xA z2EU5D0klv!0En_0J#tW5MHs)g5?1B#=ys0(3HT;1o~HkE+Cw~|Zc%}Ah9DlAU++v$ zTvda9RO^Ipbpv%d>kWC||_oBrf-$;DX7#|1Bu-K5u05&EI%2i|2X3u>qn- z+mvxpfYCbShgOQ~d>TO71y`JFJ!Z#@$r+@mlKHVbt!sM;D=g@M33+W540=UW(x2lY z$3pH?wSWbia9AeV8o&JgrxWVg+4MM^b@W~(cxp}k;AK=}@5q5= zGMRaq4h`!c6#~+)U0-hNwRE-W6m6OK3OKjxW8(A{z#`x7L&pb5jmp<+4-CI~ z=Hqc{fKMprO-gm$I6~{D+>5|-?--&v1l&>COG&3gOm`Rl z0}w(??d}_S#yECpe0S2!lZTspV2qUcJkM|f9O@9I>2n|;be|+_Hm;vSo5+9-8aW-| z{RN6b)}eK^>sE1}Qd`tzazAJo2-T=%tUEqN_9G5TG37O^vkyPaH&#n}D8JUPKU|>u z;^0~)0pI9J{htwdz9JTUU|DX#aj5JhYRcc@*!Jpoor-X==8<$f+-LIJj0R5aA3MTr zUanxH3;yyQ%qmrK-@O6Y@;j!^@`~ULovIIVRtNIPF&8)ng!DHh?=7|g>AR6eJrc1X zIPF>@8$nd0;sWpv*mPk5rX=*fZ4~-D5op)+Oq5mhP*F~q?sOktV?wUp44B63>snt z7r!2uk{eC!#j=j22#(iP79PkMG}6by2WM1N;xYs44XV`3D0MhYpx@lq$reMzg$uYk zM7h1Q6~?j9I;c68tTPn<#1Y^XlDc{QRclt3BcrG^C>`)2W+$<+XffDu*8d09Hws-T zFgDi%{#^%odbRch5G^w1>->7901fxB?1IDj&h@TOpI_GTdfSm(Vc7LD>75GLSGT$|4Y8=p2*?{&uJ@73?~qumgl&BJPSfRBtZ;coR_XzhDPTn=Y( zE99eJiF{fI^m_#S{)`&oeJ5xi_%!~>rZjbWdQvSkq`m}#<|dJ@RCxRIWo89FzHRw2 zvHKgzxNsLMXYf1Jjgeoe!@4g3?~yRSUb^|Ht2Ss`rNa?iJe zLp_px*xl4!*NI^GJu^zir?cdpGEN=}o46WY7jd+l)1`i#Q6KF(H>h%CC&Wm1$o$9= zHTCMlN>`ak-1-OMFV;V&qbjCyRo>tw$6%vgh~YD;Ve2qxVg0JIWa*M9h@>`!W5D&E z*za=bf!dw~lK88Xit)BWHs5H<`-|i_tG})lPd#ey9 zoy63nVM4sFk?Orm1kck^hu9$88o8yII}JerS24lKZ=~}hV5P;b8OukDPGU~=`8Bs$ zX_Z=Ie^H1O-U*K1MA_oC#)OK z?s%0I&ZcFVE0iLJlL0=1VhbZC-*;A&DYsljT~FgwXXQNN>pgv^gQFl6l_B@fp|tF;5u|NHIE2UeTQVz2KEA06CRO>0y{G3)^T+W zRs6(uT}!3{e8^pl+L0`l7{}g~+fyz$p}NhAg!9p%%LxYU<=N(KxdGE=6wxOYP}v_nC9E^~Nw%8Uaj9@x`DBRLfZA=S zQ(>u1U-Q4j2o}9WTQ?_rd6#%N0(d!#*K!9%FisG_{7n?pMHOzk?EC&L&s@^hW5E9d z7*tTno-_>|84jJ3zrQ&aK$toS#x*40A2TvsB%I7Rsw{Z9u@+mOS%f$veR2-C^*I|o zo40ex21m5M)_=DZWkY^g^nooc!d%TxPS1%CVJfK**;e4A^{vqxSo-T=(^&)~0V}7M zT&b#?U$aL?$IOGasx^qNHz_r}D8*Peh2At0YJ$nHCi}#|+j32G3;31{R$TDjp@h%@ z7wvpg^I@JNDM{T?)@zLK7@}R!SUG<|bQPPdbtyD{t_LJFQx>}W1*x|mOUt9m^w{yI z<6O|A_*F2`)^a8I^0TRfE4lY5rMXCKI2k`;ppV4CM#)`7F%E2Jp)zATMk#X8Qf(YN z+~cYTw$9R>j(Hk!CA^37nGtIz+%9`|qn>!M)_`_mGL{v`)dC^mlF*ZdkbJk!Ul+X{ zI`+1RST$g3suDNe=WByR)(ySDV;icfH0oTLkN*K;3pVQ~{-$@gah1AG!rqZdhtvnK zyM)Bq&c08heH=&dMCpoez7f2_)56o=m`TjZ+kzk_6?zDt!*D+R*m933@M>t!dcra7 z!2{4?TS2felS8Yk9+c8>1XGI1OZ-5djDP~8p8# zf3zFp$ZOctwMDKNKJ3TpU$9gl-;+npM<9r1XP*YX;pF(A@h&n=%fy;yGkJGWm@5sl z;{fT6h|+E047HvHlXUL{&by77d}bm!i-^{tbs2e^-r-8)!^KCJhl2@RZ74p>#*q49 z`L#}=!5X4bF5o1{km3H|!d$OGP+^wRmD2hj_X7|A08>*|fe;2X?BzyV=x0iK$wgF%|7xB_?E6fC@s_yL%C&LnyjDWZBQ}H#h zR1^mFTB}NlE3BrLGtihKrBI;sCtjkpK-PjO6+0UFEX-X+tkw@>yO^!ke1yo7X61Mq z&8(pB1NDtoGnVNwj!jbTFflG^4rR*+h)rIlZ~Q43lDX<_ zGIBq%7%GjW!44%c4NNr;iXW5p9JrZ3`+>wT$bO=eAOYJB zP)X>NI-+mk^&90H(Ow|BMySzuw=n1?nX)g8ok|+otWCqGRbr2-^0eH>Gi7M@Od+(1al(nL*ScRcBI|F{7u)XVgR*d8czV z!=f|5VafOZ08-=7ZprrmT01yDvJd|Ny1pgSSh$srA;~c=LQUD zP_YU%1H-84kCs9^Yn zc02bPoP~j?(@SjuxMk!lt|no5Sigo`l)3YMrH%Xx`FWV8BjLk54jzw~Lx>e+3^f{} zC`+WPzY7Ppg5K}cH9YlBe$2qag#gS%MNDm~N=V?MfU0T`cE1wNR|E#RF9p9FjOJe5 zy}^B0oeB&eGVgt3%ixAD9k%*>cg134gpa;b{{Ry5Id{L*aSDe})n9Bg$=I5jrpHj&UDDk* zMbaGB8RWVA@}+3*Us}Dz?XcDndYszi8HV-(%W&!)G}JRo+S?553w$x3;h7tjqbJBs z)OFzaeqz!VMC|URUe+LNZ^TV@CQ#`eKXRhgTD$Xzc3CsOQo_pyTzSMQN(cb^9-x7n z{l*D_irOB888;OyHP#+Uk&s?z-^}V5u+zs{Lu6Vzn|oKedBlOxv3c1==Uafe>v00W zV1>R9?h0gI=zF@@@;?Q|C-f*>>O%h_@R? zjlfAOLzD{_^|QHsVQ{Ewk20xZgTP@9UP^=!QlW=1rDDu-_^C%85|b#d^72QmyX?d? zmWIQ38~1q2_=~PT`w>$aK3!j^R9VHjCODXOAh=;x*LjQ$r-(j?rBlCixOS+I&!|RX z?YL)u%p<4Mq&&#yMTs?)n9K3FuGREg-PB-Y4C(z$ANY}a^GUDU0^XNn+PT$YUsT+B z4uR_|B`G}D?7dE^F$mmgd`6_uFqTy5#k&} znN>A}WlJ1FG&|ferv@xEaXmA+mQqJj=Yqmwl(884Ld0DAMr}%wD(G@LGXXB)5G1}H zDg+5pn3Zrb2APUxjRp@Yfm*psv1HZW(h#z^3cIVZ89f0uVY`bjJ5RWY6|?jZ7MgVn z(4|K`@e1rO%t8DGd!3LCm#{eckc1T2cX9DNxw7518IGQNksW8xeYTMn;0IOZk*jy;(8q z8kzGmS|8Tq8izSXtIu%+-oxB4Y1$j8#!u>~^+BOwRh5eGHMNWRWxH3_t%m7g;%uK$ z-4Sf^VIf|vEj+ywOOV}m&l%jE4YHDGQ3~UAQ#0CmiZf1)9;I@PsW4F2Sy@m}cMUOv z`;Dea+$zkIu8&lgX;rnA4pSC^SXiE;&--v7LBw3~ZH~y}cM*byj0c_^Y%rRp2MyfX zR->0iRACD$uf)fpbi285_Y4xyp2h_;D0< zumkdVjx0o~R)6XYxSC?2^~szq`nZ`xaKhKj#+pK}QM+2`(zDyTndk-9ay}v5A%W!- z{t&pzf!ch_RwYWu)8=8w(f^Z$E%b`g)NNoohO8HVKQf~rP+a^)MJ21R8&qHOEp-i5sB~e!;x7mm-5=B_JDV61 z3j(9sHkhNiK{iKnR7_Vfa0rKZhs|xcF>nmlR$e%>4v>K9ieDlmTSriknu#FJU}JLb zG6(PsxpS#g!xr-_CEr{XLS7=?9ZM`Fh5_)BipXNvsu8;eKJ7JB5zn#8!W9c;sM$qK zM40$K^D}y`nODlWU}h?d6FzT%h(!uP!Z9f|1s#!u2$afSsX?^|Oisx0m#vJb29e^< z{{V@ITL=(re^_8(LisJ~Wj3qoR2~IX$#}g&J`Iond}1Vy?qhR(y4&V! z;XpQp*u(K?1gd|`D>&aN{-FU52fuSZBFBPs>H=T0co;$uQ9CM@@2>HKhWvQdu6?!EI zr9pRMIFu!$qX1wkTP%aY`6~f{MGAtTlK6;a%9|P72Af-ZYA)1O!cvBlQ7?z}aW8;b zKN0WT!xHB-WqpsX0oHlT?kF_@9t%dgf*?B;EKEv7)S=UMV;*x2^+|jzarBO$oly9v z66+ZpuaQ#K2Tx)?rE>-~Z^}&^G@^y>$NI#m6EnEK-|jLL8nDEmS1E<;()}Kqbs=2{y(WBke zYG3}lxQhe~c`P$p-s+4btW@&$vg0EUny=~-;O&TP2EkcA+sK?%-RHyKz4qnj)%n2~>S#EQs&!B8!5%YtyRRfJyA zG9wH}rGoFEmi8`ZWAPPw5BHWM0<;ehZ6FhaVu#3P4UjQiNA3-TnLt!B96*SakPJc^ zOM<43LsmLHk@VW?7|}_G$`u8}6=v5khyk0vIO#tPSpMQe z$uaWg0GxquulE&-k70TtD$-rLJ;TcN#0nPsbVo{`A&%xYE}FDbebl+>-HY1DV8a*` ztBpLF$+%KRVhm*IYT>g0s&I=48t5_PgUNv^$!4HF?qA)y01MGsW0@tl;xiXf>vOis z#$eQ3H5iw(EUUOtyBHX1%6JXoTA>RavE+wS2{o14QsSU@jM!>Lj3@G{TTadbxo zfeA7|wH^=4*B1dW@R;FgF2gFw2r`!`i~(938(t$&tJenV>n$TVM{WtvXHkJS_nD0INfqDY$Z4cgk zQR{!SU3|(gjjAVfVqVdUtrz3C*&Hf}KQO(#{{V9tL!s=96xl|qgo$V^k=vv78<`DJ zZ0i8}*Y6X?Mr_;cqA}@|4$<6ULoiU<*F_kHL)(MJ*YN>rfGXmeP2DBXNVgbWQP?+~L|JVKZJz+R&#sL)Z?4IR+pUuBw5r_8%j@I1XoKInr8>LF2% zG0WJ$DDho+fIK7LQ8E~^%%d{~nA@3Mo!u7T4vPHy!A~J-aVK8b!JLBX`4VhKSg; zS1w{KWyDrWvDgp8sRUpmkGy09GZ!BeN!)hK0y-sUFe4J}?oq`dq!oanI=MV_8ZBIp z;9e3{{K2j6E@ki(eL_Cb{S-he++T(giQhM)M7ZvD9N8Ar;YU zsQV(U0@#sWP>Jer7WkC9qaLpi;-{9ojCz~LQIt&kiMd3`j666VA-W?(fR({v%m|Xu zW@`!)uHeZRnVf<{92j-M5iaF45uWzi*e=-Ai^#XFku>%QtuL{pa;I0S-w(cpA4yFj+4SY zcEVt#nlaT(Y#hj1P-*@fnOL%`=4PT({{Vk5EQdF>f3gF0Y=d{7h+=>^ay+M#`k20x zVUL-CFXq7(H!52iNxyFHH>;iEr5;*Oj8UwUl%dn;VTg?AWKNG z{MNljX=~dIAvY2d^DbW$GiV70Up>L*kRiqLk;H3>l)6+F5J3Yi5NlaoQQOwLOx|yq{WKjOczN4tK zVZ%OR5GLS;yj4uripGC_Ar^pVOrIoulUnutnTe%)E%$>2YgpdE&CQd*gq2W%$q3C- zh?olj8gM1B@EDlkh|$K}8Ivot)xiDg4y3B$*H3Cj%6>NiXc+rKaY5_R|q!xe)q zg9*4okar^9kx12iiTQY8Gwovr^@GhQNH|F|9OO2$^d!OJ);k z87XhkJJBY8K|{-T4YW$wD`2v}w@Sh81D%os1og)y!yE+?Jw)>x9KuB}G3q==G0oxx zAi@PGaE1_-!b*WERIVyAxm>}6Ea1A7KQmI}D-em_CR+rGO(yV_Di_4W8FKllQT!Mv z*h~GT(dH>n5FJA(K?4R435ZK!eId1rhc`6doh+b_f*g~9wwBP=-M=HjN{p)&ef{lJbrC4*~!!W!;x#9#_-2e{1FgRT&;(F~%5I zFnEpxVjxn)zY{iA?I|ezkYT!>j^KwRL+TH`mj0j*s#W@jn!4ExoOJeoxLH&iUkJfV z0dR^U*6sG2AuJ>UE&&aTzK&@j-`*jibv% zVF3c5xDheH@UmPOFn=~`R;Dt{VpuY~?F+bRG(yf}kGSGfr{+3tcxP0@8irAf7&7Lu zs_OcU!R>F`P63jND6pkRU)ij5f$0dcwWkxo@&s6dm*hSw9*Fcuk{6m-GTK0jZ(fK zp|QWXhbSC4C&?;Qc~dWj$u5zYi9#L=%oiCV5hfT^5hKK5NWw#*nP72TxI^+a_@hod z5`kS!PNJ~^N0%9u1dLcbBthh&5H-zFgDZgo68``Ob;h!KP5?pYFT9wkx82GiA`V1+jMSIGy!Gh!K zKh#56vUP2;{nVxRU-cQRHA3ej(0F6Qp!ox|Ts|jJ6xjzFb3HyGI|0-sl)tteNLmzo z_i@k&HIHS>kc}O|)l9u0QA@O=sYM73p$PnJt=QQMgmi*DS(r>HZY+FMBHr)ZRt{z2 z2sxEs3at?_8^Rz27`S-d4x$K1w7x|RDM#!;{))FG!=I^f>U?MlAAy9N(0<{p#Aho4x9|PI>=i!8%27K9MZC;&29a7Iv}`xL zOuPW8*Q_zB-A#N!RuG|q9WQ?q0V+64fijt0pA` zplUhBbCrZBB(g2dnU~F`GLAaKi-~d32_AU0Mi2>tUjc)`_%Y?&sd*KF2ETDwAq^5N z2S|Egg&k^v@>f#7g=M8YoA`@V3%7{uRL)yCv?16)=YYc}2Mi4GyOjxCj&2R{P)hd< zu_ttDqUbG8a;!kJf?Zhr%k~we{^Qj4C1dOT!}uAgYtY94n>?(HUK%W8YVXV+q%L{o z-laknLhcrzGYvEj=yOr9jVDq?QxE~>MFpP%}>m{z9BFGg)ELfM>oYpl`0R8 zP_mCuMM@@Vq>0BCmIVt1_XGgDc6wQ`?4B$E`jjv;fJk?UDt|H1twIbUxQB~^xHyEU z+#y_qGgwN2_!w%=qJQtV4g_#;9F$8qwbQyTXxrki7F{^NT& zrb)dMJ3i)~gn|r%7PfrHW+bolAY2rAvWr*o0iKjiDSctoB*K#NTsKcRWjsQLWJFjs z0!Ike#&D`<7?4E7b#djujG3f(jV%iHk+sgDmhCK;eiKsv01M)!OZyBzJxYs3{{Xv~ zZ3;&sea6=|VN~Nj$Veq|s8$By05XMOvoRF9ZH;XJ{njxm@3;q9Ln>u8Hlk1<7;qRc zX5+yF1`i1cS$sS}vV0z*pOvXq(CFwcF&Pjb-VV1I*6TFIpcQ~WiE8A-aCha@Ff=;? z237eXyr+K3ooK8WBU?n$qV&_|VTi(wufz#0(#l^<*5xAl>GAWaT#I{19F-34&`|=4 zM;dR$KqE=51r0FU(BAz*=H!DqCYALcwRoHbtBVz=B?Uzw-^>i?t(o*x4F3R1x-Xax z=u&jEQ*x=cgX$0rYHI?7>q%SA1=?k zjcS7TU9l5Vbx_+2aRRbcQz+Rs;-?}_gZhMI?`jTf?g`0EOs-`Q?t1dzzlIzB<7OCf$aw%vDmC;x`;Js0pD(=N|*q$+=JqD}D`B3dV#ocvAlWnCgy~ z6?*}tUL}uVy0N@IeB3@)yZ|)QT~sf0FOuYG$lb;-X=RQ}R_M8QO+>W28o_P4xE<@8 zet-`>ub3HpkAH~F8}m_D;uOudM1D=j7jTPK)~q4!26|)EIw@m!8M|(~eZ?&<<7Mr-pvh>F&J!&}dd-;QrWggpV8)RsE_(=N$Iw2ejQg3(5eFlhO~HD~QycM|M-sOf60||VALc3&t~{<{j!CD31>6@1Z-ZDBHnFuK z#OhyN#u@b}F-Ha*1DlJ==4ZjOsc*v%L?NM0q1w>4(k?D0420rrf^~(`#(A5n6mTQnyNh9j$%Cq5>c?LfdTZ)Uq-FvNd}6jz?JT3moIn8@a8G zd`{2YYF{_OFB1kXUfCL7DLmrv80G^Aa!6_ma^+`^2|v6pse@oIh;|jmV+Pdj;nsy` zmFB*q!9wkAJxT{o>-Z)cDf){R1^Z`b#8z(nd6-q8+2!)VR&=P<@SUD!&=SH?${aTq zABKyC0C*0d6}4?J+kHyoO}_C(eM}}csdZ9^+p-)2-sO8jwhH9mxQBdIma|3W`6VR? z(O%#26Mh>^67LPh>dkjkk;n~_qk6S~&{z4Hqp)KmX%?y~ua9358(~HI2ahtQ*0Xa) z%c3zfe>#kW5HQ}g4|)_fT*1J?q6#8e+{-jAKxXWMKv_aP$m&1((*?{~9`OnyT8=@d zD;zL^YzmsiY%oc20dk?2%db+6tV`8FA*Tlh9BNc}UTzS@D7QuX)T=OZ8x9!SgiQ;x z{L8+DyTd49m8x`-&1RZi{{TMaC}dWD)Y(rDwh4#v1mSaZ)s>j8fD^N(sl}L$-NIUJ z!d>Ifa0ZL5(L1!4V{Mj4!%g^yT70W3hdZw74~|NeEiC23oJyc|O=yLcg3c9$s20MX z5Zi$30foDld`k{D0f5U781Nvn);Tj2aj3H~RU=jCE&&?=0ttg3ET;#=TR?kepw=Tx z8E77pFzKb&uj((cMqttE{{T>Fa<;3uBGXY;pDe+nHSCXF2goK9)Lo)heq7lBYIQrO zY^i7@9iglEl?{V9)mL!xpf1*7eJ}ME9DKX}rY7%;_WVUync^<5#H;|&`p$y1OH+p- zz4!4e%!k6~*Kqqmme$$i{{RrGosj2U%TA`vKXA=jXIGp{oknyA%w+X~$a8c?BtfuR zX&o>kl^B9k1V)Mwt-$mWGSBczEcTyI z@|BzB+YMHbm?qWV&-X5UK4QutzrRpNN()uvKBMpjQ+170t-!SCvNGt+`Y#6)voqiTj_$Ph_~dVHAi}fT8b>>Ij=B+fK|7|aaVpLU#Wa0 z5QD_=245t(p2z_hUD=ze5eNfbqW(VOg4&rc%Ru6)wVGg%0v)8#nJpW%{Z7Tx6*0Ih z11F}>Fsxa2^uc&lQwFO~EwH~IAL0Z>>jLEXE;u~_dDQO~Ro1f&`PQt?Js##!k7$iO ztq+*BplL7z09S}2U8EPD{SO4hxU-Y9;sd;7M8mec!!JXGt)YDULlE~R2HoMJIVhED zTvf2f08+Nue_5T%%AQNM>TlW_m@WuU1x7|AJe-$lIv>0y)@u+X90hmw}I}z%5Q*dq(52CRc zkayvX-&ZjhzJttbSgc=jIdRz(O5B3?>OYhQ3dvl|hej%s{K_ex9ba_}#Fyw+zi;%8 zgB6jQh;m+pqpCg&`}H5V4sF+rWtb#1fXmlN+3=f%Sht^58vqp}5Oja?TG?5%MooBv zgaWf~{C&z<%7cFZhf=hT6GFo;bQM#tGWTvm`SUKVwMt!0xCDZwa0}ccTg=MJR?My6 zbjnv!MQvr=%)ZEFTYHH42>D{~H%r`7wliFV%K(5(@fHfORj;R6n8 zRH;!3s7s=C1&&9E!fxJQT}BWL{{Z3}xMKZifHHh0kQqEWNQ)0v{{Yme2Pj)5>vgMz zkcTHh>luh*!$CZ^Iyw#p=~(uGwmcahbq&r}byv86w72lipvz+i>}puMk44J@Y}Z-y z29HW^`t|iJ8h9PwcIo%~m9`8o^qnVj2+Z|?F_&xStqF0#;l`u2!&I@*)xE^+ppgEc zL`cq~Sr;l8lOloaC-_g)shQDY1w_EQd_o&0s{GdNh9f#>yu~nsP3-#h9}Jw&8kJL6 zLF_88+{YCP(erayt}0w(+uSxJ zlm?x>uyY2Om18A4u9zt?a*_EwOSq_TgHCaNC7_D1(6iE$G33;%rL<}tW7Hemz`GN? z#{Jip?%+^r2oP=MjUYp@jkITp4~<3;{+We|Q^X*$NY%$0Fy`gqEZl2iaJ_I;N~Lo? z;-4oIvzfWa>e zdxFxnT0Ma;U&^vEvb@SR(W~&MjP);_9XUcQGJft_z_EW)tK2+&#zC|pjsF0PfM*z} z6-ui_ zOaQCG^QvkrN^t6Y*O<4bWHG6%cgApK@Y+~8p*Fwii)z)$3@WNt%c0aQDk^B*F@D)OAU z2D^dXG9R*|JAgRyJ^VnmaB8}XR=#U|!_^xw+jU{>tU;-%j?GIAzzT+yO7|DBdU(jX z`hajk=M_q=xUq!5OTa7P)J`s^Tn)Zan|H;8zYwSpIZNF0n&U|FLed5i>I}vMl`#bg zYp9eSmM*%wTg)SP7q&XF6IEtuDV1?i%#fO)S`3op;N7_3wL5|+-0-j)7!0;@O+ zR0|&zTyG9uhB2S@Gj)>W?Adby6g>%ilz^2{wg@G~AlbsJqI6sv+7HxldPeR2&4D~X z8}Twm#;c-nJ6k%0t6fD|h7bs<DopmLrcrZ&d}urRG|I!j%r&C`D| z5q3B|B}m7wEI^q78H|?o*Drx!d86Hfm zcEva^;yecpW+ml=Tui7k`DJ^DU7ne?f&~aLIIRilHW40?ESHHwv%Gs9>NSr*SW)xl z5b}+A%l>Lv$J{vL?{1vjAucGF%n)HcT(h_;0aIV74NZAPHn#$;>0F98M0%<+rCTB1EW8fK9eB1;;NAQsM%kvf+TR3dAsiH659G=?O&A7$1%D zAQHwbz8E|n0Er0-l>xxQRZ5ieC|}g+SuA1f6u?_{qPM!Jl*c9pQ4Jm+sgUil>)dC7 zuEDhPQ7}?&?B26+-sr$s?e}uj?00qx?TyD^N&}AC^C~ZS2nYh6aCbHONGNK@Oi&Im ziG9j-7ZAq~;yuCq6$d?yN*GQU!@xd_7H^EQX;QtIUTuD%uPi;MYAbI*9@&f?;q1mf zrAnx62u2YVG2AhM6xi-91%*>lMyI61iEu7z@b?G+jF4Z{N}$yp9}q0m_w^9myHcVB zONG9ZOln>>l*W*ADD9kLO0QV51yFrJmsx`wTx`TeBi3JC`;P66{8Tiuh%rQ95;yk- z%OZsH2|5;*hpbE_3Z5A;M;)X%BEc3TgUOc&yr<#Ai14t)-wL@>pg@Ko@lYT{LVfoS z$mPhTc^>S{)Nq{5C@>ZFOk8D~aS9^^yVZQY=3_!)%SrU>)M-2dY%LIS_SMd>2)J`k zWT-C{Zdlo#s$3f5%p-Er3kNV^2Z0yEQnUCJl#^qjO|z`G5f18%sqOy&Q_HBB5up({ ze*pggxKS|~j^PX(K&UK@Vum@?(Cwi=fy2Ic;sam1-mDttwjei?=Eh7JXzfb@2NWmc)!o7(aI~m)s2^-)@nvt#GWlHfz*zmV z=mrD+!Ke_G34ehq3uHioCUDHm3l2d`mb&zi*XB6Ks8`4vYcWb}3NEjv4RW}AB|qX1 zaolHM0{+K$50oIwhe>+Hu`Q8cS>1K@3hx2YEw2*}CPx~N1h|36PDD@C7WKy*7SsAAqvZxx@=zxdl znOTSC0iX@Q`HT1(^%}Asrmw=G;Khr@@IM1$%Ysz6j=r2_5M zOvNX=%q4MA`Z0e|so6_L(k%o6*p1@~ET^bbn?o#>S$$Njyqyf|WWJ~YIad{k$>vF| zd5A(I28JE$X1j*pMW|YX3x%+Q5P#qZ%a<#TDy?HkIdyydY*Lbth}Ks)mWQ7MeO;RYxzn`#Y0z%|rQ zplTFOW||aB8zrTXIN4dhF=bXXRo14vJ~2DM9Z#^j0X|qt9n;4aRUx2!xq&eJD+#7W zU@_Kn7RV@xaZyDF#ZbW1CdzGtGCOy?E#NuO25&D&1y3nV&j|u#Pm~!donO<$wK`vB zMG$)-zji_1rg&vdzZE@X)RpagGWv*)B@|HIVW@sd-R=p@1yv}2FPRRn7AFm z%zOtN90Wv8GUGh~dG6lfEog+D9CZ(Y+}WPe0Cx<0E*Mo}67NtS#K1mO9~RO96)Z)F z)y#N?5SPGmvf&pj<9~=s_>^D>1dm26#A1hI2QY<98)a0~DdFl_aVd;f8-oPza|TfY zHkh*FrRf&NXN?*~Vq)?*;*MOz5)3lb96^P{o0q}=0Ep>!2`g%%hk#0jOM-AC33M~) zwB`d(l6w-6dYHPtn+t>*B9J<;xCc<7HJ7U+DJ*xkZULfv%RHf!cbQoB%EjU-aAM+C zL_EdakF=j1e63E!ZE5C6&5NIpzupz z##CXID~gFaLZH-mgdPbg;E_TybtTc$jM~owjp3v( zxL`p_{{XmL7)Cg#9NfF$Nr;doO5r1$toIOVS_pvx7vKyzR1{BTZ}kIyemjJjnzJro zJi!Lgetd>A;5<+;=J9bMq{ei{kEBcO8yciW6wZs?$8FiZn&WCL@*s zDI?)bGaCK=rCb7^;x*i;f+9aqq-34pG_Wp!+US?W=(Y)D}zw=E=N-On0TII!WbrVG0T-1aLlPv^D+1`