From 355f8310f6cf0ab6eb3f6793770714e99569eb7b Mon Sep 17 00:00:00 2001 From: Mazhar Islam Date: Fri, 9 Dec 2022 10:00:54 -0800 Subject: [PATCH 1/7] Added a walkthrough how to load test AppMesh on EKS --- .../howto-k8s-appmesh-load-test/.gitignore | 3 + .../howto-k8s-appmesh-load-test/README.md | 66 ++++++ .../howto-k8s-appmesh-load-test/cluster.yaml | 15 ++ .../howto-k8s-appmesh-load-test/config.json | 20 ++ .../configmap.yaml | 7 + .../howto-k8s-appmesh-load-test/fortio.yaml | 57 +++++ .../scripts/analyze_load_test_data.py | 94 ++++++++ .../scripts/constants.py | 10 + .../scripts/driver.sh | 74 ++++++ .../scripts/load_driver.py | 213 ++++++++++++++++++ .../scripts/request_handler.py | 55 +++++ .../scripts/request_handler_driver.sh | 12 + 12 files changed, 626 insertions(+) create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/.gitignore create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/README.md create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/config.json create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/configmap.yaml create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/fortio.yaml create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler.py create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler_driver.sh diff --git a/walkthroughs/howto-k8s-appmesh-load-test/.gitignore b/walkthroughs/howto-k8s-appmesh-load-test/.gitignore new file mode 100644 index 00000000..15b6da23 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/.gitignore @@ -0,0 +1,3 @@ +./logs +.idea/ +scripts/data diff --git a/walkthroughs/howto-k8s-appmesh-load-test/README.md b/walkthroughs/howto-k8s-appmesh-load-test/README.md new file mode 100644 index 00000000..4f608565 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/README.md @@ -0,0 +1,66 @@ +# AppMesh K8s Load Test +This walkthrough demonstrates how to load test AppMesh on EKS. It can be used as a tool for further load testing in different mesh configuration. We use [Fortio](https://github.com/fortio/fortio) to generate the load. Currently, this walkthrough only focuses AppMesh on EKS. + +## Step 1: Prerequisites +1. [Walkthrough: App Mesh with EKS](../eks/) + 1. Make sure you have "appmesh-prometheus" installed. You may follow this [live docs](https://aws.github.io/aws-app-mesh-controller-for-k8s/) _Guide_ section for further installation support. + 2. Alternatively, you can follow this doc: [Getting started with AWS App Mesh and Kubernetes](https://docs.aws.amazon.com/app-mesh/latest/userguide/getting-started-kubernetes.html) to install appmesh-controller and EKS cluster using `eksctl`. +2. Clone this repository and navigate to the `walkthroughs/howto-k8s-appmesh-load-test` folder, all the commands henceforth are assumed to be run from the same directory as this README. +3. Make sure you have the latest version of [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed. +4. This test requires Python 3 or later (tested with Python 3.9.6). So make sure you have [Python 3](https://www.python.org/downloads/) installed. +5. Load test results will be stored into S3 bucket. So, in `scripts/constants.py` give your `S3_BUCKET` a unique name. +6. In case you get `AccessDeniedException` (or any kind of accessing AWS resource denied exception) while creating any AppMesh resources (e.g., VirtualNode), don't forget to authenticate with your AWS account. + + +## Step 2: Set Environment Variables +We need to set a few environment variables before starting the load tests. + +```bash +export CONTROLLER_PATH= +export CLUSTER_NAME= +export KUBECONFIG= +export AWS_REGION=us-west-2 +export VPC_ID= +``` + + + +## Step 3: Configuring the Load Test +All parameters of the mesh, load tests, metrics can be specified in `config.json` + +`backends_map` -: The mapping from each Virtual Node to its backend Virtual Services. For each unique node name in `backends_map`, +a VirtualNode, Deployment, Service and VirtualService (with its VirtualNode as its target) are created at runtime. + +`load_tests` -: Array of different test configurations that need to be run on the mesh. `url` is the service endpoint that Fortio (load generator) should hit. + +`metrics` -: Map of metric_name to the corresponding metric PromQL logic + +## Step 4: Running the Load Test +Run the driver script using the below command -: +> sh scripts/driver.sh + +The driver script will perform the following -: +1. Install necessary Python3 libraries. +2. Port-forward the Prometheus service to local. +3. Run the Ginkgo test which is the entrypoint for our load test. +4. Kill the Prometheus port-forwarding after the load Test is done. + + +## Step 5: Analyze the Results +All the test results are saved into `S3_BUCKET` which was specified in `scripts/constants.py`. +Optionally, you can run the `scripts/analyze_load_test_data.py` to visualize the results. +The `analyze_load_test_data.py` will +* First download all the load test results from the `S3_BUCKET` into `scripts\data` directory, then +* Plot a graph against the actual QPS (query per second) Fortio sends to the first VirtualNode vs the max memory consumed by the container of that VirtualNode. + +## Description of other files +`load_driver.py` -: Script which reads `config.json` and triggers load tests, reads metrics from PromQL and writes to S3. Called from within ginkgo + +`fortio.yaml` -: Spec of the Fortio components which are created during runtime + +`request_handler.py` and `request_handler_driver.sh` -: The custom service that runs in each of the pods to handle and route incoming requests according +to the mapping in `backends_map` + +`configmap.yaml` -: ConfigMap spec to mount above request_handler* files into the cluster instead of creating Docker containers. Don't forget to use the absolute path of `request_handler_driver.sh` + +`cluster.yaml` -: A sample EKS cluster config. This `cluster.yaml` can be used to create an EKS cluster by running `eksctl create cluster -f cluster.yaml` diff --git a/walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml b/walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml new file mode 100644 index 00000000..875d671a --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: basic-cluster + region: us-west-2 + +nodeGroups: + - name: ng-1 + instanceType: m5.4xlarge + desiredCapacity: 10 + volumeSize: 80 + ssh: + allow: true # will use ~/.ssh/id_rsa.pub as the default ssh key diff --git a/walkthroughs/howto-k8s-appmesh-load-test/config.json b/walkthroughs/howto-k8s-appmesh-load-test/config.json new file mode 100644 index 00000000..57121675 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/config.json @@ -0,0 +1,20 @@ +{ + "IsTLSEnabled": true, + "IsmTLSEnabled": false, + "ReplicasPerVirtualNode": 1, + "ConnectivityCheckPerURL": 400, + "backends_map": { + "0":[] + }, + "load_tests": [ + {"test_name": "experiment16-qps50000-t10-c400", "url": "http://service-0.tls-e2e.svc.cluster.local:9080/", "qps": "50000", "t": "10s", "c":"400"} + ], + "metrics": { + "envoy_ingress_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)", + "envoy_2xx_requests_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_response_code=~\"2.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)", + "envoy_4xx_requests_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_response_code=~\"4.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)", + "envoy_5xx_requests_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_response_code=~\"5.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)", + "envoy_memory_MB_by_replica_set": "sum(label_replace(container_memory_working_set_bytes{container=\"envoy\",pod=~\"node-.*\"}, \"replica_set\", \"$1\", \"pod\", \"(node-.*-.*)-.*\")) by (replica_set) / (1024*1024)", + "envoy_cpu_usage_seconds_by_replica_set": "sum(label_replace(container_cpu_usage_seconds_total{container=\"envoy\",pod=~\"node-.*\"}, \"replica_set\", \"$1\", \"pod\", \"(node-.*-.*)-.*\")) by (replica_set)" + } +} \ No newline at end of file diff --git a/walkthroughs/howto-k8s-appmesh-load-test/configmap.yaml b/walkthroughs/howto-k8s-appmesh-load-test/configmap.yaml new file mode 100644 index 00000000..c36eeca9 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/configmap.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: scripts-configmap +data: + request_handler_driver.sh: "/scripts/request_handler_driver.sh" diff --git a/walkthroughs/howto-k8s-appmesh-load-test/fortio.yaml b/walkthroughs/howto-k8s-appmesh-load-test/fortio.yaml new file mode 100644 index 00000000..804aa60c --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/fortio.yaml @@ -0,0 +1,57 @@ +--- +apiVersion: appmesh.k8s.aws/v1beta2 +kind: VirtualNode +metadata: + name: fortio + namespace: tls-e2e +spec: + podSelector: + matchLabels: + app: fortio + listeners: + - portMapping: + port: 8080 + protocol: http + backends: + - virtualService: + virtualServiceRef: + name: service-0 # Replace with one of the fishap VS + namespace: tls-e2e + serviceDiscovery: + dns: + hostname: fortio.tls-e2e.svc.cluster.local +--- +apiVersion: v1 +kind: Service +metadata: + name: fortio + namespace: tls-e2e +spec: + ports: + - port: 8080 + name: http + selector: + app: fortio +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fortio + namespace: tls-e2e +spec: + replicas: 1 + selector: + matchLabels: + app: fortio + template: + metadata: + labels: + app: fortio + spec: + containers: + - name: app + image: fortio/fortio + imagePullPolicy: Always + ports: + - containerPort: 8080 + args: ["server"] \ No newline at end of file diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py new file mode 100644 index 00000000..d9a7fed9 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py @@ -0,0 +1,94 @@ +import csv +import json +import os +import subprocess +from pathlib import Path +from pprint import pprint + +import matplotlib.pyplot as plt +import numpy as np + +from constants import S3_BUCKET + +DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +DATA_PATH = os.path.join(DIR_PATH, 'data') + + +def get_s3_data(): + res = subprocess.run(["aws sts get-caller-identity"], shell=True, stdout=subprocess.PIPE, + universal_newlines=True) + out = res.stdout + print("Caller identity: {}".format(out)) + + command = "aws s3 sync s3://{} {}".format(S3_BUCKET, DATA_PATH) + print("Running the following command to download S3 load test results: \n{}".format(command)) + res = subprocess.run([command], shell=True, stdout=subprocess.PIPE, universal_newlines=True) + out = res.stdout + print(out) + + +def plot_graph(actual_QPS_list, node_0_mem_list): + node_0_mem_list = [float(x) for x in node_0_mem_list] + Y = [x for _, x in sorted(zip(actual_QPS_list, node_0_mem_list))] + X = sorted(actual_QPS_list) + xpoints = np.array(X) + ypoints = np.array(Y) + + plt.figure(figsize=(10, 5)) + plt.bar(xpoints, ypoints, width=20) + plt.ylabel('Node-0 (MiB)') + plt.xlabel('Actual QPS') + print("Plotting graph...") + plt.show() + + +def read_load_test_data(): + all_files_list = [x for x in os.listdir(DATA_PATH) if os.path.isdir(os.path.join(DATA_PATH, x))] + qps_mem_files_list = [] + for exp_f in all_files_list: + result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.path.join(DATA_PATH, exp_f)) for f in + filenames + if "fortio.json" in f or "envoy_memory_MB_by_replica_set.csv" in f] + qps_mem_files_list.append(result) + + actual_qps_list = [] + node_0_mem_list = [] + experiment_results = {} + for qps_or_mem_f in qps_mem_files_list: + attrb = {} + for f in qps_or_mem_f: + if "fortio.json" in f: + with open(f) as json_f: + j = json.load(json_f) + actual_qps = j["ActualQPS"] + actual_qps_list.append(actual_qps) + attrb["ActualQPS"] = actual_qps + else: + with open(f) as csv_f: + c = csv.reader(csv_f, delimiter=',', skipinitialspace=True) + node_0_mem = [] + for line in c: + if "node-0" in line[0]: + node_0_mem.append(line[2]) + max_mem = max(node_0_mem) + node_0_mem_list.append(max_mem) + attrb["max_mem"] = max_mem + key = Path(f) + experiment_results[os.path.join(key.parts[-3], key.parts[-2])] = attrb + + # for research purpose + print("Experiment results:") + pprint(experiment_results) + + return actual_qps_list, node_0_mem_list + + +def plot_qps_vs_container_mem(): + actual_qps_list, node_0_mem_list = read_load_test_data() + + plot_graph(actual_qps_list, node_0_mem_list) + + +if __name__ == '__main__': + get_s3_data() + plot_qps_vs_container_mem() diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py new file mode 100644 index 00000000..01b100b1 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py @@ -0,0 +1,10 @@ +URL_DEFAULT = "http://service-0.tls-e2e.svc.cluster.local:9080/path-0" +QPS_DEFAULT = "100" +DURATION_DEFAULT = "30s" +CONNECTIONS_DEFAULT = "1" + +FORTIO_RUN_ENDPOINT = 'http://localhost:9091/fortio/rest/run' +PROMETHEUS_QUERY_ENDPOINT = 'http://localhost:9090/api/v1/query_range' + +# give your s3 bucket a unique name +S3_BUCKET = "mazharis-appmeshloadtester" diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh b/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh new file mode 100644 index 00000000..a3303cfb --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh @@ -0,0 +1,74 @@ +err() { + msg="Error: $1" + echo "${msg}" + code=${2:-"1"} + exit ${code} +} + +exec_command() { + eval "$1" + if [ $? -eq 0 ]; then + echo "'$1' command Executed Successfully" + else + err "'$1' command Failed" + fi +} + +check_version() { + eval "$1" +} + +# sanity check +if [ -z "${CONTROLLER_PATH}" ]; then + err "CONTROLLER_PATH is not set" +fi + +if [ -z "${KUBECONFIG}" ]; then + err "KUBECONFIG is not set" +fi + +if [ -z "${CLUSTER_NAME}" ]; then + err "CLUSTER_NAME is not set" +fi + +if [ -z "${AWS_REGION}" ]; then + err "AWS_REGION is not set" +fi + +if [ -z "${VPC_ID}" ]; then + err "VPC_ID is not set" +fi + +# Install python3 dependencies +exec_command "python3 --version" + +declare -a libraries=("boto3" "numpy" "pandas" "requests" "botocore" "matplotlib") + +for i in "${libraries[@]}" +do + check_version "pip3 show $i" + if [ $? -ne 0 ] + then + exec_command "pip3 install $i" + fi +done + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +APPMESH_LOADTESTER_PATH="$(dirname "$DIR")" +echo "APPMESH_LOADTESTER_PATH -: $APPMESH_LOADTESTER_PATH" + +# Prometheus port forward +echo "Port-forwarding Prometheus" +kubectl --namespace appmesh-system port-forward service/appmesh-prometheus 9090 & +pid=$! + +# call ginkgo +echo "Starting Ginkgo test" +cd $CONTROLLER_PATH && ginkgo -v -r --focus "DNS" "$CONTROLLER_PATH"/test/e2e/fishapp/load -- --cluster-kubeconfig=$KUBECONFIG \ +--cluster-name=$CLUSTER_NAME --aws-region=$AWS_REGION --aws-vpc-id=$VPC_ID \ +--base-path=$APPMESH_LOADTESTER_PATH + +# kill prometheus port forward +echo "Killing Prometheus port-forward" +kill -9 $pid +[ $status -eq 0 ] && echo "Killed Prometheus port-forward" || echo "Error when killing Prometheus port forward" \ No newline at end of file diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py new file mode 100644 index 00000000..80be45c9 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py @@ -0,0 +1,213 @@ +import io +import json +import logging +import os +import sys +import time +from datetime import datetime + +import boto3 +import pandas as pd +import requests +from botocore.exceptions import ClientError + +from constants import * + + +def check_valid_request_data(request_data): + logging.info("Validating Fortio request parameters") + if ("url" not in request_data): + logging.warning(f"url not provided. Defaulting to {URL_DEFAULT}") + request_data["url"] = URL_DEFAULT + if ("t" not in request_data): + logging.warning(f"Duration (t) not provided. Defaulting to {DURATION_DEFAULT}") + request_data["t"] = DURATION_DEFAULT + if ("qps" not in request_data): + logging.warning(f"qps not provided. Defaulting to {QPS_DEFAULT}") + request_data["qps"] = QPS_DEFAULT + if ("c" not in request_data): + logging.warning(f"# Connections (c) not provided. Defaulting to {CONNECTIONS_DEFAULT}") + request_data["c"] = CONNECTIONS_DEFAULT + + logging.info(f"Updated request data -: {request_data}") + return request_data + + +def run_fortio_test(test_data): + fortio_request_data = test_data.copy() + test_name = fortio_request_data.pop("test_name") + logging.info(f"Running test -: {test_name}") + fortio_request_data = check_valid_request_data(fortio_request_data) + fortio_response = requests.post(url=FORTIO_RUN_ENDPOINT, json=fortio_request_data) + + if (fortio_response.ok): + fortio_json = fortio_response.json() + logging.info(f"Successful Fortio run -: {test_name}") + return fortio_json + else: + logging.error(f"Fortio response code = {fortio_response.status_code}") + fortio_response.raise_for_status() + + +def query_prometheus_server(metric_name, metric_logic, start_ts, end_ts, step="10s"): + logging.info(f"Querying prometheus server for metric = {metric_name} using logic = {metric_logic}") + prometheus_response = requests.post(url=PROMETHEUS_QUERY_ENDPOINT, + data={"query": metric_logic, "start": start_ts, "end": end_ts, "step": step}) + if prometheus_response.ok: + logging.info(f"Successfully queried Prometheus for metric -: {metric_name}") + else: + logging.error(f"Error while querying Prometheus for metric -: {metric_name}") + prometheus_response.raise_for_status() + + return prometheus_response.json() + + +def prometheus_json_to_df(prometheus_json, metric_name): + data = pd.json_normalize(prometheus_json, record_path=['data', 'result']) + try: + # Split values into separate rows + df = data.explode('values') + # Split [ts, val] into separate columns + split_df = pd.DataFrame(df['values'].to_list(), columns=['timestamp', metric_name], index=df.index) + metrics_df = pd.concat([df, split_df], axis=1) + metrics_df.drop(columns='values', inplace=True) + metrics_df['timestamp'] = pd.to_numeric(metrics_df['timestamp']) + + # Normalize timestamps + groupby_column = [col for col in metrics_df.columns if col.startswith("metric")][0] + metrics_df['normalized_ts'] = metrics_df['timestamp'] - metrics_df.groupby(groupby_column).timestamp.transform( + 'min') + logging.info("Normalized DataFrame -: ") + logging.info(metrics_df.head(30)) + except KeyError: + logging.warning("Metrics response is empty. Returning empty DataFrame") + metrics_df = pd.DataFrame(columns=["metric.", "timestamp", metric_name, "normalized_ts"]) + + return metrics_df + + +def write_to_s3(s3_client, data, folder_path, file_name): + response = s3_client.put_object( + Bucket=S3_BUCKET, Key=f"{folder_path}/{file_name}", Body=data + ) + status = response.get("ResponseMetadata", {}).get("HTTPStatusCode") + + if status == 200: + logging.info(f"Successful write of ({folder_path}/{file_name}) to S3. Status - {status}") + else: + logging.error( + f"Error writing ({folder_path}/{file_name}) to S3. Response Metadata -: {response['ResponseMetadata']}") + raise IOError(f"S3 Write Failed. ResponseMetadata -: {response['ResponseMetadata']}") + + +def get_s3_client(region=None, is_creds=False): + try: + if region is None: + s3_client = boto3.client('s3') + elif is_creds: + cred = { + "credentials": { + "accessKeyId": os.environ['AWS_ACCESS_KEY_ID'], + "secretAccessKey": os.environ['AWS_SECRET_ACCESS_KEY'], + "sessionToken": os.environ['AWS_SESSION_TOKEN'], + } + } + s3_client = boto3.client('s3', + aws_access_key_id=cred['credentials']['accessKeyId'], + aws_secret_access_key=cred['credentials']['secretAccessKey'], + aws_session_token=cred['credentials']['sessionToken'], + region_name=region) + else: + s3_client = boto3.client('s3', region_name=region) + except ClientError as e: + logging.error(e) + return + return s3_client + + +def list_bucket(region=None): + # Retrieve the list of existing buckets + s3 = boto3.client('s3', region_name=region) + response = s3.list_buckets() + + # Output the bucket names + print('Existing buckets:') + for bucket in response['Buckets']: + print(f' {bucket["Name"]}') + + +def create_bucket_if_not_exists(s3_client, bucket_name, region=None): + """Create an S3 bucket in a specified region + + If a region is not specified, the bucket is created in the S3 default + region (us-west-2). + + :param bucket_name: Bucket to create + :param region: String region to create bucket in, e.g., 'us-west-2' + :return: True if bucket created, else False + """ + try: + s3 = boto3.resource('s3') + s3.meta.client.head_bucket(Bucket=bucket_name) + logging.info("No need to create as bucket: {} already exists,".format(bucket_name)) + except ClientError: + # Create bucket + try: + if region is None: + s3_client.create_bucket(Bucket=bucket_name) + else: + location = {'LocationConstraint': region} + s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location) + except ClientError as e: + logging.error(e) + return False + return True + + +if __name__ == '__main__': + driver_ts = datetime.today().strftime('%Y%m%d%H%M%S') + print(f"driver_ts = {driver_ts}") + + config_file = sys.argv[1] + BASE_PATH = sys.argv[2] + LOGS_FOLDER = os.path.join(BASE_PATH, "logs") + os.makedirs(LOGS_FOLDER, exist_ok=True) + + log_file = os.path.join(LOGS_FOLDER, f"load_driver_{driver_ts}.log") + logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO) + with open(config_file, "r") as f: + config = json.load(f) + logging.info("Loaded config file") + + region = os.environ['AWS_REGION'] + s3_client = get_s3_client() + create_bucket_if_not_exists(s3_client=s3_client, bucket_name=S3_BUCKET, region=region) + + for test in config["load_tests"]: + logging.info("Writing config to S3") + write_to_s3(s3_client, json.dumps(config, indent=4), f"{test['test_name']}/{driver_ts}", "config.json") + start_ts = int(time.time()) + fortio_json = run_fortio_test(test) + # Write Fortio response to S3 + logging.info("Writing Fortio response to S3") + write_to_s3(s3_client, json.dumps(fortio_json, indent=4), f"{test['test_name']}/{driver_ts}", "fortio.json") + end_ts = int(time.time()) + + logging.info(f"start_ts -: {start_ts}, end_ts -: {end_ts}") + + for metric_name, metric_logic in config['metrics'].items(): + metrics_json = query_prometheus_server(metric_name, metric_logic, start_ts, end_ts) + metrics_df = prometheus_json_to_df(metrics_json, metric_name) + # Write to S3 + logging.info("Writing Metrics dataframe to S3") + s3_folder_path = f"{test['test_name']}/{driver_ts}" + file_name = f"{metric_name}.csv" + csv_buffer = io.StringIO() + metrics_df.to_csv(csv_buffer, index=False) + write_to_s3(s3_client, csv_buffer.getvalue(), s3_folder_path, file_name) + csv_buffer.close() + + logging.info( + f"Finished exporting all metrics for {test['test_name']}. Sleeping for 10s before starting next test") + # Sleep 10s between tests + time.sleep(10) diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler.py new file mode 100644 index 00000000..e934931c --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler.py @@ -0,0 +1,55 @@ +from flask import Flask, request, abort, jsonify +import os +import json +import logging + +import aiohttp +import asyncio + +app = Flask(__name__) + +async def fetch(session, url): + async with session.get(url) as response: + resp = await response.json() + return response + + +async def fetch_all(backends): + async with aiohttp.ClientSession() as session: + tasks = [] + for url in backends: + tasks.append(fetch(session,url)) + responses = await asyncio.gather(*tasks, return_exceptions=True) + return responses + + +@app.route('/health', methods = ['GET']) +def health(): + return f"Alive. Backends -: {os.getenv('BACKENDS')}" + +@app.route('/', methods = ['GET']) +def wrk(): + if(os.getenv('BACKENDS') and os.getenv('BACKENDS') != ""): + backends = os.getenv('BACKENDS').split(",") + else: + backends = "" + + if(backends): + responses = asyncio.run(fetch_all(backends)) + for i in responses: + print(f"Status = {i.status}, reason = {i.reason}, real_url = {i.real_url}") + + retcode = 200 if(set([response.status for response in responses]) == set([200])) else 500 + msg = "backends Success" if retcode == 200 else "error when calling backends" + else: + retcode = 200 + msg = "Success" + + print(msg, retcode) + + return jsonify(msg), retcode + + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.INFO) + app.run(host='0.0.0.0', debug=True, threaded=True) \ No newline at end of file diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler_driver.sh b/walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler_driver.sh new file mode 100644 index 00000000..8379c4c4 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/request_handler_driver.sh @@ -0,0 +1,12 @@ +echo "Entered handler" +# sleep 600 +# Install dependencies +pip3 install flask +echo "Completed flask" +pip3 install aiohttp +echo "Completed aiohttp" +pip3 install asyncio +echo "Completed asyncio" + +# # Run flask server +flask run -p 9080 From dd2e6c878976f85a471d837881cd78227f5c1e18 Mon Sep 17 00:00:00 2001 From: Mazhar Islam Date: Fri, 9 Dec 2022 10:00:54 -0800 Subject: [PATCH 2/7] Added a walkthrough how to load test AppMesh on EKS --- walkthroughs/howto-k8s-appmesh-load-test/config.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/walkthroughs/howto-k8s-appmesh-load-test/config.json b/walkthroughs/howto-k8s-appmesh-load-test/config.json index 57121675..98c75e6c 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/config.json +++ b/walkthroughs/howto-k8s-appmesh-load-test/config.json @@ -4,7 +4,9 @@ "ReplicasPerVirtualNode": 1, "ConnectivityCheckPerURL": 400, "backends_map": { - "0":[] + "0": ["1", "2"], + "1": ["3"], + "2": ["4"] }, "load_tests": [ {"test_name": "experiment16-qps50000-t10-c400", "url": "http://service-0.tls-e2e.svc.cluster.local:9080/", "qps": "50000", "t": "10s", "c":"400"} From e3aa71a53f8243e51e45c64e76b9bd670edf928b Mon Sep 17 00:00:00 2001 From: Mazhar Islam Date: Wed, 14 Dec 2022 15:23:02 -0800 Subject: [PATCH 3/7] Addressed comments from PR review --- .../howto-k8s-appmesh-load-test/README.md | 19 +++++++++++++------ .../howto-k8s-appmesh-load-test/cluster.yaml | 2 +- .../howto-k8s-appmesh-load-test/config.json | 2 +- .../scripts/analyze_load_test_data.py | 11 +++++++---- .../scripts/driver.sh | 11 +++++++++++ 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/walkthroughs/howto-k8s-appmesh-load-test/README.md b/walkthroughs/howto-k8s-appmesh-load-test/README.md index 4f608565..e036ab4e 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/README.md +++ b/walkthroughs/howto-k8s-appmesh-load-test/README.md @@ -2,11 +2,18 @@ This walkthrough demonstrates how to load test AppMesh on EKS. It can be used as a tool for further load testing in different mesh configuration. We use [Fortio](https://github.com/fortio/fortio) to generate the load. Currently, this walkthrough only focuses AppMesh on EKS. ## Step 1: Prerequisites -1. [Walkthrough: App Mesh with EKS](../eks/) - 1. Make sure you have "appmesh-prometheus" installed. You may follow this [live docs](https://aws.github.io/aws-app-mesh-controller-for-k8s/) _Guide_ section for further installation support. - 2. Alternatively, you can follow this doc: [Getting started with AWS App Mesh and Kubernetes](https://docs.aws.amazon.com/app-mesh/latest/userguide/getting-started-kubernetes.html) to install appmesh-controller and EKS cluster using `eksctl`. -2. Clone this repository and navigate to the `walkthroughs/howto-k8s-appmesh-load-test` folder, all the commands henceforth are assumed to be run from the same directory as this README. -3. Make sure you have the latest version of [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed. +1. [Walkthrough: App Mesh with EKS](../eks/). Make sure you have: + 1. Cloned the [AWS AppMesh controller repo](https://github.com/aws/aws-app-mesh-controller-for-k8s). We will need this controller repo path (`CONTROLLER_PATH`) in [step 2](##step-2:-set-environment-variables). + 2. Created an EKS cluster and setup kubeconfig. + 3. Installed "appmesh-prometheus". You may follow this [App Mesh Prometheus](https://github.com/aws/eks-charts/tree/master/stable/appmesh-prometheus) chart for installation support. + 4. This load test uses [Ginkgo](https://github.com/onsi/ginkgo/tree/v1.16.4). Make sure you have ginkgo installed by running `ginkgo version`. If it's not, you may need to install it: + 1. Install [Go](https://go.dev/doc/install), if you haven't already. + 2. Install Ginkgo v1.16.4 (currently, AppMesh controller uses [ginkgo v1.16.4](https://github.com/aws/aws-app-mesh-controller-for-k8s/blob/master/go.mod#L13)) + 1. `go get -u github.com/onsi/ginkgo/ginkgo@v1.16.5` or + 2. `go install github.com/onsi/ginkgo/ginkgo@v1.16.5` for GO version 1.17+ + 5. (Optional) You can follow this doc: [Getting started with AWS App Mesh and Kubernetes](https://docs.aws.amazon.com/app-mesh/latest/userguide/getting-started-kubernetes.html) to install appmesh-controller and EKS cluster using `eksctl`. +2. Clone this repository and navigate to the `walkthroughs/howto-k8s-appmesh-load-test` folder, all the commands henceforth are assumed to be run from the same directory as this `README`. +3. Make sure you have the latest version of [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) or [AWS CLI v1](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv1.html) installed. 4. This test requires Python 3 or later (tested with Python 3.9.6). So make sure you have [Python 3](https://www.python.org/downloads/) installed. 5. Load test results will be stored into S3 bucket. So, in `scripts/constants.py` give your `S3_BUCKET` a unique name. 6. In case you get `AccessDeniedException` (or any kind of accessing AWS resource denied exception) while creating any AppMesh resources (e.g., VirtualNode), don't forget to authenticate with your AWS account. @@ -16,7 +23,7 @@ This walkthrough demonstrates how to load test AppMesh on EKS. It can be used as We need to set a few environment variables before starting the load tests. ```bash -export CONTROLLER_PATH= +export CONTROLLER_PATH= export CLUSTER_NAME= export KUBECONFIG= export AWS_REGION=us-west-2 diff --git a/walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml b/walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml index 875d671a..43ee0281 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml +++ b/walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml @@ -9,7 +9,7 @@ metadata: nodeGroups: - name: ng-1 instanceType: m5.4xlarge - desiredCapacity: 10 + desiredCapacity: 4 volumeSize: 80 ssh: allow: true # will use ~/.ssh/id_rsa.pub as the default ssh key diff --git a/walkthroughs/howto-k8s-appmesh-load-test/config.json b/walkthroughs/howto-k8s-appmesh-load-test/config.json index 98c75e6c..a3422d34 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/config.json +++ b/walkthroughs/howto-k8s-appmesh-load-test/config.json @@ -9,7 +9,7 @@ "2": ["4"] }, "load_tests": [ - {"test_name": "experiment16-qps50000-t10-c400", "url": "http://service-0.tls-e2e.svc.cluster.local:9080/", "qps": "50000", "t": "10s", "c":"400"} + {"test_name": "experiment_x", "url": "http://service-0.tls-e2e.svc.cluster.local:9080/", "qps": "5000", "t": "10s", "c":"400"} ], "metrics": { "envoy_ingress_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)", diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py index d9a7fed9..55c7a22d 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py @@ -2,6 +2,7 @@ import json import os import subprocess +from collections import OrderedDict from pathlib import Path from pprint import pprint @@ -28,7 +29,6 @@ def get_s3_data(): def plot_graph(actual_QPS_list, node_0_mem_list): - node_0_mem_list = [float(x) for x in node_0_mem_list] Y = [x for _, x in sorted(zip(actual_QPS_list, node_0_mem_list))] X = sorted(actual_QPS_list) xpoints = np.array(X) @@ -69,7 +69,7 @@ def read_load_test_data(): node_0_mem = [] for line in c: if "node-0" in line[0]: - node_0_mem.append(line[2]) + node_0_mem.append(float(line[2])) max_mem = max(node_0_mem) node_0_mem_list.append(max_mem) attrb["max_mem"] = max_mem @@ -77,8 +77,11 @@ def read_load_test_data(): experiment_results[os.path.join(key.parts[-3], key.parts[-2])] = attrb # for research purpose - print("Experiment results:") - pprint(experiment_results) + sorted_experiment_results = OrderedDict() + for k, v in sorted(experiment_results.items(), key=lambda item: item[1]['max_mem']): + sorted_experiment_results[k] = v + print("Experiment results sorted:\n") + pprint(sorted_experiment_results) return actual_qps_list, node_0_mem_list diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh b/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh index a3303cfb..14401ceb 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh @@ -39,6 +39,17 @@ if [ -z "${VPC_ID}" ]; then err "VPC_ID is not set" fi +# Check creds +if [ -n "${USER}" ]; then + check_version "isengardcli version" + if [ $? -eq 0 ] + then + eval "$(isengardcli creds "$USER")" + status=$? + [ $status -eq 0 ] && echo "Ran isengard creds successfully!" || (err "isengard creds error.") + fi +fi + # Install python3 dependencies exec_command "python3 --version" From 322f0b719e78cb5b0b272835bba1ec990358eab3 Mon Sep 17 00:00:00 2001 From: Mazhar Islam Date: Wed, 14 Dec 2022 15:27:14 -0800 Subject: [PATCH 4/7] S3 bucket name need to be unique, so added a tag for it --- walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py index 01b100b1..2f019c02 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py @@ -7,4 +7,4 @@ PROMETHEUS_QUERY_ENDPOINT = 'http://localhost:9090/api/v1/query_range' # give your s3 bucket a unique name -S3_BUCKET = "mazharis-appmeshloadtester" +S3_BUCKET = "username-appmeshloadtester" From a4971d4577804b877390d450d0495f1cf2921b7b Mon Sep 17 00:00:00 2001 From: Mazhar Islam Date: Wed, 14 Dec 2022 15:27:32 -0800 Subject: [PATCH 5/7] S3 bucket name need to be unique, so added a tag for it --- walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py index 2f019c02..6e9441ac 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py @@ -7,4 +7,4 @@ PROMETHEUS_QUERY_ENDPOINT = 'http://localhost:9090/api/v1/query_range' # give your s3 bucket a unique name -S3_BUCKET = "username-appmeshloadtester" +S3_BUCKET = " Date: Wed, 14 Dec 2022 16:40:34 -0800 Subject: [PATCH 6/7] Added few more details in readme. Made result analyzer more flexible --- .../howto-k8s-appmesh-load-test/README.md | 27 ++++++++++++++----- .../scripts/analyze_load_test_data.py | 19 +++++++++---- .../scripts/constants.py | 2 +- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/walkthroughs/howto-k8s-appmesh-load-test/README.md b/walkthroughs/howto-k8s-appmesh-load-test/README.md index e036ab4e..297c4fb1 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/README.md +++ b/walkthroughs/howto-k8s-appmesh-load-test/README.md @@ -35,12 +35,27 @@ export VPC_ID=.tls-e2e.svc.cluster.local:9080/`. + For example, based on the above `backends_map`, if we want to send the load traffic to the first virtual node `"0"`, then the `ulr` will look like: + `http://service-0.tls-e2e.svc.cluster.local:9080/`. + * `qps`: Total Queries Per Seconds fortio sends to the endpoints. + * `t`: How long the test will run. + * `c`: Number of parallel simultaneous connections to the endpoints fortio hits. + * Optionally, you can add more load generation parameter by following the [Forito documentation](https://github.com/fortio/fortio). + +* `metrics` -: Map of metric_name to the corresponding metric [PromQL logic](https://prometheus.io/docs/prometheus/latest/querying/operators/). ## Step 4: Running the Load Test Run the driver script using the below command -: diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py index 55c7a22d..ba99971e 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py @@ -14,6 +14,8 @@ DIR_PATH = os.path.dirname(os.path.realpath(__file__)) DATA_PATH = os.path.join(DIR_PATH, 'data') +NODE_NAME = "0" + def get_s3_data(): res = subprocess.run(["aws sts get-caller-identity"], shell=True, stdout=subprocess.PIPE, @@ -29,14 +31,14 @@ def get_s3_data(): def plot_graph(actual_QPS_list, node_0_mem_list): - Y = [x for _, x in sorted(zip(actual_QPS_list, node_0_mem_list))] - X = sorted(actual_QPS_list) + x_y_tuple = [(x, y) for x, y in sorted(zip(actual_QPS_list, node_0_mem_list))] + X, Y = zip(*x_y_tuple) xpoints = np.array(X) ypoints = np.array(Y) plt.figure(figsize=(10, 5)) plt.bar(xpoints, ypoints, width=20) - plt.ylabel('Node-0 (MiB)') + plt.ylabel('Node-{} (MiB)'.format(NODE_NAME)) plt.xlabel('Actual QPS') print("Plotting graph...") plt.show() @@ -67,9 +69,13 @@ def read_load_test_data(): with open(f) as csv_f: c = csv.reader(csv_f, delimiter=',', skipinitialspace=True) node_0_mem = [] + node_found = False for line in c: - if "node-0" in line[0]: + if "node-" + NODE_NAME in line[0]: node_0_mem.append(float(line[2])) + node_found = True + if not node_found: + raise Exception("Node not found: {} in experiment file: {}".format(NODE_NAME, f)) max_mem = max(node_0_mem) node_0_mem_list.append(max_mem) attrb["max_mem"] = max_mem @@ -80,7 +86,7 @@ def read_load_test_data(): sorted_experiment_results = OrderedDict() for k, v in sorted(experiment_results.items(), key=lambda item: item[1]['max_mem']): sorted_experiment_results[k] = v - print("Experiment results sorted:\n") + print("Experiment results sorted:") pprint(sorted_experiment_results) return actual_qps_list, node_0_mem_list @@ -93,5 +99,8 @@ def plot_qps_vs_container_mem(): if __name__ == '__main__': + node_name = input('Enter the node name (or press enter for default node "0"): ') + if node_name != "": + NODE_NAME = node_name get_s3_data() plot_qps_vs_container_mem() diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py index 6e9441ac..4d064ee3 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py @@ -7,4 +7,4 @@ PROMETHEUS_QUERY_ENDPOINT = 'http://localhost:9090/api/v1/query_range' # give your s3 bucket a unique name -S3_BUCKET = " Date: Fri, 3 Feb 2023 16:21:36 -0800 Subject: [PATCH 7/7] Updated readme. Clean codes and added python dependencies as requirments.txt --- .../howto-k8s-appmesh-load-test/README.md | 230 +++++++++++++++--- .../load_test_flow_dg.png | Bin 0 -> 50672 bytes .../requirements.txt | 7 + .../scripts/driver.sh | 33 +-- .../scripts/load_driver.py | 83 +++---- .../howto-k8s-appmesh-load-test/vars.env | 5 + 6 files changed, 247 insertions(+), 111 deletions(-) create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/load_test_flow_dg.png create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/requirements.txt create mode 100644 walkthroughs/howto-k8s-appmesh-load-test/vars.env diff --git a/walkthroughs/howto-k8s-appmesh-load-test/README.md b/walkthroughs/howto-k8s-appmesh-load-test/README.md index 297c4fb1..a73c1a28 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/README.md +++ b/walkthroughs/howto-k8s-appmesh-load-test/README.md @@ -1,23 +1,146 @@ # AppMesh K8s Load Test -This walkthrough demonstrates how to load test AppMesh on EKS. It can be used as a tool for further load testing in different mesh configuration. We use [Fortio](https://github.com/fortio/fortio) to generate the load. Currently, this walkthrough only focuses AppMesh on EKS. +This walkthrough demonstrates how to load test AppMesh on EKS. It can be used as a tool for further load testing in different mesh configuration. +We use [Fortio](https://github.com/fortio/fortio) to generate the load. This load test is for AppMesh on EKS, therefore, we +need [aws-app-mesh-controller-for-k8s](https://github.com/aws/aws-app-mesh-controller-for-k8s) to run it. Note that, the load test runs as a part of the controller integration test, hence, +we need the controller repo in this walkthorugh. Following are the key components of this load test: + + +* Configuration JSON: This specifies the details of the mesh, such as Virtual Nodes and their backends, a list of parameters for the load generator e.g., query per seconds (QPS), duration for each experiment to run and a list of metrics (and their corresponding logic) that need to be captured in the load test. +The details of the `config.json` can be found in [Step 3: Configuring the Load Test](#step-3:-configuring-the-load-test). +* Driver script: This bash script (`scripts/driver.sh`) sets up port-forwarding of the prometheus and starts the load test as part of the AppMesh K8s Controller integration tests. +* AppMesh K8s Controller: The K8s Controller for AppMesh [integration testing code](https://github.com/aws/aws-app-mesh-controller-for-k8s/tree/master/test/e2e/fishapp/load) is the +entry point of our load test. It handles creation of a meshified app with Virtual Nodes, Virtual Services, backends etc. It also cleans up resources and spins down the mesh after +finishing the test. The list of unique values under the adjacency list `backends_map` in `config.json` provides the number of Virtual Nodes that need to be created and the map +values provide the backend/edge connections of each node’s virtual service. These services corresponding to the backend connections will be configured as environment variables +at the time of creation of the deployment. The *Custom Service* looks for this environment variable when re-routing incoming HTTP requests. +* Custom service: The custom service `scripts/request_handler.py` script runs on each pod which receives incoming requests and makes calls to its “backend” services according to the `backends_map` +in the `config.json`. This is a simple HTTP server that runs on each pod which handles incoming requests and in turn routes them to its backend services. This backends info is +initialized as an environment variable at the time of pod creation. The custom service script is mounted onto the deployment using *ConfigMaps* +(see [createConfigMap](https://github.com/aws/aws-app-mesh-controller-for-k8s/blob/420a437f68e850a32f395f9ecd4917d62845d25a/test/e2e/fishapp/load/dynamic_stack_load_test.go) for +more details) to reduce development time (avoids creating Docker containers, pushing them to the registry, etc.). If the response from all its backends is SUCCESS/200 OK, then it +returns a 200. If any one of the responses is a failure, it returns a 500 HTTP error code. If it does not have any backends, it auto returns a 200 OK. +* Fortio: The [Fortio](https://github.com/fortio/fortio) load generator hits an endpoint in the mesh to simulate traffic by making HTTP requests to the given endpoint at the +requested QPS for the requested duration. The default endpoint in the mesh is defined in `URL_DEFAULT` under `scripts/constants.py`. Since fortio needs to access an endpoint within +the mesh, we install fortio inside the mesh with its own Virtual Node, K8 service and deployment. See the `fortio.yaml` file for more details. The K8s service is then port-forwarded +to the local machine so that REST API calls can be sent from local. +* AppMesh-Prometheus: Prometheus scrapes the required Envoy metrics during load test from each pod at specified interval. It has its own query language [*PromQL*]((https://prometheus.io/docs/prometheus/latest/querying/operators/)) which is helpful +for aggregating metrics at different granularities before exporting them. +* Load Driver: The load driver `scripts/load_driver.py` script reads the list of tests from the `config.json`, triggers the load, fetches the metrics from the Prometheus server using +its APIs and writes to persistent storage such as S3. This way, we have access to history data even if the Prometheus server spins down for some reason. The API endpoints support +PromQL queries so that aggregate metrics can be fetched directly instead of collecting raw metrics and writing separate code for aggregating them. The start and end timestamps of the +test will be noted for each test and the metrics will be queried using this time range. +* S3 storage for metrics: Experiments are uniquely identified by their `test_name` defined in `config.json`. Multiple runs of the same experiment are identified by their run +*timestamps* (in YYYYMMDDHHMMSS format). Hence, there will be a 1:1 mapping between the `test_name` and the set of config parameters in the JSON. Metrics are stored inside above +subfolders along with a metadata file specifying the parameter values used in the experiment. A list of metrics can be found in `metrics` defined under `config.json`. + + +Following is a flow diagram of the load test: + +![Flow Diagram](./load_test_flow_dg.png "Flow Diagram") ## Step 1: Prerequisites -1. [Walkthrough: App Mesh with EKS](../eks/). Make sure you have: - 1. Cloned the [AWS AppMesh controller repo](https://github.com/aws/aws-app-mesh-controller-for-k8s). We will need this controller repo path (`CONTROLLER_PATH`) in [step 2](##step-2:-set-environment-variables). - 2. Created an EKS cluster and setup kubeconfig. - 3. Installed "appmesh-prometheus". You may follow this [App Mesh Prometheus](https://github.com/aws/eks-charts/tree/master/stable/appmesh-prometheus) chart for installation support. - 4. This load test uses [Ginkgo](https://github.com/onsi/ginkgo/tree/v1.16.4). Make sure you have ginkgo installed by running `ginkgo version`. If it's not, you may need to install it: - 1. Install [Go](https://go.dev/doc/install), if you haven't already. - 2. Install Ginkgo v1.16.4 (currently, AppMesh controller uses [ginkgo v1.16.4](https://github.com/aws/aws-app-mesh-controller-for-k8s/blob/master/go.mod#L13)) - 1. `go get -u github.com/onsi/ginkgo/ginkgo@v1.16.5` or - 2. `go install github.com/onsi/ginkgo/ginkgo@v1.16.5` for GO version 1.17+ - 5. (Optional) You can follow this doc: [Getting started with AWS App Mesh and Kubernetes](https://docs.aws.amazon.com/app-mesh/latest/userguide/getting-started-kubernetes.html) to install appmesh-controller and EKS cluster using `eksctl`. -2. Clone this repository and navigate to the `walkthroughs/howto-k8s-appmesh-load-test` folder, all the commands henceforth are assumed to be run from the same directory as this `README`. -3. Make sure you have the latest version of [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) or [AWS CLI v1](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv1.html) installed. -4. This test requires Python 3 or later (tested with Python 3.9.6). So make sure you have [Python 3](https://www.python.org/downloads/) installed. -5. Load test results will be stored into S3 bucket. So, in `scripts/constants.py` give your `S3_BUCKET` a unique name. -6. In case you get `AccessDeniedException` (or any kind of accessing AWS resource denied exception) while creating any AppMesh resources (e.g., VirtualNode), don't forget to authenticate with your AWS account. +[//]: # (The following commands can be used to create an ec2 instance to run this load test. Make sure you already created the security-group, subnet, vpc and elasctic IP if you need it.) +[//]: # (Follow this https://docs.aws.amazon.com/cli/latest/userguide/cli-services-ec2-instances.html#launching-instances for more details.) +[//]: # (```shell) +[//]: # (aws ec2 run-instances --image-id ami-0534f435d9dd0ece4 --count 1 --instance-type t2.xlarge --key-name color-app-2 --security-group-ids sg-09581640015241144 --subnet-id subnet-056542d0b479a259a --associate-public-ip-address) +[//]: # (```) +### 1.1 Tools +We need to install the following tools first: +- Make sure you have the latest version of [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) or [AWS CLI v1](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv1.html) installed (at least version `1.18.82` or above). +- Make sure to have `kubectl` [installed](https://kubernetes.io/docs/tasks/tools/install-kubectl/), at least version `1.13` or above. +- Make sure to have `jq` [installed](https://stedolan.github.io/jq/download/). +- Make sure to have `helm` [installed](https://helm.sh/docs/intro/install/). +- Install [eksctl](https://eksctl.io/). Please make you have version `0.21.0` or above installed + ```sh + curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp + + sudo mv -v /tmp/eksctl /usr/local/bin + ``` + + ```sh + eksctl version + 0.127.0 + ``` + +- Make sure you have [Python 3.9+](https://www.python.org/downloads/) installed. This walkthroguh is tested with Python 3.9.6. + ```shell + python3 --version + Python 3.9.6 + ``` +- Make sure [pip3](https://pip.pypa.io/en/stable/installation/) is installed. + ```shell + pip3 --version + pip 21.2.4 + ``` +- Make sure [Go](https://go.dev/doc/install) is installed. This walkthorugh is tested with go1.18. +- Make sure [Ginkgo](https://onsi.github.io/ginkgo/) v1.16.5 or later is installed. + ```shell + go install github.com/onsi/ginkgo/ginkgo@v1.16.5 + ``` + ```shell + ginkgo version + Ginkgo Version 1.16.5 + ``` + + +### 1.2 Installing AppMesh Controller for EKS +Follow this [walkthrough: App Mesh with EKS](../eks/) for details about AppMesh Controller for EKS. Don't forget to authenticate with your +AWS account in case you get `AccessDeniedException` or `GetCallerIdentity STS` error. + +1. Make sure you cloned the [AWS AppMesh controller repo](https://github.com/aws/aws-app-mesh-controller-for-k8s). We will need this controller repo +path (`CONTROLLER_PATH`) in [step 2](#step-2:-set-environment-variables). + + ``` + git clone https://github.com/aws/aws-app-mesh-controller-for-k8s.git + ``` +2. Create an EKS cluster with `eksctl`. Following is an example command to create a cluster with name `appmeshtest`: + + ```sh + eksctl create cluster \ + --name appmeshtest \ + --nodes-min 2 \ + --nodes-max 3 \ + --nodes 2 \ + --auto-kubeconfig \ + --full-ecr-access \ + --appmesh-access + # ... + # [✔] EKS cluster "appmeshtest" in "us-west-2" region is ready + ``` + +3. Update the `KUBECONFIG` environment variable according to the output of the above `eksctl` command: + + ```sh + export KUBECONFIG=~/.kube/eksctl/clusters/appmeshtest + ``` + If you need to update the `kubeconfig` file, you can follow this [guide](https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html) and run the following: + ```shell + aws eks update-kubeconfig --region $AWS_REGION --name $CLUSTER_NAME # in this example, $AWS_REGION us-west-2 and cluster-name appmeshtest + ``` + +4. Run the following set of commands to install the App Mesh controller + + ```sh + helm repo add eks https://aws.github.io/eks-charts + helm repo update + kubectl create ns appmesh-system + kubectl apply -k "https://github.com/aws/eks-charts/stable/appmesh-controller/crds?ref=master" + helm upgrade -i appmesh-controller eks/appmesh-controller --namespace appmesh-system + ``` + +### 1.3 Load Test Setup +Clone this repository and navigate to the `walkthroughs/howto-k8s-appmesh-load-test` folder. All the commands henceforth are assumed to be run from the same directory as this `README`. +1. Run the following command to install all python dependencies required for this test + ```shell + pip3 install -r requirements.txt + ``` +2. Install "appmesh-prometheus". You may follow this [App Mesh Prometheus](https://github.com/aws/eks-charts/tree/master/stable/appmesh-prometheus) chart for installation support. + + ```sh + helm upgrade -i appmesh-prometheus eks/appmesh-prometheus --namespace appmesh-system + ``` +3. Load test results will be stored into S3 bucket. So, in `scripts/constants.py` give your `S3_BUCKET` a unique name. ## Step 2: Set Environment Variables We need to set a few environment variables before starting the load tests. @@ -29,6 +152,8 @@ export KUBECONFIG= ``` +You can change these `env` variables in `vars.env` file and then apply it using: `source ./vars.env`. + @@ -44,10 +169,11 @@ a VirtualNode, Deployment, Service and VirtualService (with its VirtualNode as i "2": ["4"] }, ``` - where the node names are `"0"`, `"1"`, `"2"`, `"3"` and `"4"`. + where the virtual node names are `"0"`, `"1"`, `"2"`, `"3"` and `"4"`. * `load_tests` -: Array of different test configurations that need to be run on the mesh. - * `url`: is the service endpoint that Fortio (load generator) should hit. The `url` format is: `http://service-.tls-e2e.svc.cluster.local:9080/`. + * `test_name`: Name of the experiment. This name will be used to store the experimenter results into S3. + * `url`: is the service endpoint that Fortio (load generator) will hit. The `url` format is: `http://service-.tls-e2e.svc.cluster.local:9080/`. For example, based on the above `backends_map`, if we want to send the load traffic to the first virtual node `"0"`, then the `ulr` will look like: `http://service-0.tls-e2e.svc.cluster.local:9080/`. * `qps`: Total Queries Per Seconds fortio sends to the endpoints. @@ -57,32 +183,64 @@ a VirtualNode, Deployment, Service and VirtualService (with its VirtualNode as i * `metrics` -: Map of metric_name to the corresponding metric [PromQL logic](https://prometheus.io/docs/prometheus/latest/querying/operators/). +### Description of other files +- `load_driver.py` -: Script which reads `config.json` and triggers load tests, reads metrics from PromQL and writes to S3. Called from within ginkgo. +- `fortio.yaml` -: Spec of the Fortio components which are created during runtime. +- `request_handler.py` and `request_handler_driver.sh` -: The custom service that runs in each of the pods to handle and route incoming requests according +to the mapping in `backends_map`. +- `configmap.yaml` -: ConfigMap spec to mount above request_handler* files into the cluster instead of creating Docker containers. +Don't forget to use the absolute path of `request_handler_driver.sh`. +- `cluster.yaml` -: This is optional and an example EKS cluster config file. This `cluster.yaml` can be used to create an EKS cluster by running `eksctl create cluster -f cluster.yaml`. + + + ## Step 4: Running the Load Test Run the driver script using the below command -: -> sh scripts/driver.sh + +```sh +/bin/bash scripts/driver.sh +``` The driver script will perform the following -: -1. Install necessary Python3 libraries. +1. Checks necessary environment variables are set which is required to run this load test. 2. Port-forward the Prometheus service to local. 3. Run the Ginkgo test which is the entrypoint for our load test. 4. Kill the Prometheus port-forwarding after the load Test is done. ## Step 5: Analyze the Results -All the test results are saved into `S3_BUCKET` which was specified in `scripts/constants.py`. -Optionally, you can run the `scripts/analyze_load_test_data.py` to visualize the results. -The `analyze_load_test_data.py` will +All the test results are saved into `S3_BUCKET` which was specified in `scripts/constants.py`. Optionally, you can run the `scripts/analyze_load_test_data.py` to visualize the results. +The `analyze_load_test_data.py` will: * First download all the load test results from the `S3_BUCKET` into `scripts\data` directory, then -* Plot a graph against the actual QPS (query per second) Fortio sends to the first VirtualNode vs the max memory consumed by the container of that VirtualNode. - -## Description of other files -`load_driver.py` -: Script which reads `config.json` and triggers load tests, reads metrics from PromQL and writes to S3. Called from within ginkgo - -`fortio.yaml` -: Spec of the Fortio components which are created during runtime - -`request_handler.py` and `request_handler_driver.sh` -: The custom service that runs in each of the pods to handle and route incoming requests according -to the mapping in `backends_map` - -`configmap.yaml` -: ConfigMap spec to mount above request_handler* files into the cluster instead of creating Docker containers. Don't forget to use the absolute path of `request_handler_driver.sh` - -`cluster.yaml` -: A sample EKS cluster config. This `cluster.yaml` can be used to create an EKS cluster by running `eksctl create cluster -f cluster.yaml` +* Plot a graph against the actual QPS (query per seconds) Fortio sends to the first VirtualNode vs the max memory consumed by the container of that VirtualNode. + +## Step 6: Clean-up + +After the load test is finished, the mesh (including its dependent resources such as virtual nodes, services etc.) and the corresponding Kubernetes +namespace (currently this load test uses `tls-e2e` namespace) will be cleaned automatically. However, in case the test is stopped, perhaps because of manual intervention like pressing +ctrl + c, the automatic cleanup process may not be finished. In that case we have to manually clean up the mesh and the namespace. +- Delete the namespace: + ```sh + kubectl delete ns tls-e2e + ``` +- The mesh created in our load test starts with `$CLUSTER_NAME` + 6 character long alphanumeric random string. So search for the exact mesh name by running: + ```sh + kubectl get mesh --all-namespaces + ``` + Then delete the mesh + ```shell + kubectl delete mesh $CLUSTER_NAME+6 character long alphanumeric random string + ``` + +- Delete the controller and prometheus + + ```shell + helm delete appmesh-controller -n appmesh-system + helm delete appmesh-prometheus -n appmesh-system + kubectl delete ns appmesh-system + ``` +- Finally, get rid of the EKS cluster to free all compute, networking, and storage resources, using: + + ```sh + eksctl delete cluster --name $CLUSTER_NAME # In our case $CLUSTER_NAME is appmeshtest + ``` \ No newline at end of file diff --git a/walkthroughs/howto-k8s-appmesh-load-test/load_test_flow_dg.png b/walkthroughs/howto-k8s-appmesh-load-test/load_test_flow_dg.png new file mode 100644 index 0000000000000000000000000000000000000000..b91f035e5c0a1648c5a39e47743eb0cca98f6e75 GIT binary patch literal 50672 zcmdSBbyQbd*EYPBltv__RZu`c5Rh&$5D*C!>6Vi2P*PDq5fMp66hTTrxzebr=6b56AV-KtgNGmp;q_}W$_RqNh9JjH zak1eiQZ|(t2*QY5myuL)ikcg6S5Z-q5!fl(x!7o@KKnJ|ON0mKCd(a>Q%+vkxa^S`u$Rebw?jaf{-GWLZ~;j2jc% zNhQ)_r2G6LA|WLuRFirnATKjcz+&#nVu%gx zj)aUi`78tJOIrfAkF>0zCtGwXcI4oB_OlnN==h+mubwh8jTus}{{8IVZNU3TNn5r!oFKe+k--lE3k%?XMvpX&u{TF5k^KASD&7!n^+OXaCd2QL`9C-gp)F5CKznvH;E%NaVkCc_N87IX_;=fAU*vB6v!nld_U|_S z9`OI0lO~uFl8}(looP(`>eZ|6?tBZ^jX6nEQ`1Gy6M6A0A0(fx2M1IO7T`rhMh5DZ z`$^)KRZgm%<~OIZv9%pII46H&v>At)ChG2CKtNQeWI}2x5wB5w2u`%3*!I~EA3lV< zeOq1GX%YDB*^f{~u?wD#M)hwW=jRKBm2{yKH$GvY6d~p=LwWOrU!Sy^S{X`#xE z8+d+1ba+gdpA=(L^M#KdtiWs*?7xYBVX7%seSmdE5B?0-t?1Cz`G=(*_lgSr!Fyr+ z#DxnNY#N`j`wSZ>|LGeUjM7E{@8xF`TeSyyG*thwI@WLBzA^epsHnXDRUAV6^x3l~ zB**XFyC=$7_aEzFJxNB!MLbJMbpbiy{{0d0{aYHDhFvz;m2G%xtmB6y97d`PaiZE+)v7cUYXJ9ccv z{`T$Lxi%Be$~QCbayaeI=U5C@>2MFP&9ot2=Wf0k4pY~fS)b}DaurC`D(ao>D)?c} zFOB>7)vJ?{vHlxd>$6?ESssoaX;;=B2Vk>ziZe4$?VM84a-HXsk&_#1j22EzOOv~L z6&qe_H?5SD)|X?&K!7dHRbb8~EUZEQK)koNuP?)}jv{iiO{%sfUhWvesu=V7Fy}7E z6~_hIt1oDLhlh3i7sFq_#w^kbCSz47eV9GEG2i!=#}Lm>h%PB9DcxyV52j&lXO)0K ztUq3|@nujD5v{lftx;yr5)ueB*V@(=ZK+{Qr4kU} zQi*!#2eD0wxJ)EuWZoAwz7O|io2>{YcRYi&0OMnXp4d(`hDm3<;nc)~Wyv&d`cPav zILWnTZ*Omtx3%Sj%M|kFjo;jyITZo+$4{Sn*QN2<`o3Fan)VXnWXJKCb#mY`jh7XF zg8mX|-c(SyYH69;=FCRPrS)3+Q`8KN3XCu0iY z#z)5!SdEC&5(55=>XM#MR5I+^$M4?Jj89GV^~;q%%$7>)(w}OK=9hlqg2&c-j2|%{ zDyAD58FAQeesV)eDdbipzXc6ajo}upfD_*m0F!u^gCf%YLB8Dy__egU8d_)B^Q~f` zNFiDXiwSd~KX1&gAorw7ZEY>1PwB6YX9u^iBiImPvYr@KQW#v_vC5eN>)`XkrmLfT> zs;bIA%f`j!PjY;Bb1}y?mhd!xH{rg^TOnb_0-pWhV6;BZMNP}sIalK(^-~J zPn%Gyz#`FkOq#ZRVq#)CW^HZVoojt&&84W{qbp5&@lv9R@pJGROg^Q)_>@tW+H$DX z1_UH>xzMRRd-jacC!Kl$3xTguiMm`SVi4nK5_#KT1izyMZLwkh5_vy!&~aPn7mSQM zNlD4twcjGgAeq+Hw_Xt~$cBuQ7vSLmGD|C5!DCT~GFg4s+NuoZ92;x3>Le$J13ME= zAKlfhj%0P{!i&=ab5EW*bb=Ywj%e}m@twbLLGtEJd>b1Z5*ivd?RYG53$glQ_uX-R z4mLKa>(`$Wur)T?&GqFFgJ?50q?W?HeED*B(GYk7!(o$$4`C!Hq@<)s$jPxv9j3c- zt;e{H>Z#g0I%+n1GW2m8V?^um4L06Lax)WX8TVCXA@EcefoNHOyBcd{hHvAF#J_FtjUUG;$hxdhni zoQX(jPo^;}+y~jf=E1?Bno!ohLeBH&t4EeN<5tDktO}R?8dhM3pXcJj(Q#!UpRa#K zavbasPEebpaWeXZ^hp(*?1{80zJPH{Qu1+>z!{&?x0m#rJ3CptpYf+vkcQUySdUd6 zb8^C-gB3x{{hjRhov+}t_4cr;UrHn)U4M?^p**pnh(W!KxQ`z(!#Oo=oE$ekePVe~ z=+t5)7Pyh0YRvv?cGe>G0T!~ny!=+!k@H<-aS?x3 zh(6xzEI5|B`Ig#*N~JpVU7M{E8UK)e%9w&?&SF>BdL*?4>E+u28dDgl9kxAg2R?C1K4G1b_?phi9)} zp8_j|-_F6W!r0ZBFI^%9L0MU;LBl;j~?vr@+6);caDjX5yAGB927mc z%@TTj@KBc5Veam8I$?_W`FUI-qRPpJNKz6KuhCJxQxp`noA^XTZ;T~6Q#Aa2d@!M% zA5Bffa8y;-*T3$QE+{#HeEj?wJ%!=LcXJ;xaZupHYcEO*nnWR8!If8RBG$!7do zdwaWGfhg*NXlXeXW!vB!gVRq%T^+UHwokXb0e1lafsS{59`Y>{`WUh!*Y+tsk*C47 zfX~x`qO%ewS3s99Nq$vnb(h>kUKBAK!$8tCir3wy7`SNe5XVh!vMU+B^LHgvEAQPx z-i&G_2D$YmY+CeIqd>V`ot}{G(xppp`OGLDK72SHJ$Njxu#i>?7ZV3(tUE*ht$-Ch z_!6*65(WlhCd}5>)(`+;04mzsRS;r-O^g0q(mS8zACnw^|NeblqNZl)Gm>p3{^`w=q)Z4SCBVw1yBuUX(wV{ayqG*@fD!*t=o3D~WK zgoIZd=BVJU{N~+^R{4o-Ga*q?&y$ka0ORS7SHHaCymEH3DV7*KBK(l!K`x0DF8Y56 zmqG+w`itF#;aECvb?Gz_1)6dLQ3P2ru2(K+d04Z+6ZnV;dvV9u>}hd5C8n-)G-)xF5%&!sJT8+C zHx>LIkMNtEG@@3?dVgKQIOJZw-G$Myv6mqsj0{b6b>1Y$Vb^j;$n#*TsHsWeJ_a*E zO5tGE<%t*{3;qr#ED|kUmsN#EPq3q@Ar!uW*l9@uX2YlC>~`mb^9lC#gCZAl^)#*Q z6HND&E^u%_Xk5t7$XL3t4uLPat06SuVi2Ht(GAfbGTdiX;!ARzm|n!I#UyK&LnxZa z%Ux+EzZjxKo%*+z1}uZ+09RFh`S$JP#KZ*8Yia`QuMjd;#1`sD$t%XR=UzIUBQ?3! zy&!Y;eM+vaN&I%E+qTnuR7#?DR(XQGoy8_gnjHLd<#2MbsWyvTeN;i_Ra+OI+nR4} zgRGX%Q3fFxE3CO$>1Rd72&~%`m$mZ%?+BPMA@(*9*quAF0e+u9LS-kwJ18kDKY>$pe0-eo z^5s+DH5@``T=Cd~8*jImfLW8-g7t!1b?(ubw~Y~V@3mYhXfcsdQ=>ro;s7B(5wxAG zZc@H|`|;jhqj&rQOPiTwLc#odEb@zS(ZY@j*kOT`TxdjTXlQsh`(fBEWR?({eC4BG z@VDE`C40XJ6>f|}2@|eu^*F`E>_e(Tnv)0E*yJ{ZM>#b56Knv00ZjvH;}4dT zmDTOfwdP4I+@6dC$X8um?L~4Nz<`vSn+UjmW9M-r2g zq*PQ+X?biLBGKgqS-i51Pi{E&N#iTjy}2lT=MLHE=%|4bkMFtk){uwv) zH+=Pces8{g@?Kpb$#DpBY|gg{FBW1T0l~pJ)C;FhojRL#W8!=wgo2b3#!4p$39%6P zVQEv7VyZqE=!##^SmldQvaT0b&nU!-&0kIwp~=b;TRVlu4CDF`+^z89F{PWfu^<~| zKTIy`0Uki;vBSV30I%x)3?N;}gi|B_>sJEg(QxT8@HU3Er~D-i=ejdQErTJ_+;%cI z|4P6nC@A=CNGLkBDO_yQz-8s*?&;#vhT+5M4Atj%y*d}mZw&0ZYPvL}3~^ROd%6xK z7%rZ3^n2YoT5G9&rzA$w7b2N7`nu?uY+knY_4V`Y?9#TjnQok7V)QV9vha5i#zn&4 z$PUi3{)N2L0+tZ>=XT9HsB3D{acUKI&(=(|R|ay+=d{*|P!$QO@t-Pij~_c*h67MqOvZyvPI@TeXZQ%v>;IxOlp z76l}EYWL00B5~G-5evKN2dh>2?htdiwdfPF_3q!!Z+(KdO+mEJteB+jNhNS_;eC|@ z@7_hb%a_Y-k~?R3G^KsEyJg!`#GmJ{`|FBraOYlL*sYwIWr&(D^t@9sU7-;EMWZTC zDpeN+dkO}IhF~v=iHRPrp?-eYklUh#%!fl09Y83U-u;cik5?hnR`^y*@k+#N7v-D- z+Xuw&dh`n-yPa*@3ZC}SiD~b;Uo+iLJt?4l*I;w6fs&$z_afnLu|sZ9l1~HT)oZz< zQm_#6j!;(1|H!l4zJDK5g)uI_E_ApvF|4%TQS5mzlOo)I`;OwnF83g_l6NEN11@|G zmfNk(0`qU!cZaMnrU@Dx<2pk|A_BVe!0Wqu2S=gCf2*vn@XECm;CGa=Jsh2 zCF-2snfxp@ejaMr7R^!Gqidp(t2faHk>SBJ`vo~0ao;Rt2Yru*a0~$;0I=Xu~Gx{cr?TU;C$oPvB&UHYqc?~=YLjwdMJxnp<@(yjTw7D9->+;b%v z66NjCP2Lte^;+@4gk8j9Oava1Te6vK8E)-ZG(n~=IH%q4uBqMpJVXc#%-%ONHKPgG zd!xcAwcSVo&YGJKe22f@ZoJ+ig??67;&c5vpzDETXj`@3j}fr2&gyS)77{AOh_k`rCwfyCUAcZ z4)EbvPcVGGV(!>whyCo^35F}0leOXsl5+aIy=)8i1@c{;kFr*=(+4iz;UB^9-!F*V zpyt%f_fZtzfBhn*kHtr65Qm>%T|-*h``bQ3sGtx`a`WcRmtkSPKYz-6=e&jczK|Fr z1$=v+ojup*5+4G#cJBf+F=^e>Si0}O7MrxP1ndT{b@tfUyA~fuUf;TApZ~n`1~bKR zgeHjMirc|pCvD{BSC5aL9i})T5fM+rwOpS52VQBL0-Id<@ljuvMTp$vV*2kDF<@1CE@lk$t)D~U;EN8y0+fS4 z6AlH*;OA9gQ#K2y}0n*{PfIn0vGz{9QPC<3f+ZxclA1Y7HQ% z@I-z&ka<6wvW`?%T=?1>jDNUq|9M`onc%X=Ivwn?R4$8{f}Gi`@+#N18>}fvnbBR8(=zcG;f@XpI{@OoB|*k?z$`d^835POtFaA6`) zEZaGiW%U#iXEbFRS%oimuFP$l#j<|TJV#6}Wlfy_{oee%XzFH*TEsWGp6?L769Y+i zJUD;gXPng9mg=X4iipoFy_Z)v%ZqLt>W4OrIhGKUIXcFCbKYAyRFK+r&n)k8S%~Q@ZXYJBiOOTQaWP|LQq@kxJzn7QKDJQ^|_e z+*@B*$uGDrbsX}&?^7#^=l!T+jCO2v72oh$9=}<<8^>XI=zOQOcpC2fW&fvm*!QV-VLM(5*FQs@hEDI_k!F_O&9C%Qy95gK}^p%I$v`#Kt#cCsU?!3LH z-!0Ft;*XDQuBIXgXRwA*+dIy%V-h)w6F1mBt#bbI!T)Manjw*OV$pcyW{J3vxvH=N zg!~RsSqYlLN=I?iS ziV|GVWZ;U(JBBn5>QwrAMq6#>;~xq!2n_Yjr)%!)EBh}_C=}$`gacZZ8MprDQic`f zoMt`jW9Of5++><)3?kzXj@{PgJ%+r&=53TMG~CRuYN>@?T3z+e{Vb*jzj9;82c|QO2GP8?Dqtio$<9zcw!on(6Jj~3& z;K9h9+UHrY$v;%3Ewmdx@HF=C%IdunuKHge!?xp6o`6WWx_+GdqB6jO-|HD~DR3zf z-Pd1!&~beMa%Awy`YRP3Upq2KwwNl58%d9A&2d8;)b4j7+L7 zJovjy%tK{#T%<3}?s8rKW7&}URT&8iSJ6ke_o9C~=^rhC%XPNjTm651X`~A)a5Nf@ zNiDs-zn1L<#GMcz7CGKHT#krrX=xc*-b5bsEj!rR#ntpb=MPNK-Wiq;&7<~}J(pPQ z+BCyszrJZbqQ_*}{}ouJUHv&aXS`X4e>pdQu|&h{`^&%D1-E|HI|FE+o|dS6cZ(Za z6z8y%dke=;_b0xm#B#?h{Y=y1!Tit&1C`$v_3F~WyBABu1m%=+icSUt!8TcA78YjG z@Wt-zMO&ma)Xmo%F4KGCU|nSZZTu6W}OxXiqP9Ksx?}p294*WTzirSB?lhTu{*Y7Mp*E6l9gFX>$!IIYSEYe zhfZcEQr#T06=&@$TY_>jGEN*k)(vpkRz+Z>%Jpw|`8Q~{K4|YC-PmE7l$_{LntWtN z_xVo0%)Kn@_p(;Q5s~marQdApG&@i3bl~Xuy3S4AhOo3eZbkmu+4=^+qo!78Au)VA z?e7hUxBaT?bJ0_A`KVtuN*Q;1Cyp)6q^(Shr71*Xz*L*}6yqaH6J5rJUfZA3DbpWn z3Vl^$@8zJ7ktuotvr#$jv+cxk_9Pi=Q)PB5Pk+AVIqg+=nVMTVeZ;+3-SO@dTPw4f zh?k}-G1GV#*Jwn@HMiUuvgb}C#CP)^V*L8Gs1Hk7`aBO{uE;F!sIT~N#}`{fg0nzS zA18)x`=pKH_(*leovo>*maWAS7m9f8XXS4!gpVO(m5OhebhK6quBbmcaYkzkqtmh; zK~z)*D;M=y(HV~+Iw1Vis2~%X$BX;X^VQRUM0M0asFr^)=scqeT#r1=teWrTOGL*A zGyOI+_kmc6$b|Hs?x;$Xs3&Q7+NnCF*6!}J;SrHrM6g)n#tSZ$W)(QI$GBk*fgt+N zp+4>HQW-b(0FSN9s{b`k2}NDGASr}KkUs&2WaVoBzrpog#~s!P6AO#5!!Dw=eT?#@ z${&WgckC%|bCs}_H{>@QUhwyr7k>LfSk-aD%)8u^{cwNk9{!`8NqTto(%#|$E@EP3 z~&l<(rDXd!fddTg(Za(<;S+8jRYT&}_hFo7j(>@YHHU$N61am|cP%^3M7LLblH9atW9B2_#Ce%Q-~M_249ueuAx!XY&Ub^I}>LE^>Ec zkS52wHtApNz(Gx4!PU@ZsrJRC*`Z3iM&BAiQ-QT)GbO?034s-NutzX$ zp7Hk1j=H#4g&gFi<(ag?--zrAG#R4fS762Yjb&g8xgYk&>SSb{u_#wV9mvS5O^eRk z?wc_LcYB-Gz%0=vN6ld4DzkSj^7F5){kQ*3ib(AXB#oW*kKs4 z3|P(724J08Cyu9LhF$D)=*1*OsP>gf2`{2_RBH;J~ezcWa;zu!(*_A zsjI0}u!&IAjULwQA0u<`pa55?sUxnnx0mg(HI44MMvB^=n8T~6oDb2m-Al_3NGa@Q z=S~@XHqzT^)%N)`DxGdw9Y*x^avEY`lCt=ez3HqLTRriT6F#4wKH2rM3vu^l&H}!6|eLX2vX{U3qt6Lh+T!Y1W@A=5nee5>pzl`qrP?_ z(d2Oz)moH@bQ~5rvhuaOz31sl>Cio;?X&?LwC^orm16=;hn*~Ld=ZhtQ>kySGH=Sn z+*0}iRvaD;4nxa*w&!KEu$8*^joZt7)Pq^csaWr>9`~Zvnq=q}ymS!VZoeWbK9J3T zjQ1o+v|T%U@n#u&T0=-$-aYG^2lCnTz4oV3XJYjahs+Efd~0qAytZuP^=--nx$N)R zx(6S^_&whfZ?6aDx+UD@%pCPz=WjX`b2#~HQ6MfmC8aO!mFI!R7LJ2LDvsA{zDVdj z%OcY^aA$%ivy6dN=UHm3m;#v(XuU4mNzpgV2-3tHghKp?FsDG~W zcvvP`5L`tA1 ziL(>IIp%$+haI>56p-O$wJx`^y|YVb6A;}QbK>c1Bx6!^y1g z*@A8dS{9{f#A`}=;CmY&#`myR&i-{`GrjQ^Z^e`?YjW6UgFEpY1NK5K)+e}EEz zqd&nDJf>OXff4!rfO7C|E`ga?@3Yea>^oUv3r^Sh(B1Ym{LgMHtO%q`UN}yu+`s(l zA?=Rr5wiQg#cKZt`0N!M`XY3Tpn&f$01QF=zaZrQf((7D9RX;{KwjzQKcZnq)!eaj z9Y7bQ<>Sk6{s&m1vvu?l8s(;Iy~6t%VmYdS;_~a{|8pKhAanPxr;lP{yqAkGzX5>Q zVZwfgh2z5TI%Y^mpZik!7cTY}3;EZm{tnRoe_v#I$0Z7tMU^lNEPv0+N}N7PsQ1TZ zs8!RuEmu%(C70~5yX`GkA@mC>Vh3CL1gH54{D~RVdPHMO-xIH0DOzj23G4^(R^h^f zK$(MPrBsJIuDDpsp3;=$Pb|%ym^=KGHO32MT3jB>__$0vh^*H0ZcT7`K1DyL;xRyT<*JUs0VWdYuAowRkZ#_?vo|d# z4$?lLKcsNczyAQWi>l*`0GBe8%Rt32>u@>ben^ zm9=*1p!Ze5mb*j$0d$v%gG1g~99wy8Ga`Ui7ba-#$qh?OPRKSI@z%V4(GUjoCinQ{ zB-<7ri}9Ab(X0jfWIj5T2H>ZY%b!#8jw}4CVU3jmE&-I0Js=we?%Qdl2IA&M_*6+< zeQJeESNWvBm=jG&lX;Pa`@UGH>z}+0PojLR7@ZU@nqURSifva$i|yPwENOot;Hla^ zD>`o5OFc?@{dsv;-o2x5kvH#8M8^ntfT>rq0SFD%&8LOk@aII?i{$dFx$zqu9CL-t;?dg0@@!Iw4FZKI1Gn|%3Jr-Jqe=PDkzbY6qwjl;S<0x2_F*W@{ zTdpvEp_j^0OLTg9bTeWHqohd&UHgEqUqSH(YMF$5u5oFQw1QStP3_IER06g=u{)l_ zFg0j)a;tb9f$UJgUskBd9765wqdM^_@UM~152nihKdna0j)?Gw>NSv02AN#w`7FSna z1IjVu?gNj8EYmhdAJ{E&@JoE7(YAj(3fsGEa?18s`vqs>*I)vmDbrbaZrDT*?f1bmx1s!9n5T z;+DR-sDVgqc!Eab?Pb%G@Ldu*x)&Nbmd6lC=Gs4};ll&U%o|>tcspK6Ny#fQ@jNQ^ zWMYzxA$3M_QSIP>a zmyUzei;uXS=8Qu08lJU=Z!l#!9KVvLD!mF)5Okid|!L45&oo#LHe>;;>DHWg|nv^xoY z?0uTTcROb6&YwR&Z5<8V{zP54J}gzP$G#Al1k+GoN1V;t^zFMTYC!aX${E%P6=ZQe1>% z0&M5?J}3@AF!ux2#c((=HTBJ+21poy3$86)A1=fAn2-Qm7syTA!11lfD;OFkxCI;5 zz0uvcR!5ZF0_%9Y_19&}yQt$B6MfeA(pLSb5AhkBzv z5DOHa!Qog{aNS{=Mh+3EU_?{f?kVMgNXCZ*RUd*N ze!RyU_#4z4MW6$HRpb)edrR}-T=jCsFD{HZ2!!xtzIV!FNg-m&pYO)Sb1jvBR z(u#~W=?C*;m$3ixf4(00Y#{cRu(PuZuG1`X=~$3Jby{#1LT-adJAU6@O&IqVF*!Lo z5%HtS=hS0MI%TS?3Q-}_;`HHwe5G-j4)6Cx&-dl%um9?B_fRC-|0O=4`2|>FG1$1z z=|yQ!3K{gfpf&?2fFPe0V=KQ~^wFl?!Dl-o7lb-3gWb8zCW9@|VkFWQXFnQ|itM@- z!6QpAUvN?-i~ZFLYF=#M?m@Ev`Y9sqsX3$=aGHlz$^M20D9t{kr;~tC`nMi~2WIg> z)&a<$+l+#0KVAzxwdHL6MkhbveKMM{H$6*aR7@ban z1qGW1OLtAr84w1^5n7o7O5F9_uMX)w>LTeg4BOk=D40Bc{P&Djx zgGJDYzW@4FzF0x{nukgka49&%^gqGXa2|?-IuGCii1?AMrE=m4+g^9cEOAbg$CAPt z@3F!_3y#X1N*2sKbDJkyMRxMx-0;U)x_{ScB23+!Uc12IBKXc1FTx}fW?d^PDtdf~ zy~`NIOSIC$$r=iv$UFwX-#{_?bb*ED?Jd+*zEVJy3x9)LTy7S^b5WZ zRVUDaAK+xXFX>l8JfQ=22wA}j0Nvy1AyApJX_`nyY`&uiMC?(aQKF{KjA=p6_wOGP z6LB{ehc@qe!dJXOui)R5os|`rpWi!ge?LV{MMVW9z8{AyTNAOuKyd*pH#%74x*~Wn z5fxy9+6^=wwzDjuK@SQX2to4U5Zu+-xiQ0$lAM&Bd|N}qY%{k)uPkTCqi=o@^DjIJ zv(0fo3l~I6-Pva9Y2J9qHf0&;Luy9w?zeIB?M!PMb7>Xg238!0nu8)OE8Bz^IWEGa z0c}Y$kO}fHreTqbi;LICh@M7|C{Njr)RS2@6E}K-L1_wF@)Kk%aR~{sL|@)QfQ^p??_U0=&S6RxY^g&XTjN*lrq-*UYbZ;z3aU;7*<%5Y2V#|#$7%* z*=pwbJRo$T06}}1D7q1Z>=O5S>;A#!(&j;YFIKT~MtYgFDpGsHJJWNGmzGA5^ul&>Gi!tf)+<6hsR1I|*3i za`R9lcC2|jK{LyQ^v2DbhKXQHEA@e8IucNb1lAD@=OZi%G^uG}dU&u80omy>`-!=g z1b8ujALx1XQH%?cSUL|*IZ%Ef!XA6Mv<}CeJfvg9FL6e288e|25Q%K0}?!F6K7u18->tW7GKG$46-v+d*Lm z$&UjL62w2Y?zU13OFhH~+Ytz#tx9gZLvTDV8vrY`V4|M)D1!JK)t7;)^K{ThK8q(t z5)xkJ)-5!0k`u7x{5bpIEhodT43q`!jos2b>T7Mp7c_7F&B;t zO*KNH))?S%BJuF}I0%X`hDSOYvNE;Vp5Xb5<;oQ>t8{sN7&&%s9p^x6o#C%Us#J+) zj1!#Q{L&?2`)hZiu2}!D%u7|@fA;igM#Rkzw@4h%C8B3=m#Hy(e!=C$WJ%fCt~#y~ zjww43MIugDf+yjXZvjgau=)D>t_;B`9IAe`wz~Qed*YP>w=W$J!z8&cUc99xc%GHD zY9fyLsFuKhyG2Uz+O>eW>kiFR%;-tASb2Qjor zPsOpo67{3NP~>}@9!~@tE;nW582y%%{*Fu zI9cqwJ&iYSQ$uMBROI0y=y}*=s!jp8B-is$%%~~GVzoD1+k=`S5DV5jlEm5|i6)fT&EMns6+A1Rk*_QlKbQ!Bu1|rwQXzKC z!7-J~7@q+YUJRl@51XW;lx*Qf%Q0qYVi3-qUw*P!)c0F+_g$PiV8V40W(e%iPDQWdELu2 zDg(9N!3g9UPDxBL5P|!C`RdhiP^-qpF*Iz*m6J23YNyf`VU}ju&_YXd0MMJj7{#z3UQ0h+j1eC{Pjn?9aZf+QkluKU@OU zWl3Kla6|h~LuWdcd>3tvj6R8Y_wZdtO54BOnFx`em;VCZ!_18RRcNS1qu9g4)GuE$ z3f3IDguBejFT=JsUG;WUR44!h$^^9C0T7i(jzqtB@dD+j4x|7Tyq1(Lpcw>c41qJm zTS(t~ZTQ$m6Y@*`6?!i>rGy|x@bWKbgsPA}3vilmtS4&8Ux$U&NVNC&a}3!I7CO@x z)l7~};+LpnKt=X%R`;)H{ip}6<4_z3?EY#GoXQjWi^aRCBGPbT0nS4Z@VA6a^n;t# zx0OynY&YA=sr0@m&7tH26iEsZvIWOP%=E(M9(V2R=vW`Zb|r8d*?buHN1?)0d^b(q*Mt`AC{P{t7mZVz=>Z=eA#!1o%z>kRCyto9*gwhfjDiGWwdKi3tJ+LTC2%zRY~fYFe5@7SiQ0c!&-@UjWr2~;YH+DEp$CFlW( zhsc{|vvxvgkr#r3Lx?3I3c)}iG6W9x0 z2l9QNlCk!#sYhZ|#z~3(C!KkbTFL*9769hcz=h9kV%I$TKIF{MdR*vPx2QJ!9mMfD zi|b>x2{~qFX4W&U=in^J>2il&r4gSC`j_FOS){VK57_^mL1JcS$Ctu&F6U)dge~@` zlDPmfUscWVWsir9e-SUZ#ny-(rObh6CxJKtqU0~1W0VIdSC|w<*CBPVk~*^H1lfjt zc%B(e`~9z4$PP{F*IVu^ijXi1Kw#;l5edj70?2{_@y1y3w0yoG6i|4z6T*U!!(*gf?%Rcz)`Koj?96qRKO+7qmBR6loCziSXeSLjbe&Nr8bgNcF zPXsHYUo=GX+f6rX5fWdAGIsw1NDOLNX;_s&zqHmvjx;30Yl?y}Mb)*nQhIvKr!{ge zva<)OW6%tv$<)dqxTc;K%JS4a`u_cnvGY7Ru%Wg|(QOyU_GKWNc_5&-kxQ zJIs^pm3@V^lwY99VKh_!`$B|&2~xm*|J}UIWfSYa=>cM(-?J0B@htyh4$!+8{+klOCTFI}`VVmcJd-!Y-$ED|tRRv5Zyw_hS%Hq0 zHT1vPlK<<0&G%&DK=o2mfh62ZfvN+JX7j&z3z*HI(Chz2S{&U{@ju4=FNqG&KHw2r z;wt_Y^8AnCqSsOUyAdU(2@o#-N9S`yU!nI-g#K%V)vhTANi$sg7h?m`I>P(?%O^?m z{7@Ovf3G7t`^{rnI%UV96dKEGK1e`ZoB;|50ZrfT5v4Ks1G9fmaqHM?s68s#t1J0A zbhtk>*5rA>8{>K4?Bnk*4RsU@2O|tBj=|W}f0NWl6C!GGwA!adZubs3nB|RUu5$P&PR~^Ep;+;3AFBe>x#^`MjyIY8Ed$@D)ebh&^g0Edcb88zm-{kndCRU3Oq>IB`;wAgG;)m+!#h@-*vso z9DZRGIe$AKCB%e5ABUSqrfQpU7V4i?YvkU6iwgq(l6i@*fL+J{HJ4URi5kh*Pvhjd zW30?pWiGPC0pR6y_|8qBRpD+YU?*}NF{NvAwRzv}7%MTsQ#EVjf#}Xf7Y&rtZYR9) zwwclHlfQ!Qb1|=+19y4U&~OWbGZt70P?73?edo>!UjH%~sv%?6M)`vDcj%2MvNFX! zP-k@19g&?gU^6k$E^Axp(cAAbT^B@KIR|ROrfXqxi(tK~e$;g||QDZt-7 zzI&^N{ZW;o3^1L~+(=RJ)rh{LP8Mr|6P`>kf z{IK28#=?OUPlJOR2MRBbf#wM)I5#L3ha9IeSQsYSE#MQGwSOdgyi9X~Sbn)J_TUs` z9PdK<9QAfkdvwHSX&)WR6 z|57i3ACSRup0p!(opZFAYvo+;J@OP5LTVmNYHI4F0$Fd&;Dc*|3ZFJ_fzyUGsk%qp z;?0XtnFjw%^2}<-aQsDXcG}rH)mYVu-S*B6M_iX=u&B zv;s`8g0S=i(1I``I5;{6K3Jmj*wUD^WR53ldE_@ew%@?5CL-(YcOWAM`QOCkv^ymyX@$>=?H$!n;`h0nw>k*DHtMmh~JH4-PP>X z-C8r7#PFFgA(IuC&pwVhOP&{KyS>wSCA`To_juHCqyXPXJyo_FZjN_>07p^No46fDG*Z%|>YP=_4 zWyJ=PinO%PfxFO!hyX|%AjL9IpcS%E-j2>bU{j#3Q4*otpBQG@?=ZH=k@y)~-Ku++ zyq%+JQ$5jqz|8aMr_Woe#aHUNc~xRXSa%<>SbZnhbTaiTe@}I~cCxl~x5+iCf3D&k(VR)qd)glvTNUfjmJ%SAz7lKjW)H29~8F&+$3KVup}+bNwZX zKc6x3ZuYv)7j>JZEozm>d5R;9P{Jw{-DoLH47rIkV^hGS8~H)oz-`U(`( z%;}`{HWD@r42-=iaK#21&p})JyPMISwr5q)3T!VgL|$Ir>ypt3CEPLL4`n}`DrnR* zobpr+uJoZ~b9Z-t@HPNStMz)m89J>_5@Je900W0GLZLSB@EN8z_4UuRKjg!d^iuxf z1Hmlh{{rFCk>E74;RE|mu4Mw7PguvogvS_0JOd8p7E)U^;W`oM-u-Mm$nksKNcaIo zH6!1&pyp34<$lF2{Ya?7h8#-eTMelyI++&x^+Cz!A@3BbGYC1QtA4w zTe1+pK^^s6F?I0Mrz27c1qG2^`1ttcV0yVqUm!l}F@l?Dpk@p~(0VSwV@@mMC#dvM z3pBk6TL(h2mbNx;m^nypNhB!W+AdpIKu5ZpYs=L<9`LAA#cdlF##Gk`ZbFz<8?69N zGtyd0!~;amP~L=IS#i*}Fi?PAWrCE#nuNe@QxMFZKZOU0Bz-rx{zG91@aJ zb=!J%Ei{D!2~uCF$LZt)>VZld+VGNd5dY)m#NpI<%0!ou>vaud=VVv@F(mlWj^kxw3CP{n;%2*@P-=H`Y^ z?a~tWDLgE2ftRja!L6i%BE5k3@0kb)3EvkN(?cl#AThv-kCl~`3HLZ8&8vKCo100f z^nop$+zk#2nj~I@nU*5Koa@cT2Uo&Bf0}_I$0?wztIIayGB-DO!t8jX{l>T!qeyv2 zk%W}*(Ol7sy7)ld@=_CUky3VcS#>Uud`FY&T176tQ26^EctDVaD}7Nxg<>FM*BlEYj2JB*+5BTW4z=^0h5yu4he{?seSB&f5^DW7;ooD9{eef#eDQid%bWFaYx zlK2)D7hx_p?*I*c3W{8H@5r4(D<#3dRslQ!x;#$#ORlK`aMW<5f$LtHZaH&4>;^dM zSp2ZHH9NR%0lgmT`6bB2-``4@HD(-pT3Po~s-z6_o6hJN!Llw1E{L=b^jyul9?feURBi#6h1KAwz&!&{0S=Je z_OqF>imK}SqN0Hbhe?ymG;j~mz-U2%j#Z#3w+^I|aq)6jRx{<=EgYPvZIhh(E+2 zRqyJFc76O-ENB5saY63B`%8v9bgaoqNpVW&<62DmUGlIC*Kg1K`X!~VP6;=k0C7W? znh1AC-Nz-j)kW{h0Mrzze}#+<&-eKABEl+<-3<%w%5;cyo(Bh?5ftpXDx;Za^6Y)61mWTee^1D)R}Iks^&cqQ$aV@3t|W+0OCv@? zUcdH%46Ckz0p|UbH+=(1Dr^N-5lEL*MbC15Pl>0vHWrV7rX@v3}wFc?lgI zMin-4xG19vX9qNDD--MBaKv7A&Da6t7+N0|EnWd(hhe1{vDQI_w>$ zv=~1H$GblE!5ex0<#}j!b~Xv-F`$7T35jXwH_6G`)iY32OWwc#&ez$(z~G~=(k0KQ zeB|$$Q+)z{+<1_!prA0BJOW_c#N1p444aY-h*_QI%gaK1d|^J%CSa|c}#r*0CeXsUi7_@AO+W5AZ>Bcv9V3=6mV7nXsZt& zf$QGPgQjl4tyD3BaP5=7S~7YxP^tp=+AkngC6%=S!98V5xHuw?Du1Iy_u)M(E~z86zIK>u58IuRsk^MkTGP8WyfGMWx<# z@8Le9Dy^n5&pvb1rLbLFxQ+^3MgLFL|Bt#i565zC--d6EN&}UlGDL%-5-DT~NhCw= zQZlPl3Xw5WA~TgtnKC32QZi&rhGa@Yndc$%Jm0?k(t6gjp67jk@B97tZQr)7ZCf(j z*L9uOc^<=l?8kmgykF_y?!KO;Ma*hIJrl41QF*C z=Q2dtViVTYLH1cJnF}H6SpQZ!I-O)s`eDvrM?arMB zquDcKw{P6|lE3FTG9OP~LenfsE4RCLg(t6*v-1@qcAjn*6kYs_ zpA8QToC3r}L>%^vgExzdi|;tO&m2E{y9M#&9dAYkhU?X=Rf^Yd-!AJLEu(XGXZrLy zCT0y$FoVq?Kx;eO5Ye(k^^E*M=MQR!(1E)uuXd9VVNpMyDF}CRC$mTJ9o-cnDaZj} zL!2_|Gur0DrVHGgA-6nb-c(n21tF3b5@MrY+}qbD0h5Xcx@~6w?lOwMw{yZnLJFW) z*L?HdDcx=4~7_$S$7?Mu*5K+#Iae zSB$lp`la7kAlHd19q0)lMHC-}19CW!b_V4|ACYT^$pzO;csCfKTmVnvQdOl4o2-NT zBBTQ$$OYd88>%;6TNX~Z`ESoc>hyQ)Azxo}bxvBa*_Dj7K8}V;u zQ=8Ak(ux&_x_nGqj$YEMTM7zR!6Q8h3W8Oco1Z6WgA%mg8XG-RQwbHfp*^DX5himJ zH-~O>yRUwE-VGENlCiajX&JU;T|AkKS4ik4I~foD;$r-0XUW!k=H?uTbcw3$=Zp)`S3$D z>kzr@vzZ4)6<#iF-oIJ{@ez|ALseajJJX{E&C!C~8~5&QgQ@J6Y_`7kLnUOLADD9) z7eszteBFgaSP2!@FgClWGy_h#?D~|--bL-Xs-%(dGvF29Q_wOrO?=&MU{N%gCy{C&i zg3xBU@j(sC6#4%BRuq{;Dk$Z_0!{TU+}K^|8_glv$moJYf)H12p3?5YlgHFS5gVp4 zqM`qw=7nF4!rviCfr3(}a&L_Lesbf+sNWk#!?+=ND891bG0 zecSlbYOd6-fZT<>h`hVsvB_fZ^w^$W+yWAx1WcxMpl;qCW+L!tYts??G`+^CIqMS` zqPLq_vphQ7%(HQq$9>ZwG!pgkqqQ2lN%kQ~$AuPJv^R`HepB1nm=?~_BkjQhDq=;< z%n0^`L0w`Y1l8cmX>r2q-WW^|HL34%7qOjtmLx~DPhSaHs+-$7Qr=fTZh!zs1O;8; z;|P<2phGFX2HeX}d3hw9_5!^{R8DNbp<>#;{S(ASBFJ0_iot`l^!7s47(ib~d{T}h zyb1z5)SLk1tj7*V%`C0_d$Y4rGQ+C?9<5Z`&Od^1O8d9LgLddVXL0_S%D}~VSp6gf zr;u!2y}Cy;%kn{37%Lvb!}TUXi+D18DuM`TG6(G!W_Tg>ge#}09coDgJVC>7Fb_hM zv`kEdgw>HE;tNFACzI;NM_x_U_pB$p&!_+h<}=wX^P<>TYwbtYik$D9=a=2V!V>Dw z6u@_N?$~=GY89L9+#1mEWZ8!}yYD{LHj-o6Qarw9SXa6WvR;;xzv*FZM!EvD#@3k_ z86_27{iS_mM4%mhV5MFRH6M6bk0c}Beu+HD@yI8&;+q(bz(WStC;A1I?XSs3Mm)ID zHAPGae=W(+p*<6slJYF+5vth{iHY8vcI5rrY{I$x*6Zo%F|Z#m_f>@RQnf6&@UCy- z6M61am=a(xD!mPG)Rwbjdztj`b|jFr{$bNl32y4UbC`_eVpS0n(lIw-iy=a_34o5T zuP;@j^|Gaa9Xob}q#AK8wz?^?FZdF&F|?~_9&qs0==(8kD6|=X{R~pkH@$l>_!jk) zg;(ajz%-*4^O?weIvCL7d&tze?Y8UuPz_@>pRpau@#)zJdo(c1;#&GC(aBx#@)`%k z|Ah7-vmRw()|@l98}TI2aqY5aF9zTvK@B|#VMEf{>Ll3hW(qdByn=#>j>`G-`Ob1W zI_yOB%gF*nNy^OZ7?Y{EM8`ZW7ilJ13deFS?vJz3WYF{5@&b8 zb$kMgTc&?~`_`IbTS54|cDAKaw$WGibFUayNMHRZxfk?AH#aw%#_u=yH%XhJYDOf; z0+n_0c^X^Gu$9K&mJCTsUb`+O>HK&@&XGlqn2O9x|BAAS8-htWa{cR>+lR==3a_z~ z!Tm65NZp8{;_+fyy@e}Du@F%00QyJ@q49%FAE)1Z*SR1}$!a-yXK23(l7YF4GTe3+?XVpUL48{0V?)?V0ZZSXl$>4ki`ZU;-II>^U=*s8e9*Zlp zlauKH1`xk^GyG_E63+PHd#8Y-LDb2(qiEw$?rK2>u8R7e4LD%T?9Wfv>wT1zF1W!z zcDEDe;Y(?0N6s_I0TC3-i*GIwl?a@FNV4uec+k`1Io>xMqKMrBQ<7#jp^OCgixd<} zk|c+<>Lp*koHQ|c*A#F7#rwtyzQc#TlbQ+R1859kHT>Zzz%WI{#St~Po`i?{0GvQz zZn9Ng=6W>#!lug#KG0B$=lRyM+i=oPKZUxSSmzh?FwNQhgO7XqEfat&hzA7P0d6u# z7K)_2-yJbFS*P`o_YuGgN{nV%-oeCGgaPuSt(uNbOJl?@BG&$l1y2T<^6=y^OR^CmABK@a#YWgQtYHa>2{cu6p26R#GC1?;=#jjmJ+35C(mx_II=FJdPUR%Lagc}S09<2hUV)`v?(kh6Ekz?R(nM!IoLZb zI^{TW<1-K5M|cKOYYRw}LV4C@g~EE%e#dC9c+s*-+Nh)U=?(25n>P zj-2qs#Ki49^i(F2BPMLrT$)qA8MHA5)_6dPb5Jvk7M z_Amvbn}_LWkojd!;ng@DVB6y@K_(ZiK#|eWPQgBxY3P+#dARiytb2-8b#z+KOP_Us zTyUWQ>|S>o`teL5}`}E*cgH^t-fQ)6MTuChDhFjMQjlBP zw0QC|i`g(a^l2VvX?_3xtL$*#Jb+Oeyq+Ef#E57) z(`bT9<&437Awz`nqjQr69s*uSU4XZQUu3$WZ+QiO3@8LFtAos@qU(~IYHP1gf4pa5 zF<$bO%*(U^72Uhs+v3eGnWzq$Ite(fXBRIBxd;KDIGM>e&T?0->}2dgxT_u;*z@9@ z_Z&R~Ly@x!(=I7b>sM7bMrKC!M64&MK@NwSTf5|5Bv^*QA2TxvscE=Af{F@eB3QMA zbT*UTf+o85q`*`^_tRu~?Y0HhyMKKZZnwVZ_TpBzASwjoNh8imR+I%H0$#+!5=pLS z5Z{@Vt&s!;o>1#!(j%y@H*d?_ad`>L~krRE35n;%jaZ?-gEC>g9PI z;I$o7@@uuI>GQW}t)ss$FHkTAG?0#gq1(LRebB0P^txPNI6dHf1eHrGvf?%_+%4!T z?HtnbWxu_ZgzOdswH7ad3QejmWw_gda#Q2*HPGA+BMCY_JJF{?%0sS<-Wu=ZWL}s* z0Ft^lZcxLYLQ>A+0C^zj$%>dBU3qy=nK}bF{)NkD%0&L0qvv*)1F-)+y}jYikPv&^ z9s*Q`IO*>4!@cH?;5ePSCGpg5M4g^c3CpiO(oHkCLP)faO zxBd0DUDZ6guU#wYs{#sG-i*fr6o7Q%4g}^@BNt->gU_hZ&{rwKxc&ZpD@WtnwY^$* zbUfKn-N7+dkA*;di%urc^9uO&8GWP_;r2{~Zl$GbMf*`A9YiC> zef1=WOAo!j1|VT@F@taSV1K{qb~nXSp2qtUkJBC9yoJl@>WIG*UHSVM1IhL;GRWez zKR?UloWmsn_cMEVpukuq~_~`Ik3qjU&TTk%2B*j zpN69>z??rZeMZglwq4Y^v3bB|Q?=PA1&gbev2kZqQ-Ev(&IHV|Va>y&U^GV?oL(Q8 z7#}EXfL5)08u~{NOO~`As^+;H2YO#d^(~*>5diT@c0XOUdNn&LA86n}Q0sU;t^m?{ zEcN{-Xd*u9SF+;O@gF-jy1+UG*NHlY_Go({kY#D$O9*~FPe&1(06cB88}C}VBlu37 z`0c-SQoa~Wc3s{4F7w31_i00`^M!zs!qzpSJ7B~6>*ej!zn|=uA$XKp0;T%$#hw$N z97>r(L@`1-=fF{BtfN*NQJ$E1wY5uX8&~%eFOtfb7;;ZoT9_TFII@ZFYjg=1 zat?gatW&10fv#Z=5AbY|!`aE3SWXl)8241LXllig+rgV89eFt=znWLA9Ce@p$`E&i z?j09Vje>!S;b8P#1Q&Qr8TXvH-=VpSxnn@{#f2^T8eyXMi$ZhaEMJuc3smlIVapc) zMz*ltf^9*U-w)?DcU(Z_57-GBnRB_3_kbCTvr=eaAi=AwyfO4Uo!}ofD7+L6Bc7C? zrh5u`N49fEJVhA?+Q{{;rE#DRAKNQD*w=n8VoYp!bw=B7O@zgkauDH!KOVWIFTj;a*cQ>!mx_uog=*I&%rFR6EDI^5qzOD=MWzY7n2w2w z8ViP6U087^VrN(Z7s&{R(@_$dnye(>l`#shq!4L=vpAY&Kc5DOk(!34W5pqKH6Z!% zVG}Qc9y`hp>7+(r#p0Ui#uBhKy(eOTnJC8K;h`_(NIo@}&3=Iw_34v^pT|av z*-V4A?&@}z2~{<(nnU>*#TVy;%WxmT&L$nPr`Q}R!XsJsmLtSt6+8*uS8tiSacBmt zy3x74!0U9Bb6*`w>7N2i3KYuH0iJ}iR_;Iy4vr@436+G49l&x>8(7Em{us@+qooEP z)YNnBShQ?LSFe)D10c67rRnGZY#%r;b}5eT%Djd*L&ZL9J@XaiDgu)_W$7&^| z?~M!$53kiT^7V6bmF&W9A;~K@0s8-1T8i8t;|P`fFaJW%Z>UWGVt__i4(eYT-MOPb z3cQ!A)JIMu$`FV5Xm*09-hTY{NMc{WF(i_NnId{=xmf*S7pyv;?LWa399)1^3SCP= zUK;%-I=Vnd_ne;1qU8DC7&$n>{2I!s2| z{VDLksc``6c}x%^x3Cs5h~XlmzM#Xu!%7$gT4E+k!bJ}tF1o0U2{)~b$r1!$0BE^Q zc895{X=ryYooVzpQC`!C9QFk|SQ~-m{KewVoOG7fsqkk2i%2odJUBOsi7QvEFa|T8 zbdvDld+GXtCz?d)4vOm@VVX$=JiKtPwyDI`nKFtiKqd-$=*k-a&8Os!V(#B@t*}SV|8d03|!D-1_ zw~k(tFMV8zGlGMtp>&r8tt?_fo1Ep-rlUI*i7#Ni zrvTn>BG@$q_o&3}LD!g}kGt*hAS z|9M+y*5I$G^TQDb=Cfh@SZ*b})-HW(;sZ@$IRE~@Gi!+Znc0eu{d)zT66@hO!imLt zx{OT5&PdFH4c9wT)5Y}J%Jae0r5ocWHxnE2 z%cqG4yQuS7O_-L>#Lle!_m3QRQ9JIk*|d&+>Ex}gIvMOAC|D5MRT8vLmFyAiY-jFp z92AJI6~_vc>{?sB7QcQUSKj?!KRUUY*Kt>;J9Ck^P)HQ>ndRAV?E!X|Q6DGwe=Y&OOP$~ISZQ=~^l>`>o#*@6h}VFdnnB=2 zdGOD*Fgaq22YBm(EC2V8{(rno#@%Jx4_-+vt1I8dOziz;y8rPxiPuLRE+ttg{*P5B z7OTzlKVS6!VGl_1eUtLsMV-OnGOD<#XaCxNue#IK7cNW^v?kJMPNlsEmq+YZE@ z8WDmGTKVtINxZ%<4#=obWp=S=WoABwiU43o;3%TaTEKD9*?&8S2jv-a^VH(OF@L24 zz~w+@4rTSQwD{j$*HUl613(rKn=?b2loZK9#)nmh&~DkZ=@EVncY=Xqo_TpE0Wm`hBLdp|B+R$)f|)>X ziVs`nA0+_jMSVe}_49C1V3XeD^mNB~r8vs=g&@mSH8pL_OEKBX%)B8~p?G>H2xQKZ z1jdR~Vq$t)IBN~*1)x#X9}zi2NT3}O2>{VaBxXTl5>Pl232^~32HZf1h?R+&E$;=i z^nRC-0(z`>xf5Y^Un>q#z>HiAS9mHGrm z*qXI#=>Rt&xAypm-GH^`gvf_sqXRM%e1h2gz)|ccc1+E{1e|<=UOjG(PALuupa7n* zG2;{!4W%i-9is#?rnh(BzA;K#OhUp&V8n!^2U^sRh=_QC**0!IF){HXGIE8Ejt=6C zQzj;S;ys{kW3K^OCu*5MZHXrXT?s?WjwMi@Y=K2?XOW1}!m;w}^R!2i6z>tVQihf* zG=2ne7~6>?&#?#Tl7nn4dM$qr)M-RSlDRSj-F$p{jK&fg_O7Sz*DEY2*mdH>4&)eT z&Ydenrb`tlUxzb*{!(M0jXBw(5$(M@fY$(~qWn^iz1@)NMMIBTZ5r>=ylpc0KnP2y zjwIXy0rLgMEr4i2!YH{15DD3WL&6~W8nb7h2g5M`wYUKT;hBoUJ!@TkPhL<}`Ja^eIr0*~m;%$+IZF&_BItNH4~2YN6|dbqU!9Gy9P z_A_epcu$0A1=0Tnj;Mlj>v&Hk2rEEWfH8t>ON9Y56p9^KFA)$x00UYIP31V_#MGQ4 zM;s8m60M3L%%FM+5V&Kp0Yel(ld;pJz%dfmY}fS-sOJhAw>p$h*Ggia zh0#QJ-+zP^GdxTkSXK_=i!o>mXuV*~aM<6DTjD`8f!ULE-;d9vaG z3(=*mtggO+1c+RW7~=z;SbDl(^KpZt0`|^M>j^7QdvwLaPb5FNYdSmj>K-D4SOW}^ zSWdzuqU7RmljD|REMa?46C^$dL|p+hXkELgb8MjAQi9HT_WlNV99#3iryN{&QRjr z;e|cgII#IhLf(eWn>%p0h&p#<4_G}j@5Sej5^Ppnmx1H_wgBu*%Y~Wb!co{>m}^YR z>KsuMv8ONtFY#2E? zB#yZz7eQWP$8sZCU8}7X8zN@A1I*OE$+rw#sD~5vYq0~~%S9!IgO}jFR)SoBf?_1P z?NN0@rRuJgIuZFxWbOVE6!v$J(t7}o2wEv=vv$CcLGW8J1y%%t=1um_H?U>NZXdf3 zL6M8c&cWH;$X~=9v zW@a|%I9IVkSbo+vl)^4{~Ec`in;vXLv*LBk017} zMCKrhD?j;<=>7i`NB+;bgT$to&9Em^k3Y}+f4EF@+@hkQ3r78JUA?`rJkL)BvUd?V z(%&!OGh1=4{bHScp`zW`R%RiSRe4t;Ogg{N5s)f&^3=6)>diAC72(7g4>ya7B{MTK z7eHqSwCQgR4gLN5zV0|MjU?^QYtCQGWgam7S!~42*Y&_iX3OFx`xzJe8&v^E7w=?;JibL-ch5>8wJq3! z_=^SzixK^0;6$@gMqHpX7M~P+N}g^sA)DM>Te|VU)N+D4Gyd>7vcByUF^XF}SITpa zJ7q2PXWQ;%H$K)~{=|pk!Nh(S{M=G3UZv`MQ6<_3p`?%9#BSNFgn-?DCy8w`(2CoQ zcvF07w(`(GRh(to!a|R@42lov2YA(}ikVt@=n5kma}W>CFNDPY^v4&wQQ!f)GY>Fj zmgTFtM+lcul`F3+H{SUt`)pe)a(`+AT((&F&yr+<7=^!Lom2%Y*KMmCfw?PQ4h*|* z`Plu!-L=awxW#h5ER()t3B;+HXbaRj5Uh`Qq~#tB0ub{8jY`08+I)XPU?}ak9W*#? z#?WctgMTt@cx2I_tt*FlzCiv6aDDn$F`NCss2zpU%M*Kv-U5O7nKebdEsu9mTeaU_ zILfqv7+M)V@!e^9kpQ^%fbyPkILRJ{9Tn|k2L28Cj{>*j!*t+L!3gH&E{*zlSLOrx zjl1yj8zXjLr~CU+9;}@wFUktTi^p8w5{*mvZFcrgZO5^4t=w9B&PU5)LY$Vw_|+P| zAy@x@Juc9cKOdmTFEjc>#&P5@BIyp>Hdjoo9{1ys{OHPk1ChT)W<{jql3d~Ibkwql zJ^y%rdxM7BxuET?@m$^)&#EsPDu8p#DsBfrRcHA4cSS^oUX|XWe)B_1hh~!gx?f^v zyK&43+h8JTj^TQ$)@ns@v){S9p4$qI>)pJ4+X+o|uE*&SzJuwX8-HE66I{6CPm;iF zv)R%NQGS9HS%e@TghG_2R8*?(8?DtIJG-Skw{h?BrCgbv|Dx0q95 zfiGxBjReg8#RVANq`KJlmH$n4HjCpfB8oRMo$mP(VlhQ5VvQ#R}twk|E;NASo`RIlD zyV-3VtRGrB3hBBVp8cR4tsoB;tEyhqDW7f0s!n~*_-8KwxB@IBxB~!B*})iYGD4`e zh{E)0Cw|v(gSihbG;~1_B04UagLHTB(pv|6Ok({vr=A#=JUV(8`4C1e??z)3=5G=L zrAaCp8U!fY+^p%*;VyX%r8C_tMNA+j5F8z7HR|GV2)~b5gXjuE^9N{FXeH3Kv}8hn ziogkFpwr;)BT@rt-4mP=N$^?Q@|_zC5}@`4lZDkFLP3B4B!?dA^~iw;QidM-I|1UZ zfT@I9V_w6h%a>6=+YIxFpc4_~O447ANI6X?M=m!a5nWw>n1o0mzSY(BybW*cJ8NKP zdZxqgU3a6A@x+uuDsbInd;`kSNNONBmthIfey7d zPom0=A=%p)89_ioa7H4;L_qE==ar7&L*fBQmtR0Y4}=>$K0?3Zm~+9{i%2aKH8LL& zV}`+R>F$oi2u-<{D2HyhS-l6Z1$n;uJwm7mJA19(1|s)F2?dNK=o0&2r=f=UDEO-e zCKr41;@YOw+z9@`!R7#a&65{J^mZsQ;!H!m6@(--+`4EcttBBlB7_1L7Kovo&z==H zYbz@gdQK?Qxq?6Q^`_Q+E;0k_clivP*@kQB&Bk<6dW)N;TNdV8M$;ErLW9q);v2R0`OHGse<7V~W2u%my4E3=3V+qQWG5B!}3Phsof%uOI;u2y|9*cXiDx^4fV99@&H1 z91uKS0h$;+HL8=agrt+;k)RJ>KAoTeZ0h9o%Pf1H_ z_-!~C{a-oP9%J<@l<8@kSQNQ8*^v00-(pEv?v{eC(X4ee{ZdhfZ6;SxfPY)>phkN0 zh?mCb*qcmN8N=a%_Neg~GiKr-6B*Q<gth^oR7L1V9?xlk|V0I4;nrgFt!l= z3?Kw;nqN1f9$UaXQ)%dDFF1B~TT|ud7d5|2n!5Q0vU6CMj5`AihfA+?jlZY4+c|Z3 zXL!J;WIOy)@R(f_P9%|8p4ry#HVR}2i6=4U5Pk*3mC?$`(3~$o0Z#MhZb(4e zv-RXH*xAe56dSf+|DsQoMAXRY8(4x9(_f3zzv)+0jU{w)J$da|ksn|{B)!0WSRZ1} zHu!T%*btwcF-)Eor@`{-o7&oojhPlcG(=vmISd{o*ov9vgObL^Z=0rmjw|yPtHmNB z*o{2Wz8N$%SpPdAEiCKkM@29}yri-X<_Uxkli%{Ff)RZHHMgu5VNV-Bn$aK)0*>+i z%oiMRH{|HBYbg29VX7~|1;QNmLaSe&aSm)-#WN3vhicqNcKm+qoA>TC3--1zJWa2c z#N^KF2)Yk->D(B&qwS*KG&wUcAn7yR1@;Sd$;m$mQCy%MW~E)0Uz5P{8f7*hAF$ap z%fq?RsDjXRq()VXAX<>V0p^5tB}(5gG2!+UA+1^C&KY&6h^?k!Ag0v=p-QdP6(nNg37YDqI6Y(guc0-V0D$4ohgxzs7cmjkf+dPA>^drx zD~_x_2J30@Q*mhG<*}-H`=JyY^X{!V?tJ{m9d<9JR$pV$Vl@54Cs^xKSz*(zZ2M0? zEngVatQLDon4FfDhG&J0p_oMDACU*d4X|${ZK#C8b*5X6a=?`#Fb6gldP1xv{q93y zVK($P#@qo90v)BxlyBWF1vs_9eSsfQL0u7i9)g#Da|x`;i1Ts8h&u!)P=97el8?QG zRxr>hKBH9v>>es~v;6vXv+5B>ReH=to)NO0;|KDwzDV9}JByIol{$+WHeK;8g6QPPd&uj*b&Jp2{R?0Q!tvAnL$LW`)7@f+d-2UTen)phMg9*PB!~ z*O)8bPG12bC8PBRW7VS`(r}N?fK_F<&JTW)l-ljl?Rb%Y`?m5LFzUegCTRnn&dQOD z$OG6$MFP@`+g8{9J!x$?N6*4io~9aKgYvg=y<*sUKdqj{#+!7~U*8Ye%X}AKN+=!F zmz!7tR>x>OtT`_u`HLQ2y0?e7%VA0(Lu}U1KYG8R8Px?S)eZ-CQAD`^FzX< zeVn(IV+BoZ7|=1W#0a!ff)?F7R_Mt)1y&~GJ`R961V8@VJk{B1Zsk}4Qr8|xh&rx4 z7K%3#eNMPw4!8u;EtKsS(r~1pN-1cTGL_mFAjG4UQ-G3tdfthY@ja|Odo&$eu%Q*{37Xv-#au>Wx2!u3Z*YeuGLmjbHq=<8m#J@F{K z=XW))VV&u6#`0GPFCd@$hUIm+^H-o;qv+Qezk@mVpU&w31oHh1A~n%m8`tsvQVX58 zd@k5&cbZ2C!C&At<}CSx>-@ke_ZVlee;;s2^Gypuqf|dNE+FGwJA6Ji>+GL?!i*N0 zHleVou9m_1FFJJ2>$RWv3OAVV^IvdhQMr{%BqYlLJ|8+M=MDtY5ZckZPn>vebRTdf zP9N+sjqeUpV@PRG$2i&9{pY^%4rv(~JhaJANNS6V?iv3dM@*_}sYinyFTs@Z?A|$4G0qyB$ zrxVdYKv3!D%PK2-gd{F>@hyCY-dNqPQEI99>3WR9omaR2P<+XrU19ZRq{RNq+B zGE^n@CyxCx;Q`RB{aiIqRc^irBeIjuswYfGukB!zY6T+22whoiPl>3ili&_PhLFT8 zEY^}`W%^aJ3;Bb{4tl)}M6=+`OuB}Pag|M6Z0r{fA7b!;#c_SlHb~c+`&`q^{rK^@ zu-efg=T3xN%3b<$^s)Wh)oB--h$o%I+2P_ieU3Os0IjN%wh6u*>3VhZt3} z{R@V}_O1*)^Okj8OCj@X)om=9LodqiFHEgVZJNAc|0e^RU~tDH@X_mAD}t>ngWMA) z2f2na)_ChF21Z7rKmwB%7#K+SAdr$RHT1nJRj#4282!4NqLhQi!S^-nzUB>Ib?%&3 zY2(4}Nz5hw{rk7&m+Pj?LTbTv(=o(E>#nM7COtGG%s{NJ@pb>%vehH#qaidp%lpc9 zn+Zv63nW0^7NTdvlNY7T;+wYP9VZN0?v0RRPjyrLsyPxBwp|=5ToOwAdHutRBSFqJ zDRwUT!&ADYMylFfaW?|yLvLNS;@WY=E^`GV8>`H@ehc%;zFSSJ4~jCldu&z&mq|>d zt%~=bSxEB-rJJ{It&VKLRpd5RNvtyeSTQH|zip%WX&<{o@EV;;(x}n^AYKO)6 zr9(2+EW+%kmsmCyMg%uhO}uXI%MqThSE`<4F%-{s_w~DP`D>wJn~idac3Pm3#XFPD zx72c_@C>~J@3XIWHv({8y0>>_uFyz}m2kPVfBZN<&p*@XAoo5D*_m##Kn?K}D9O^6 zt%!mM>>N(wOG!y`iHu}c;T6f32!K3!fxmd!3jjfdp7Cgj1%Pt^&4|w64rhL=V5U3) zlH)uRbL&wH!MnB*2lhKN*N@TgC@msTal(cG>v+cfxs~p@@e9$!N&TRPtZU_wE^cDb zJE@xMG{QV$D{xdT^Err4vt=UNTQt;%MSmChks^fb9<+LnyoZ+OkY!{j-F;^KVKsAGQ73K`AKqi=WN!IFr(>@mXuF%8oG*R=+{#1{r>gKs~8*|sR*v%gT zflB%?i$1aG^eeOk5A_wUBRN#$TnuPXt82J-bG1VAuZLKq)~V|F*qhk~%BRAgR#1)y zx7<=2ZqI+%kwu9)%l$Ue_<_tFT3H^2!1JQ||8f=QFJ43s9)gRW{uo*k{$K3uASx2n zi01MwsCm5oG73J*V?IgTc@LcLhR+@d4YpnLY za$kD(pgb41!r?%ZOd|DR-~N{ozrC=PQ@-8FYh)vkHB%ugGYb=8K0e0>57J^p!BG2| zeiyyJ=@?_7<@NS38dH9z+}v6@z9Q>bko9OgacUm>`>)A+wz>FY4pUJOZz-OIc#-G{ z6+{sSKc@xw2wgdpY>1!*#Qoj^6#h09F(E{`VG8R(NJ+snHM+b`OADZ(*Iy?yh*kkN z1wC|`g@g>e6CY@cQ(9NK$@ucA^OV`)#(KMb&yMNKuP`u9Y3ca*#}a_4hSJB~Mdh}$!N;4)KEW5hZyk zQOU>Cp~FCX;Lg4|lokPAjM^gTIZ>vnr(j_rz@!HVy9n9HV<1#0DFS@MjL!41t`O6u zrmy1X210{ZOUiS0fA17E<>a2dAb~>w>8;M<#i7h$C3W?IclJa^-*SCz`oV`$`;Mj^ z)8DfEJoY(v-1DXuJ%&XR`~Rd^1UoI`FAVmm{w8#Ao;l<2N!!xr&~8LK$g~l>g8?6+ zX>Y-q1{ymo`LAEUi2MU->-2#P&s4)u?_c~v;@f{vXDK28 z-IwOFiRImqo9DUw7-acW3AF;1{oDS~El~Bmlyc)qVGgo-tP{vPGZuf62pcvSb;SJU zTMg9SgR%=lgyTE3bBt&qe_O5+S>1;}z$lx6^?o201uq8E(DOch&Qv4+s(KVe&Mn^0 z+0tIVENr(aYY^b&{Up6lZv$TX@WJ18NjVhLfR__hLZm!5`%*K5`xbHw5lf#`Z;@JW zs)p80EygD;EhMBrmkjonBmUw`V*c>$|v74mm8a`K- z9O5+{RgYZ=Dt*^)(MNLN3^l{&DbP88EzP;OVZlYo>Q|k+?X@d!cTF=^H|KuQm$n!0 z`badl{vkn1CVrdgVFT5UO=PfiNxv>U<6!t=Y0LeTY~j5_txI=w=B`!b-4OR)Y%pAC z8q$)#*;48weJ(!x&r#-22enf5#=lCD!JzNwQ6jTk%jthym#h&lf@mH zDUAk4>9vx^Z*S{sK5n&6J5g_Yp(Jw$jbZcgD$ zbN}*N<*s2}kld`Gj7N4|YU?$vU}<4Cv7~~Jrm}{E9hV9XHHQbXchAf$84Wi-jNd=L z;3KXgtYTPWXcSxj`hD8K??+7wr_Ei;t{=}FxO#@KY@~Ed>Npou)suA|?+>5f@802{ zAYbONfosPp+GoM})Q8jPuASuE&8~jRi8jM^yV43>Cu5qlldXGZ7bsuCj&nwT7RoRa zShTw!dcpeXP<6?=mg?~8mY%-tH}A4Y9i-+O*{H@wT17uOJa(XnjnW#_8CWC} zZsfa1^u4)mS;IYZk@x12i&=Csl{OJ|sbxwPG)n{NO$NbQp7q@}iyD~Slm75D`~CfE@|BGL_`LpoPe&oMp`9u4cp}PDtCUJ*Ni1fx zK4$yV=<@lqy3H~1#rE<`-kJ^kQpFWEbdyJ!MaP{l-qwoE)Yre6_anFN!E@WE(XZ@} zN-Xg%3R~+>tFr5NO{pf$iX`uf%@ep`OQI$J$nF0;NlS)XeD&td!OQ-bcgBCGIqMfC z2dyPs$Btb?`>GyMt$;UXo_Cp-*5m$G;_fZkpHY z^04S$*}2}4)WEd9iqV4C^B=72OD>34Rxz-5x9y$&*^;X}A3xP$QC@M&qGkCu7A`eX zz2H__O5AL>nb4~OC+EitiHDfGwXdtEr}W*ynjb%ULFwWv>J8EltHQvQZR?${UiRM; z_u|D#3|{&MYzW`szx8APoyK%!sYe^CvZtH-#}w;$aaWgf7cv8jq>`5Q58biWZ9E_H zIw&{zjiNz)4&TMQGb@3t3&E$Vzcla+L>`&q8X%|;<^Zd}ZFn~HVpdX`HEswQ)>sM+|HKH$CiZ%sT46?_TQ`L38afhcD>7Iu)EIh-}aREG#vSQ zXUV6}AIj}{oHh29drH<0c?~f~QPnP{A~qqxkM}e_kJo5CTUM6W=UM}U+|$}2we5B0 zqC!(_26jy`S(7(hrs;4#Z9e;iOH!RweL>M+L&r8&2@~!bT)rPbVpESQhv)&#rx*rx zK~3$ovEUmZhTw`RJzTT#mgziTXryhMNHSOUoXl!Hy0fpSiJDAJN31@ z6kEXu4MlI)WRIbt39))9r3XeMr#J43jkT|H0l#v)o=wBhk1!O!zcNLET6Hd5Od~?+ z45aK#=HGsl*3=C1 zIA5=!VPb=0Ln4tT9=EsaZrA$(*tehWjd~@DQh)<_w(F6BJJ+QyUA(8FzPr7cXDK&5 zZE%R!YEp5ptng;rZ*hx%ZJLMZ8y;BK1!?^ss9o3yd=lG_Xo=8E^I{hIZdZ;Uw>K;1 zavHfCgd`c|#B~R%x)f*MvGB3l8FTjOS1$X8KOgn?4iSeKf%#6R&urisun%z`pj%S2 z)Kk~fzc4`#*Ye1K!F zn?c#3J2@sQnjel0-O0Rvsxfz@;_xrU`~P-9n@7@8I4S!QN2~WP!FDbNuo2JXO{KUP zwRm)~*tBKvM~mDTKGgeds)<+AZrDop zyZ6H@!r4j7R!M%}YWN26ycSB|XhC{0ezI4#+HU-+OX{$<^2cEf+3z9KVZ#%|ef_;Z z9dZ||_3Y`VEp5j|WWQ&e)t}TdbuqiarXdujp`yJcQeC^v+}eKG!v1kR`1qW~pIU{Y zbBs-I#seyT&0ks?J+XLo=kl^+kIwUmUtEd|7S3+L&0!fF44?6h z@(A;<5nW8GzdN!RYa5z*?_#{gf=}3`;|du^*O*(jDD1WC(BRgh`gy5mnmH~4G(MA5?g5H!r&Er+}R?F z@P(MM0MY0gVgI)YAA)hUbbrw6OT{9C5u&zVl*rn-3UpTW=T?-bWt6A#hy?|oM+ zv*cKHYx$dh8B1(NmvD+uRE;b40hX}$w5{D-kLShR46Eqz&lfwJ7QTe@s(JmxC$Y5} z3HU4>Jstn&!Ahb@j_z}7XBq^GU6>M1FUL^Lu~dkmuz?v{tRDT>1y|86U&x_{mQ2f3 z!~_v3V_OnQmoRAeQYRm5T}3kNzrhkYNpq5x4t_9n;||=(^0(}$s5xL|AHGVw((l2Vd2LVomJciUnnnj z@!t8l@WA@j)%|;IA3tieO2U`D5pKDz_r-fEwUqD&T^p9%XvDA8vEzpDJPY10R;+3; zIIH^FLhjMNZ3-XQg4IN>Sa?~V?4rNVKl_WZMtH9j+u*NPs3PorWv5jV*WzNC<<{#LHE1!M~@xGr5bay(KoGIt0bE!QA-!0 zqjkOPa;%+r`h}L?0SWgh{p@#UwO#haB_}IO=Y}Lcn~)$@fu~F}mMX3BWlLhq9@)9@ z3kJ``jAIQI<|Br_?2O`h?PESOEsdA=ZEkhmFxJ&^@bj3)l%J1%?qadEUAOvR^L)vL z7K_&RQe1Og_psPX&Al@1&6SF*fiId5Z2u@U=<=NJj{hcGiZt2dQfDqBMHTFDs(KUPlV?77)>W%wt1@0G=OgM-bNW>b$FH5p=tjL}jui#t@1PE5>m#O^wR zRct)XvSMW6jg_3jq8(X~$)?zzi<$EKDkbjb(ZMA8%I9p;c4}#M(o?GZldEOF*KW2T zRy!v^^dML4!OJ;nZN?8%?JgzCR+fEBWnt@Eywf!`{zD@xwrcZ|dbSE*ifzx~WBmL- z58X0)V{>I%bS4n%d}DNHJQ?qM-e70T(p#xqZpZa29uoUFY7^Dt=bV|jxe4mhDq7#l zZ;F%gs)28YaLquL`C0W(woKV)X7fM`A*~Ar9B=Q<7qNauCJ-}@E_Ooo~{pz)w zK1|PdC3oz=lbm*baYjj2mMt2;g_rsiy#cn~EMyKN#sSm&U*zqY4-@--bW_%ax zm6NYr5&K!y*5xb8NRCkqR}dWi(pwWBfG?X>covVD9lrq+pPmq9<~p}po$YweQiQEv z8|P+ul)HO=-nl@$j>7ld@pzeQx*MZvloxV; zk8EYmhzVJ^_AT9Xf7vfnbEWP4_oB1!%*xy!y!Z~Ma!zyV2oBP>`G&7%6v@ygxv5ak zD+BRjrS^+Eu{-zMf9Jp7@WpA-drV&Gz~RIG%MmOzD+Y&NNo_#%<%j9AdY&NQG>I{* z<&tJt6SPPi3ZY^hmM_~nZs%JlQ)wDT4K(#LQ)#&gXmquHeSz0e$YN+CPesPjN1NqIGY!JI z+#Vm>tFfgP?Tigv0(Keta5_R-EW-1q;k4uQsrtP2G&D_O&lLI+w>qKE8rwd@?tIY= zf+_$$$Z!9;6Pjfi;}s)cLxlARD{Iu??K1%b@%++E3Il8nUiW~1&*&*adiWWy`nl29 zUoH^UY*nAn9tm1d|Cw^@^%7`~1?kPY+qYpCN@(BmooUQ4%dVdDrD5IH?#7Ht57VI& zYA&#Tw5P(Txt?qlZ7#@;Ljoc;YpSCE@U-e$KkSUAd)4Qr-1gh&4#h*?TT#yFW8c0y zP5ix;k7{?LS<0rnd)J(#wc?09CD!L?VbykQOigLKkV!OWB7fVhC-EtWf3v0pEO_w; z0gT;^*i*aASJ4*~&?G&nQ_JPZ-I`S9lBbG?R-X^PRuP((A=={&_vjj$g$Ns&&cn#}?u_vs1XVV6zBdxFR5FfZ*FXejq^Vs6j($YcSje4G&m<4Hj6l-#f z1rO?Z63Nc_^N~vXf3(>qsC|$l#CG-^5Ea#c^3OnI)WUj+y=qrCuZ7S1gT6NgZ=9Cj76daHZFR_&00B48P;(Tv-8mUkN_1~y8l7D{eD=6jWuvuM)`-#SLDTVLh)GDgJsvMm)Om@y zkLr^%m=_UG&wcUCt(^Anx%T$XlI%^Um7;Udv)oh2K-RSCqDK-9PR4vPctOMj#Menl zsu-bY270;dceI}Pg@n98d%?Nq{E&n{SO4ys2c~xD8fBuHZX?SH^3!``ow7|9Ew)He ze_;gEyC- zra zCmHb=Q}E(2&cUEH?}2r1gbVR>5WZp8^F#%YTed#?ewrsQk>a8Zhc|;3ib?2^f<$@~ zFp1j7dvOO#zXKRfF&|Vz7hV1QmG{@ZK*iZ3yIEw!0iebQfI5dXvn~}e0aK7!?dPAM zg=DdoP%s4qArE{&R5ZJ3k$mwY*MS4F;7@{f;Y9o$HbqTc9XBtpGV*GL*g} ztEfbwD{Q;oy@oq!anaGIfJ*&t$+dr_85P#r^MU}NWmfAHB!R%0nAyCA-0B2yg3>zs zVvA4&=BK^vm;x#DT`p#E-t}S;t`+;_vp4q4dB`~pWlw9$Vu-5mk2E7KS@-o^7)K5m zXgGU1EWhn7loVc~ZSGyui!p54J}BPeeb=F>F)G!F8#5>!r8~bquC|NrtxYBMFKu9?0DOsb%*0t1V?CuSpNS0#4mF04~-6SrW zsYDcN3Nx8&>|2X;O?Dx&R7f;R8H(uhJa~L=pC7(|!1wX^`a!)PZ!`0rb6)4QK3~t* zRNsrsE5I!0tY_$g;)y$upK5PYmC!p46q+&=w{1BE3ZI&!e}+MSYP3U2w@^Y)+q*yx_*-;Ya6^C zQ#)H`I{F(dK*3EDAqt{VOc5G#pgoThY@a=kfuRck7H2t?b%s24*E`VuPwJ}n($%YM zRH!npdSl62PJJkrF)RH=lN6*j4G236d~eQQB9jSl%*nBi)>f&pVityOJgvnyOvO<> zI?`Xi<)I%RH4~DVnebLv&Q0gAZLxDAkXXvnJ*5j2^EytpTW#n-Ul>UhqE##hWQ|Z9 z&Xr;?if%y79~Wgn^E{GDUAgsN_>Cl`X{4AUN*fJGFVTTaA;N~N38(mdBm{dk{e_M87d?ThYTnZKTz(l0xh#^ zHbl;2V0ShS3?rE{!Nah*B z8ghn!2sS|T-P8>p-?&CBk`8-Z! z8RkDE`m?{oLKUfcytD8d91qj5zRI?4AkUEPs3MW)Xdd01us0gW|7%oHvNHBOqZ zDhltiEdwS83KLW(Nb00v`B;I9u}U%+?@Ki{J`3fv4Nt-c}0k} zmX;mVDR*v!TK$;i*_=f@84wVieiJBIPTwzQVc&`MWQECZbTa>4}TRg=f z1_`O({qJ@>@N>FS)Bon|>D9he^vq^}p3`u$qj-2W zm3h;v+z~w1fsqAuEv2LCr|&<%lsh+UFv&YSvPnBT1eRO#M9&|=;o@isO#{BaRb&c_ zP#7?L_@b;waBvB+xF9!l6!P$U01A|*69O2xG!|U9=22*6M#&_ap>J19#v*?yc*56A z6?#Z4+&G=D_8a(}F}xnl3#dm2yn2yalWA2T7TW%Ok@*6 z?p7Z5O@dBPE{K~lK}z`=NI`@dS|^WPxpoR`MZPBNt6+(|k zl-ucvjKx`6xCuY(1U9>W$yE1}HQsXkE{U+U@2xC;4U*&31DcjP$D~K`E*gFdC#60& zkJD>)#@w1%mXRwMp%5^HV$+Ttw6U(xvb0vtJ>FtPv`0XinDCyi!^*Hy z;*yg&C|SYryTxXk9jQsIhYw2!%>;&88KNn>4g>^(oz9?7DmuDFM_)v`_oPKZ!0>hC zm;WVt;nDCZfrnEgCZ@7?bOyY{ z%5LL9^SfVq%%{?-%_~bZ3Qd$Z+(lVDUaeZ`IG*-)X{jT8mQkyC2qX$u(wcrocrZA0 zEtuJ=DMzyTv=$&G?&!GLI6*hK5GP>-O5!!K!qAfl))fABtkJ}`j@yQ_5EOc# zG8_w7x=0dx`EKp%tU`=qakry9%Aa34|&M8eCb+~9C&!S6Gakqr> zdy{tZCho3XX;wpURA7kWgB3DdS9_SoDmiCW@(x|}5TFZvRE@`TDYVn%+K=(_rK!=f zdTj4gFISI`x0J~U6DGlyb8D#hgBh}TKYe~n#+3)lG3+3Kucz1 z8vAKC2~)&mwB3j|n;$)qmDkCA)eM_eSZ&qutS+{&f$e`+I=PgJhzj=F4*#STbb%m=Y(6=1t8`E06T1*qj~(TKK4ju3qmkR3wNph& zMP1dr1HLSAa!@yY?qiMeA5sL3csX$kbRg}&pzbr`G8d%TU2YhpH{cY_8!g-N$b(3kWADW**$*ATXjdcvS=; z{un`Y;RaKHBSb@FTQcGcn*xt5W7EMKh&pZozv~)X=J@${t6f zN-w#RkHfhhBNseM_Ms`OBSe-lsKNp*#UZpz zzU4X_Y~zSBj3$!Q_93vzEo-h%iNjY^ClN!XOwi-fn-`i+-M4g3@OIM*eD8ula>8Uk zznR0-ubBN7GFx1XsQgY)0ASgVi83CY7>G^QYGAiwlh6d9Ixwb&8Z(Om5aV?i5Ax)8 z#jB`=&RUcO;k)KDJUVQ&;hEjJQ|CqhLpimLoBjeY`NrbF58Lp8sHL}xU;1H#cfKeU8S*)x_RF1??$a%gI9X0UXIdhF9A-nY{Jun%ZL!fsbTWEABiFVJV8PV3 zTXG+MlcVX_4%KdA<>Y13P0jcwd5LEvSE{^K;CXlH2Fa589U#hrpdlDR#euO{_?f`L zItrf&RQx->*PAbiWK_A$+K4LNkg_hy!w8I8zt0O4sz-|ID_BFOn;bhO2EvtjxY)0+vfG;WcneZ-uCmoc|n^7R7*b^5Y%CW2|uG6K`h`-4d_ty ztc`X5G_g@pk$bV7nTB?43-C-F^)DFA{iudgXQId!mXV;#MjHd6N(<$TiB+<(d1|F1 znH6E;qMn4nfgH5V%*SI{P9fcuDOs66f1eUZ-wQmP+w@wS??SLdiMHs z{q*8ud!%7}WiaZTwQl$CIR#~;!_IV1-eN_8ibqFWBe&txr%xtkX3gLiz6}slZ%>cg z$9K=+FN1)h)ii};oo#TyZ#r=RzmV8LWEF^01T30hvwuf@3#;g3N%Wks6?lC6!I~FL z*S$%{_T=b(<0ehDt?8B8=Jw}}{g(|a&gw6B7B+0f;=(+Ma@jaTR+%(p_MlDmD3=HTFIKIFZU+Gf{W zsBqIY9eqveKb42E1ejY035T1_n%vWtFBd;xx48vTAq(DM+r+9ZE`K$`JmBb;U-a)-F>4fLKh47 zNiv)iQU!$=mw3^YZo9eJz2GULLyT&wo>~pEk78J0KO_JI&k7FiJ|p&IYpr%F-qa*A z^m6MclOqsw6jRj6DlD{x4;oFAlEblpdiQ_wGJCU!0I~$Vgj5_A+poYs40?$JKmpKn zhH{?KJrJ@wPF$HoQC;X_QP7-JfhMg$><9kb1T!3X?c!`ns!Y%uHNYc+9QkmitnBPO zzT~YyN^sFpi@=^(L%$af7LsTQfZYZH&Zt#z^R_v3v-fZk^NJ3-Aa!a@;wE?4AV31g zN81NpALiU&b3L3CeX|JQ5|Zs`xj3T~if3ZAlSR8-7S=YaNCI|^M!dfu_nucn#@#0- z$RPka-zHZ8{>hxE!k$3jSdSG zeO^?i7@#Jn@vV!+S)anc0v=%J4%^;RT24hwM&fR*&W@+&il9xgO!|qFEqE8tjRO> z8K^4#bfGVLX!kvo!i>kSo^v@WB|oC8{rVbD%k^qYS>&>7APzR&b6!NZb zke7cX)!3)F*~iI+4Z@c2jLKzAe~)CDTfn#2=o|&V@;l=iK37v^MSJIozhW!oqSRKY z9sUhRZgafSt=;?wn$qU}#M6eDhaRMKXWA*DsqF@E9LVI?+|(D{%e-Ix__3c*BV3Al zxHm^Rqwz$xKxIb|$mf!F`mVq6>b(DD>zglMiy`r|jzuXKrsExYev?yGI#4K_zQT3Y^`7kb>P!cLBM5b#%cl+Q<>`ypLxJ@bcH_+|lvm!FS} z9lG4k-FdN$q=8Vc!P9Ntd~(7^GfT;Qsaz-rIo&~S@O2M}VGX0q0jcXN?Y=|E!`BIa zN#pkQ^t2{Vmja;1yQ@pjfX(G7_^d6()9p6z9$c@=3kh`X6iX~nA3ML(zq4<&%mF4M zWl-2^xC;=D(G78n{Xq;7ev_T%x82Q8xZ!ea*Fl6A7l&3P(X6a`ZG)D~W|O5Suk@HK zrf3Y-FS4<;yavP!pc~6=?H6o-hl&);$g@H5BL^rH5pUx?`qV@T4r2a=<_IasTU}%@ zdTp(Ls!8$>{JDnYK?-Fgm$N?{7`)i@Y`s4U0acg!4()A|=%C@8=BZ$3&$CO`qv{~T zd0a44%%&uwm>twHeAWe(kjdAde%#TKMso^&;!C?84&pEabZFo9LOozY+&6D7QsB>C zRqXV)U-?ZK+tBs&TYAev1E?~cUlX2Bq4Q$oaq&8=I~?NQoiy(um1<%*9uWMqc~BE@<>DT!T_W zcE-DD8oM!bk+o#@;-5vpJCQZ89*@pFJ~l%;48F)Oy5`_IJ;?h$t&h4>+hK>o#=l+(YuS5BXWBK^N$KuZ44 b3dZNqGuBfb^q;iq$tD}`HQU2B_$mC~>F?PF literal 0 HcmV?d00001 diff --git a/walkthroughs/howto-k8s-appmesh-load-test/requirements.txt b/walkthroughs/howto-k8s-appmesh-load-test/requirements.txt new file mode 100644 index 00000000..c9603025 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/requirements.txt @@ -0,0 +1,7 @@ +altair==4.2.0 +boto3==1.26.14 +botocore==1.29.14 +matplotlib==3.5.3 +numpy==1.21.6 +pandas==1.3.5 +requests==2.28.1 \ No newline at end of file diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh b/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh index 14401ceb..61a14af9 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + err() { msg="Error: $1" echo "${msg}" @@ -39,31 +41,6 @@ if [ -z "${VPC_ID}" ]; then err "VPC_ID is not set" fi -# Check creds -if [ -n "${USER}" ]; then - check_version "isengardcli version" - if [ $? -eq 0 ] - then - eval "$(isengardcli creds "$USER")" - status=$? - [ $status -eq 0 ] && echo "Ran isengard creds successfully!" || (err "isengard creds error.") - fi -fi - -# Install python3 dependencies -exec_command "python3 --version" - -declare -a libraries=("boto3" "numpy" "pandas" "requests" "botocore" "matplotlib") - -for i in "${libraries[@]}" -do - check_version "pip3 show $i" - if [ $? -ne 0 ] - then - exec_command "pip3 install $i" - fi -done - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" APPMESH_LOADTESTER_PATH="$(dirname "$DIR")" echo "APPMESH_LOADTESTER_PATH -: $APPMESH_LOADTESTER_PATH" @@ -74,7 +51,7 @@ kubectl --namespace appmesh-system port-forward service/appmesh-prometheus 9090 pid=$! # call ginkgo -echo "Starting Ginkgo test" +echo "Starting Ginkgo test. This may take a while! So hang tight and do not close this window" cd $CONTROLLER_PATH && ginkgo -v -r --focus "DNS" "$CONTROLLER_PATH"/test/e2e/fishapp/load -- --cluster-kubeconfig=$KUBECONFIG \ --cluster-name=$CLUSTER_NAME --aws-region=$AWS_REGION --aws-vpc-id=$VPC_ID \ --base-path=$APPMESH_LOADTESTER_PATH @@ -82,4 +59,6 @@ cd $CONTROLLER_PATH && ginkgo -v -r --focus "DNS" "$CONTROLLER_PATH"/test/e2e/fi # kill prometheus port forward echo "Killing Prometheus port-forward" kill -9 $pid -[ $status -eq 0 ] && echo "Killed Prometheus port-forward" || echo "Error when killing Prometheus port forward" \ No newline at end of file +[ $status -eq 0 ] && echo "Killed Prometheus port-forward" || echo "Error when killing Prometheus port forward" + +cd "$DIR" || exit \ No newline at end of file diff --git a/walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py b/walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py index 80be45c9..b6b033d6 100644 --- a/walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py +++ b/walkthroughs/howto-k8s-appmesh-load-test/scripts/load_driver.py @@ -16,47 +16,47 @@ def check_valid_request_data(request_data): logging.info("Validating Fortio request parameters") - if ("url" not in request_data): - logging.warning(f"url not provided. Defaulting to {URL_DEFAULT}") + if "url" not in request_data: + logging.warning("URL not provided. Defaulting to {}".format(URL_DEFAULT)) request_data["url"] = URL_DEFAULT - if ("t" not in request_data): - logging.warning(f"Duration (t) not provided. Defaulting to {DURATION_DEFAULT}") + if "t" not in request_data: + logging.warning("Duration (t) not provided. Defaulting to {}".format(DURATION_DEFAULT)) request_data["t"] = DURATION_DEFAULT - if ("qps" not in request_data): - logging.warning(f"qps not provided. Defaulting to {QPS_DEFAULT}") + if "qps" not in request_data: + logging.warning("qps not provided. Defaulting to {}".format(QPS_DEFAULT)) request_data["qps"] = QPS_DEFAULT - if ("c" not in request_data): - logging.warning(f"# Connections (c) not provided. Defaulting to {CONNECTIONS_DEFAULT}") + if "c" not in request_data: + logging.warning("# Connections (c) not provided. Defaulting to {}".format(CONNECTIONS_DEFAULT)) request_data["c"] = CONNECTIONS_DEFAULT - logging.info(f"Updated request data -: {request_data}") + logging.info("Updated request data -: {}".format(request_data)) return request_data def run_fortio_test(test_data): fortio_request_data = test_data.copy() test_name = fortio_request_data.pop("test_name") - logging.info(f"Running test -: {test_name}") + logging.info("Running test -: {}".format(test_name)) fortio_request_data = check_valid_request_data(fortio_request_data) fortio_response = requests.post(url=FORTIO_RUN_ENDPOINT, json=fortio_request_data) - if (fortio_response.ok): + if fortio_response.ok: fortio_json = fortio_response.json() - logging.info(f"Successful Fortio run -: {test_name}") + logging.info("Successful Fortio run -: {}".format(test_name)) return fortio_json else: - logging.error(f"Fortio response code = {fortio_response.status_code}") + logging.error("Fortio response code = {}".format(fortio_response.status_code)) fortio_response.raise_for_status() def query_prometheus_server(metric_name, metric_logic, start_ts, end_ts, step="10s"): - logging.info(f"Querying prometheus server for metric = {metric_name} using logic = {metric_logic}") + logging.info("Querying prometheus server for metric = {} using logic = {}".format(metric_name, metric_logic)) prometheus_response = requests.post(url=PROMETHEUS_QUERY_ENDPOINT, data={"query": metric_logic, "start": start_ts, "end": end_ts, "step": step}) if prometheus_response.ok: - logging.info(f"Successfully queried Prometheus for metric -: {metric_name}") + logging.info("Successfully queried Prometheus for metric -: {}".format(metric_name)) else: - logging.error(f"Error while querying Prometheus for metric -: {metric_name}") + logging.error("Error while querying Prometheus for metric -: {}".format(metric_name)) prometheus_response.raise_for_status() return prometheus_response.json() @@ -77,8 +77,7 @@ def prometheus_json_to_df(prometheus_json, metric_name): groupby_column = [col for col in metrics_df.columns if col.startswith("metric")][0] metrics_df['normalized_ts'] = metrics_df['timestamp'] - metrics_df.groupby(groupby_column).timestamp.transform( 'min') - logging.info("Normalized DataFrame -: ") - logging.info(metrics_df.head(30)) + logging.info("Normalized DataFrame -: {}".format(metrics_df.head(30))) except KeyError: logging.warning("Metrics response is empty. Returning empty DataFrame") metrics_df = pd.DataFrame(columns=["metric.", "timestamp", metric_name, "normalized_ts"]) @@ -87,17 +86,15 @@ def prometheus_json_to_df(prometheus_json, metric_name): def write_to_s3(s3_client, data, folder_path, file_name): - response = s3_client.put_object( - Bucket=S3_BUCKET, Key=f"{folder_path}/{file_name}", Body=data - ) + response = s3_client.put_object(Bucket=S3_BUCKET, Key="{}/{}".format(folder_path, file_name), Body=data) status = response.get("ResponseMetadata", {}).get("HTTPStatusCode") if status == 200: - logging.info(f"Successful write of ({folder_path}/{file_name}) to S3. Status - {status}") + logging.info("Successful write of ({}/{}) to S3. Status - {}".format(folder_path, file_name, status)) else: - logging.error( - f"Error writing ({folder_path}/{file_name}) to S3. Response Metadata -: {response['ResponseMetadata']}") - raise IOError(f"S3 Write Failed. ResponseMetadata -: {response['ResponseMetadata']}") + logging.error("Error writing ({}/{}) to S3. Response Metadata -: {}".format(folder_path, file_name, + response['ResponseMetadata'])) + raise IOError("S3 Write Failed. ResponseMetadata -: {}".format(response['ResponseMetadata'])) def get_s3_client(region=None, is_creds=False): @@ -125,17 +122,6 @@ def get_s3_client(region=None, is_creds=False): return s3_client -def list_bucket(region=None): - # Retrieve the list of existing buckets - s3 = boto3.client('s3', region_name=region) - response = s3.list_buckets() - - # Output the bucket names - print('Existing buckets:') - for bucket in response['Buckets']: - print(f' {bucket["Name"]}') - - def create_bucket_if_not_exists(s3_client, bucket_name, region=None): """Create an S3 bucket in a specified region @@ -165,16 +151,15 @@ def create_bucket_if_not_exists(s3_client, bucket_name, region=None): if __name__ == '__main__': - driver_ts = datetime.today().strftime('%Y%m%d%H%M%S') - print(f"driver_ts = {driver_ts}") - config_file = sys.argv[1] BASE_PATH = sys.argv[2] LOGS_FOLDER = os.path.join(BASE_PATH, "logs") os.makedirs(LOGS_FOLDER, exist_ok=True) - - log_file = os.path.join(LOGS_FOLDER, f"load_driver_{driver_ts}.log") + driver_ts = datetime.today().strftime('%Y%m%d%H%M%S') + log_file = os.path.join(LOGS_FOLDER, "load_driver_{}.log".format(driver_ts)) logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO) + + logging.info("driver_ts = {}".format(driver_ts)) with open(config_file, "r") as f: config = json.load(f) logging.info("Loaded config file") @@ -185,29 +170,31 @@ def create_bucket_if_not_exists(s3_client, bucket_name, region=None): for test in config["load_tests"]: logging.info("Writing config to S3") - write_to_s3(s3_client, json.dumps(config, indent=4), f"{test['test_name']}/{driver_ts}", "config.json") + write_to_s3(s3_client, json.dumps(config, indent=4), "{}/{}".format(test['test_name'], driver_ts), + "config.json") start_ts = int(time.time()) fortio_json = run_fortio_test(test) # Write Fortio response to S3 logging.info("Writing Fortio response to S3") - write_to_s3(s3_client, json.dumps(fortio_json, indent=4), f"{test['test_name']}/{driver_ts}", "fortio.json") + write_to_s3(s3_client, json.dumps(fortio_json, indent=4), "{}/{}".format(test['test_name'], driver_ts), + "fortio.json") end_ts = int(time.time()) - logging.info(f"start_ts -: {start_ts}, end_ts -: {end_ts}") + logging.info("start_ts -: {}, end_ts -: {}".format(start_ts, end_ts)) for metric_name, metric_logic in config['metrics'].items(): metrics_json = query_prometheus_server(metric_name, metric_logic, start_ts, end_ts) metrics_df = prometheus_json_to_df(metrics_json, metric_name) # Write to S3 logging.info("Writing Metrics dataframe to S3") - s3_folder_path = f"{test['test_name']}/{driver_ts}" - file_name = f"{metric_name}.csv" + s3_folder_path = "{}/{}".format(test['test_name'], driver_ts) + file_name = "{}.csv".format(metric_name) csv_buffer = io.StringIO() metrics_df.to_csv(csv_buffer, index=False) write_to_s3(s3_client, csv_buffer.getvalue(), s3_folder_path, file_name) csv_buffer.close() - logging.info( - f"Finished exporting all metrics for {test['test_name']}. Sleeping for 10s before starting next test") + logging.info("Finished exporting all metrics for {}. Sleeping for 10s before starting next test".format( + test['test_name'])) # Sleep 10s between tests time.sleep(10) diff --git a/walkthroughs/howto-k8s-appmesh-load-test/vars.env b/walkthroughs/howto-k8s-appmesh-load-test/vars.env new file mode 100644 index 00000000..bfe59de0 --- /dev/null +++ b/walkthroughs/howto-k8s-appmesh-load-test/vars.env @@ -0,0 +1,5 @@ +export CONTROLLER_PATH= +export CLUSTER_NAME= +export KUBECONFIG= +export AWS_REGION=us-west-2 +export VPC_ID= \ No newline at end of file