From d772fdc6a49698f2b4f0fa4df9e97cf2adb6d3b3 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Wed, 11 Nov 2020 14:56:28 +0000 Subject: [PATCH 1/3] Update Alibi-Detect wrapper and drift and outlier examples --- components/alibi-detect-server/Dockerfile | 6 +- components/alibi-detect-server/Makefile | 4 +- .../alibi-detect-server/adserver/od_model.py | 9 +- .../alibi-detect-server/adserver/server.py | 14 +- components/alibi-detect-server/setup.py | 2 +- components/drift-detection/cifar10/.gitignore | 5 + .../drift-detection/cifar10/cifar10.yaml | 30 - .../cifar10/cifar10_drift.ipynb | 567 +++++++++++++++--- .../drift-detection/cifar10/cifar10cd.yaml | 31 - .../cifar10/message-dumper.yaml | 12 - .../drift-detection/cifar10/trigger.yaml | 15 - .../outlier-detection/cifar10/.gitignore | 5 + .../outlier-detection/cifar10/cifar10.yaml | 30 - .../cifar10/cifar10_outlier.ipynb | 505 +++++++++++++--- .../outlier-detection/cifar10/cifar10od.yaml | 29 - .../cifar10/message-dumper.yaml | 12 - .../outlier-detection/cifar10/trigger.yaml | 15 - 17 files changed, 936 insertions(+), 355 deletions(-) create mode 100644 components/drift-detection/cifar10/.gitignore delete mode 100644 components/drift-detection/cifar10/cifar10.yaml delete mode 100644 components/drift-detection/cifar10/cifar10cd.yaml delete mode 100644 components/drift-detection/cifar10/message-dumper.yaml delete mode 100644 components/drift-detection/cifar10/trigger.yaml create mode 100644 components/outlier-detection/cifar10/.gitignore delete mode 100644 components/outlier-detection/cifar10/cifar10.yaml delete mode 100644 components/outlier-detection/cifar10/cifar10od.yaml delete mode 100644 components/outlier-detection/cifar10/message-dumper.yaml delete mode 100644 components/outlier-detection/cifar10/trigger.yaml diff --git a/components/alibi-detect-server/Dockerfile b/components/alibi-detect-server/Dockerfile index 2ae9a1cd43..7e510553b4 100644 --- a/components/alibi-detect-server/Dockerfile +++ b/components/alibi-detect-server/Dockerfile @@ -1,5 +1,5 @@ # TODO: Add to release script -FROM docker.io/seldonio/seldon-core-s2i-python37-ubi8:1.3.0-dev +FROM docker.io/seldonio/seldon-core-s2i-python37-ubi8:1.5.0-dev LABEL name="Seldon Alibi Detect Server" \ vendor="Seldon Technologies" \ version="1.5.0-dev" \ @@ -27,9 +27,9 @@ RUN pip install -e . RUN pip install pip-licenses RUN mkdir ./licenses && pip-licenses --from=mixed --format=csv --output-file=./licenses/license_info.csv && \ pip-licenses --from=mixed --format=plain-vertical --with-license-file --no-license-path --output-file=./licenses/license.txt -USER root + + RUN yum install -y mesa-libGL RUN mv ./licenses /licenses -USER default ENTRYPOINT ["python", "-m", "adserver"] diff --git a/components/alibi-detect-server/Makefile b/components/alibi-detect-server/Makefile index d91578237a..ca9222149b 100644 --- a/components/alibi-detect-server/Makefile +++ b/components/alibi-detect-server/Makefile @@ -45,7 +45,7 @@ docker-run-drift-detector: # curl-detector: - curl -v localhost:8080/ -d @./input.json -H "ce-namespace: default" -H "ce-modelid: cifar10" + curl -v localhost:8080/ -d @./input.json -H "ce-namespace: default" -H "ce-modelid: cifar10" -H "ce-type: io.seldon.serving.inference.request" -H "ce-id: 1234" -H "ce-source: localhost" -H "ce-specversion: 1.0" curl-outlier-detector-scores: curl -v localhost:8080/ -d @./input.json -H "Alibi-Detect-Return-Feature-Score: true" -H "Alibi-Detect-Return-Instance-Score: true" @@ -59,7 +59,7 @@ curl-metrics-server-metrics: curl http://localhost:8080/v1/metrics docker-build: - docker build -f Dockerfile -t seldonio/${IMAGE}:${VERSION} . + docker build --pull -f Dockerfile -t seldonio/${IMAGE}:${VERSION} . docker-push: docker push seldonio/${IMAGE}:${VERSION} diff --git a/components/alibi-detect-server/adserver/od_model.py b/components/alibi-detect-server/adserver/od_model.py index 5f3df1429a..5a495943a7 100644 --- a/components/alibi-detect-server/adserver/od_model.py +++ b/components/alibi-detect-server/adserver/od_model.py @@ -82,7 +82,7 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: and headers[HEADER_RETURN_FEATURE_SCORE] == "true" ): ret_feature_score = True - op_preds = {} + od_preds = {} name = self.model.meta["name"] if ( name == "IForest" @@ -105,5 +105,10 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: # scores used to determine outliers return_instance_score=ret_instance_score, ) - + logging.info(od_preds) + if "data" in od_preds and "instance_score" in od_preds["data"] and od_preds["data"]["instance_score"] is None: + del od_preds["data"]["instance_score"] + if "data" in od_preds and "feature_score" in od_preds["data"] and od_preds["data"]["feature_score"] is None: + del od_preds["data"]["feature_score"] + logging.info(od_preds) return json.loads(json.dumps(od_preds, cls=NumpyEncoder)) diff --git a/components/alibi-detect-server/adserver/server.py b/components/alibi-detect-server/adserver/server.py index 4c00379441..63772e70af 100644 --- a/components/alibi-detect-server/adserver/server.py +++ b/components/alibi-detect-server/adserver/server.py @@ -16,7 +16,7 @@ from adserver.protocols.tensorflow_http import TensorflowRequestHandler from cloudevents.sdk import converters from cloudevents.sdk import marshaller -from cloudevents.sdk.event import v02 +from cloudevents.sdk.event import v1 from adserver.protocols import Protocol from seldon_core.flask_utils import SeldonMicroserviceException from seldon_core.user_model import SeldonResponse @@ -136,7 +136,7 @@ def get_request_handler(protocol, request: Dict) -> RequestHandler: return SeldonFeedbackRequestHandler(request) -def sendCloudEvent(event: v02.Event, url: str): +def sendCloudEvent(event: v1.Event, url: str): """ Send CloudEvent @@ -153,10 +153,10 @@ def sendCloudEvent(event: v02.Event, url: str): event, converters.TypeBinary, json.dumps ) - print("binary CloudEvent") + logging.info("binary CloudEvent") for k, v in binary_headers.items(): - print("{0}: {1}\r\n".format(k, v)) - print(binary_data) + logging.info("{0}: {1}\r\n".format(k, v)) + logging.info(binary_data) response = requests.post(url, headers=binary_headers, data=binary_data) response.raise_for_status() @@ -217,7 +217,7 @@ def post(self): request = request_handler.extract_request() # Create event from request body - event = v02.Event() + event = v1.Event() http_marshaller = marshaller.NewDefaultHTTPMarshaller() event = http_marshaller.FromRequest( event, self.request.headers, self.request.body, json.loads @@ -250,7 +250,7 @@ def post(self): else: resp_event_id = event.EventID() revent = ( - v02.Event() + v1.Event() .SetContentType("application/json") .SetData(responseStr) .SetEventID(resp_event_id) diff --git a/components/alibi-detect-server/setup.py b/components/alibi-detect-server/setup.py index 962a705180..2b97c6f381 100644 --- a/components/alibi-detect-server/setup.py +++ b/components/alibi-detect-server/setup.py @@ -15,7 +15,7 @@ "kfserving>=0.2.0", "argparse >= 1.4.0", "numpy >= 1.8.2", - "cloudevents", + "cloudevents == 1.2.0", # Fixes #2533 "google-cloud-core==1.4.1", "google-cloud-storage==1.31.2", diff --git a/components/drift-detection/cifar10/.gitignore b/components/drift-detection/cifar10/.gitignore new file mode 100644 index 0000000000..b466618c7e --- /dev/null +++ b/components/drift-detection/cifar10/.gitignore @@ -0,0 +1,5 @@ +event-display.yaml +broker.yaml +trigger.yaml +cifar10cd.yaml +cifar10.yaml \ No newline at end of file diff --git a/components/drift-detection/cifar10/cifar10.yaml b/components/drift-detection/cifar10/cifar10.yaml deleted file mode 100644 index cea433ff02..0000000000 --- a/components/drift-detection/cifar10/cifar10.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1 -kind: SeldonDeployment -metadata: - name: tfserving-cifar10 -spec: - protocol: tensorflow - transport: rest - predictors: - - componentSpecs: - - spec: - containers: - - args: - - --port=8500 - - --rest_api_port=8501 - - --model_name=resnet32 - - --model_base_path=gs://seldon-models/tfserving/cifar10/resnet32 - image: tensorflow/serving - name: resnet32 - ports: - - containerPort: 8501 - name: http - graph: - name: resnet32 - type: MODEL - endpoint: - service_port: 8501 - logger: - mode: all - name: model - replicas: 1 diff --git a/components/drift-detection/cifar10/cifar10_drift.ipynb b/components/drift-detection/cifar10/cifar10_drift.ipynb index a8f07ede58..4eb6371f2d 100644 --- a/components/drift-detection/cifar10/cifar10_drift.ipynb +++ b/components/drift-detection/cifar10/cifar10_drift.ipynb @@ -37,9 +37,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gateway.networking.istio.io/seldon-gateway unchanged\r\n" + ] + } + ], "source": [ "!kubectl apply -f ../../../notebooks/resources/seldon-gateway.yaml" ] @@ -52,44 +60,126 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 2, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/cifar10drift created\r\n" + ] + } + ], "source": [ - "Enabled eventing on default namespace. This will activate a default Knative Broker." + "!kubectl create namespace cifar10drift" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing broker.yaml\n" + ] + } + ], "source": [ - "!kubectl label namespace default knative-eventing-injection=enabled" + "%%writefile broker.yaml\n", + "apiVersion: eventing.knative.dev/v1\n", + "kind: broker\n", + "metadata:\n", + " name: default\n", + " namespace: cifar10drift" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 4, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "broker.eventing.knative.dev/default created\r\n" + ] + } + ], "source": [ - "Create a Knative service to log events it receives. This will be the example final sink for outlier events." + "!kubectl create -f broker.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize message-dumper.yaml" + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing event-display.yaml\n" + ] + } + ], + "source": [ + "%%writefile event-display.yaml\n", + "apiVersion: apps/v1\n", + "kind: Deployment\n", + "metadata:\n", + " name: hello-display\n", + " namespace: cifar10drift\n", + "spec:\n", + " replicas: 1\n", + " selector:\n", + " matchLabels: &labels\n", + " app: hello-display\n", + " template:\n", + " metadata:\n", + " labels: *labels\n", + " spec:\n", + " containers:\n", + " - name: event-display\n", + " image: gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display\n", + "\n", + "---\n", + "\n", + "kind: Service\n", + "apiVersion: v1\n", + "metadata:\n", + " name: hello-display\n", + " namespace: cifar10drift\n", + "spec:\n", + " selector:\n", + " app: hello-display\n", + " ports:\n", + " - protocol: TCP\n", + " port: 80\n", + " targetPort: 8080" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment.apps/hello-display created\n", + "service/hello-display created\n" + ] + } + ], "source": [ - "!kubectl apply -f message-dumper.yaml" + "!kubectl apply -f event-display.yaml" ] }, { @@ -101,18 +191,67 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize cifar10.yaml" + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting cifar10.yaml\n" + ] + } + ], + "source": [ + "%%writefile cifar10.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: tfserving-cifar10\n", + " namespace: cifar10drift\n", + "spec:\n", + " protocol: tensorflow\n", + " transport: rest\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - args: \n", + " - --port=8500\n", + " - --rest_api_port=8501\n", + " - --model_name=resnet32\n", + " - --model_base_path=gs://seldon-models/tfserving/cifar10/resnet32\n", + " image: tensorflow/serving\n", + " name: resnet32\n", + " ports:\n", + " - containerPort: 8501\n", + " name: http\n", + " protocol: TCP\n", + " graph:\n", + " name: resnet32\n", + " type: MODEL\n", + " endpoint:\n", + " service_port: 8501\n", + " logger:\n", + " mode: all\n", + " url: http://broker-ingress.knative-eventing.svc.cluster.local/cifar10drift/default\n", + " name: model\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/tfserving-cifar10 created\r\n" + ] + } + ], "source": [ "!kubectl apply -f cifar10.yaml" ] @@ -126,18 +265,66 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize cifar10cd.yaml" + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting cifar10cd.yaml\n" + ] + } + ], + "source": [ + "%%writefile cifar10cd.yaml\n", + "apiVersion: serving.knative.dev/v1\n", + "kind: Service\n", + "metadata:\n", + " name: drift-detector\n", + " namespace: cifar10drift\n", + "spec:\n", + " template:\n", + " metadata:\n", + " annotations:\n", + " autoscaling.knative.dev/minScale: \"1\"\n", + " spec:\n", + " containers:\n", + " - image: seldonio/alibi-detect-server:1.5.0-dev\n", + " imagePullPolicy: IfNotPresent\n", + " args:\n", + " - --model_name\n", + " - cifar10cd\n", + " - --http_port\n", + " - '8080'\n", + " - --protocol\n", + " - tensorflow.http\n", + " - --storage_uri\n", + " - gs://seldon-models/alibi-detect/cd/ks/cifar10\n", + " - --reply_url\n", + " - http://hello-display.cifar10drift\n", + " - --event_type\n", + " - io.seldon.serving.inference.drift\n", + " - --event_source\n", + " - io.seldon.serving.cifar10cd\n", + " - DriftDetector\n", + " - --drift_batch_size\n", + " - '5000'\n" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "service.serving.knative.dev/drift-detector created\r\n" + ] + } + ], "source": [ "!kubectl apply -f cifar10cd.yaml" ] @@ -151,20 +338,50 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!pygmentize trigger.yaml" + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting trigger.yaml\n" + ] + } + ], + "source": [ + "%%writefile trigger.yaml\n", + "apiVersion: eventing.knative.dev/v1\n", + "kind: Trigger\n", + "metadata:\n", + " name: drift-trigger\n", + " namespace: cifar10drift\n", + "spec:\n", + " broker: default\n", + " filter:\n", + " attributes:\n", + " type: io.seldon.serving.inference.request\n", + " subscriber:\n", + " ref:\n", + " apiVersion: serving.knative.dev/v1\n", + " kind: Service\n", + " name: drift-detector\n", + " namespace: cifar10drift\n" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trigger.eventing.knative.dev/drift-trigger created\r\n" + ] + } + ], "source": [ "!kubectl apply -f trigger.yaml" ] @@ -178,31 +395,71 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "35.240.18.201\n" + ] + } + ], "source": [ "CLUSTER_IPS=!(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n", "CLUSTER_IP=CLUSTER_IPS[0]\n", "print(CLUSTER_IP)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are using Kind or Minikube you will need to port-forward to the istio ingressgateway and uncomment the following" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "SERVICE_HOSTNAMES=!(kubectl get ksvc drift-detector -o jsonpath='{.status.url}' | cut -d \"/\" -f 3)\n", + "#CLUSTER_IP=\"localhost:8004\"" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "drift-detector.cifar10drift.35.240.18.201.xip.io\n" + ] + } + ], + "source": [ + "SERVICE_HOSTNAMES=!(kubectl get ksvc -n cifar10drift drift-detector -o jsonpath='{.status.url}' | cut -d \"/\" -f 3)\n", "SERVICE_HOSTNAME_CD=SERVICE_HOSTNAMES[0]\n", "print(SERVICE_HOSTNAME_CD)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(50000, 32, 32, 3) (50000, 1) (10000, 32, 32, 3) (10000, 1)\n" + ] + } + ], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", @@ -231,7 +488,7 @@ " 'instances': X.tolist()\n", " }\n", " headers = {}\n", - " res = requests.post('http://'+CLUSTER_IP+'/seldon/default/tfserving-cifar10/v1/models/resnet32/:predict', json=formData, headers=headers)\n", + " res = requests.post('http://'+CLUSTER_IP+'/seldon/cifar10drift/tfserving-cifar10/v1/models/resnet32/:predict', json=formData, headers=headers)\n", " if res.status_code == 200:\n", " j = res.json()\n", " if len(j[\"predictions\"]) == 1:\n", @@ -245,6 +502,8 @@ " 'instances': X.tolist()\n", " }\n", " headers = {}\n", + " headers = { \"ce-namespace\": \"default\",\"ce-modelid\":\"cifar10drift\",\"ce-type\":\"io.seldon.serving.inference.request\", \\\n", + " \"ce-id\":\"1234\",\"ce-source\":\"localhost\",\"ce-specversion\":\"1.0\"}\n", " headers[\"Host\"] = SERVICE_HOSTNAME_CD\n", " res = requests.post('http://'+CLUSTER_IP+'/', json=formData, headers=headers)\n", " if res.status_code == 200:\n", @@ -264,11 +523,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVmklEQVR4nO2d2Y9kZ1LFI/elKjNrzdq6tq6uKpfdbbu9NN6wkQbGDAwa0ACCF57gCYkH/h3eEA/wADKWrdHgwYPHg+2x7Gm7965eauuufcnKfbv38g/ECck8MCF0fo/f0Zd5M/OevFLEFxGxKIqEEOKP+G/6AgghOjQnIU6hOQlxCs1JiFNoTkKckrTEf3j/ZzCU++Te13Df0cZddT0I8NtNzD0DtbmlNagNT85BLZvT32/99mdwz9bDG1Dr1epQSxifrThcgloym1fXr735NtxzaQV/V+3zU6jdvnUdamHYVde7vTbcc+f2TahVK8dQ63Q7UOt1E+r66UkT7qk38TX2A/xe4+MjUBseGYRaENX09+rBLdJu4azIe//205i2zicnIU6hOQlxCs1JiFNoTkKcQnMS4hSakxCnmKmU6hkOy48O4TB0ND6hryeLcM/U3EWoBSGOUcdDHGIPm311vX12AvdELRyWnxkrQ21u9hLUZi/NQ2165oK6Xi7r36GISCqVgVp/SE/NiIjMXpjE+/p6KqXdbsE9lTOcWjo+xvdOMp2FmsT0VMrwKP7M2QF8jefVM6hlsvj2DyP93hERSSX1a6meV+Cebue7F5jwyUmIU2hOQpxCcxLiFJqTEKfQnIQ4heYkxClmKkV6OIXR7WCt2dTD8gsrM3BPvdHA72VURoyMGRUfKf2/Z3l5Be5547VXoDYzoac9RERKpXGo9ZIB1PJZPSyfNCLvsT4O87caOL3RMX7PfE5PwQwP4fTR0sVnoXb37n2oSQxfR6ejp8ZKxWG4J5XGb3VePYBaJPp9KiIShvgHODvT79VWE1fA/G9adfHJSYhTaE5CnEJzEuIUmpMQp9CchDjFjNb2jUPPsT6OQGbSOXX9/Bj3lRmdxJHQuefwofLy7DTUUiiMZzR76fVxZPjeHj4w33x8hF8zjqOC929+q66/uoYjoW9fexVqVgf/avUcattbu+p6OoUPqafTuJBhbBxH5rd3HuDXBD2V6i0cza9W8X2VTKnteUREpFjERQKtFi6oCECwvN8P4Z5MxggpA/jkJMQpNCchTqE5CXEKzUmIU2hOQpxCcxLiFDOV0mni8PVgDofYiyP6IfCXXngR7pm9uAy1mnHQ+/7jHahVm3o4vF7BvV5OKjhdsreP+9EUjYPvEscHoj/4l39V11N/jv8333n9LailUjhNNDmJ004S6emIypk+ekBE5NfX8eiKpNHnaKCAUzD9QE8Fdev4N0sYjxhr5EIQ4BTXySlOz8RFT8Ekk9hOQ0O4QAO/DyHEJTQnIU6hOQlxCs1JiFNoTkKcQnMS4hQzlZLJpKDWSxSg1srpU4E3qrjK5Ztffgm10xPcF+fpLu4Rk0roFQmpOK4e6ICxBCIi7TbWpsbxV3m4vwW1IqhWqFWqcM/6xga+jqkxqKVS+BqnZvVRDdNgXURkex+nse7fxFp5CqedNrdBCqOHf7Owi7XA6N+UTeN0TyaJ7/1WW3/NYhGniJJghIMFn5yEOIXmJMQpNCchTqE5CXEKzUmIU2hOQpxiplLyeTxd+bCCK0Ue7uhh9Du3b8E9cSPMHxijH1o1XDmTACmTVgenKSo1rNWMUQebT+5CbSCH006rS6u6YKR0/vvT/4La/OIi1FZW8RiK0VG9asKa/lwq4vRAvI+biTU6+JmARhq0Krg6JghwU7ZsDqdE6lX8mkWjciaT1advd7vWiBLcMAzBJychTqE5CXEKzUmIU2hOQpxCcxLiFJqTEKeYqZShEVzh8HBnHWp7m3rVRD6FG12dN3DzrHr1EGqxEFckVGp66qPSwqH3pFGJMzaBpzznCriB08zCC1CbBWH5jW8/h3sSMZxm6QW4CuPoGDcvu3JlTV2/tHwR7pk1qksGX7sKtRv3tqHWaeuN4zopoypFcNojjHDKb39fnw8jIpLO4DRRaRjdBzit12rhiiwEn5yEOIXmJMQpNCchTqE5CXEKzUmIU8xo7aNHuK/PvUcPoba790hdD4xD6oXSANRWlxegdnntMtT2jvQI2dYRvo7xSXzYf34JHyovjOJI7sEZfr/oWI9sb2/hiOaRMTLCGIgtv7eiR2RFRBp1/bsKcfBXoi6OGt/+Akebl1fxWI6JmSF1/YsvfwH37B/gYoVeD0dr2y18/WfGGIrcoH6NYYQjyg1jtAmCT05CnEJzEuIUmpMQp9CchDiF5iTEKTQnIU4xUylf/OIjvHEC9L4RkaW1K+p6zmibv/Ysnmy9unIBakFbPzguIhLF9fRAQ/DU4mQKT+xOJPQQuohIr48PSjdqp1ArdfVQP5rwLCKyfYiLBLKDT/F7FYehdnFpQV2PjP/vVgX3xbn3q2+gFrXwfXD53d9X1688jw/gt77CqZRHDzehls/rY0NEREpDo1AT0fNL1Sr+XTod9hAi5P8NNCchTqE5CXEKzUmIU2hOQpxCcxLiFDOVcriDUw5XX/hDqGUyem+ZEZz1kKlp3Afm1GjFv/MQpym6oZ7eiMdwqUUiaUxJjnAPJOlb4yRw/5go0N9vsIT7N53UcYVDPI2re8IIp2dEgIa/DhnM4t9sYXoWatkEvo646H2frlzGFUFDQzjF9X7rP6C2v4dTHzPlaagFMb0HlTU5vFrF6R4En5yEOIXmJMQpNCchTqE5CXEKzUmIU2hOQpxiT7YeHIFayojKVyr6+ITMCA55N/s4Zt/G0xMkN4ynRmfCGHhBnEqJjG+k3cOVBdkc3hg3xieEcX3f4CgO5acjnD5K5HDlSZTGuawwpn+2WIBTM/EE/sypgTTUcoNY63f0tNnJ0wO4Z3QAj4X40R+8C7Wvvt2EWt1o/tXuHKnrHWPkwlAB3/sIPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDjFTKVMzeFKgFgc+7rd1k/gH1Tx26WHcBVGr49D77EUnkTdqusVDr0IX3syiRt19RNYyxdxhUZ5tAK16FQPv3eNGR+xEF9/LpeDWtyoCkIToANjUnY8ZTRXS+BrrDdwlRGaVJ4x7rfqEU6z5PI4Hfj2689D7f6jLajdurOvrteruFoobTSOQ/DJSYhTaE5CnEJzEuIUmpMQp9CchDiF5iTEKWYqJYrhULk1zrtZ00PlGSPMX6sajbrauLFWs4rD8ilQlFIYwCmR8WEcei+O4AqN8SH82YJkCWqtjP49ns7jqpROsAc1MSpngr5RHQMqeII4rhaKGamUoRFcHRMGxjWC+6pUwt9vOoZLpCo1I43V01NtIiIvrk1Cbaig3z8ffICbiR0d4GZ5CD45CXEKzUmIU2hOQpxCcxLiFJqTEKeY0VoxonvJEGslcMZ3tgTCpyLyzEXcY2UwiyN1iRj+f2lU9Uhdu3kO9+QGelBbXcaR3Nl5PH07npqHWr2iX+Ps1BS+jg29R5OISHEEH7AeGcaH85NJvbggNHpFRcZB+uxAHmr9No70x8H7paxCC8HR/NExPL263sRR40ZFP9wuIjIzrvcs+uM/+j7c896HP4Magk9OQpxCcxLiFJqTEKfQnIQ4heYkxCk0JyFOMVMp77z+MtQuPvsC1HafPlXXZ6ZxKmJleQlqk+NlqCUinJ6pgUPPHeNweCyOX29wAB98HxzEKYxEGqeCUiAl1WroLf9FRF66jFMzCysLUOuFOE0Ugf/pfojTHlECf1cJY8pzr43zMyE4+B5P4udILIuvQ4x9nR7+PpIJ3Jsq6Or31biRtnnrt1+FGoJPTkKcQnMS4hSakxCn0JyEOIXmJMQpNCchTjFTKS8//wzUnruKUymty3paZKCEqyJwpxqRKIZD5XEj5D0yoPeBMaYxmP9WIRgVICLSN3oqiRGy73T0cQxLl+bgnlwap3RaDVxxE4Ep2iIiEtO1yOjPE0ZYC4zfLDRKXbpgOnQQGhO2k8b9YfyitROcUtva2IHam29dVdebPdzPKm+lewB8chLiFJqTEKfQnIQ4heYkxCk0JyFOoTkJcYqZSslZVRhZPNJgIA9eNok7QlmNpGJWKsUK2Ud66iPs4ZSIlR6wpnn3jWSQUegiEWhQNjiEK3j6AX6vILTGV+MLiUSfYB23Lj7AWpDEKa5IjB8bNJWLhXjCdsb4zKkA/2YDbWMy94Ge0hEROXqsT9K+sIqbvB3H8egHBJ+chDiF5iTEKTQnIU6hOQlxCs1JiFNoTkKcYqZSCiUczo+MapBmRw+HRx0806ID9oiINOoNqHV7eF+no1eD9Ps4FdEzKkh6xns1jbkbzQauVuiDSpfCCJ6GXSjhuTJDhTGoZdP6PBQRkQDNvokZc00Ea4UCbnh2coi/x3ZLTzmEIZ6UHRP8ucIA33NFMKFaRGR+bgJqraZ+P0ZGM7RSAaclEXxyEuIUmpMQp9CchDiF5iTEKTQnIU4xo7Xvvf8TqAWpT6F2dqYfDK6fH8M9aKKxiB3JPTjQ30tEJACn6UeM8Q7DY6NQyyTw19U41Vv0i4isP7gLtWpdj07OLuKRC4kUjpQXC/j6FxdxX6ILs3q/pcWLM3DPSAYffC9k8TWGRi8pSeiH0XsBjoQmjJELCeMaJxaMyHYRR3J7kX4IP4GDxjIyYnxmAJ+chDiF5iTEKTQnIU6hOQlxCs1JiFNoTkKcYqZSPvr5Z1AburAKtSjQ0wPXP/s53DN/AfdfGRvF6YGnT/ah1gd9Z/Ij+OB4N44PxR88wS36v3ftdai9+PxzUGt22up63JgMvbG9BbX1B4+gdvPWdagNlfSpzD/+0z+Be958bgVqaWPmxYWpWah1QSrFmjhu9X3qgd5IIiLxpNGXaAgf3M+BXlJhAqf8cGIJwycnIU6hOQlxCs1JiFNoTkKcQnMS4hSakxCnmKmUP/vLv4JaprwMtWZNT288uPkt3DM1icPrcWMMQi6LT/t3Q72l/splfO3DU7hipTmG+9j88Ae/C7V8IQe1BkilGJMTpA/GTIiItPv664mIHB6eQm1rY1ddz+fx97v/5ARqm7cfQC3extf4eP9QXb/2/VfgnvmFaahZ1SzxrFFGksJplhjqFRTDe9Ixa3a7Dp+chDiF5iTEKTQnIU6hOQlxCs1JiFNoTkKcYqZSMmns3fV7t6BWPddTKZFVPdDFJ/rrxjgGa+p1NqPXAvSaeDzC+RG+xoNtXJXyk5/iZmhnNeP96ufqeqGIUxilYTwmY8BoTPXkiZ4uEREpj+mNvLJFnFr69EP8mU8f3IBa0MUjLx7u6w3bnhgjLZbXcGqsVMxjbRiPvMjlcVVKaUC/r1JZPCk7n8e/C4JPTkKcQnMS4hSakxCn0JyEOIXmJMQpNCchTjFTKbUT3Dzr43//EGo7+0/U9XhPrxIREblxo4ovxEiX9Pu46kBAJcBHH3wMt6RTOOT94tWXoNZNF6BW7eCp14+39SqMkxM8X6XbxhUOu/ubUNvYxK/5ytWX1fW/+9u/h3u+/OJzqPXPccVK1Zhw3hI9lfX4K5zG+vTrPagNJHHaJpXGqY9EBt8HBZBKuTC/APf86Md/ATX9m+eTkxC30JyEOIXmJMQpNCchTqE5CXGKGa2dmpiC2vLCItQi0aOJSWPUQcKIyMYT+D8kAtOrRUTS2QFdSOFDzdPTeJLz77z7LtQKeeOAdRb3HrpzS++rtP4Qj1WYnFmAWtsYg5DI4Wu8tX5PXb+zvg735BfWoLa7iz/z8BDWymm9r09+EPdhOt3H4ylOnj6E2tExnoreDowiDdDgaa+C7fTG94ymUAA+OQlxCs1JiFNoTkKcQnMS4hSakxCn0JyEOMVMpZwe4fb9r/3WG1B745131PVMBh80ThrpEmscQ2iMJkiI/n69Lm6b3+riQ+onTzagdtrGB6xPj/H3+BikTHYPcdHBYBmPH5AMThPF0jiV0u3rh9E/+uSXcM/80hWozY7glFQ2jm+7PCg86LRxD6HH1dtQGyzgXkxBhIsm9s/06ewiImNjC+p6s4fvxY8/+RJqf/03+tgTPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDjFTKUMGC3kT6p4OvH1G1+r6+UyrkaYKI9BrdfDaYqzswrUBExQTob49WYWcZpidhj3CXq6jvvYNOq4Z055YlJdz48OwT0JY5p3s4V/l6mpOajt7+p9n45P9HERIiJT08aYDGP0Rr2Dv39J6vdcL8Tpr0wOVB+JSMaoduqeHOHriOt9gkREJkBVULeDR4oYXwe+hO++hRDyfwHNSYhTaE5CnEJzEuIUmpMQp9CchDjFnmydwqfsO22cwvjss/9U16MeDvMX87iBU6+HqwfaLTziIQn+e+YXZuGey689C7WlOZxmqezoqQgRkf2zY6ilc3rqYGlUT7GIiBwd4YqJK6uXofbclVWo/fM//aO6nhS94ZaISK+Bf89uF2tRH6dFJKv/1tZ4hIXFi1A73LmP3yuOq6RyA/j91tZW1PV2E/8us1N4QjiCT05CnEJzEuIUmpMQp9CchDiF5iTEKTQnIU4xUynNFm52JUbTrXd/8EN1PeziKoaEkS4JA5zSiRLGdOKkngbIDuBGV/sVnJqpVfDckNMWvv5YFjfduv/NY3X95HNcMXFxEadEXr20DLWuUbGSS+upg8ioCLIqYOIJfGuBUSMiItIKwZydAH+/8xdwKqVdxxO2ny3iapYvv74Otd0tPT3TauD7O2qeQQ3BJychTqE5CXEKzUmIU2hOQpxCcxLiFJqTEKfYDb4GcUVCyWhYVBjXT+13OrjRVdb4n0jH8HVEOVzNksnr+8I2rh6o1apQS+RxY63yEm7ItZTHVSkPNsB4+RhOEaWMxmtP97ahNjqGG6whrdvC6YFOBzf/ahgVKx2jeqPX0dN3ySxOf01Mj0Ntaw+Plj/YBt+9iLTr+LM9uv2Nuj46iq8jGh6BGoJPTkKcQnMS4hSakxCn0JyEOIXmJMQp9sH3Gj7oLSH2dSo2qK4fHOAI2IM7m1DLJnFENl3CUdIxMP5heqwE9ySNA/2jpVGoGWfzpd3Ch57LZT0CPDONo3t7+3jq9fr6XagtdBehhiLptRr+zZpNHAmtnuOotxWtDbp64UEigw+p376FR3lYIxLK5QmozTyPezGVx/V9Y+O471PWuH4En5yEOIXmJMQpNCchTqE5CXEKzUmIU2hOQpxiplJCo6V+3PB1sqcf2i4a4x2+/uITqO0f4IPjsRQ+BH7t2svq+luvvwL3nJ/j1MGNX/8Kag0wRVtEZH17B2qPNzfV9VYT92+KItyEJ1vEh6+r1RrUamBkRKOK00BGKyBJJrBaKuBD7NOLerpneHQK7ilP4xTG9NUrUBsxegilrd5USDOKFST67s9BPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDglFkVGMyBCyG8MPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDjlfwBMLfgMMxlEmwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'truck'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "idx = 1\n", "X = X_train[idx:idx+1]\n", @@ -292,11 +574,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ - "!kubectl logs $(kubectl get pod -l serving.knative.dev/configuration=message-dumper -o jsonpath='{.items[0].metadata.name}') user-container" + "!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}') " ] }, { @@ -308,9 +590,31 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e4dad391e1a648849dc62363ce407eee", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=50.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "from tqdm.notebook import tqdm\n", "for i in tqdm(range(0,5000,100)):\n", @@ -327,11 +631,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res=!kubectl logs $(kubectl get pod -l serving.knative.dev/configuration=message-dumper -o jsonpath='{.items[0].metadata.name}') user-container\n", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Drift False\n" + ] + } + ], + "source": [ + "res=!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}') \n", "data= []\n", "for i in range(0,len(res)):\n", " if res[i] == 'Data,':\n", @@ -349,9 +661,18 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Logging before flag parsing goes to stderr.\n", + "E1110 15:45:08.551094 140080817293056 plot.py:39] Importing plotly failed. Interactive plots will not work.\n" + ] + } + ], "source": [ "from alibi_detect.datasets import fetch_cifar10c, corruption_types_cifar10c\n", "corruption = ['motion_blur']\n", @@ -361,9 +682,46 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAS6ElEQVR4nO2dSY9k13GF474p58yaeiQI2YZkwIZX+mv6Gf4XXupPCF4aIAhvKBgy3KTIZrO6uyqrMiuHN18tqOU9hyAXZqBxvuULvDHfyQfEuRERYowmhPBH9mtfgBAijcQphFMkTiGcInEK4RSJUwinFCz4H3/4PUzlZiGQPdO7DeOA96BZYxwL5Dqijel9yJnY/1VGYiHga8xJLMvS1xjJPY8jjg0kNo74zoeYjg0R3/OQvvS/74djjBEclJ0L7WNmxl6rccT7tSRW9+mDdr/wd/n3P75JPnx9OYVwisQphFMkTiGcInEK4RSJUwinSJxCOIVaKZGYDmMkuW1kA4B0vdlP2BssSGLIwQgZsV+w2wOMmR/JmbVErBS4G/MAyEUGYsFE8l8cwaswBGKlkHsmToRF8u5E9I6QxxHYNyb8gvf0J2PI/iKXwf27JPpyCuEUiVMIp0icQjhF4hTCKRKnEE6ROIVwCrVS+rGHMVppAfwN9k+AKkjMuPXBQFfIqhECeSQh5GQ/YpcY82fAM4742bOc/Rjx9fdZBWPtWCa3Hxt8rr4nVS49vmdW3VMg64bYLzlzUsh7FYnNwt9HcB3E12NVKQh9OYVwisQphFMkTiGcInEK4RSJUwin0GytjS0Msd49ASxezjOy8Jpl3Eiylqyltwh75rD/JPxIsgxna2fTdLbTzGwAPWfMzLo2nQ4NAV/HGPC5mjiBsUOHj/lh3yW3twM+Vz/gh58HfB3ZmD6XmVkGstTTEj97G88wNClJlpekebOc3BtIl9N8rBa+C/HpIHEK4RSJUwinSJxCOEXiFMIpEqcQTqFWSkZ61eQknZ8Dy4G0ozEjmXLam4X1OUKnIpZIOZnB2KTCsdXFGsa2d+9hrAeuwhCJXTJOYez2Ea9U/+GhhrGxTF9/MdvAfQZiRfQ9seF6fI25pR9IaI9wn/UM/y7DgO+5IgvfC/KuZnn6jcwNFytkzPND+/zsPYQQ/y9InEI4ReIUwikSpxBOkTiFcIrEKYRT+DiGfA5jLelVE8AE6xxMcTYzK0iPGDpDm+4HqmNInvz62Q2MFZMFjN3fPcDY4xnfwbFbJrdvdzgt//bDFsaeWnxv1eIaxpbLF8ntxRxbRHTmQoetlK7Gtkh93ie3zybYWjqQqpTQ4gqYOXn7l7jdksECGdJXi00qR+jLKYRTJE4hnCJxCuEUiVMIp0icQjhF4hTCKdRK2TZ4tX8kZSSnw11y+2aB08k3JHddsJnSZAJ0yNPXWFb4tpsGp+W3uxOMvScVH3d7bItsD+nrf7/FlRtPLbYVZutLGCs32CbKZ6vk9vkKWylsSsZ+9whjXYef1WyVroLpGmy/7J6eYGyS42cFp2ibWRlwRVYFqnGKDL+LbAQIQl9OIZwicQrhFIlTCKdInEI4ReIUwikSpxBOoVbK90943sXTcQdjaLX/bIbT2iMZlhJIIybWNAzNu+hJxcTdPb6v+xM+2QOJHXpc3XMENxAX6WoVM7PpkjQom+JzVXMcm83Sv/W0IJUWA7aIKmJFRGJltV26iuRY43Ode/xenUkFTLXAv1lLKqg6MPtmUmBr5pcMZ9eXUwinSJxCOEXiFMIpEqcQTpE4hXAKzdaec9yK/zDgjGdVpjW/XOPjZRlevDyySchgcbuZ2WDpLN6JLBw/DHix/8cjzkCWpD/PqiIjDY7pLOR4xs/33ODnEcj/LUsYVuAxdif8u7Cig4JMhm7IIvDjMZ1dPTc4W9t25Doifj/OLVncTootSjCOYYIeopll6iEkxKeDxCmEUyROIZwicQrhFIlTCKdInEI4hVopxewCxq4rPJrg5Sqdhp4vsT2QoxHPZja2eL/B8OL8wdL9b7J1evSAmdnE8H3943Mcq3v8P9f02FbYn9KjFcKIrYNAet8E4pecSK8da9L9kdZLfM/TKZ6wfa5xD6S2w/eG+vqwaeRFSQoqiO107rCVMiW2yAkUTlQdfgfmUzLfAaAvpxBOkTiFcIrEKYRTJE4hnCJxCuEUiVMIp1Ar5fnr38BYe8a9WV4/T6eNq/4W7hOPOM3f1Lh9/2LzCsbmq39Ibr/+7F/wuchk6OMTvue7B9x7qOuwrVCGdMo+IxUfkwpbBw8PeAxCTiYvZ4t0Nc7N9RXcZ0WqjPoRn6snVhCquCkrUiUywTbFOZLeQ8RmyU/4fDloklW02H5Bo0EY+nIK4RSJUwinSJxCOEXiFMIpEqcQTpE4hXAKtVKWG9y0io0LKCbpVfsh4gqSmLEYroyoFrjC5ObV75LbzzW57UhiI556XZIJynmGq2o2m/RE6WOLz3XaY9uJVaWw4Az8ns9fvIT7rNd46nXdYptif8T3ZqD6pKrw+9GTiqZyghu2taxipSfN3Jq0zRLAxGszs3OD7TSEvpxCOEXiFMIpEqcQTpE4hXCKxCmEUyROIZxCrZSywmnomkyHDiF92LLEx+vBXBMzszLHafTTU7oxlZnZ7bt0FcwY0vaFmdmETIbuemwBRMPPY7XGjbCsTldvzA5kWnODqzBY86zLC9yw7fXrz5Lbb55hq6qu8fNoe1INUuNnhSa6sJk4bA5JNuB3pyBN6oxUs7Qxfd/7I66eih2uaELoyymEUyROIZwicQrhFIlTCKdInEI4ReIUwinUSulJZQGb1zGdpO2IbMQWwKnGqWs276I738NYOaSvIxY45Z2dsIVBxnXYzStcvVGTCofxkP5/XB9xxcdorJEU/kkvLi9hbHORjvVkRPyJzENhdglr/jWAxmZxIA3DenyNFvD3Z7HCzzgM+N4iqDAZiH0UyXNE6MsphFMkTiGcInEK4RSJUwinSJxCOIVma6sCh4sCLxBvwaL4/Q4vlD42ZFpzjzN144Azr0OzT24/nfA+kyUeMfCC9NO5fPYMxp5OOPN3v08/kxBwZnu1wP+pqOjAzKys8DFRBrUl059rkpFtWpy5zMl7FUBmu23wb1ZN8OL2qsT3nBnJoOLbtgH0Yhob/J5WmmwtxKeDxCmEUyROIZwicQrhFIlTCKdInEI4hY9jmJHeNyNOlR+P6b4+wDX48XAB9xeKI+4TxKYPHM7ApijxwvFnl7jPzvLiBsayEo+nGMni6/MpnbOfENuj63AhQE3a/k/JCA009bo+4dEPXY+tlOUKn+t4xrZIP6SfR17ggoTpFL+n8ykZ8zGQnlBkUf9YpGPzGf7Nip8/2FpfTiG8InEK4RSJUwinSJxCOEXiFMIpEqcQTqFWyoykqFtSkTCGdPp6KHB63SY4dV03ZPQDmSacgfEP6w2uPClIxcTVs/TIAjOzh90Bxt68+QbGui5tK+z2W7gPHWcALBEzsyxiC2MEvZjmSzyy4EAqgpozGT8QcclHVaYtk0CmchcZeQfIWIWx2cFY3uLeVBtQkLUglScjuWeEvpxCOEXiFMIpEqcQTpE4hXCKxCmEUyROIZzyE5OtcROvOOJl9vNlOp1/rHF1CRjwbGZm/YQ0tBrwMdeoSiCQEoEcV8d8/eZrGLu9/QHGdk+PMHY+py2YmoygyEpcaWGk0qIasR2xAjbAdvsB7vPDR1yxUpB3pyQlGrtd+phssnXssA03RGzprHMcu5zhd24zR+8Itm3QmAmGvpxCOEXiFMIpEqcQTpE4hXCKxCmEUyROIZxCrZSczJkoiB1RFkDzFa5w6Mj8kmKKq0iyAV/HAJyDwXC1ze3tRxh73GJL5OmAKxympPFTP6TndeQVtktyYqWUOU7Zx4hng/z1u++T28/Y0bHlCk/K3j6k59SYmX3/3TsYa8GUauZEXCzx811PsdX2bIW/TZdL0twupq0b9FuamYX487+D+nIK4RSJUwinSJxCOEXiFMIpEqcQTqHZWrY+PBgJgkXxFRkxUJDMcDSSqiP9YyK4xO0OL3g+7HHWlUxxsEjSiUWB7y2v0j9BAP2PzPizGlv8k+6ecF8ctBh9MsPXsSd9k95+jwsBHh7xgnn00i3n+He+XuBRDZ9v8PN4vsaFALnhPk0tGIeRkdkgA5uiDdCXUwinSJxCOEXiFMIpEqcQTpE4hXCKxCmEU6iV0tTYcigmeGFwEdKt52fANjAzO5IeMTWxB3qwUNrMLIK0/PoC97fZksXtZzAp28xsMsGL0bsB/we+fPE6fbz5Cu5zf38HY0PE+fzpfA1jXZtezH1/j62lb96+h7HHPV5wPpBeRrNZ2ha5RjMQzGxD1qivF8TGynDPn77H19iA35NNMB9HWSlCfDJInEI4ReIUwikSpxBOkTiFcIrEKYRTqJVS19g6uJhi66AHvWoqMoU6z3HKe4hkNEGBr2O5TtsRRcAp9JuXr2Bs+/AAY9M57o/0+ee/gbFX4HyHE7Yi2GTrocf3tnvEx3wElTpfv72F+2yf8BiEPuJXazbDz2pxeZXcnmEnxUYS2xJ740NNrn/Ek6jbPl2BxHoIRSMlKwB9OYVwisQphFMkTiGcInEK4RSJUwinSJxCOIVaKacRN05ajGTadEgfdiQt6dsOWwBlia/js9cvYey3/5S2MNjohMXmAsZeBXwdXYfT6MsVHidRg2dy6rBdch6wffRQ4+qet3fYOri9SzfrOgzY9rD1NQwtptjfYFZKBBO2DxP8qn6L5m6Y2XhkE6VxbBhxrO7SNgt7v9m5EPpyCuEUiVMIp0icQjhF4hTCKRKnEE6ROIVwCrVSdj1O2Y87nJafg6O2xH6ZTPFMjhfP8ATlf/vXf8b7vXie3L7d4xkfNsNWypdf/R+MnU74ecQ7PLW7B421zmd8vLtHXB3D5pAcztgKaqZpSyqQBllFhTtrlWSeixX4Os6gKdsRzN8xMws9juU5tlkiqTxhjeNYEzWEGnwJ8QkhcQrhFIlTCKdInEI4ReIUwik0W/vtA84ybkq8kHdZpGObOc7+viIL2J9fLXHsBi8qn83S2cTtDzjb+Z9f/A+M/e83eFpz17LW/jhTNw7pjGEDFlebmdU9jg2kd89Q4mfVo1EZGcmS5vhcTYZjNSuaAMdkk9RLUGhhZhZJlnQkPX/YaIUeTDEfSfY3kOeI0JdTCKdInEI4ReIUwikSpxBOkTiFcIrEKYRTqJXy5v0exq7nWNefXaQtjGDYmlmShe+bNbZSJjMc+/Z9ulfQn/7rK7jPl3/+K4zVHbaP2LrmCFLvZrhXzUBazgzkP3UIpJ8OPWY61V8Qm6Jnrw+xdDJiwZRgUfx0ghfLZ6Q/D3n0FsmzIi6LoXXv0zmZol1QqSXRl1MIp0icQjhF4hTCKRKnEE6ROIVwisQphFNofnd7xCMBRjDd18wsH9OTqAfsltjVCvejqea4r8/dHl/jF199ndz+339+A/c5kfsaIq4syMjU7oHk5buYvn7WvJ91o2ETlCOptMiRvZFhC4PFMjJCYzrBv/WkAvuBaelmZgOo7DEz68n3pyPeUkWsvatNemJ6Re55HHDVEkJfTiGcInEK4RSJUwinSJxCOEXiFMIpEqcQTqFWSkMaUz00ZPxAm07nl1d4ovFQrGHs9hHbJc3dexj7y3fvktv3TdrqMTOznNgDGUnLk8ncrCrFQGXEOLKW/ziWkUZSOWm6ZSiWk3EME9ywjVWesKZhLXjnmF3SEquKVfesyBTz6ys8AiQHZlZ9xGM+giZbC/HpIHEK4RSJUwinSJxCOEXiFMIpEqcQTqFWCnEHrCPVGwWofnis8X/BxwOZM3GHG429e38LY9tjuqFYS6oRRlbVQSwRVpUykv0CmORcENtjZJ26SOVJJPNGMmCZBDKFmjY1C6y6h02UTr90LXkZA7G/1pfYEmGxwwHbIqf9Nrl9OB/hPrMZKckC6MsphFMkTiGcInEK4RSJUwinSJxCOEXiFMIp1ErJC1yRMJA8eg1slocTTqH/5bsPMHZ/wGnoDx+xlfLuYzrl3ZCx7S2Zh4JsDzOznI1nz7A9k4NKFzYSPTBrhlSDFCX+PSOwblhbqkgqLcYeV/705N56UH1SVLgC5uLqGsbmyw2MPe2xRbe/x+9jbM7J7RlpvcbmueB9hBAukTiFcIrEKYRTJE4hnCJxCuEUmq2NJCMb0HhfMxtANnF/wpOt2+4EY7f3OLt6OD7h2Cl9zA4fzkirGjh12cysqkjPH5KoQ1lZsm7cStL2v6hwRpb17kGjGooc79ORrPe5Jj2myHs1B2MQLq+v4D6LFc7IstEVpxHnogeSbe7a9L1NSyynwwEvikfoyymEUyROIZwicQrhFIlTCKdInEI4ReIUwinUSmG+AlvkG0GPmIb0gWGp9yywtDazZ9LX2DEvhVlEZMF22+FrDKQvUQmsiowslg9sUTy5taqcw1gBJkoPpF9R1+J7LgxfSDXDi9hXy/TIjqs1HuVRTfFrfCL23Ujenb7FI0AGYCEdgcViZtaT9wOhL6cQTpE4hXCKxCmEUyROIZwicQrhFIlTCKcEOnVZCPGroS+nEE6ROIVwisQphFMkTiGcInEK4RSJUwin/A0FKch2HmWlvwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQ1ElEQVR4nO2dS28k2XGFIzMrs54ki2Sz2TPjeUljCYIWkgD9ZwMGDHvhhQ0YNuCFf4C9MDyQMAt5hFG3+sEmi6xHVqUX9vKeM+4CpxVuf98yAzcrKytPJRDnRkQ1DEMAQD7qP/UFAEAZxAmQFMQJkBTECZAUxAmQlJELPn/+XKZyP9Qsb1VV7/cDj7iNx975Y9YdfzfMp1WPfCXv+VFUz8ixmri+vi6ekDcnQFIQJ0BSECdAUhAnQFIQJ0BSECdAUqyV0jSNjGGlPBbiPtrbe9y9H5wdIULDoNdU1mjR11gdZaWYT/o/bqUoeHMCJAVxAiQFcQIkBXECJAVxAiQFcQIkxVopLjX8oVopP8T3Os6cefzrcGc8CMvE3w9nl+hVlTnn+zayHhOsFID/JyBOgKQgToCkIE6ApCBOgKTYbO373wT+uGTJKPt8Zznqrv2wP8jYzsX6vYxtd+VY3/f6Og76fPPZVMa6kS6omIy74vHaPIs/xK98zLP/2HrhzQmQFMQJkBTECZAUxAmQFMQJkBTECZCUoze+f6gcmw73LX9cVGw4Nyv2Znv47qBX3m+1LfLwsCke73c7uaZp9HXMptpKaRr92FXifeF/lyOfU3PODCYib06ApCBOgKQgToCkIE6ApCBOgKQgToCkWCvFVgI8ss2SpbLAYXvwGAujP7x7pch6qy2Mu4e1jt2XLZGIiM3OVZiUr39UG7tkUq4giYjYmuqYsbmRcvrDu0+S+F9F5aoEFVm8OQGSgjgBkoI4AZKCOAGSgjgBkoI4AZJirZTNZitjzmapRPrdr9H/E8dWJLipzIqDa6xl7JKtaXa1MdUga1H1sRJVIhERL9/eydjrtyv9WcaeCfHd3L/3pNXRtyczGXt6cSZjVxfL4vHZdCzX+EHZ714R9H0cU8tyzCfx5gRICuIESAriBEgK4gRICuIESAriBEiKt1JMArg1TZoaYX0MJudtih/C9JGKyiWpxccdBl0xcdibeSJmbsjWNMLamFhdl+9j1+p5IovJ5J3PFxGxNbNS9iK2N/djZP7am1ZXrOwrvVCZd2Njtdk3jHFSHrvwxFk6x1Rd8eYESAriBEgK4gRICuIESAriBEiKH8cwmcvY3ma61CZqnSWtTMxtRq9NTEVc/yPfG0nHKhMbmTTeuCv/BNNOZztnY52tFQOqIyKicmMQRJbX1Q7UZkL1yKRyR42OqZDOk0fU7nepzbPjzumyvDLwuOla3pwASUGcAElBnABJQZwASUGcAElBnABJsVbKX/zlX8nYyVz3iLkUPWI+urqUaz66upCxi5OFjFlE+tpmtV2av3JWhPYcRm0rY63YIF7V2qaYGH9jb2IH81/ci5uyMZv9H9Z6LMTq/l7G7u91D6TtpnzOyhQrONvp9ETbgefLU73OPN+qF9bgeggdscmeNydAUhAnQFIQJ0BSECdAUhAnQFIQJ0BSrJXy5vVbGdut9biAWkxynjbaHjib6kqLE1OFYfsLib4zzqYI09+mMtffhE7nN3YiQPkLmMkPtkpnZ8ZCPGweZOztqmx9vHpzI9e8ePlKn+/2VsY2ZjL3IJ6dzlTAOFvv6sm5jLWmcmZunsfGPAePCW9OgKQgToCkIE6ApCBOgKQgToCkIE6ApFSuodW//Ns3MtiYZkadSFG7Rlet6ahUDboy4nDQMWWZuEZXg0mTu6qDg6kGqY11o9LyLl3vJn3vze/pxkncC2vs7l7bLysTG/a6iiT2psFaX17nijpMIVEsFtoSub7WVVKnC23PqAZl7hpd7PJsWQzz5gRICuIESAriBEgK4gRICuIESAriBEiKrUppBq3d1qTzG2UriIqDiIjV2qXsdXXMaqWbRd2JJlMbM1Dk5k5fx7d/eC5jr290FcbYVNUsREXF2aluanZxvpSxqyfaHnjyxDRROy9Xb1yc6SZYgy2dMSFjs6jJ4u6zBtP8qzE+y2Q8lrHeDgMqx9wMGOulCHhzAiQFcQIkBXECJAVxAiQFcQIkBXECJMVaKX/zt38nY+NWLz2Zl62DpUnLLy907ORUz7to5+W5LBER47p8HXdvtDXz3Uttl/zHb76Vsd9/p9eNx3pWyrmYA/PMzJUZPtPWweVC38d5o5uQLUbi93TNxEyVy3a71et2O71OxAYz4WYwFUG9sW32r5wFo6uCZvNp8fjUNgXT78EnpyfF47w5AZKCOAGSgjgBkoI4AZKCOAGSYrO1//rvX8vYYqYzU2dn5c3cz66fyDX7Wm9G7w86u7eY6UzuZFrOqp2a7N5Hz/Rn1ZXO4H31xRcydmYmc19fle/JJ8+eyjWXZiJzZ+ZT7O71GISv//CiePz1zRu55vWNznrfmoKEjZheHRHRiwywGq0R8T2Twzu9uX08MQUJ5jc7vywXCSyX2jkYqWx4RPzo2VXxOG9OgKQgToCkIE6ApCBOgKQgToCkIE6ApPgeQge9eXk21u3qT2bl09YH3Z/n+X9+I2M3btPwpbZnzoVNMTX2y1efP5Oxn3z5iYw1ZsSD2/RciY3lDw/6Xv32m9/I2OtXr2Xs7a22N27vVsXjK2O/rLd6uvne9ItqzJTqyaRsfUyFLRYRMRtpu2Q6MetOyhvOIyKmC22lVKNyAcGm13bgeqeLBBS8OQGSgjgBkoI4AZKCOAGSgjgBkoI4AZJirZTuoFPl10ttpfz4R58Wj7c6gx63b7QFMOt075vLE30dFydly2RhxhnMTXq9MdUPezMSQE2Njoh4eyssjK22Um5MVcfGWDqTC207za8+Lh63FpGZ2G2mdURjKjS6rhxrzb3vOlOVYtaF6Uu03enf7Fb8Zi9f6mf4cNA2S/z6F8XDvDkBkoI4AZKCOAGSgjgBkoI4AZKCOAGSYq2Uzz/7TMZ++YtfydjPfv7T4vGm1vn1g2ntPzap8olp4NQqC8a02j8YC6Af9DXuxUTmiIh2YsYxTMtNoWZL3eDr6Z+ZMQg7MzXaNDarhWXiJkof9jq2t6MatE3x8FCeRr5el49HRNzc6iZkDw9l2yMiYm0qf9zzGMI2q8zz7SqTFLw5AZKCOAGSgjgBkoI4AZKCOAGSgjgBkmKtlKef/kTGpuflKoaIiJe3IsVubIraTF2ue1POcqdT3oOYsXIwFSS9sUR2e93wbGPtAR3bbMvXeG8aa/WmWVRvmky5SdT36/Lnrc2E6sqUnhxMgy9ll0RErEWs3+nrGBkLozWlUBNRAfPfMTNjRVTV1NYu0baTgjcnQFIQJ0BSECdAUhAnQFIQJ0BS/DiG6YWMffOtnmq8Xr8qHu/dJmqXzDIbto9IgsVgsrUuy3gYdCZ0v9eZ0J3JeG635Syp2xxemeuvTEp8EKMfIiI24hq3Oz3p+2BuvuuZc9jrc6rvNjITu+vO9DIyfY6i1++mQ63v8b5S07f1Z/lMrljzzisA4L2AOAGSgjgBkoI4AZKCOAGSgjgBkmKtlH/+p3+QsaY2G9Wb8qbhRkwEjohozMb3ZqR78DSNacWvWvvb9v16w/PIjBFwYwuqSn+3TvQzaid6zf5g+vMYm6I3ls5BeFmHndlIbzaj972+DmdlVVX5OnpjpfS9vo791lhLE219jCr9HDTdpHi8Hel3nXk8JLw5AZKCOAGSgjgBkoI4AZKCOAGSgjgBkmITvP/4938tY9WgdT0alVPNI2Ol1JW+lNqMT3BTkttx2TLpxqY/zFhPyu4m5UnZERFdp2Ntp885ElZQbfriVNUPUDkjqk/cGmuJmL/9xtginej5MzYjLWYz/VzNjA03FpZfRETX6Ps4irJ1U5uKpmpPVQrABwPiBEgK4gRICuIESAriBEgK4gRIirVS5toBiH6nG1A1TbmyoGlMNcWDrix4uNft+20DKtlkyjQMq3TqvRIWUUREPdJWSt3qWDVSFTymcsbZR6ZKx01XVpMVmlav6Sb6OmZTbVOczvV9nM7K9+rSrDk/1bH5XN8PU5wUo1Y3L5uOyzdrMnMVUtruUfDmBEgK4gRICuIESAriBEgK4gRICuIESIq1Ukw2PGoT7Mbl1PZMpMkjIhrzP3F3o+ey3JjY6n5VPN6bplUh5mBERIw6nV4fz3TlTLeY6s/ryrHbtbaqtrsHGetDr+uMTTQbl1P9p3P9O1+c6t/zyfJUxq5M7Py07N8txmYOyaDttP2gLbrGlM7Mp/q7LZaL8pqFXjMWmnDw5gRICuIESAriBEgK4gRICuIESIrN1t68eCFjzchkJ7tyhm8/uZNrZmJNRERtNrePQmdX27qcXa1GOuuqRjhERJyLLF1ExCcffyxjX3755zL2xedfyZji9q3OULsp2u6fuBM73ydmxMDE9DkaiXsf4cdJ3G/KU9FXq/IE8IiIEH2HIiJmyxMdOzuXscXyTMZOzsrZ5vmJfj6UJhy8OQGSgjgBkoI4AZKCOAGSgjgBkoI4AZJirZQq9Abx+1vd1+dhuC2vqY39YjYhV6bnz86MH9iKcQGDOV9jWuoPKzMy4o3e3D5/oTdEfyo2S//44kp/1kxvot52+j5ujM2y2pQ309+8LRcPRES8WpvYRsdWxko5TMUIDbNZfm4skenSbEY3BQmtucfttLyuaXWfoOqI0da8OQGSgjgBkoI4AZKCOAGSgjgBkoI4AZJi87uzEz2PYbPRvWp2azEl2fTn2Rl7w1kpQ5jqh0FUpZhpDG6Ssxv9sFppa+nNc13d8/Lr3xaPP53pqojO3I/b0FbQm0rbTn8Uv82LWp/vptPWWC96AUVEjC+XMrZ4Wo7Nz7VdMp6bqeITY4mYCefdRNss0jIxVuHgRn0LeHMCJAVxAiQFcQIkBXECJAVxAiQFcQIkxY9jmOl0+F2nqw42wko5CGsjQtseEWEHURsnRQbdP5IpSolDb9r+28oZHXsQzs2dmfQdJi3/nbGrvu30dbyYlh+F7Zm2FLpL3Txr8eRCxmZn2kqZLMpNslxDOeeNVaYJmTtn07x7rDaVJ7U5n1zzzisA4L2AOAGSgjgBkoI4AZKCOAGSgjgBkmKtlLbTk5BrM0MjRNZ4OGjf4+DsEueluJC1Wcr0xpt5MLHXptHY7w7GFunL1SwvzTyXvtX3/o/6J4u1mUQ9vi5bH8tr3Wjs5FxXzoxnR1R1REQlLIfK2R6tfoybkY5VtXmG7fNYpjaWTk1VCsCHA+IESAriBEgK4gRICuIESAriBEiKH+DQ6BKNRmfDYzIrp737Xp/PuCw25uyZSlWDmPP1Jh1+7xqDmTHra9ckqy1bML+f6DXzM10t1J3rSpHlE90k6+KiHFss9PlGpgnWodOP1mAqNKq27AW1E92MayJml3xfrDMNvlpj9yh7xlozR8CbEyApiBMgKYgTICmIEyApiBMgKTZbq7KuERGn57rN/WRa1ny/0xnIvdsUb2JiePX/BMuH3SZkl3EbmSyjKxKYmUzjQvRpOj3VWdLLU73h/Oy03IMnImJmJjmPJ+XsZNOaURgmQ+36RdlihaZ8/10PnlqsiYio3YZ520PInNP0JVK4sSHyc955BQC8FxAnQFIQJ0BSECdAUhAnQFIQJ0BSrJWyvNBpebNnOPa78kgAMzQ6XOZ9MOMMKmOL1MIWaYxd4lLobatT7xNrpejYidg8fmpGYcwnZmp0pzds12Yz9yBsha2xGwZnKZj7aC0pMdLAjUc4dsO5s+j2e90Tat+XY1Vl7B56CAF8OCBOgKQgToCkIE6ApCBOgKQgToCkVIPzMADgTwZvToCkIE6ApCBOgKQgToCkIE6ApCBOgKT8F3zpIIhq7MiJAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAARZklEQVR4nO2dyY4sZxGFI6eq6urq2237AhLzJEBixYoX4D15D96DYYNkAcb4+rZ7rsrKkcVl+Z+D3AsIWd+3rNCfndPplOL8EVGt6xoAkI/6/30CAFAGcQIkBXECJAVxAiQFcQIkpXXB3//hjzKVuy6TXNfUVfH3w34v11xfv5Gx3aaTsa7R/19UrK10hrotn/p/jqdjTaOP2ZmD6og+3jLP+kTMut1mK2ONvI8mm2/ulV3nVoll66KP5xyHutInWdfu22SOKS68bvQLUpmbdXXYFoN8OQGSgjgBkoI4AZKCOAGSgjgBkoI4AZJirZSLrdHurGOVSENvjN2wa3Vs2ywy1tbGwqjK65TV8+F4MhS1sWCMyxJNpaO1OBdnD6yzvh/jNOqYsT7WpXyObePulY4tqz7HylgOIawPfbSIdTV2SeXeU3tQfUxx3e5+vAa+nABJQZwASUGcAElBnABJQZwASUGcAEmxVkobuvIkKl0ZUYvEd1cNcs2m1sfbmnS+sm0iImqRDm/M/6TKVD8siz7H1dgDi7FSlK2wuqqURRsLs6lYOU5HvU5YMK4iaNPp16cx1SC7na6OCWF9qEqQ/0ZlTRjNaqwgXTqj36vVPLOI8v3gywmQFMQJkBTECZAUxAmQFMQJkBSbrW1MtnZd9QbrRmwQb8328GY9y1htsqSmhVCoJG/jMn9mw/NiNpUPOhTLpG9zt9kUf6/MJup50s9lGHRG3CSiYxLX9vyiM7zuftxcHWSsc1le8Yq4wQSuv1Bj+gS5e7zM+h4vYqP9YgsLvn5PJb6cAElBnABJQZwASUGcAElBnABJQZwASfEb30UPnoiIZdWp5lps2q4XneZvjDVTm03ItekfozYbr27ju9mwPc/6/OdR349zL0OyZ05lLIC205vRJ+PpPDy/yNixL1tZbsTAxlgRl3vT58jcK1lcYN4B22+pNv2bTH8hN/JCvXPzagojsFIAvjkgToCkIE6ApCBOgKQgToCkIE6ApPiqFGMdrJOOhUgpr2asQj3v9HmYdH5r0uGrOI9KjB6I8FZKbaylMUzvnqP2Uk7CwujPukqnac1jM/djMBbGo6g+eff+vVzjpor/+le/kLH9XvcQ2nbimLO2iBYTc1UpjbFZXBWMGq/hqlIWMwk+4tvFX/lyAiQFcQIkBXECJAVxAiQFcQIkBXECJMVaKUuvqxiWs7FSxI7+2aS8540+lXWnqzCqjbFFRHVMZcY7aEMkYh5N8y8zbbppTSMpMS7g/Vf3cs2TabrVu05jrgpDVE18/q8v5Jrn50cZe/funYz97Effl7Gf/OB7xd8/ur6Sa1wjusZUirRuUrmxWVTMvFa2YVjEz4u/8uUESAriBEgK4gRICuIESAriBEgK4gRIirVSTs/PMjaYrlWjqLQ4mxkZGzMPZTZVDK4yQhWz1J2bQq3z4cOsU+/GZXHzwWMUs02qRt+r2Ty2uydtf93eaXvm/q5si0yjaWpmrLHPP9dWyl/+9GcZ+/ZH18Xff/nzn8o1P/rBd2XscmveOf0aRGMmYqvJ7dvWVLmYqpTv/fq34u8AQEoQJ0BSECdAUhAnQFIQJ0BSbLb2b3//TMbOg+5x059Oxd83phfQ7d2djLmMbNfq2OGi3JdoYzJ4bgzCInrHRESsJjabY55EX5/nQWevn086g/rwop/L8Ww2em8vy79vLuSaxfSR+ujmRsYOW3OvzmWH4B+f/VOueXrUG/B3G1000Zns6mL6Z23EuotdeUp5RMRspoD/5nfl3/lyAiQFcQIkBXECJAVxAiQFcQIkBXECJMVaKX/+66cytor+PB+C5dhiphN/+u5LGWuNlaLS2hER+52wUswaN/qhbnRavm715vzVjUgQ07cXY7/0g95E3bsx2qbJzbYr2wCV2QDehOnttGjrYBjKVltExNub8sb3T250D6HVvFcvL7oQ4P5LHXsyRR/NpvysW2vNuO5UZfhyAiQFcQIkBXECJAVxAiQFcQIkBXECJMVaKV886HTydqN34NfC+jDteaJZdbAyvXs2ode9HMuWQ2W6+rStsQcafR5VrWPzqv8HVqK1f1Vre2AYtF3iqh8aY6XUVTnV3xlr6WqvK1Z27V6fx6Jj1VKuBnl60v2PXDXImzflapuIiMm4gfcnXd3z3ItxI8Y+chPTFXw5AZKCOAGSgjgBkoI4AZKCOAGSgjgBkmKtlMFYAKuxNy5EU6XtVldubDodq4xdEqY6RlWYbC90en02ls5gphM3xoJpGtP4STgm59FMqDY2S2uaZ3XGClKjCfZmqvhhp5/ZWzOJ+jsflytPIiKmc3lq9+2tHu/wcPeVjJ3MM2uNtdSYCpPzqWxlzaIaK8K9pRq+nABJQZwASUGcAElBnABJQZwASUGcAEmxVspWzM+IiNiZqpRZzJk4mZ3+da2P5xo4hWmedXP1pvj79Ucfm+PpFPrprOdnTJNp4OTmqIiJ3qtJy0+mqsatcw2+2m3ZMtlt9XPZmQnhO2NFtMYZa8V7dXOl7Zf+WTfqen7SlVVn5WNFxHQqWzoREUfV/Es0/oqIWKlKAfjmgDgBkoI4AZKCOAGSgjgBkoI4AZJirZSLva4scKPgp7GcRnfzIvreVHyIOR4REftLbfcsVfnyTmdd8XFxof/W4aDvx2CslHXR9kYtZqJUV6Yx1agbfC2myVRbaetgI/yNy41+RS432h7YbfW6xTQhG8Ssl8E8s2nS1/VgrJTRWCmLqQoaXsrHnHr9XOpOV/fINV97BQD8T0CcAElBnABJQZwASUGcAEmx2drdxUHGzB7q2F+Ws5pt4/6c3ijtNrdvxPTqiIjDVfn893s9DqAz2cnGbPRWYxUiQk76johYVMZQbIiPiKjNtOnabIqfRz1Rep3KsY3J8JrbEbW5ZjcyQu0Pb1r9XDY7M96h1dn3h/v3MvZy1JnXTryOtekUVLniDQFfToCkIE6ApCBOgKQgToCkIE6ApCBOgKRYK2V/KPfgiYjYmo282658WDc1+sLYNluTKq9Nir1V52E8gNrdEdsGRqfK19X0FxLFAPNo7JdR2yXVos/DpfPHodwfaRx0f57Tos+jNxvfL439tRU9i0ztQOxNQcIPf/wTGdvtdXHBnRnx8LEocrA9idwFCPhyAiQFcQIkBXECJAVxAiQFcQIkBXECJMVaKU2j28urMQIREZNI57u+Q7Vp31+3ep2rVlDex2ymcs/GblhNhca6mhEJbnzCXK7QqEzlhloTEVGZPk2rm8wtbKL+lX12OjeOwdz/6Vw+x9X4WC5Wm2qhb719K2M//fEPZawSfZ+++PJWrnl60paUgi8nQFIQJ0BSECdAUhAnQFIQJ0BSECdAUqyV8nzUDaHcxOO6LqfKH5+e5JpH0zb/8EZXx7y5uZGxpilXwVSmSMRVl7jYvOqp1/OiYyHGJ1SrsUsmbYl0tbYVLnb6mbViDMWFec7O0nFWymJsuFFYQZO55uGsJ6Yv5hxbc69CTa+OiFbYdxeu2majYwq+nABJQZwASUGcAElBnABJQZwASUGcAEmxVorrZzWaJlPTWLYcGlNd4tLQbtZIf9K7/buNsAHMvyQ3x2M1sTBNvCpjwXRionRnmpB1OzPPxT00cx/VkJK205VJi5lhs5rqHlfR1IsJ1kdj6w2DtlLOvV53Oup3xxRQxXZbvifb3YVeg5UC8M0BcQIkBXECJAVxAiQFcQIkxWZrJ9NzZh50xu1KTJS+MZvUdzudFQyT+VtMz5xxKE8nXkz2dAkzUdpkZN0kZzciYRSt/adeH280KdlOjKCI8JOXG7UJ3Jy7GwsxjXqz/7nXU6P7czm7qn6PiHgxm9Qf7u9l7GQywCeT5Z3FM7u+vpZrDld6ZISCLydAUhAnQFIQJ0BSECdAUhAnQFIQJ0BSrJVy/YluV+/sDZWWH83G8elZp94rYwFYe0BYDputmbBt+uy4CdVDf5Sxx8cHGetfypuvh7O2GxYzUdr1djoc9ITwq8vylOfK7JV3vXvcZnQXezmWbZG7ez1p2lkpsxkZcT7qe/zyYo4prJTbr/Q5OstPwZcTICmIEyApiBMgKYgTICmIEyApiBMgKdZKcdKtTP+YSaSNx0GntRdjs1TGtnFWilrnKjf2l7oPjOuB5Gyi2VR2nESFxt3te7lmENU2Edo+itDVQhERbz/5pHw8McU5IuJs7J7+pKs6Hh/1WA5lpZzNNU/GLhlMBczZnGOYid7Knnky40ac7aTgywmQFMQJkBTECZAUxAmQFMQJkBTECZAUa6U8PtzK2JsrPW1aVZGMk6486Y+6qmM0VQzTqGMq5d2I0QMRfpLz5UHbLIdDuaojIqJr9G3edOUKmU78HuGrY4zDZe2e26/Kz1rZYhERvR114J6nqU4SE6ydbeMahjkrZTUN7FZRefLhmOXrHgb9fi+mukfBlxMgKYgTICmIEyApiBMgKYgTICmIEyAp1koZj2aXvZjIHBFR1+WpzK7yZFl0rD/pZkvPz48ypiY5b41N4So+zoO2DkaT6neVIqOYKdK2+tEsi45Nxq46vmjb6UE8m8k9MxmJWM3smNFYDqpZ19jrc1+NTzEbK2h2s16MfbdIm0U/Z+PeSfhyAiQFcQIkBXECJAVxAiQFcQIkxWZrBzNN+PTspiSXs7WryWbVJvfXNvp/SKsmMkfEupSzastq+hWZ85gmHbu/1/fK9Y/pRZbXZUlX0zfJjcmwMXHMxWyyP5kMqrquD6ehz6MSoaYqv1MREcPsCiNMttY8z3XV71wl3u/avN8meS3hywmQFMQJkBTECZAUxAmQFMQJkBTECZAUa6Xcm5EA/W4nY42wPlwK3cWm2Yw6sJvpyxubZ3O8aTIbnoU18yFoxkKYxj61ulf6L9nzcBvO3TmqkQauEMCNQXD490Cco7kuZzupwoIPh3R3WcfWtWyZVKHtHjPVQq/5+ksA4H8B4gRICuIESAriBEgK4gRICuIESIq1Uv712ad6oelxU4lKEWsBmPS6w/VmqUSwrnWa3Ke8jV3iKm7MQatZp98Vi6kumcU4g4iI/uT68JSfjSn6idVMf15M757J9O5ZxbU528NW8Nj3ypy/WaeurRYWS4S+LgdfToCkIE6ApCBOgKQgToCkIE6ApCBOgKT4ydbvP3/VQVVvpKbRtkHVmsoN0VDpvx2zEROlVdXMh7/lWuq7VLmpYjCWg1rlLCJnzbh7tenM417K6wbTqMuNVXCjDuy9Uo3GrLXhGnWZSiJjb7jRIauwUkYzKdtZXAq+nABJQZwASUGcAElBnABJQZwASUGcAEmxVkptZoosJh1eidDi5k+Y5kiuVGRddGwRFR+TO57t+fSKgRcR0Zi/1worqHLlMe483HOp9eNWFsZ41naJmwFTv9J2moVlspoKElfx4QZKL8b6WExjsEU0iHOzhSasFIBvDogTICmIEyApiBMgKYgTICmIEyAp1kpx6WtXoaHy13aNHQ7yujHryu6pjP3i5prY0zenOJuU/SpS7O5vNSZYi0qcDzEzgl1UdrjKkzCVIqYWxFaYyKoUa5eYKhc3g8fNejHN6CYxP+bcH191Hgq+nABJQZwASUGcAElBnABJQZwASbHZ2rCZSzsHofiz629jRxa4mE0al4MqIxjhR0bMZsSA6xEz2zEUIqMsV0S05rm0XadjWz2NXGWbXZbRZl1Nht3Pky5H3TNToyQiImY32dqscz2ERrXh39wP9zwVfDkBkoI4AZKCOAGSgjgBkoI4AZKCOAGS4nsImU3Uzt6oVarf+B4u1WzT0Latj0htm03Uq+0ro2OTSdm7DdbzVE7nOwvD2U7dRtslm1nfrKYpWzBuQrUbg2B5zQN1Ixxc3yTzl9y1Db0eQ6Enab9ulIeCLydAUhAnQFIQJ0BSECdAUhAnQFIQJ0BSKpeGBoD/H3w5AZKCOAGSgjgBkoI4AZKCOAGSgjgBkvJviII0D0ByTCAAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "show(X_corr[0])\n", "show(X_corr[1])\n", @@ -379,9 +737,31 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9708c321c1a04cef8ed9877067159405", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=50.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "for i in tqdm(range(0,5000,100)):\n", " X = X_corr[i:i+100]\n", @@ -397,16 +777,24 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res=!kubectl logs $(kubectl get pod -l serving.knative.dev/configuration=message-dumper -o jsonpath='{.items[0].metadata.name}') user-container\n", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Drift True\n" + ] + } + ], + "source": [ + "res=!kubectl logs -n cifar10drift $(kubectl get pod -n cifar10drift -l app=hello-display -o jsonpath='{.items[0].metadata.name}') \n", "data= []\n", "for i in range(0,len(res)):\n", " if res[i] == 'Data,':\n", " data.append(res[i+1])\n", - "j = json.loads(json.loads(data[1]))\n", + "j = json.loads(json.loads(data[-1]))\n", "print(\"Drift\",j[\"data\"][\"is_drift\"]==1)" ] }, @@ -419,14 +807,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl delete -f cifar10.yaml\n", - "!kubectl delete -f cifar10cd.yaml\n", - "!kubectl delete -f trigger.yaml\n", - "!kubectl delete -f message-dumper.yaml" + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace \"cifar10drift\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete ns cifar10drift" ] }, { diff --git a/components/drift-detection/cifar10/cifar10cd.yaml b/components/drift-detection/cifar10/cifar10cd.yaml deleted file mode 100644 index 102a093ecb..0000000000 --- a/components/drift-detection/cifar10/cifar10cd.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: drift-detector -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "1" - spec: - containers: - - image: seldonio/alibi-detect-server:0.0.1 - imagePullPolicy: IfNotPresent - args: - - --model_name - - cifar10cd - - --http_port - - '8080' - - --protocol - - tensorflow.http - - --storage_uri - - gs://seldon-models/alibi-detect/cd/ks/cifar10 - - --reply_url - - http://message-dumper.default - - --event_type - - io.seldon.serving.inference.drift - - --event_source - - io.seldon.serving.cifar10cd - - DriftDetector - - --drift_batch_size - - '5000' diff --git a/components/drift-detection/cifar10/message-dumper.yaml b/components/drift-detection/cifar10/message-dumper.yaml deleted file mode 100644 index 7bbd4e0a98..0000000000 --- a/components/drift-detection/cifar10/message-dumper.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: serving.knative.dev/v1alpha1 -kind: Service -metadata: - name: message-dumper -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "1" - spec: - containers: - - image: gcr.io/knative-releases/github.com/knative/eventing-sources/cmd/event_display diff --git a/components/drift-detection/cifar10/trigger.yaml b/components/drift-detection/cifar10/trigger.yaml deleted file mode 100644 index d37befc877..0000000000 --- a/components/drift-detection/cifar10/trigger.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Trigger -metadata: - name: drift-trigger - namespace: default -spec: - filter: - sourceAndType: - type: io.seldon.serving.inference.request - subscriber: - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: drift-detector - diff --git a/components/outlier-detection/cifar10/.gitignore b/components/outlier-detection/cifar10/.gitignore new file mode 100644 index 0000000000..9d4c9d601e --- /dev/null +++ b/components/outlier-detection/cifar10/.gitignore @@ -0,0 +1,5 @@ +event-display.yaml +broker.yaml +trigger.yaml +cifar10od.yaml +cifar10.yaml \ No newline at end of file diff --git a/components/outlier-detection/cifar10/cifar10.yaml b/components/outlier-detection/cifar10/cifar10.yaml deleted file mode 100644 index cea433ff02..0000000000 --- a/components/outlier-detection/cifar10/cifar10.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1 -kind: SeldonDeployment -metadata: - name: tfserving-cifar10 -spec: - protocol: tensorflow - transport: rest - predictors: - - componentSpecs: - - spec: - containers: - - args: - - --port=8500 - - --rest_api_port=8501 - - --model_name=resnet32 - - --model_base_path=gs://seldon-models/tfserving/cifar10/resnet32 - image: tensorflow/serving - name: resnet32 - ports: - - containerPort: 8501 - name: http - graph: - name: resnet32 - type: MODEL - endpoint: - service_port: 8501 - logger: - mode: all - name: model - replicas: 1 diff --git a/components/outlier-detection/cifar10/cifar10_outlier.ipynb b/components/outlier-detection/cifar10/cifar10_outlier.ipynb index ed1c3ac5f6..4647a2275d 100644 --- a/components/outlier-detection/cifar10/cifar10_outlier.ipynb +++ b/components/outlier-detection/cifar10/cifar10_outlier.ipynb @@ -38,9 +38,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gateway.networking.istio.io/seldon-gateway unchanged\r\n" + ] + } + ], "source": [ "!kubectl apply -f ../../../notebooks/resources/seldon-gateway.yaml" ] @@ -53,44 +61,126 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 4, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/cifar10 created\r\n" + ] + } + ], "source": [ - "Enabled eventing on default namespace. This will activate a default Knative Broker." + "!kubectl create namespace cifar10" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting broker.yaml\n" + ] + } + ], "source": [ - "!kubectl label namespace default knative-eventing-injection=enabled" + "%%writefile broker.yaml\n", + "apiVersion: eventing.knative.dev/v1\n", + "kind: broker\n", + "metadata:\n", + " name: default\n", + " namespace: cifar10" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 6, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "broker.eventing.knative.dev/default created\r\n" + ] + } + ], "source": [ - "Create a Knative service to log events it receives. This will be the example final sink for outlier events." + "!kubectl create -f broker.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize message-dumper.yaml" + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting event-display.yaml\n" + ] + } + ], + "source": [ + "%%writefile event-display.yaml\n", + "apiVersion: apps/v1\n", + "kind: Deployment\n", + "metadata:\n", + " name: hello-display\n", + " namespace: cifar10\n", + "spec:\n", + " replicas: 1\n", + " selector:\n", + " matchLabels: &labels\n", + " app: hello-display\n", + " template:\n", + " metadata:\n", + " labels: *labels\n", + " spec:\n", + " containers:\n", + " - name: event-display\n", + " image: gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display\n", + "\n", + "---\n", + "\n", + "kind: Service\n", + "apiVersion: v1\n", + "metadata:\n", + " name: hello-display\n", + " namespace: cifar10\n", + "spec:\n", + " selector:\n", + " app: hello-display\n", + " ports:\n", + " - protocol: TCP\n", + " port: 80\n", + " targetPort: 8080" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment.apps/hello-display created\n", + "service/hello-display created\n" + ] + } + ], "source": [ - "!kubectl apply -f message-dumper.yaml" + "!kubectl apply -f event-display.yaml" ] }, { @@ -102,18 +192,67 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize cifar10.yaml" + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting cifar10.yaml\n" + ] + } + ], + "source": [ + "%%writefile cifar10.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: tfserving-cifar10\n", + " namespace: cifar10\n", + "spec:\n", + " protocol: tensorflow\n", + " transport: rest\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - args: \n", + " - --port=8500\n", + " - --rest_api_port=8501\n", + " - --model_name=resnet32\n", + " - --model_base_path=gs://seldon-models/tfserving/cifar10/resnet32\n", + " image: tensorflow/serving\n", + " name: resnet32\n", + " ports:\n", + " - containerPort: 8501\n", + " name: http\n", + " protocol: TCP\n", + " graph:\n", + " name: resnet32\n", + " type: MODEL\n", + " endpoint:\n", + " service_port: 8501\n", + " logger:\n", + " mode: all\n", + " url: http://broker-ingress.knative-eventing.svc.cluster.local/cifar10/default\n", + " name: model\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/tfserving-cifar10 created\r\n" + ] + } + ], "source": [ "!kubectl apply -f cifar10.yaml" ] @@ -127,18 +266,64 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize cifar10od.yaml" + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting cifar10od.yaml\n" + ] + } + ], + "source": [ + "%%writefile cifar10od.yaml\n", + "apiVersion: serving.knative.dev/v1\n", + "kind: Service\n", + "metadata:\n", + " name: vae-outlier\n", + " namespace: cifar10\n", + "spec:\n", + " template:\n", + " metadata:\n", + " annotations:\n", + " autoscaling.knative.dev/minScale: \"1\"\n", + " spec:\n", + " containers:\n", + " - image: seldonio/alibi-detect-server:1.5.0-dev\n", + " imagePullPolicy: IfNotPresent\n", + " args:\n", + " - --model_name\n", + " - cifar10od\n", + " - --http_port\n", + " - '8080'\n", + " - --protocol\n", + " - tensorflow.http\n", + " - --storage_uri\n", + " - gs://seldon-models/alibi-detect/od/OutlierVAE/cifar10\n", + " - --reply_url\n", + " - http:// \n", + " - --event_type\n", + " - io.seldon.serving.inference.outlier\n", + " - --event_source\n", + " - io.seldon.serving.cifar10od\n", + " - OutlierDetector\n" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "service.serving.knative.dev/vae-outlier created\r\n" + ] + } + ], "source": [ "!kubectl apply -f cifar10od.yaml" ] @@ -152,20 +337,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": { "scrolled": true }, - "outputs": [], - "source": [ - "!pygmentize trigger.yaml" + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting trigger.yaml\n" + ] + } + ], + "source": [ + "%%writefile trigger.yaml\n", + "apiVersion: eventing.knative.dev/v1\n", + "kind: Trigger\n", + "metadata:\n", + " name: vaeoutlier-trigger\n", + " namespace: cifar10\n", + "spec:\n", + " broker: default\n", + " filter:\n", + " attributes:\n", + " type: io.seldon.serving.inference.request\n", + " subscriber:\n", + " ref:\n", + " apiVersion: serving.knative.dev/v1\n", + " kind: Service\n", + " name: vae-outlier\n", + " namespace: cifar10\n" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trigger.eventing.knative.dev/vaeoutlier-trigger created\r\n" + ] + } + ], "source": [ "!kubectl apply -f trigger.yaml" ] @@ -179,31 +396,79 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "35.240.18.201\n" + ] + } + ], "source": [ "CLUSTER_IPS=!(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n", "CLUSTER_IP=CLUSTER_IPS[0]\n", "print(CLUSTER_IP)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are using Kind or Minikube you will need to port-forward to the istio ingressgateway and uncomment the following" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ - "SERVICE_HOSTNAMES=!(kubectl get ksvc vae-outlier -o jsonpath='{.status.url}' | cut -d \"/\" -f 3)\n", + "#CLUSTER_IP=\"localhost:8004\"" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "vae-outlier.cifar10.35.240.18.201.xip.io\n" + ] + } + ], + "source": [ + "SERVICE_HOSTNAMES=!(kubectl get ksvc -n cifar10 vae-outlier -o jsonpath='{.status.url}' | cut -d \"/\" -f 3)\n", "SERVICE_HOSTNAME_VAEOD=SERVICE_HOSTNAMES[0]\n", "print(SERVICE_HOSTNAME_VAEOD)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Logging before flag parsing goes to stderr.\n", + "E1110 11:58:05.548886 140347082585856 plot.py:39] Importing plotly failed. Interactive plots will not work.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(50000, 32, 32, 3) (50000, 1) (10000, 32, 32, 3) (10000, 1)\n" + ] + } + ], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", @@ -236,7 +501,7 @@ " 'instances': X.tolist()\n", " }\n", " headers = {}\n", - " res = requests.post('http://'+CLUSTER_IP+'/seldon/default/tfserving-cifar10/v1/models/resnet32/:predict', json=formData, headers=headers)\n", + " res = requests.post('http://'+CLUSTER_IP+'/seldon/cifar10/tfserving-cifar10/v1/models/resnet32/:predict', json=formData, headers=headers)\n", " if res.status_code == 200:\n", " return classes[np.array(res.json()[\"predictions\"])[0].argmax()]\n", " else:\n", @@ -247,7 +512,9 @@ " formData = {\n", " 'instances': X.tolist()\n", " }\n", - " headers = {\"Alibi-Detect-Return-Feature-Score\":\"true\",\"Alibi-Detect-Return-Instance-Score\":\"true\"}\n", + " headers = {\"Alibi-Detect-Return-Feature-Score\":\"true\",\"Alibi-Detect-Return-Instance-Score\":\"true\", \\\n", + " \"ce-namespace\": \"default\",\"ce-modelid\":\"cifar10\",\"ce-type\":\"io.seldon.serving.inference.request\", \\\n", + " \"ce-id\":\"1234\",\"ce-source\":\"localhost\",\"ce-specversion\":\"1.0\"}\n", " headers[\"Host\"] = SERVICE_HOSTNAME_VAEOD\n", " res = requests.post('http://'+CLUSTER_IP+'/', json=formData, headers=headers)\n", " if res.status_code == 200:\n", @@ -269,11 +536,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVmklEQVR4nO2d2Y9kZ1LFI/elKjNrzdq6tq6uKpfdbbu9NN6wkQbGDAwa0ACCF57gCYkH/h3eEA/wADKWrdHgwYPHg+2x7Gm7965eauuufcnKfbv38g/ECck8MCF0fo/f0Zd5M/OevFLEFxGxKIqEEOKP+G/6AgghOjQnIU6hOQlxCs1JiFNoTkKckrTEf3j/ZzCU++Te13Df0cZddT0I8NtNzD0DtbmlNagNT85BLZvT32/99mdwz9bDG1Dr1epQSxifrThcgloym1fXr735NtxzaQV/V+3zU6jdvnUdamHYVde7vTbcc+f2TahVK8dQ63Q7UOt1E+r66UkT7qk38TX2A/xe4+MjUBseGYRaENX09+rBLdJu4azIe//205i2zicnIU6hOQlxCs1JiFNoTkKcQnMS4hSakxCnmKmU6hkOy48O4TB0ND6hryeLcM/U3EWoBSGOUcdDHGIPm311vX12AvdELRyWnxkrQ21u9hLUZi/NQ2165oK6Xi7r36GISCqVgVp/SE/NiIjMXpjE+/p6KqXdbsE9lTOcWjo+xvdOMp2FmsT0VMrwKP7M2QF8jefVM6hlsvj2DyP93hERSSX1a6meV+Cebue7F5jwyUmIU2hOQpxCcxLiFJqTEKfQnIQ4heYkxClmKkV6OIXR7WCt2dTD8gsrM3BPvdHA72VURoyMGRUfKf2/Z3l5Be5547VXoDYzoac9RERKpXGo9ZIB1PJZPSyfNCLvsT4O87caOL3RMX7PfE5PwQwP4fTR0sVnoXb37n2oSQxfR6ejp8ZKxWG4J5XGb3VePYBaJPp9KiIShvgHODvT79VWE1fA/G9adfHJSYhTaE5CnEJzEuIUmpMQp9CchDjFjNb2jUPPsT6OQGbSOXX9/Bj3lRmdxJHQuefwofLy7DTUUiiMZzR76fVxZPjeHj4w33x8hF8zjqOC929+q66/uoYjoW9fexVqVgf/avUcattbu+p6OoUPqafTuJBhbBxH5rd3HuDXBD2V6i0cza9W8X2VTKnteUREpFjERQKtFi6oCECwvN8P4Z5MxggpA/jkJMQpNCchTqE5CXEKzUmIU2hOQpxCcxLiFDOV0mni8PVgDofYiyP6IfCXXngR7pm9uAy1mnHQ+/7jHahVm3o4vF7BvV5OKjhdsreP+9EUjYPvEscHoj/4l39V11N/jv8333n9LailUjhNNDmJ004S6emIypk+ekBE5NfX8eiKpNHnaKCAUzD9QE8Fdev4N0sYjxhr5EIQ4BTXySlOz8RFT8Ekk9hOQ0O4QAO/DyHEJTQnIU6hOQlxCs1JiFNoTkKcQnMS4hQzlZLJpKDWSxSg1srpU4E3qrjK5Ztffgm10xPcF+fpLu4Rk0roFQmpOK4e6ICxBCIi7TbWpsbxV3m4vwW1IqhWqFWqcM/6xga+jqkxqKVS+BqnZvVRDdNgXURkex+nse7fxFp5CqedNrdBCqOHf7Owi7XA6N+UTeN0TyaJ7/1WW3/NYhGniJJghIMFn5yEOIXmJMQpNCchTqE5CXEKzUmIU2hOQpxiplLyeTxd+bCCK0Ue7uhh9Du3b8E9cSPMHxijH1o1XDmTACmTVgenKSo1rNWMUQebT+5CbSCH006rS6u6YKR0/vvT/4La/OIi1FZW8RiK0VG9asKa/lwq4vRAvI+biTU6+JmARhq0Krg6JghwU7ZsDqdE6lX8mkWjciaT1advd7vWiBLcMAzBJychTqE5CXEKzUmIU2hOQpxCcxLiFJqTEKeYqZShEVzh8HBnHWp7m3rVRD6FG12dN3DzrHr1EGqxEFckVGp66qPSwqH3pFGJMzaBpzznCriB08zCC1CbBWH5jW8/h3sSMZxm6QW4CuPoGDcvu3JlTV2/tHwR7pk1qksGX7sKtRv3tqHWaeuN4zopoypFcNojjHDKb39fnw8jIpLO4DRRaRjdBzit12rhiiwEn5yEOIXmJMQpNCchTqE5CXEKzUmIU8xo7aNHuK/PvUcPoba790hdD4xD6oXSANRWlxegdnntMtT2jvQI2dYRvo7xSXzYf34JHyovjOJI7sEZfr/oWI9sb2/hiOaRMTLCGIgtv7eiR2RFRBp1/bsKcfBXoi6OGt/+Akebl1fxWI6JmSF1/YsvfwH37B/gYoVeD0dr2y18/WfGGIrcoH6NYYQjyg1jtAmCT05CnEJzEuIUmpMQp9CchDiF5iTEKTQnIU4xUylf/OIjvHEC9L4RkaW1K+p6zmibv/Ysnmy9unIBakFbPzguIhLF9fRAQ/DU4mQKT+xOJPQQuohIr48PSjdqp1ArdfVQP5rwLCKyfYiLBLKDT/F7FYehdnFpQV2PjP/vVgX3xbn3q2+gFrXwfXD53d9X1688jw/gt77CqZRHDzehls/rY0NEREpDo1AT0fNL1Sr+XTod9hAi5P8NNCchTqE5CXEKzUmIU2hOQpxCcxLiFDOVcriDUw5XX/hDqGUyem+ZEZz1kKlp3Afm1GjFv/MQpym6oZ7eiMdwqUUiaUxJjnAPJOlb4yRw/5go0N9vsIT7N53UcYVDPI2re8IIp2dEgIa/DhnM4t9sYXoWatkEvo646H2frlzGFUFDQzjF9X7rP6C2v4dTHzPlaagFMb0HlTU5vFrF6R4En5yEOIXmJMQpNCchTqE5CXEKzUmIU2hOQpxiT7YeHIFayojKVyr6+ITMCA55N/s4Zt/G0xMkN4ynRmfCGHhBnEqJjG+k3cOVBdkc3hg3xieEcX3f4CgO5acjnD5K5HDlSZTGuawwpn+2WIBTM/EE/sypgTTUcoNY63f0tNnJ0wO4Z3QAj4X40R+8C7Wvvt2EWt1o/tXuHKnrHWPkwlAB3/sIPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDjFTKVMzeFKgFgc+7rd1k/gH1Tx26WHcBVGr49D77EUnkTdqusVDr0IX3syiRt19RNYyxdxhUZ5tAK16FQPv3eNGR+xEF9/LpeDWtyoCkIToANjUnY8ZTRXS+BrrDdwlRGaVJ4x7rfqEU6z5PI4Hfj2689D7f6jLajdurOvrteruFoobTSOQ/DJSYhTaE5CnEJzEuIUmpMQp9CchDiF5iTEKWYqJYrhULk1zrtZ00PlGSPMX6sajbrauLFWs4rD8ilQlFIYwCmR8WEcei+O4AqN8SH82YJkCWqtjP49ns7jqpROsAc1MSpngr5RHQMqeII4rhaKGamUoRFcHRMGxjWC+6pUwt9vOoZLpCo1I43V01NtIiIvrk1Cbaig3z8ffICbiR0d4GZ5CD45CXEKzUmIU2hOQpxCcxLiFJqTEKeY0VoxonvJEGslcMZ3tgTCpyLyzEXcY2UwiyN1iRj+f2lU9Uhdu3kO9+QGelBbXcaR3Nl5PH07npqHWr2iX+Ps1BS+jg29R5OISHEEH7AeGcaH85NJvbggNHpFRcZB+uxAHmr9No70x8H7paxCC8HR/NExPL263sRR40ZFP9wuIjIzrvcs+uM/+j7c896HP4Magk9OQpxCcxLiFJqTEKfQnIQ4heYkxCk0JyFOMVMp77z+MtQuPvsC1HafPlXXZ6ZxKmJleQlqk+NlqCUinJ6pgUPPHeNweCyOX29wAB98HxzEKYxEGqeCUiAl1WroLf9FRF66jFMzCysLUOuFOE0Ugf/pfojTHlECf1cJY8pzr43zMyE4+B5P4udILIuvQ4x9nR7+PpIJ3Jsq6Or31biRtnnrt1+FGoJPTkKcQnMS4hSakxCn0JyEOIXmJMQpNCchTjFTKS8//wzUnruKUymty3paZKCEqyJwpxqRKIZD5XEj5D0yoPeBMaYxmP9WIRgVICLSN3oqiRGy73T0cQxLl+bgnlwap3RaDVxxE4Ep2iIiEtO1yOjPE0ZYC4zfLDRKXbpgOnQQGhO2k8b9YfyitROcUtva2IHam29dVdebPdzPKm+lewB8chLiFJqTEKfQnIQ4heYkxCk0JyFOoTkJcYqZSslZVRhZPNJgIA9eNok7QlmNpGJWKsUK2Ud66iPs4ZSIlR6wpnn3jWSQUegiEWhQNjiEK3j6AX6vILTGV+MLiUSfYB23Lj7AWpDEKa5IjB8bNJWLhXjCdsb4zKkA/2YDbWMy94Ge0hEROXqsT9K+sIqbvB3H8egHBJ+chDiF5iTEKTQnIU6hOQlxCs1JiFNoTkKcYqZSCiUczo+MapBmRw+HRx0806ID9oiINOoNqHV7eF+no1eD9Ps4FdEzKkh6xns1jbkbzQauVuiDSpfCCJ6GXSjhuTJDhTGoZdP6PBQRkQDNvokZc00Ea4UCbnh2coi/x3ZLTzmEIZ6UHRP8ucIA33NFMKFaRGR+bgJqraZ+P0ZGM7RSAaclEXxyEuIUmpMQp9CchDiF5iTEKTQnIU4xo7Xvvf8TqAWpT6F2dqYfDK6fH8M9aKKxiB3JPTjQ30tEJACn6UeM8Q7DY6NQyyTw19U41Vv0i4isP7gLtWpdj07OLuKRC4kUjpQXC/j6FxdxX6ILs3q/pcWLM3DPSAYffC9k8TWGRi8pSeiH0XsBjoQmjJELCeMaJxaMyHYRR3J7kX4IP4GDxjIyYnxmAJ+chDiF5iTEKTQnIU6hOQlxCs1JiFNoTkKcYqZSPvr5Z1AburAKtSjQ0wPXP/s53DN/AfdfGRvF6YGnT/ah1gd9Z/Ij+OB4N44PxR88wS36v3ftdai9+PxzUGt22up63JgMvbG9BbX1B4+gdvPWdagNlfSpzD/+0z+Be958bgVqaWPmxYWpWah1QSrFmjhu9X3qgd5IIiLxpNGXaAgf3M+BXlJhAqf8cGIJwycnIU6hOQlxCs1JiFNoTkKcQnMS4hSakxCnmKmUP/vLv4JaprwMtWZNT288uPkt3DM1icPrcWMMQi6LT/t3Q72l/splfO3DU7hipTmG+9j88Ae/C7V8IQe1BkilGJMTpA/GTIiItPv664mIHB6eQm1rY1ddz+fx97v/5ARqm7cfQC3extf4eP9QXb/2/VfgnvmFaahZ1SzxrFFGksJplhjqFRTDe9Ixa3a7Dp+chDiF5iTEKTQnIU6hOQlxCs1JiFNoTkKcYqZSMmns3fV7t6BWPddTKZFVPdDFJ/rrxjgGa+p1NqPXAvSaeDzC+RG+xoNtXJXyk5/iZmhnNeP96ufqeqGIUxilYTwmY8BoTPXkiZ4uEREpj+mNvLJFnFr69EP8mU8f3IBa0MUjLx7u6w3bnhgjLZbXcGqsVMxjbRiPvMjlcVVKaUC/r1JZPCk7n8e/C4JPTkKcQnMS4hSakxCn0JyEOIXmJMQpNCchTjFTKbUT3Dzr43//EGo7+0/U9XhPrxIREblxo4ovxEiX9Pu46kBAJcBHH3wMt6RTOOT94tWXoNZNF6BW7eCp14+39SqMkxM8X6XbxhUOu/ubUNvYxK/5ytWX1fW/+9u/h3u+/OJzqPXPccVK1Zhw3hI9lfX4K5zG+vTrPagNJHHaJpXGqY9EBt8HBZBKuTC/APf86Md/ATX9m+eTkxC30JyEOIXmJMQpNCchTqE5CXGKGa2dmpiC2vLCItQi0aOJSWPUQcKIyMYT+D8kAtOrRUTS2QFdSOFDzdPTeJLz77z7LtQKeeOAdRb3HrpzS++rtP4Qj1WYnFmAWtsYg5DI4Wu8tX5PXb+zvg735BfWoLa7iz/z8BDWymm9r09+EPdhOt3H4ylOnj6E2tExnoreDowiDdDgaa+C7fTG94ymUAA+OQlxCs1JiFNoTkKcQnMS4hSakxCn0JyEOMVMpZwe4fb9r/3WG1B745131PVMBh80ThrpEmscQ2iMJkiI/n69Lm6b3+riQ+onTzagdtrGB6xPj/H3+BikTHYPcdHBYBmPH5AMThPF0jiV0u3rh9E/+uSXcM/80hWozY7glFQ2jm+7PCg86LRxD6HH1dtQGyzgXkxBhIsm9s/06ewiImNjC+p6s4fvxY8/+RJqf/03+tgTPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDjFTKUMGC3kT6p4OvH1G1+r6+UyrkaYKI9BrdfDaYqzswrUBExQTob49WYWcZpidhj3CXq6jvvYNOq4Z055YlJdz48OwT0JY5p3s4V/l6mpOajt7+p9n45P9HERIiJT08aYDGP0Rr2Dv39J6vdcL8Tpr0wOVB+JSMaoduqeHOHriOt9gkREJkBVULeDR4oYXwe+hO++hRDyfwHNSYhTaE5CnEJzEuIUmpMQp9CchDjFnmydwqfsO22cwvjss/9U16MeDvMX87iBU6+HqwfaLTziIQn+e+YXZuGey689C7WlOZxmqezoqQgRkf2zY6ilc3rqYGlUT7GIiBwd4YqJK6uXofbclVWo/fM//aO6nhS94ZaISK+Bf89uF2tRH6dFJKv/1tZ4hIXFi1A73LmP3yuOq6RyA/j91tZW1PV2E/8us1N4QjiCT05CnEJzEuIUmpMQp9CchDiF5iTEKTQnIU4xUynNFm52JUbTrXd/8EN1PeziKoaEkS4JA5zSiRLGdOKkngbIDuBGV/sVnJqpVfDckNMWvv5YFjfduv/NY3X95HNcMXFxEadEXr20DLWuUbGSS+upg8ioCLIqYOIJfGuBUSMiItIKwZydAH+/8xdwKqVdxxO2ny3iapYvv74Otd0tPT3TauD7O2qeQQ3BJychTqE5CXEKzUmIU2hOQpxCcxLiFJqTEKfYDb4GcUVCyWhYVBjXT+13OrjRVdb4n0jH8HVEOVzNksnr+8I2rh6o1apQS+RxY63yEm7ItZTHVSkPNsB4+RhOEaWMxmtP97ahNjqGG6whrdvC6YFOBzf/ahgVKx2jeqPX0dN3ySxOf01Mj0Ntaw+Plj/YBt+9iLTr+LM9uv2Nuj46iq8jGh6BGoJPTkKcQnMS4hSakxCn0JyEOIXmJMQp9sH3Gj7oLSH2dSo2qK4fHOAI2IM7m1DLJnFENl3CUdIxMP5heqwE9ySNA/2jpVGoGWfzpd3Ch57LZT0CPDONo3t7+3jq9fr6XagtdBehhiLptRr+zZpNHAmtnuOotxWtDbp64UEigw+p376FR3lYIxLK5QmozTyPezGVx/V9Y+O471PWuH4En5yEOIXmJMQpNCchTqE5CXEKzUmIU2hOQpxiplJCo6V+3PB1sqcf2i4a4x2+/uITqO0f4IPjsRQ+BH7t2svq+luvvwL3nJ/j1MGNX/8Kag0wRVtEZH17B2qPNzfV9VYT92+KItyEJ1vEh6+r1RrUamBkRKOK00BGKyBJJrBaKuBD7NOLerpneHQK7ilP4xTG9NUrUBsxegilrd5USDOKFST67s9BPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDglFkVGMyBCyG8MPjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDjlfwBMLfgMMxlEmwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'truck'" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "idx = 1\n", "X = X_train[idx:idx+1]\n", @@ -290,11 +580,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res=!kubectl logs $(kubectl get pod -l serving.knative.dev/configuration=message-dumper -o jsonpath='{.items[0].metadata.name}') user-container\n", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Outlier False\n" + ] + } + ], + "source": [ + "res=!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}') \n", "data= []\n", "for i in range(0,len(res)):\n", " if res[i] == 'Data,':\n", @@ -312,7 +610,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -328,9 +626,32 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVkElEQVR4nO2daYzd5XXGz92XmbvM7hnPMDM2HmOwDWZxWUyIoOAsVGmbVE2XtFLVfiJKN6lF6qcoaqO2qpR+qAQlrdSGbl8igkApdQIhJkAswGBsY8bL2DPj8Yxnv/v6//drK53nKHal5Kh6fh/fR++979x7n/lL57znnEgYhkII8Uf0Z30AQogOzUmIU2hOQpxCcxLiFJqTEKfELfG5F78HQ7mL596F+1bnPlLXu138ds/+/Teto/zUiEQiULMi20cf/STUxiZ2Qu3rf/036vp7H56Fe26duQ1qje0NqJ05fRJqQdBS11vtBtxz9syHUCttrUGt2WpCrd2Kqesb6zW4p1LDZ+x08XsNDfVDra+/F2rdsKy/VxtukUYd/3Ze+PYr6o+OT05CnEJzEuIUmpMQp9CchDiF5iTEKTQnIU4xUymlTRyWHyjiMHQ4NKKvx/Nwz7E3T0Ht8QcPQs3i1MVFdX1mSj+fiMjxN47d1HvdZqQ3/vaZZ6AWieqpm60t/NknEimodYpZqE2M78D7OnoqpdGowz1bmxWora3h88eTaahJRE+l9A3gvzndg8+4XdqEWiqNf/5B2IFaIq6fpbS9Bfe0mjdeYMInJyFOoTkJcQrNSYhTaE5CnEJzEuIUmpMQp5ipFGnja/atJtZqNT0sPzWDqzOeeOhOqN1sn6M7b50Er9eFex4+8jjUlq7qqRkRkSd/+Vd/8oP9z7ME+t9WreqVDyIikQ4O89erOL3RNL7PbEZPwfQVh+Ge3btuh9pHH30MNYngczSbevVJId8H9ySS+K22SytQC0X/nYqIBOB7ERHZ3Kyq6/UaroC5mZ8wn5yEOIXmJMQpNCchTqE5CXEKzUmIU8xobce49Bzp4IhnKplR17fXcF+ZV994zzoKPofR8wfx9Nf/EmolI6yWM15zdAxHoh/94m9CbWN5SV3/2p8+Dfd84vB9ULMi26XSNtTmr+jnSCbwJfVkEhcyDA7hz2N+4Tx+zbQeNa7U9QipiEiphH9X8QT+feTzuEigXsc9i7ogWN7pBHBPKmWElAF8chLiFJqTEKfQnIQ4heYkxCk0JyFOoTkJcYqZSmnWcPi6N4ND7Pn+IXX97jvvgnsePXK3dZSbAqUVMtO43883vvmPUGtemr3h9xIRee0//gVqIgl19f3PnoM7HnngCH61BL5UvmPHGD5GqKcjtjbxBfz3TuK+T3Gjz1FPDqdgOl39c2xVcH+emPGIsUYudLv44vv6Bk7PREVPwcTj2E7FYgFq+H0IIS6hOQlxCs1JiFNoTkKcQnMS4hSakxCnmKmUVEoP84uItGO4RqOe0acCz5VwlYvF7CZOU8z04ZQOqlj57T/4KtzzlT/8fag9+9w/QO1HF3Ho3UZPfZS3SnDH7Nwc1EZHB6GWSOCve3RCH9UwBtZFROaXF6D28YdYGx7VU20iIpfnwefYxhUfQQtr3TiunkoncbonFce//XpDf818HqeI4mCEgwWfnIQ4heYkxCk0JyFOoTkJcQrNSYhTaE5CnGKmUrJZPAH6+hYeCXBhQQ+jnz1z+ic81v9mZnzcUHELfMRAP06/3DNZvOHXExF54jMP39S+33nqT3QBTJoWEfnR8R9AbXJ6Gmoze2egNjCgV01Y058LeZweiHZwM7FqEz8T0EiD+haujul2G1BLZ3BKpFLCr5k3KmdSaX36dqtljSjBDcMQfHIS4hSakxCn0JyEOIXmJMQpNCchTqE5CXGKmUop9uMKhwsLuNnVtct61UQ2gdMeZaNB1m/8+peg9p1//RbUUFVKLKGHwv8vfOHBw1DrPfAo1PbcdkBd/+D4K3BPLILTLO0ursJYXVuH2oED+9T1W/fsgnsmjOqS3vsPQe3UuXmoNRt6mquZMKpSBKc9ghCn/JbBnBoRkWQKp4kKfWjaN26IV6/feEUWn5yEOIXmJMQpNCchTqE5CXEKzUmIU8xo7cWLJ6B27uIFqC1du6iud8s4mpUzJlRbow4abawh/urpP74pzeJLf/Z3UHv+L74Mtae//H11/aHnn4N7Vrdw1HXf7VCSx2f0iKyISLWiRxMDHPyVsIWjxmfefgtqe/bisRwjO/XCg7dP/BDuWV7B/ZbabRytbdTx+TeNMRSZXv2MQYgjylVjtAmCT05CnEJzEuIUmpMQp9CchDiF5iTEKTQnIU4xUylv//AY3jiyF2q79+mXuTNG2/ztcsU6CuTC7ArUUArmP0/gKcmfOox7CH3tudeg9q0/fwpq++/Ck6gvLerFAGjCs4jI/PVNqKV7r0KtkO+D2q7dU+p6aPz/rm/hvjjnfvw+1MI6/h3sP/opdf3AQXwBv/4OTqVcvHAZatmsPjZERKRQHICaiJ5fKpXw99JssocQIf9voDkJcQrNSYhTaE5CnEJzEuIUmpMQp5iplOsLeFrzoTs/C7VUSu8t02+07nns6ONQ+8Yz/wa1hQsbUGsFeh+Ycoj7uXz7xQzUdoyPQe2r//Q61K4uXIPas+//QF3vLeD+TesVXOEQTfZALTCqe0SAhrMe0pvGvXumxiaglo7hc0RFT6kd2I/HTBSLOP31Yv2/oLZ8Dac+dg7j77ob0cc/WJPDSyWc7kHwyUmIU2hOQpxCcxLiFJqTEKfQnIQ4heYkxCn2ZOvefqgljKj81tZ1dT3Vj0PetQ6O2Tfw4GLJ9OWglgpA07AG7loVGp9Io40rC9IZvDFqjE8Iovq+3gEcyk+GOH0Uy+DKkzCJc1lBRP/bIl2cmonG8N+c6ElCLdOLtU5Tb6y1fhVXHw304LEQn/vMUai988FlqFWM5l+N5qq63jRGLhRzNz4xnU9OQpxCcxLiFJqTEKfQnIQ4heYkxCk0JyFOMVMpo7fgSoBIFPu60dBv4K+U8Nsli7gKo93BofdIIgG1ekWvcGiH+OzxOJ5o3IlhLZvHFRrDA7ihWLihh99bxoyPSIDPn8ngqpqoURWEJkB3jUnZUWNCeBjDZ6xU8RySSKCn1FLG7620itMsmSxOB37igYNQ+/jiFaidPrusrldKuFoomdAndlvwyUmIU2hOQpxCcxLiFJqTEKfQnIQ4heYkxClmKiWM4FC5Nc67VtZD5SkjzF8uGY26Gvo8ERGRWgmH5ROgKCXXg1MiQ3049J7vxxUaQ0X8t3XjBajVU/rnuDGJq1KaXdwwTIzKmW7HqI4BFTzdKK4WihiplGI/ro4JusYZwe+qUMCfbzKCS6S2ykYaq43n89y1bwfUijn99/PSS7iZ2OoKbpaH4JOTEKfQnIQ4heYkxCk0JyFOoTkJcYoZrRUjuhcPsFYAd3wnCiB8KiK37cI9VnrTOFIXi+D/L9WSHqlr1LbhnkxPG2p79+BI7sTkONSiiUmoVbb0M06MjuJzzOk9mkRE8v34gnV/H76cH4/rxQWB0SsqNC7Sp3uyUOs0cKQ/Ct4vYRVaCI7mDwzi6dWVGo4aV7f0y+0iIjuH9J5Fv/gLT8A9L7z8Pagh+OQkxCk0JyFOoTkJcQrNSYhTaE5CnEJzEuIUM5XyyAP3QG3X7XdCbenqVXV95xhORczs2Q21HUPDUIuFOD1TBpeem8bl8EgUv15vD7743tuLUxixJE4FJUBKql7VW/6LiNy9H6dmpmamoNYOcJooBP+nOwFOe4Qx/FnFjCnP7QbOzwTg4ns0jp8jkTQ+hxj7mm38ecRjuDdVt6X/roaMtM2Rh++DGoJPTkKcQnMS4hSakxCn0JyEOIXmJMQpNCchTjFTKfccvA1qdxzCqZT6fj0t0lPAVRG4U41IGMGh8qgR8u7v0fvAGNMYzP9WARgVICLSMXoqiRGybzb1cQy7b70F7skkcUqnXsUVNyGYoi0iIhFdC43+PEGIta7xnQVGqUsLTIfuBsaE7bjx+zC+0fI6TqldmVuA2kNHDqnrtTbuZ5W10j0APjkJcQrNSYhTaE5CnEJzEuIUmpMQp9CchDjFTKVkrCqMNB5p0JMFLxvHHaGsRlIRK5VihexDPfURtHFKxEoPWNO8O0YyyCh0kRA0KOst4gqeThe/Vzewxlfjg4SiT7COWofvYq0bxymuUIwvGzSViwR4wnbK+JsTXfyd9TSMydwrekpHRGT1kj5Je3wvbvK2FsWjHxB8chLiFJqTEKfQnIQ4heYkxCk0JyFOoTkJcYqZSskVcDg/NKpBak09HB428UyLJtgjIlKtVKHWauN9zaZeDdLp4FRE26ggaRvvVTPmbtSquFqhAypdcv14GnaugOfKFHODUEsn9XkoIiJdNPsmYsw1Eazlcrjh2fp1/Dk26nrKIQjwpOyI4L8r6OLfXB5MqBYRmbxlBGr1mv57DI1maIUcTksi+OQkxCk0JyFOoTkJcQrNSYhTaE5CnGJGa1948btQ6yaOQ21zU78YXNleg3vQRGMRO5K7sqK/l4hIF9ym7zfGO/QNDkAtFcMfV3VDb9EvIjJ7/iOolSp6dHJiGo9ciCVwpDyfw+efnsZ9icYn9H5L07t2wj39KXzxPZfGZwyMXlIS0y+jt7s4EhozRi7EjDOOTBmR7TyO5LZD/RJ+DAeNpb/f+JsBfHIS4hSakxCn0JyEOIXmJMQpNCchTqE5CXGKmUo59tqbUCuO74Va2NXTAyfffA3umRzH/VcGB3B64OriMtQ6oO9Mth9fHG9F8aX4lUXcov+xww9A7a6Dd0Ct1myo61FjMvTc/BWozZ6/CLUPT5+EWrGgT2X+/Bd+Ce556I4ZqCWNmRfjoxNQa4FUijVx3Or71Aa9kUREonGjL1ERX9zPgF5SQQyn/HBiCcMnJyFOoTkJcQrNSYhTaE5CnEJzEuIUmpMQp5iplF/5td+CWmp4D9RqZT29cf7DD+Ce0R04vB41xiBk0vi2fyvQW+rP7Mdn7xvFFSu1QdzH5slP/zzUsrkM1KoglWJMTpAOGDMhItLo6K8nInL9+gbUrswtqevZLP58lxfXoXb5zHmoRRv4jJeWr6vrh5+4F+6ZnBqDmlXNEk0bZSQJnGaJoF5BEbwnGbFmt+vwyUmIU2hOQpxCcxLiFJqTEKfQnIQ4heYkxClmKiWVxN6dPXcaaqVtPZUSWtUDLXyjv2KMY7CmXqdTei1Au4bHI2yv4jOuzOOqlO++gpuhbZaN96tsq+u5PE5hFPrwmIweozHV4qKeLhERGR7UG3ml8zi1dPxl/DdvnD8FtW4Lj7y4sKw3bFs0Rlrs2YdTY4V8Fmt9eORFJourUgo9+u8qkcaTsrNZ/L0g+OQkxCk0JyFOoTkJcQrNSYhTaE5CnEJzEuIUM5VSXsfNs179zstQW1heVNejbb1KRETk1KkSPoiRLul0cNWBgEqAYy+9CrckEzjkfdehu6HWSuagVmriqdeX5vUqjPV1PF+l1cAVDkvLl6E2dxm/5r2H7lHXv/LUH8E9J95+C2qdbVyxUjImnNdFT2VdegensY6/ew1qPXGctkkkceojlsK/gxxIpYxPTsE9n/v8F6Gmf/J8chLiFpqTEKfQnIQ4heYkxCk0JyFOMaO1oyOjUNszNQ21UPRoYtwYdRAzIrLRGP4fEoLp1SIiyXSPLiTwpeaxMTzJ+ZNHj0ItlzUuWKdx76Gzp/W+SrMX8FiFHTunoNYwxiDEMviMp2fPqetnZ2fhnuzUPqgtLeG/ua+IteGk3tcn24v7MG0s4/EU61cvQG11DU9Fb3SNIg3Q4OnaFrbTg48ZTaEAfHIS4hSakxCn0JyEOIXmJMQpNCchTqE5CXGKmUrZWMXt++//uQeh9uAjj6jrqRS+aBw30iXWOIbAGE0QE/392i3cNr/ewpfU1xfnoLbRwBesN9bw53gJpEyWruOig95hPH5AUjhNFEniVEqro19GP/b6G3DP5O4DUJvoxympdBT/7LKg8KDZwD2ELpXOQK03h3sxdUNcNLG8qU9nFxEZHJxS12tt/Ft89fUTUPvd39PHnvDJSYhTaE5CnEJzEuIUmpMQp9CchDiF5iTEKWYqpcdoIb9ewtOJT556V10fHsbVCCPDg1Brt3GaYnNzC2oCJijHA/x6O6dxmmKiD/cJujqL+9hUK7hnzvDIDnU9O1CEe2LGNO9aHX8vo6O3QG15Se/7tLauj4sQERkdM8ZkGKM3Kk38+Utc/821A5z+SmVA9ZGIpIxqp9b6Kj5HVO8TJCIyAqqCWk08UsT4OPARbnwLIeSnAc1JiFNoTkKcQnMS4hSakxCn0JyEOMWebJ3At+ybDZzCePPN76vrYRuH+fNZ3MCp3cbVA406HvEQB/97Jqcm4J79998Otd234DTL1oKeihARWd5cg1oyo6cOdg/oKRYRkdVVXDFxYO9+qN1xYC/U/v35f1bX46I33BIRaVfx99lqYS3s4LSIpPXv2hqPMDW9C2rXFz7G7xXFVVKZHvx++/bNqOuNGv5eJkbxhHAEn5yEOIXmJMQpNCchTqE5CXEKzUmIU2hOQpxiplJqddzsSoymW0c//aS6HrRwFUPMSJcEXZzSCWPGdOK4ngZI9+BGV8tbODVT3sJzQzbq+PyRNG669fH7l9T19bdwxcSuaZwSue/WPVBrGRUrmaSeOgiNiiCrAiYawz8tMGpERETqAZiz08Wf7+Q4TqU0KnjC9u15XM1y4t2TUFu6oqdn6lX8+w5rm1BD8MlJiFNoTkKcQnMS4hSakxCn0JyEOIXmJMQpdoOvXlyRUDAaFuWG9Fv7zSZudJU2/k8kI/gcYQZXs6Sy+r6ggasHyuUS1GJZ3FhreDduyLU7i6tSzs+B8fIRnCJKGI3Xrl6bh9rAIG6whrRWHacHmk3c/KtqVKw0jeqNdlNP38XTOP01MjYEtSvX8Gj5lXnw2YtIo4L/totn3lfXBwbwOcK+fqgh+OQkxCk0JyFOoTkJcQrNSYhTaE5CnGJffC/ji94SYF8nIr3q+soKjoCdP3sZauk4jsgmCzhKOgjGP4wNFuCeuHGhf6AwADXjbr406vjS8/CwHgHeOYaje9eW8dTr2dmPoDbVmoYaiqSXy/g7q9VwJLS0jaPeVrS229ILD2IpfEn9zGk8ysMakTA8PAK1nQdxL6bhIX3f4BDu+5Q2zo/gk5MQp9CchDiF5iTEKTQnIU6hOQlxCs1JiFPMVEpgtNSPGr6Ot/VL23ljvMO7b78OteUVfHE8ksCXwA8fvkddP/LAvXDP9jZOHZx678dQq4Ip2iIis/MLULt0+bK6Xq/h/k1hiJvwpPP48nWpVIZaGYyMqJZwGshoBSTxGFYLOXyJfWxaT/f0DYzCPcNjOIUxdugA1PqNHkJJqzcV0oxiBQlv/DnIJychTqE5CXEKzUmIU2hOQpxCcxLiFJqTEKdEwtBoBkQI+ZnBJychTqE5CXEKzUmIU2hOQpxCcxLiFJqTEKf8N5ft2KqQX3QpAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'truck'" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "show(X_mask)\n", "predict(X_mask)" @@ -345,16 +666,24 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res=!kubectl logs $(kubectl get pod -l serving.knative.dev/configuration=message-dumper -o jsonpath='{.items[0].metadata.name}') user-container\n", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Outlier True\n" + ] + } + ], + "source": [ + "res=!kubectl logs -n cifar10 $(kubectl get pod -n cifar10 -l app=hello-display -o jsonpath='{.items[0].metadata.name}') \n", "data= []\n", "for i in range(0,len(res)):\n", " if res[i] == 'Data,':\n", " data.append(res[i+1])\n", - "j = json.loads(json.loads(data[1]))\n", + "j = json.loads(json.loads(data[-1]))\n", "print(\"Outlier\",j[\"data\"][\"is_outlier\"]==[1])" ] }, @@ -367,7 +696,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -383,9 +712,22 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABGoAAAERCAYAAAAjakGjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3debxld1nn++e39t5nHmseqcpQlQSSEAiEAEG4IAZUGidanFulr7fbsa/j1dYL0o6tdmvrFYXWtqHbiVZEASEyY4gxQOahklRVUpWazzyfvdf69R/70Lc65vn+KnvVyVnn1Of9etXrBXnqWeNvPeu3f2fXeUKM0QAAAAAAALD2srU+AAAAAAAAALSxUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARLNRscCGEnw4hvOdi/90L2FYMIVx5MbYFQAsh7F955uor//8jIYTvWuvjWkshhNeEEI6v9XE8XQjh7SGE9631cQCrgVr0T1GLgOceteifohatPyzUrDMhhH8RQrgvhDAfQjgVQvjdEMKI9/djjL8YY3zbhWz72fxdAJ17ts/xM+QfDSF8pRePMb4xxvhHF+dozUII3xtCeDiEMBNCOB1C+HAIYfBibb/Ecd20ciyTIYTxEMKdIYTvXuvjKiOE8LqVaz0fQvhkCGHfWh8TNi5q0UU7rg1Vi0IIXSGE96/c3xhCeM1aHxM2NmrRRTuujVaLbg4h3LZyLmdDCH8eQti51sf1XGGhZh0JIfyomf2Kmf24mQ2b2c1mts/MbgshdD3D368/t0cIIOXZPsfP8bH9k5oRQni1mf2imX1LjHHQzK4xsz9d7f1eQM7LzewTZvZpM7vSzDab2b8yszdezGN7LoUQtpjZX5jZz5rZJjO7yy7ytQa+jFp0Yfu9gJwNV4tWfM7Mvt3MTq31gWBjoxZd2H4vIGcj1qJRM/t9M9tv7TExY2Z/uJYH9JyKMfJnHfwxsyEzmzWzf/60/z5gZmfN7HvM7O1m9n4ze5+ZTZvZ21b+2/vO+/vfaWZPmNmYtT8MHDWzr1yJ/a+/a+0HIprZd5nZk2Z2zsx+5rzt3GRmnzezSTM7aWa/bWZd58WjmV251teNP/yp0p8LeY5X/v9/MbN/d178NWZ2fOV/v9fMCjNbWNnWT5z3vNZX/s6nzOxt5+V/j5k9ZGYTZvZRM9t3Xiya2feb2aNmduQZjvnHzOwD4px6zezXV+rKlLUn970rsX9mZg+s1IlPmdk15+UdNbOfNLN7zWzJzOpmtsvM/sfKtThiZj8k9vs5M/sdEX+NmR03sx81szMrdeq7z4t/jZl9aaVWHjOzt58XS9W/t5vZn5nZf7X2pOEBM3vJeXH3POxpNflpx/x/mtnt5/3//pX7fPVaj13+bKw/1CJqkapFTzv+42b2mrUes/zZmH+oRdSiC61FK3/3xWY2s9bj9rn6wzdq1o9XmFmPtX/a+r/EGGfN7MNm9vqV//Rmay/WjJjZfzv/74YQnm9m/5+ZfZuZ7bT2qvXuxH5vMbOrzOx1ZvZzIYRrVv57bmb/xsy2mNnLV+L/uoPzAi4lF/ocu2KM32Htl+SbYowDMcZfVX8/hPBmM/tpM/sGM9tqZp81sz9+2l/7OjN7mZk9/xk28Q9mdmsI4R0hhFeGELqfFv81M7tx5dw2WXuCVIQQDq7s50dW9vthM/vrp/107FusPTEYsfYk66/N7B5r16XXmdmPhBBufYZz6rN23Xm/Oncz22H/f537XjP7nRDC6EpsztoL1yMrx/CvQghf97R8r/6ZtSdbf7KS/0FrL1ZbCCG70PN4Bi9YyTMzsxjjnJk9vvLfgYuJWkQtAqqAWkQteja+wtqLQJcEFmrWjy1mdi7G2HqG2MmVuJnZ52OMH4gxFjHGhaf9vW8ys7+OMX4uxrhsZj9n7dVR5R0xxoUY4z3WfsBeaGYWY/xCjPGOGGMrxnjUzH7PzF7d2akBl4wLfY4vpv/LzH4pxvjQyn5/0cxueNrvPvmlGOP4M9QMizF+1tqTmReb2YfMbCyE8BshhNrKy/d7zOyHY4xPxRjzGOPtMcYlM/tmM/tQjPG2GGPT2hOXXmtPXL7st2KMx1b2+1Iz2xpj/PkY43KM8bCZvdvM3voM5zRq7ffXycS5N83s52OMzRjjh639k7arVs7rUzHG+1Zq5b3Wnjw9vYY9Y/1b8bkY44djjLm1f5r35dizOY+nG7D2T9/ON2Vma/7v3rHhUIuoRUAVUIuoRRckhHC9tT+7/vizyVvP+B0m68c5M9sSQqg/QzHbuRI3a39VzbPr/HiMcT6EMJbY7/n/Nnne2h8kbGVV+DfM7CVm1mftsfSF1EkAl7gLfY4vpn1m9pshhF8/778Fa/9U44mV/6/qhsUYP2JmH1mZgPwfZvbnZvaImf2ltX8S9vgzpO06b/sWYyxCCMfsf/8W3/n73Wdmu0IIk+f9t5q1f9L1dBPW/knTTjN7WBz62NOu8/k17GVm9stmdq2ZdZlZ98p5ne8Z658T61n5N+XP5jyebtbaXwM/35C1v0YMXEzUImoRUAXUImpRUmh3Ev6ItRfALpkaxjdq1o/PW/vfK37D+f8xhDBg7V8S9fGV/6S+IXPSzPacl9tr7V801YnftXYhOBBjHLL2VwhDh9sCLhUX+hzPWXsB9Mt2PG07qW/Cne+YmX1fjHHkvD+9Mcbbn+32Vn7K8nFr/7K6a609gVo0syue4a+fsPbL2czMQgjBzPaa2VPOfo9Z+9+Cn3+cgzHGr36G45i39rX8xgs5bsd/t/ZXc/fGGIfN7F12cWrYBZ/HM3jAzvvpVAih39rX9pL5mi+eM9QiahFQBdQiapG08k2nvzOzd8YY33sRjmfdYKFmnYgxTpnZO8zsP4UQ3hBCaIQQ9lv7Fzcdt/ZXzFLeb2ZvCiG8YuXfQ77dOn8AB639y6ZmQwhXW/u3igMQnsVzfLeZfXUIYVMIYYe1/z3z+U6b2eUXuNt3mdn/E0J4gZlZCGE4hPCWCz3mEMKbQwhvDSGMhrabrP1V2DtijIWZ/YGZ/UYIYdfK135fvvLvtf/MzL4mtNtNN6z9y+uWzOx2Z1d3mtlMCOEnQwi9K9u6NoTwUufv/4SZ/YsQwo+HEDavHOsLQwh/coGnNmhm4zHGxZVz+tYLzEt5tudxvr80s2tDCN8YQuix9ld8740xqp+OAc8atYhalBJC6F6pQ2ZmXSGEnpUPlsBFQy2iFikhhN3WXgT77Rjjuy7S8awbLNSsI7H9y7F+2tr/pnHa2r/M6piZvW7l3z6m8h8wsx+09i96Omntr9mfsXaReLZ+zNoP8Iy1/50hLWSBC3CBz/F7rf1vf4+a2cfsnz5fv2Rm/zaEMBlC+LHE/v7S2m0v/ySEMG1m99uza9U4YWb/0trdD6at3VXu38cYv/zLyn/MzO4zs380s/GVfWUxxkes3dr1P1n7J0xvsvYv+lt2jjM3s681sxus3RHgnJm9x9q/9O6Z/v7tZvbalT+HQwjj1m7h+OELPK9/bWY/H0KYsfaCyJ9dYJ70bM/jablnrf3TsF+w9nV/mfH7JLBKqEXUooRHrN1FZ7e1u+Is2HnfBgAuFmoRtUh4m7UX4N4eQpj98p+LcVzrQYjx2XxTDBtJaH+tcNLa/3zpyFofDwAAAAAAlzq+UXOJCSG8KYTQF9q/++DXrL3ie3RtjwoAAAAAAJixUHMperO1f5nVCTM7YGZvjXytCgAAAACASuCfPgEAAAAAAFQE36gBAAAAAACoCBZqAAAAAAAAKqKugu/+4N/Jfxd1/OEvuLGzRx6SO85zf9e/9/vvkblrJYQg46l/Rnbra1/jxnbt3S1zf+nf/7ob++J9D8rcKw9e7cYWp8Zl7gP3f0nGi+IZO8qZmdlyc1HmPvjAfW5sevKczF1a1h3Fm8s1NzY+Ni9zZ+f9427ler9bt26S8dFNA24sjzMyt9X0Y4sLeux94C8+qgdvxb2+9s/1CYpnLxsclKlhzw4/NqnvSZzTY8ky/7LHvNDH1dVwY++/529l7ltu/noZn7p5jxvrP74gc7O7D8m4UiyKmpCor/VdO2U8zs25sXxySuZWlromIfFzliLX8cyvkcncEm4r/nxd16Kv6v12WYvikv+OqG3ZLLdd7PNrUXb0pMyNC/p9K8dSpsdSqPvztfff91GZ+5ZXvUXGz92yy40NP67ra+3uR91YbLZkbmz6cxf5bJhZfed2ve0Zv3NsPj0tc9fquUwSxxXEe87MLLb0vUjVfr3xzn99w3qvRWZmtw5/j7wAxYw/h6nv9ecBZmb5Nr9zcnb4hMxVdbB9YGL+k6pH3d1u7GWfOi1z/+GtL5Dxxb3+OXedTcyNjvrXJJ+clLlyHJedGy37Hx7yc/qzVpnjKvNsllLV40rw6hHfqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAiqir4PTEuEzePLLJjcWt22VurA+5sdtuv1fmvv4V18u4cu/jx2X84H7/uD/7uds63q+Z2dUHr3Zjv/mud8nckPl94Scn9X1qNLrdWGukT+bu3bNDxlutZTe2uLggcycnZt3YuXP6nOpdPTJuoeaGRjf718PMrKffP+6p6QmZ290jHykrYsuNNer6uKanJt3Y8lKUueteTJxf5t/vYtYfZ2ZmtadOu7HW9LTOHfLrmJlZbPr3u5ibk7lZf78b+/o9N8nc+mV6HA7fddKNNXeNytzrP7/oxu55qd6vBb+Ope5x66kTHW+7NqrPKZ8Qz7U6ZrP02CxDbTvmJbdd+LG1POeqyxPXXdSifEy/17Il/32az8zo3MFBfVziuIsS207Wov16rGz+wpgba272a6CZ2Wvv9Gv3x2/Qz7wc44W+x6laFOp+Haxt2Sxz83P+9UhazedWXJMYE/vFmgndYv5/TH8eys6ec2P5oj8PMNPzl5Rifl5vW8Ruf2GXzK1do/fde59/TeZfuFfmxl8ZdmM9b9LnFJf92r+qc6NhPYfNJ6f8YNl5QIn54KWEb9QAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFaF7qTabMry85Mfn50WrMTPbf3C3G/uqV75Q5sYSbbteeOW+xLb9FoSvuuX1MvfEU7rV3dd+wzfLuBIL/5zn5nRrzdDyWwQvzOnWxUuJMdDX67f3Hh3ZJnOvuPz5buyhhx6RuRb0cS0t+a3whod0286G6O43Ne23AzUzi6bHfSHu48SEbte8ML/k7/dS72SnWg0n5KIF95N/fp3MvezndIvK2OMPpl2/+6TMPfONfkvcDx76jMz9Zwf859JMt79cfv52mXvXj97oxrr7HpW5+awY42XbTSs1/TMJ1U43ptoxr5WyrXgv+aLRGfUuLku1yS4+rlvD1n6gR2+732/T+9L33CNzv/Qmv+3snzz0MZn71oOd16LZb71Z5n74x17rxnoHDsncfMqv+2WVqhnquQ6Jn60m2oqvllDzW9KbmUUxB23/BWpRxxJzdCnxDilEC+7xvzkoc7f+aGJOJt7JU7+mczf/gD+eXvNXD8jcT3+F33LczKxYWHBjM7svk7mNP9zhxvpH/e2ambVOn5HxUtTzJeY+Zrq9e1zyP5NckNV67lN1cjXnmquAb9QAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEXIBuqtRd33PbT8XuTdXb0yd+qc38v+E5/7osxNCSF0nPtTv/Qrbmw60fN9MLHtnbt2u7HXvvXbZe74qRNu7J0/+VMy9ytueqkbi4lzmp6ekvEnn/CPq6vRI3O7uobc2Jat/rUyM3vy2KN62z19bmx2YU7mTk/7Y7Pe0GNraMjfr5nZwsK8G8tbMtVarcKNdXd36eSNLjGOO3X5T87I+KHv2ynj+/52yY099qvPl7n95+52Y2++4Q0yNxttyHhx7RX+fu95SuaaGGut6WmdW6I2p3JDrebG8nNjnW87NbZS57RKY9NC6ucsfr0ws9U7ro2u8Oc9ZlZuLAmN75PTNXv8u7fI+O5PLruxv/+pm2Vu17F/dGPfess3y9xssx6HrRuvcmMjH7hXb3t0xN/uVKIWlbGatUiJibG3RmJe4pkwoxaVEFuJiaNM7vzdtvln9Rzj0Nv8+b2Z2ZV/6s+thn5B17rWYX9u9OlvfpHMDaN+HTQzW3rR5W5s8x/cIXNrmze5sdbYuMyV7/PUc5/59SYlVY9CYx1+toiJuc86wzdqAAAAAAAAKoKFGgAAAAAAgIpgoQYAAAAAAKAiWKgBAAAAAACoCBZqAAAAAAAAKoKFGgAAAAAAgIqQPdCW5nUb44Fev/3y0KatMvfFL7zBjb32lhfL3DJS7ah7L7vajf3H9/yBzF06fKjjfX/yT/+bzDXzW+Hd/TUPy8xXv/wWf6uNpszdsWOXPqzot7KenNCtjb/4Jb/1Zr3RLXP7B3Xrv1buX+vl2UmZWxPLl1u3+u33zMzyXLf+Gxv3r1dmurV3ve4/riMjwzJ33Uu1H0y1zBVqV13pxhb36Ot68PdOyng857dk/LuHPyNzXz/23W7s0Dfo5+Oml+hadPw/+GOtt1e3xgwt/9kKRxJta+t+HYtN/eykWonKNqWJ9rC1Eb/Nbz4xIXPXrLVsmTbRq6lEq9B1YRVrUf2yfW5s9ho9p7ryPydq0bg/jj/84Kdl7te8/E1u7OQbdsvcga8/JePFu/32r+HW62Ruz7hfM7KnTsjcIN6nyZbHq1mLNo26sTzV4hf/u41ei+wCxmqJ90Dtiv1ubHFrr8y96rePy3gh5kYfe/TvZe4br3qVG5u73H+Xm5md/PZFGR/+qF+PRm7xP7eamYUF8Xkq0QY76/bnRsVi4p1S4p2TtJqtrtXYLDOvKjsnW63j6rAe8Y0aAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKiIugp2i77uZmbN2qAbW+gdkLlHphdkXDk0ofuYHxztcWNB9Uc3s+/6kXe4sR/6Nz8sc3/v3f9Zxv/+8XMyrjXdyMzktMw8dOSIG9u5c4vMbTTkELGde3e4sV0iZmb25KljbuyR+/yYmdm2nVtl/OiT4lo3C5lbLPvxvJ7L3J6ubhnvrvvP1MKi3vbQ0JAbq9f1fuErjvpjLdvm1zgzs0feOSrjB773lBt745WvkLl1e8SN7d50ncyd+OFxGW+8wR/jjVMzMjdMz7qxVtS1OTaXZbwUVduD/plEPiVqaOKdkZS4Jh1LHVfinC2KepPVdG7h54as5PWquqjfH6U2Pe0/e93jutYMv3dKxide788h3njglTI3G/Jzl0b1/d56qz//MDOb+rbtbmzT3RMyN8wvurFW4vmIrZaMlyKen1DTz1Y+Ls45cU6pba/aOadqXKqeqFqUqnNi3xu+FpWVuC+h6Y+X2rweSz/88b+V8d+4+oVu7I2X36yPa59fM45/pX7vHfiOh2X8xA++xI01TukaGxb9+U2qHhVLSzJeith3EJ9JzMrN2bIe/7O4mVmx7L9XZE1IKTtnWyWd1iO+UQMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFVFXwb4+v1e9mdmZyZYbe+zYMZn74AP3y7hycM+exN/ovB/95k1+3/cb9410vF0zs6/66ld1nPs93/8TfrCl+9z//Wc/5cb2XXaZzD141UEZ37x52I1198jhZcND3W4sa03J3Lklvca4MO+PgYXJGZmb54turKe3IXNnp/W2hwaH3Fh3T03mLi833dj8/LzMXfeKvOPU0OiS8dj061jIC5l71b+dlPGiIcZLobcddm5zY0NfPCFz48ErZHx+m/9s9t5xVm+7Wzy3g4Myt5iddWOhpsd/bPn3KSVkofNtZ/q4UmMziOuVEpc6f5eVeWbK5Ja5T+tCjJ3nBj0O87FxN9YcuFzmTn2THmdBhFP3rNg26sb2//FxmRuvOSDjU1f47/JNHz0nc63Hn69lfX0ytRDvzFDX7/nY1HMui6K2Z3pepMZXqpYk64WoZaGhjysuJ85ZKVOLSjxvG74WmVmo6/sWuvz5T7GwIHNbT/if44or/fmJmdl/eNHNMp6N+McVl/RYy0f9Z/vq3zolc+OB/TLeHBDBCf25JI76n4dqiblRPuN/dkjd41LjXNWqhNRxFYv+Z6n0xvW7UtaFMu/oVRTzzuog36gBAAAAAACoCBZqAAAAAAAAKoKFGgAAAAAAgIpgoQYAAAAAAKAiWKgBAAAAAACoCBZqAAAAAAAAKkL21hrZtEUmP3bskBs7efSIzO1r+G0EZxKttb7tW79Dxv/qv7/XjYVEy69aI9GKtYRvesVNbmzgutfK3ANXX+fG7vnsR2VuLfit7pqJdmFnz43J+HXXXePGrjyg24nu3bnVjQ3c/CKZe+/DT8r40qLftnOpodvRFea30C6iboN36pRum9wl2msOj+p2h2ZzbmQh0WZxvctEG1azRBvAVPtB0Tr0sW/T+z3wg7o17am/8Nvb7/p/9WHFw/62mzdcKXNrs7q95ZbP+uM0Br1+H/r91pj56TMyV29Y7zfZ+lu0t4xFol2jeC8k29YulWg922G7RrOL0EIYHSnT9jk1xi364+HoW3TqwdvOyvih9/jv1Cv/UI/DxoNPuLH5l+j3fGNWvzMve7/fgjsfn5S52XV+fS2OPyVzy6gN+XMEM7N8etqNxWbnrXTL5JqZhZqY36ZqpGobvprtgyHVturPacWUPxbLtDE+9jq/vbaZ2eWf1nPSx37mWje263O6HvV94n43Nvs6f7tmZl3TTRnf/0G/5uSJz0Pxqr1uLDx6WOZaJp5N9dyaWdbbK+PF7KwbS86NhFV9rkuMzcrWow7PiW/UAAAAAAAAVAQLNQAAAAAAABXBQg0AAAAAAEBFsFADAAAAAABQESzUAAAAAAAAVAQLNQAAAAAAABXBQg0AAAAAAEBFyGbjjz9+p0x++PHH3NiJk4/L3Hxmzo0NhiBzY6IX+WKz8/7rv/pTP9pR7EJ8x8/8jht73y/+gMz9qR/4uBt75fveLXPPTo65sWueL1Pt9QevkfG52QU3VuR623F52Y09cMfnZe6Bq26Q8e27R9zYHXd+RuaeOj3txprNlsxdXPDPycxsYmLGjfUO+MdsZlbEwo3NzfvP00ZQLC6u2rZDd7cb2/u3upYsvOlGGd/5Dv++PPGzus7t+P2r/GCRqIFbBmR88LOn3NjJb00887v9fV/5y+Myt1hacmOqHpiZxRkdD3X/dRbzRDFS+xXHbGaW9fXJeLHg18hQb3R0TGZmsdXsOLe9b3G9WrrOXcqK+flV27YaS5v/QY+V4pXXy/jz/905N/YVf/WAzP3Ut7/EjXWP6dqc9yeO+/En3NjhX75J5vae8mvonqc26/3OzLqxVC3Kp0vUokTtltut1WQ8tWVVB7MufZ+iKDdl6mt75+K8UhPJS1zrpP8uL0vNjbrH9fyl9rw9Mn7wN4+4se/6tJ7//9HrXuXGBh44I3Nb24ZkvLjnITf26G+9TOZuvtv/7sO2J/X1KMYn/FjinZOao6zW3Cg194mJz0uW+WMoVYNNrAOUrUdVmxvxjRoAAAAAAICKYKEGAAAAAACgIlioAQAAAAAAqAgWagAAAAAAACqChRoAAAAAAICKYKEGAAAAAACgImR77js+c5tO3u63j73imutkbu+y32p4SrRNvBCPHTrtxlKtvf/2zkk39oabdPvkd777kzL+3l/4fjd27Q23yNzDx/32a61cn9OTZ/y2bz0DT8nc4aFRGb/8iv1uLCbWARcm/ZZzD//D3TI3Lvjjx8zs2lvf4Mauu/5yfVx3+e25H3/sqMzt69NtkYdHVMtQ3VJuetq/j0tLq9cytgpCo0vGYzPRyk8Rrfwas7oVX0i0Wo33POzG9r3zoMwt7rnLjWXXXy1ze+46qbd94HlubPvv+/s1M4s3+vtOtlSUG9bXMuvpkfFSLdyD3yYyNfZU+20zs9Dl55dqq1myTWSpfHG9UvdxvSvTjj3r8dvdmuk2rPUFfV27jo/LeOuI3wb7M29M1KLjfsva+p7dMjeb1fO5sGuHG7v8J3Sb3toL/DlokmrhmhjDq/o+Emq7/WtlZtZ6Us/nVAvuZP1czRbatODuWDY4KOPFnF9Tsv5ELRPP7tCTeg4ep2dkPB/z69Uf3ni93vbyWTeWqs8m6qCZWW3bVjd24IfvlLnN19/oxlJzo2JBPH+peiTaqJsl5hnqXZ5yuT+PNDOLDzwi42pulJxHrOIcZNXmRh3iGzUAAAAAAAAVwUINAAAAAABARbBQAwAAAAAAUBEs1AAAAAAAAFQECzUAAAAAAAAVwUINAAAAAABARbBQAwAAAAAAUBF1FTxz7JxMftELv8aNdXf7vejNzDbV/Njrbn29zP2P7/pjGT/22LgbWy50v/mZuODG/uKDvTJ3x55dMv6OP/q0G3vq2EmZ+3t3f8qNDQxvkbljs3NuLOvql7lFsh+9iBc6c6BnyI3t37VX5vbU9HFlNuvGrrv2Mpk7MjLixj648DGZe+rkhIzv3uaPkTwsytxGw39cp6enZe56lw0NyHgx5Z9/1tenN97t14TGnY/o/c75z5aZWRDbjg88KnOznh4/eOiozM2XmzIevvSQv9/Nm/S277jXjSUeeS0EHe7q0vmL+vmR26433FhsJa5l4rhis9XRMbX3LXIz8RK9EEXeeW7yvbBxhT07dfzIMT/Wr9+3WeE/QcPvu0PmthLjob5ntxuLiWenNjrqxoqJSZlbzM/LuInaXRsZlqn5A6I+p54PNf5TubFUpetY/tQp/RcSz3RsiRpb5pwTtTupTD0pu+91LtvsP5tmZkFdnz07ZG59fMqNDf7FXTI3L/Q9rQ378/9CfCbj3JwAACAASURBVGYxMwt1fy5czMzI3NRYy0+fETvWY63xMf+a5CXqkZwLmpnV9Lbj0pIIJp49cdzx0SM6N0EeVzJZHHeqJpQ45+S8aRXmRnyjBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqwm9Gb2Z9A5tkckO0C5+cFL3ozax704gbm28VMndxUYatd3TQ32+R6K++6PdIj/JqmS0252W8p9ffQBaWZW6R+bkDm3fJ3K447sZqvaMyN3aJfvJmVgT/nEPeL3Ozmn9Ojf4umds7oOOtpRk3NvbUaZm7uX+rG3vzV98qc++656iMzy7493lx6azMXVpYcGMjg/7ztCEE/dzGVsuN5dPTetMNfyyFRuKhz/TzEZeWVic3iuJ7AaIosXFKX6/VEmqJ6yHu8WpKHVfW3S3jubqPiXEt77O6ianc1L5D4mc4hf+e3OhCU4/D2PRrfH5W1/hSEvek9dQJP1iynqyWuNzUf0GN4RJjNPXMJ5+9VRJbJa6HmcW8xHMrxkio6/fkqtZuNXYT79gNYVG8X8ysmPfn6PGhR2Vu6PLnRrFI1IzE85dPTul8odR4So2J1Xq2E9cjiHlE6O/Tm56d0/tWdSFV+8X1iIl3YZn3StaXOGcxrpNzyVQdXK0xkJrvOfhGDQAAAAAAQEWwUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARsqfezuddJpND5q/zLC7qFq+np/1dd41skbnNlm7NHBoNN7YwO6u3Hf1zqtd1G9ZWTcf7hobc2LbNkzI3jvutmZcTLdJC4Z9Tb2+vzE11siuiaIucaIGWNfyNx5peQ5yd89tvm5mFwm+v1i3GrZnZ9Fm/fXdvn25Z/xUvv17GH3n8CTd2/4OnZO7stN+Cr6vRI3PXu2JK3+8yVDvdZDvUMm1t17LFcYdtAldzv6mWm6Vay6aoFpSJNqTFwqLedplrrQpwqt1niZa5Wbf/DjUzi7mo3eJ52giK0yVabJdpx76W214jqWc+2UZbbVuM/2TdXysh8bPV1DulTJveErU76++X8WJOtBcu02J7Ld+xz5F8bELGy7Syjkui9XfZOUSJd1spq9V6uSxR62KiBbu8T2arNt8Lmd5umUudav2t5jepMV8bGZbxfEqsX5R5z3b4DuYbNQAAAAAAABXBQg0AAAAAAEBFsFADAAAAAABQESzUAAAAAAAAVAQLNQAAAAAAABXBQg0AAAAAAEBFsFADAAAAAABQEX4jcjOLQfS5N7Om6HM+PzMjc7t7e93YzPS4zF1O9JSfn/b33Ui0QB/s73ZjW0c3ydyhTf0yvnXEP+e8rvu6L3T713p83y6Zu5Sf9IPNeZmbt5ZlvCj8C5pnhcwNDX98jWwa1fvNE8ctxubwsH8fzMy6gt/rfnJmUubG5qyM33DNDjc2MuiPPTOzv/mbj7mxs6fPydz1LuZ5x7mhLsucxZY/Viz6Y8HMLHTre2aFnx9bTZ2b2LeU6dotd6uuR1lB/GwgUZutSIyBIDaQuJZlxlds6TpX6j7GMseVuI9ijBRL+h2bqXGfJZ6JdS4u63eiErq61mzbKl7M6fepRTHGy4zvlMRzWea5lfVC1SmzdC0qQx1X2f2WqkWd5xZzczKu3qPJZ0Lcq9DQz8RGUOoZSM0T1Hgr+9yrmrKayhy3ejZT8dQcRLyvS8/JSp2z/3wljyt1vcRxxWbn78KUfHJKxsvUjeS8vgN8owYAAAAAAKAiWKgBAAAAAACoCBZqAAAAAAAAKoKFGgAAAAAAgIpgoQYAAAAAAKAiWKgBAAAAAACoCN23NtGauV748eEeveO9w37brqsvH5G5Az26vXJNtBObm9btlRfn/bZdvf267dZVB3T77r379rixrLFP5s5O+se9d+dOfVxHzrixoU36Rm0aHZLxel20/Ex0hIuiM2BPf5/MbS3qtnCZ2Hcj0+uTi+a3pt28ZUDmzs7rNqdzk6fc2O6tW2Xu173pq9zYBz70dzJ33Uu0JVUtuMu0EJRtiM0s5okW9D0if0HnyuNOtT1MXS/VfjDT2w6qHWpNP1uy7XPpdp+r04Iy2UZ0NdsTryY1RhLjS41N9SxuBMl2uOLaxUTb8063a5ZuYxybop6UaPusWiubpc9ZjpdUm+zgP3shUcfkfVzN9tspshat4XEpJdrwmukxkqwnNTGRLNO6er1YzblRVYnxFrp0a+VkDU61LO9QqKVae3c+B1m397GiVGvwVOtu9ZmhWO6sdTffqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAiqir4KtffqNMvvz5L3RjJ556Subu3rXJjR08cIXM3bF1m4zXot+vfmZmUuYuNefdWMj87ZqZDfT36/hAjxurdfXK3Ebh93VfmDsrc1987T43tv/gfpnbLHTf9yjW+lpFS+fW/OtZa8ihac3FKONF0993Vtfrk6FH3OdE7lJTX696reHG8mU9NrduGXBjt7zqpTJ3vcsGB2U87PRrQn7ocb3x6I+lYnFR77eux2lc9p/bmOf6uOSG9fhPprfEOA2J9fvCP+7Qo++TiesZGl0yNTb9a1laLESs3LVeM1lNx8V9TJ2zGruxpev+ehfqfg03M6ttHnVjrVOn9cbVPVP3y8ws6PnJaolLS+Xy1XhJnZMYp9nQsEzNJ6f83XZ3692WPGcl1PwxEFNjoNSOO7/Wq1kjk/WkEPtezetVFYn7lo369Sg/qz87ZD3+Z5bU3GitqDnXBVmlMZMN+/fBzCyfmHBjqbnRaipVj9Zq7lSmliWk5qGxJfbd4X75Rg0AAAAAAEBFsFADAAAAAABQESzUAAAAAAAAVAQLNQAAAAAAABXBQg0AAAAAAEBFsFADAAAAAABQEbK37I3XXy2TX/Aivz33wrW6xXb/8JAbE41SzcwsptrRiRbIm/p36G2LpavUqlZR6CNviZbRlmjrvLS04MauuPJ5Mre3y28bvjDnt6g0M4uZbj9swY/HkGihLVqV5Yl7XKiWjGa2vOBfr7zQbdSzur/vLDEKZsb89u5mZk8cOebGXnnLi2TufHPGjfWpluIbQDE7K+PZ8bVpw5lsHVrVVsWy1WrnbYDLtMZMtt8u03441RZxvbbgFkKmr1dtu/8ubJ08pTe+Aa/XhUqN09aZc51vvExr2OQYX4etilPnJGpCsZBoH6zq2Cq2306JLT0XXDOq/oZyP/OtbRpxY/nYuE6+FFpwK4lnJD/XeT1a1RbcZd4hVX3/ZKKVdWpupHLXsCbEXDxf4pjNrNyzWabFdqoeJd6FWb//GbGYm0ts++KPTb5RAwAAAAAAUBEs1AAAAAAAAFQECzUAAAAAAAAVwUINAAAAAABARbBQAwAAAAAAUBEs1AAAAAAAAFQECzUAAAAAAAAVUVfBXtFL3MxsoKfbjfX3yU2b1f3+60WiDXlI9FfPRLyIhcwtmn68SPRHD5le92qZv+0s1TJe9IUfGNmk95v7+80L/z6YmVmhDyya348+S51U7sfzeiOx38QgaS27oVD4x2xm1i2uSSPX97h/UV/PeHrBjZ09fFrm7rlqjxs7l83K3HUv8ezFZf9+41lK1Fd1L+LS0kU+mAvbr5mlj7tTJa7HqkocV2y1dHxuvvNd1/33e2q/G17i/YKLaK1q0WpS9aSitSgk5nqpmhAaYr6XOqdMzLkuhWdRnb/ZpXENqkJc62Ju7jk8kOdGqCU+75QZeyXme6Gh1x/ikj6uUnMYVaM7rM98owYAAAAAAKAiWKgBAAAAAACoCBZqAAAAAAAAKoKFGgAAAAAAgIpgoQYAAAAAAKAiWKgBAAAAAACoCBZqAAAAAAAAKkI2Gx8c3iSTY63hxuaXlnXu0pIbW0rkzs3qfvTLTT9/aakpc1utwo01mzq3KfZrZjY/P+/H5mb0cRX+cQ1uGpa5g8MjbmxkcIvM7enqkvG8EOccdC/6zPz44GCPzB07o6/14sKsGyuKUZkbzD/nIvfHrZnZ0GC3jO973nY3tjCvx3Us/Os1PNgvc9e9rCbDobfXDy4s6G0Hf7065rlOrenjUvkhCzq35d/v0NDPZZLYd6jL14KFHn+Mhx793JqoY3E58c5YWNSbXhTPZtT3cV2KUccTz0w+Pd35vsUzY0GP640u6+tzY3KMmq4Jqh60k0tcd3U/zcwK8fwkxlly1w1db2SumJ9kA52/E2PiPlnivVCI+W2S2HZqDKRqt8xP1RO13cT1SGmdPNVxrnoHx+i/bzYM9WyaWej239epd+56FOr+59LS2651/t0GOUc1k3OypEI/u4X47JmqZVFtO/V8pd4NKj9Vj0Q8Na6TdbJM/S5RRz18owYAAAAAAKAiWKgBAAAAAACoCBZqAAAAAAAAKoKFGgAAAAAAgIpgoQYAAAAAAKAiWKgBAAAAAACoCNmj6gMf/IhMzhufdWMTE6dl7uzUOTeWJbpbpdp3nz7t7ztPtDHbtHWbGxvdslnmdtd0y6+58Uk3dujRh2Tu9KzfbnrvZftkbq3ht6sbGtTndNllz5PxPXt3+LmX75a5m7r9dnSDPbrFXjE8JOMmWjY2c93islb31y9r4pjNzLbvT7Q7H/JbJTYTLYRroiPzpk2J67HOZb267XOynarKbXbeojLZiE+0zoyx83aMsdXsOHdl536omWgBu+zvO46Nd3xIq6pM6+JVaLd4wdRxJ44r1TpetQNNtQGW7XjX8no9B2ojwzJezM75wUQ709jq/Nql7rduzdx5e+X0fnWtUq1QU21UrfCvZ5mWz6WJtrRZV2JuU6LVdbKFexmq1W6JFtFmVqoleel34TpXS3wuKaZm3Fjy2S1T51OtmRNjpuNtJ9pcp+Y38riirkeqXuUTE3q/q0nMI5LzBHU9Gonnuui8HqVqf+jyPxDJduRmZiHxHZUStW418I0aAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgI2aj8tk/eLpNH9lzlxmI+K3O/dPsn3di+PXtk7pbNm2X8qeOn3Fgr0QO9b9OIG1vOCpl7+vgxGX/dTS93Yzdc/wKZO7+06Mayhu43f+TJJ9zYoUcfl7n33f8lGR8ZHnBj3/hNXy9zX/mCg26sK+o1xD0798r4cq3mxkIWZG4Roxtrmh4/WV3Hu0d63Fhvps+5qC27sYbMXP9aL/HHiplZ435/jOfjEzK3Njrq507o3KTMH4e1zZtkaiH2HXM9zuo7tst466RfIy1RI2MiroTubjdW27ZV5uY7/PtkZpbN+89H/uChxIGJZy8mrvXuXTIe5xfcWGp8hbr/ZKfGQGz616O0EmNgvVu86YCMd//9Q24szs3J3NrIsBvLJ6f0gaWoWjTQL1Pz6Wk3lhpnqeejdeKkv+3CfxebmdnSkh8L+j2vnq3a7h0ytxjq09teELXo0cMytwz1LjMzMzH3ycfGO9+xGFtmZlHdp7LEfO1SMP+yy2W85+/udWNxqSVza0NDbkzVhAsins/Q1SVT1XiKS/rdpGqsma6zqXoUl5syrmQ9/meDbLueG8UBXY9MHFeZehSbevyExGdTdZ+LmRmZmw0O+jGZaVYs+HMyMytVU9QcN3W9PHyjBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgI2TvrLd/ynTK5e5vfpnJ+RrR/NbNH77vHje3coVsvZ4k2xr09fku55UK35Tp4rX9Oozu3ydz5Lbo14te+8SvdWN9gr8ydE+25C92F0lrRbyu+2PK3a2Z25oxu2fjEkRNurK/Pvw9mZqeOj7mxow88KnOzRX3ch0+dcWM3fdVLZO6+/X470Wau26tlPbqtoDX81oGhSLRuC35uV9Ct49e77DN3y3gh2vzV9+l6kp887QcTLV5NPFtmugWsbC1r6fbLMnduXsZV201LtFSMi/5xp9pqqvaw1ko8W4/4LdjNSrYLTbTglql9fltNM7PinF/nQj1xrVezxTY60vjYXTIetmx2Y9mVz5O5hXrvJWpRsl6IFvSp3KxftO9OtDLNz5zThyVqRpZoG67qXOjVc6pQE/PIPFHXT5yV8VKtruWOy72P8olZN6baA5uZFYk5F9ZG94f+UcZre3a7sXz7iMzNv/igH0y0ZA/qXW9msSXqRqINtmrNrOYnZmbFgh7H8jmo6XM2WUfFXDChOJeoJ4m4POdkTfHvRUhcj1S8mPXrUeq48okJP7WRmIemqH2L96iZWdbnt0ovZuc6Ohy+UQMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFVFXwe4uvY5z6OH73dj01CmZG0Vv9ubyssydTfQiD6IHek+37mXfnJ9xY1Nn/WM2Mzv95DEZ/8hHP+LGJmb8/ZqZTc1OubHBoSGZOzy6yY31D3XL3OPHT8j4ti273VjP0DaZ+9kP+ddj/NF7ZW6+3JTxx06ddmPH5/S1PnDNATc2PNQnc4dHh2W8t6/Hz+3XY7PRU3NjfX36Pq57ol6YmcWlJTdWnDmnc5stPxgSa9lFnti2X8tyETMzq23Z7O92ZlYflrgeZvp6maifZmZZtz/W8okJmau2nfX2ytRiYUEf1+Cg2K0+J3W95LUyMzudGF+p/NWS+fXCzCxk+poosaWemc63uxEUk/67Oqvpe1Lkop6UrEUW/Xgxp+dU9cv2+cHEu7gYG9fxxUU3lqvabGah4U9hY2JOJbe74B+TmVlMzFGzPjFPyPR9DHUxLVfjw8xyMfZSisXE+FESz3zo6tJx8VzI97Ppd2yqBl4KinH/nVxLzKtaIh7q+trGlq4LMjc5N/I/0xRFkdi4Pudift6NyWfTzIKYw8TE/EW+U0vKevzPHWGgX+bGef+4g5gLml3AfHCVpMZPsi5EMYYS79l8arrjXA/fqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAipBN4WfGTsnkT/zVh9zYsVPHZW7W9Huz33uv6ENuZhaCDLdUP/og+qOb2W1/8wk31tXQPeNveNGLZXy5a9CNTS/Ny9zDT55xY2NjD+n9LvrnfOLUUZl75Kje9ktedKMb+6Hv/79l7p13fN6NtabGZO700pKML1h0Y4fvOiZzP/uFk26sv96UuY2umozXuv0xNNjfkLl79u13Y2/+xrfKXP8ubQy1oSE3FvNc5oaauGdR1wvLZAmVYuGPUTOz/Jz/DGT9/Xrby3qc6mR9XMXi4qpsOzZF3Taz+o7tMt466b+v8tfo2tz4x0dkXMmnE+8rIdT1+AmiXhTz+p1hhR73qaGtyONWz9MlIPT2urFickrn1sU7oGQtStUbpXXkCX+3qVqk5mMpqTG8JOKJeWKqzin13btkvHX8KT9403V626cn/WDinMKCrs2x5b8X5HvQzCz4P9dV2zUzi4n5Wud3wuQ1yXp7ymx5w8tP+58rzMws88dE6p4nqbGceDbVuz40umRu8rjFcaXmknFmpqPtpmSD/mdHM7Owc5uM548e9nOvuULm1k6Jz2KpmjGVmBuVmYSUkXivrBb1OUXhGzUAAAAAAAAVwUINAAAAAABARbBQAwAAAAAAUBEs1AAAAAAAAFQECzUAAAAAAAAVwUINAAAAAABARch+jju375TJB/Zf5sai6bZb9cyP1xJtzLKaXl9SbSi7enQrSWv47fx27dotU19z660yPtjX58aGe0Zl7oP33+PGDj32uMzdsXu/G1uM+lrWev1jNjO7/9DDbuzBQ4dkbt/+a9zYiRP6eoyO6Pi2Lr9FX9+A3z7VzGz8lN+KdOypx2Tu2XOnZXwx98dms9Dj/uSk/7i+4nWdt/5bD1JtjFWLZNXi2Mxki8Bkbopq55jrGhl6xL6zRA1sLsu4bGkq6pSZWTE3p7etdivuY+qYW6fPJjbun1NtUbcIVueUuh6lWssmlLnWayWI2rshiJa1ZmbF7KyfmqgnMYp6kWiFGlLtX0W9SbWdlc/tcqI1c6I9t9p2aiwlW9Sr/Yo2vql20q0Tfnvg5H6buu63njjmxrIe3W66TN1Pja+4LLZdotV5UqoluTjuDV+LzJLXp1hY8FNT7ZXVbku0czdL1BxVBxNSz0BqTqfGTEg8f/lZMUdJPCOqDhaq7beZZcn63fBjTZ2rWqGn3oWp9tvqfRh69ee0XLX+XqP222amP08M054bAAAAAABgXWOhBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIv3G7mY2fHZfJN7/sFW7sFa9+tczt7vb7r9drev0oy3S8EH3Ma6b7vjeX/f7rC8vzMnfs+BEZH19s+rFz+loffuxxN3bijOhzb2YD23b5we4emRu6+mR8ubXkxm779Odk7r4rrnNjezftlrk9mRy61tfodmNLizMy9/D0A25sYHBI5uaxJeOnJmbd2JYt+2XufNMf15/49J0y923/8jtlvOpiS1/XUPfHg4qZmYVuf6yk9psSY/T3m6hzcXnZjyWOK+vRz3Wx7NeiYl7XOQvBD3V1yVR1Tqsp3PWgjMfMfy8kr0cJyfEljsvEe87MLNT0u67U2A7+2I2L/jthQyj8OYJZohYlno/QI2qReGYvRCa2bYVfp8zMioVFNxab+pkOjcQ5izqYfPbE8xEyv06ZWfL5kRJjQLrvERlW46dY9O9DWXEp8dzKWlTiepQlalE+MfEcHsgaEXMMMz2/yQYH9LbVOyRVjxLzGyUk5i9xzq8L+eSkzk3MQeRzMOvP380Sc83E+9jyzp+h5Dmp+v7wYzI36/M/AxYLCzI31Bs6Lt93iblimRpcQurzhJpXtY4d72iffKMGAAAAAACgIlioAQAAAAAAqAgWagAAAAAAACqChRoAAAAAAICKYKEGAAAAAACgIlioAQAAAAAAqAjZZ6q/T7RzNLOxab9V4Jfu/YLM3bZt1I1t37ZF5jabui3cxIRoz5Zob1gv/G3vvky0uTazvaODMv7UoZNubG5Wt0bctn2HG+vbPCJzaz1+S+l50XbTzGznzufJ+KkTfruxc2NTetu75txYSLQcnF1KtAas+2O3mWjr1t3b78dEa2Izs+Wxs/q4Mr9d3fbd+/W2l0S7Zn251r/EdVftB+OcP85S21atCc3SLXNV69pStyxxPYpUq9VVGjCxmWijLlpUJttFp9oxivaxyW0nrmen+zWzUm0k5fVq6u2Wab+dbkHpj/tUS86NLoo2q/n0tE5Wtai3V+830c50NdvMK6n23Wa6fbcknq1o+rkMDTHGy7SuN1uzWpR8bmUbdt2uXNaiVWyVmxr3clynavMlQL2T83NjOllcv6xfz41sSY+JYlHMUaIei6s54Q0Nvx6lapl8thPtt9XzlZKqKaoupHJlzUjch9T1yidT7wZB1clVHB/ZoP6cn09MXPx9XvQtAgAAAAAAoCMs1AAAAAAAAFQECzUAAAAAAAAVwUINAAAAAABARbBQAwAAAAAAUBEs1AAAAAAAAFQECzUAAAAAAAAV4TdXN7Puhu5lv7Q46cZuv/3jMjc2F93YUF+vzG02dd/3xYUFN1ZPrE3t27/XjV178/Nl7hXP2yXjk8eOu7FTE+dkbldvt7/fzTtk7tmzs27suquulbkvuO4qGf+T9/1XN1a3LpnbnPPHwPKyHzMzi61cxq3HHyO1bv9ampntv+xyN3bm2CN6v1lNhnv7/X1fc81Bmbs479/HvTu36eNa50JNX9fYEvf7ysv0tmfn3Vgx58fMzLLhQb3tRkMk61pUTM/4wWZT5mYjwzLeOn3GD8Yoc5XaQL+M5zPinBLqO3Wdy8/6NTTGoDde4pxXU2wur8l+s82bZDw/O+bGYkuPzQ1PjKX65ft17rJ/7aKY15il71ns9mtRyPVcL8747564oN/VyTo362/bQufPbdbfp/dbphbt1nO9/IyoRamNF2Juk7oeq2jNatG2LTIej5/wY0U16/pzSoyn2sErZGpYWHJjqXok5z5mlm0Sc7rEOI+L/nHlY+P6uLLEttX7q0w96kvUo3k911Rq2/X8v0hcE52c+KyllLheSWVySxxXGBnSm1b3Me/sWvKNGgAAAAAAgIpgoQYAAAAAAKAiWKgBAAAAAACoCBZqAAAAAAAAKoKFGgAAAAAAgIpgoQYAAAAAAKAiWKgBAAAAAACoiLoKzi8k+rpn/jrPrW/8WplaLM+5sVqzpXPzQsZjreZvu94lc3v6/V73pyYXZO7M5CEZH1/wzyv09MjcR+4+7MbGPn9W5l5+2VVu7KVXHpC5ywuLMt7b1e3GYrMpc+fFtrOaHJpWBBm2hcIfI/Vcj699ey53Y4uzYzL3+UP9Mn7nF77kxk488YjMXZjzn5k4PyFz17vY0vdMyR870vmOQ2KgzcTOt11m31Hvtziln9vkeQmZqlUN/dxaED8biLlMzccTY1zUfSsS9ymxb6kokZuQ9fv1pEjU5qxXv1OKef/9np/R7xQpMTYvZa3DR1dv42Pjq7ftMhK1JtQbbiw2l2Vu1ufP11JzqrC45O+3pecu+ekzetvd/rwo5Prno8WiqCeJZ6vMezKlNjLsxvKpaZmr7rGZWcz9c249cUwf2KUu9S4XYyY/9PhFPpjqiybmCVayHomaExLv47Ak6pF4PszMikTtV/VIzskscc4l5pFlZYODbqyYndXJiXMOdX+MtI4+GKs5RgAABhhJREFUqbe9CvhGDQAAAAAAQEWwUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARspdq/4BuZT0sOgUObj0oc5dEK7KexPpRV9DHFXt73Vh3n84tFv22XjMzugVhrW9IxrddMeLGrug7J3MfPSLa6AXdbq7R57dme+qkbjW2ectox/HlBb+dtJnZ0tKUG5ub061nl+Z1+7Xmkt96tt7jt/Q0M9u+a6sbe+LkaZl7+knd7nBx1j/nxx+4W+Zu3uwfVxzdJHPXvRItKC1LtGPM/G1nA7rdumwJbSbbQseFBZ26KJ6BxDklW0aL6xUaqRrpH1dItYct08o60aJStqZNtgj2X4Wp1phr1o46cS1T7btTLXOlWPihVWwRvN6pcZaKhy79XCZrUZe434l7lo9P6G0rqZbSov2rbCtrusV8WNatdOVznXqmE+1dizk995FUbS9TP8sS77KyNbA27M+dY+I+qn0nczeCEtc+WY9Um/ke/WyGPj3PlvdNzX3MLB+f9IOpZyQRjyKeul5yzrbclLllnu2onk0ziyXqUeqc5X5Tc6cSgnqfJZ6JrDcxdvvFvD/xroxibaNIzPk9fKMGAAAAAACgIlioAQAAAAAAqAgWagAAAAAAACqChRoAAAAAAICKYKEGAAAAAACgIlioAQAAAAAAqAgWagAAAAAAACpCNkifnzmkswt/nacRBmTq6dNTbuzRB4/K3J56r4x3DY+4sS3bRmXuri3Dbqye6XWtzcObZTwv/NjiwoTM3bZtyI3t3rVJ5p48dcqNHTr0kMzdv3yZjC+JnvEzM/49NjObnz/txqanpvV+52dlPF/2+9XXuvtl7gP3b3Fjy0vLMnfbtu0yvvv6a/3crTp3y9YdbqwncU7rXowdp4aGLHMWl/17mk/qMbxmilyGQ12fczbi17m4sNjRIZmZZf26NufquU7c49Q5WRD1OYria2YxF9czNfZCSMT948p6umVqMTfX+X4TYySqeGLbod4QscR9upTVajJciPepqZhZqRqZHEtl6m+qFg0O+kH1XJpZFMcdehO1aHJSxpVQS/2Ms8uNyFpjlnxupUyPr6zLf25TYzOfFrU7sd/UOecTev4rpcbuBlcb8j8bmOmaEhM1JbZafjA1Txgb13FlFetRattZX58fTD27nW7XEs9XQmqOa4X/fMZWU6bKMZCSutb9nX9uydX4StUjMec3Myvm5zs5pFXDN2oAAAAAAAAqgoUaAAAAAACAimChBgAAAAAAoCJYqAEAAAAAAKgIFmoAAAAAAAAqgoUaAAAAAACAipA9vYpl3X4tE+s89aZujzXU8NulfuGOT8vcU6fPyXho+C1Pb7rpRpl7y8tf4sampnSr3nu/+A8yPrfoX89DTx6TuYePHnVjC4lWYjH6LdJ6hrbK3OnpGRmfmfDvxdy0brmoGrfVa7qt2/CgbnW36zK/rfjo5p0yd9suvw32rhddJ3M3Del2c12iBWYt0R7TgojHDb7mWqLtaKpNq3WLFsmqnamZxR36+QminXtx5Em97RIto1MtFfNzYzIuiXtRzIp20ma6hbbpFtqFqJ9JZVq4lm0VGv37WCzq9qiqtbFsa2xmIdH6W7Xjjc1Ey875BT+YyN3wVMvoEq3Lazu2yXi+dVjGw7JfE+JDh2VuqoWrzC3081GqNbN6LywnjlnVIvHMmpWsRYl3mYqHTNeiVN0vFjtvL6zGbujy25GbmQX1jk1JtESWLe1LtFNeL1KthlfrGtQO+HNsM7PWlgEZD03/uLJ7H5W56vlL1lg5BynXmjmId6ocp2YWGv4zFJv6HqfarEupeiRKjjpfs8Qc1syKucR8UShTj9Tcx0x/Nk09T7EpanDUc1zPBv90BwAAAAAAsH6wUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFARLNQAAAAAAABURIgxrvUxAAAAAAAAwPhGDQAAAAAAQGWwUAMAAAAAAFARLNQAAAAAAABUBAs1AAAAAAAAFcFCDQAAAAAAQEWwUAMAAAAAAFAR/xOZsqCJI+qDbAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "plot_feature_outlier_image(od_preds, \n", " X_mask, \n", @@ -401,14 +743,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace \"cifar10\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f cifar10.yaml\n", - "!kubectl delete -f cifar10od.yaml\n", - "!kubectl delete -f trigger.yaml\n", - "!kubectl delete -f message-dumper.yaml" + "!kubectl delete ns cifar10" ] }, { @@ -435,7 +782,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/components/outlier-detection/cifar10/cifar10od.yaml b/components/outlier-detection/cifar10/cifar10od.yaml deleted file mode 100644 index 584144641a..0000000000 --- a/components/outlier-detection/cifar10/cifar10od.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: vae-outlier -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "1" - spec: - containers: - - image: seldonio/alibi-detect-server:0.0.1 - imagePullPolicy: IfNotPresent - args: - - --model_name - - cifar10od - - --http_port - - '8080' - - --protocol - - tensorflow.http - - --storage_uri - - gs://seldon-models/alibi-detect/od/OutlierVAE/cifar10 - - --reply_url - - http://message-dumper.default - - --event_type - - io.seldon.serving.inference.outlier - - --event_source - - io.seldon.serving.cifar10od - - OutlierDetector diff --git a/components/outlier-detection/cifar10/message-dumper.yaml b/components/outlier-detection/cifar10/message-dumper.yaml deleted file mode 100644 index 7bbd4e0a98..0000000000 --- a/components/outlier-detection/cifar10/message-dumper.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: serving.knative.dev/v1alpha1 -kind: Service -metadata: - name: message-dumper -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "1" - spec: - containers: - - image: gcr.io/knative-releases/github.com/knative/eventing-sources/cmd/event_display diff --git a/components/outlier-detection/cifar10/trigger.yaml b/components/outlier-detection/cifar10/trigger.yaml deleted file mode 100644 index cdcf59e998..0000000000 --- a/components/outlier-detection/cifar10/trigger.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Trigger -metadata: - name: vaeoutlier-trigger - namespace: default -spec: - filter: - sourceAndType: - type: io.seldon.serving.inference.request - subscriber: - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: vae-outlier - From a49bcde1dc4c21300120d4567f466ce907e3e442 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Wed, 11 Nov 2020 15:00:54 +0000 Subject: [PATCH 2/3] remove debug logging --- components/alibi-detect-server/adserver/od_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/alibi-detect-server/adserver/od_model.py b/components/alibi-detect-server/adserver/od_model.py index 5a495943a7..fb04fa6446 100644 --- a/components/alibi-detect-server/adserver/od_model.py +++ b/components/alibi-detect-server/adserver/od_model.py @@ -105,10 +105,10 @@ def process_event(self, inputs: Union[List, Dict], headers: Dict) -> Dict: # scores used to determine outliers return_instance_score=ret_instance_score, ) - logging.info(od_preds) + # clean result if "data" in od_preds and "instance_score" in od_preds["data"] and od_preds["data"]["instance_score"] is None: del od_preds["data"]["instance_score"] if "data" in od_preds and "feature_score" in od_preds["data"] and od_preds["data"]["feature_score"] is None: del od_preds["data"]["feature_score"] - logging.info(od_preds) + return json.loads(json.dumps(od_preds, cls=NumpyEncoder)) From ecb584085671c54924beac8520297ef2bcd90d01 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Fri, 13 Nov 2020 07:55:02 +0000 Subject: [PATCH 3/3] update istio and knative versions in notebooks --- components/drift-detection/cifar10/cifar10_drift.ipynb | 2 +- components/outlier-detection/cifar10/cifar10_outlier.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/drift-detection/cifar10/cifar10_drift.ipynb b/components/drift-detection/cifar10/cifar10_drift.ipynb index 4eb6371f2d..604f1aab82 100644 --- a/components/drift-detection/cifar10/cifar10_drift.ipynb +++ b/components/drift-detection/cifar10/cifar10_drift.ipynb @@ -16,7 +16,7 @@ " * [Seldon Core installed](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html) \n", " * Ensure you install for istio, e.g. for the helm chart `--set istio.enabled=true`\n", " \n", - " Tested on GKE with Knative 0.13 and Istio 1.3.1" + " Tested on GKE and Kind with Knative 0.18 and Istio 1.7.3" ] }, { diff --git a/components/outlier-detection/cifar10/cifar10_outlier.ipynb b/components/outlier-detection/cifar10/cifar10_outlier.ipynb index 4647a2275d..82e4f98467 100644 --- a/components/outlier-detection/cifar10/cifar10_outlier.ipynb +++ b/components/outlier-detection/cifar10/cifar10_outlier.ipynb @@ -17,7 +17,7 @@ " * [Seldon Core installed](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html) \n", " * Ensure you install for istio, e.g. for the helm chart `--set istio.enabled=true`\n", " \n", - " Tested on GKE with Knative 0.13 and Istio 1.3.1" + " Tested on GKE and Kind with Knative 0.18 and Istio 1.7.3" ] }, {