examples/models/template_model_with_metrics/modelWithMetrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model with Metrics\n",
    "\n",
    "## Dependencies\n",
    "\n",
    "```pip install seldon-core```\n",
    "\n",
    "## Summary of Custom Metrics\n",
    "\n",
    "Example testing a model with custom metrics.\n",
    "\n",
    "Metrics can be \n",
    "\n",
    "  * A ```COUNTER``` : the returned value will increment the current value\n",
    "  * A ```GAUGE``` : the returned value will overwrite the current value\n",
    "  * A ```TIMER``` : a number of millisecs. Prometheus SUM and COUNT metrics will be created.\n",
    "  \n",
    "You need to provide a list of dictionaries each with the following:\n",
    "\n",
    "  * a ```type``` : COUNTER, GAUGE, or TIMER\n",
    "  * a ```key``` : a user defined key\n",
    "  * a ```value``` : a float value\n",
    "  \n",
    "See example code below:\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mclass\u001b[39;49;00m \u001b[04m\u001b[32mModelWithMetrics\u001b[39;49;00m(\u001b[36mobject\u001b[39;49;00m):\r\n",
      "\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32m__init__\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m):\r\n",
      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mInitialising\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
      "\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32mpredict\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m,X,features_names):\r\n",
      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mPredict called\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
      "        \u001b[34mreturn\u001b[39;49;00m X\r\n",
      "\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32mmetrics\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m):\r\n",
      "        \u001b[34mreturn\u001b[39;49;00m [\r\n",
      "            {\u001b[33m\"\u001b[39;49;00m\u001b[33mtype\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mCOUNTER\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mkey\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mmycounter\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mvalue\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[34m1\u001b[39;49;00m}, \u001b[37m# a counter which will increase by the given value\u001b[39;49;00m\r\n",
      "            {\u001b[33m\"\u001b[39;49;00m\u001b[33mtype\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mGAUGE\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mkey\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mmygauge\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mvalue\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[34m100\u001b[39;49;00m}, \u001b[37m# a gauge which will be set to given value\u001b[39;49;00m\r\n",
      "            {\u001b[33m\"\u001b[39;49;00m\u001b[33mtype\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mTIMER\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mkey\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mmytimer\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mvalue\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[34m20.2\u001b[39;49;00m}, \u001b[37m# a timer which will add sum and count metrics - assumed millisecs\u001b[39;49;00m\r\n",
      "            ]\r\n",
      "    \r\n"
     ]
    }
   ],
   "source": [
    "!pygmentize ModelWithMetrics.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## REST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:0.12 model-with-metrics-rest:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5c2a9f9337acefa041df77b1bf476fa69b46acb81dfae4a8bc0ffbbb12067be4\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"model-with-metrics\" -d --rm -p 5000:5000 model-with-metrics-rest:0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[6.543 4.872 6.89  0.208]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "  metrics {\n",
      "    key: \"mycounter\"\n",
      "    value: 1.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mygauge\"\n",
      "    type: GAUGE\n",
      "    value: 100.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mytimer\"\n",
      "    type: TIMER\n",
      "    value: 20.200000762939453\n",
      "  }\n",
      "}\n",
      "data {\n",
      "  names: \"t:0\"\n",
      "  names: \"t:1\"\n",
      "  names: \"t:2\"\n",
      "  names: \"t:3\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 6.543\n",
      "        }\n",
      "        values {\n",
      "          number_value: 4.872\n",
      "        }\n",
      "        values {\n",
      "          number_value: 6.89\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.208\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-tester contract.json 0.0.0.0 5000 -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model-with-metrics\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm model-with-metrics --force"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gRPC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build -E environment_grpc . seldonio/seldon-core-s2i-python3:0.12 model-with-metrics-grpc:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "85ac344a48c56931ff3bef98ca02bb0d9f7e7198619fee933faf92d62c594080\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"model-with-metrics\" -d --rm -p 5000:5000 model-with-metrics-grpc:0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\r\n",
      "SENDING NEW REQUEST:\r\n",
      "\r\n",
      "[[7.607 2.626 3.027 2.034]]\r\n",
      "RECEIVED RESPONSE:\r\n",
      "meta {\r\n",
      "  metrics {\r\n",
      "    key: \"mycounter\"\r\n",
      "    value: 1.0\r\n",
      "  }\r\n",
      "  metrics {\r\n",
      "    key: \"mygauge\"\r\n",
      "    type: GAUGE\r\n",
      "    value: 100.0\r\n",
      "  }\r\n",
      "  metrics {\r\n",
      "    key: \"mytimer\"\r\n",
      "    type: TIMER\r\n",
      "    value: 20.200000762939453\r\n",
      "  }\r\n",
      "}\r\n",
      "data {\r\n",
      "  names: \"t:0\"\r\n",
      "  names: \"t:1\"\r\n",
      "  names: \"t:2\"\r\n",
      "  names: \"t:3\"\r\n",
      "  ndarray {\r\n",
      "    values {\r\n",
      "      list_value {\r\n",
      "        values {\r\n",
      "          number_value: 7.607\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 2.626\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 3.027\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 2.034\r\n",
      "        }\r\n",
      "      }\r\n",
      "    }\r\n",
      "  }\r\n",
      "}\r\n",
      "\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-tester contract.json 0.0.0.0 5000 -p --grpc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model-with-metrics\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm model-with-metrics --force"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test using Minikube\n",
    "\n",
    "**Due to a [minikube/s2i issue](https://github.com/SeldonIO/seldon-core/issues/253) you will need [s2i >= 1.1.13](https://github.com/openshift/source-to-image/releases/tag/v1.1.13)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There is a newer version of minikube available (v0.35.0).  Download it here:\n",
      "https://github.com/kubernetes/minikube/releases/tag/v0.35.0\n",
      "\n",
      "To disable this notification, run the following:\n",
      "minikube config set WantUpdateNotification false\n",
      "😄  minikube v0.34.1 on linux (amd64)\n",
      "🔥  Creating virtualbox VM (CPUs=2, Memory=4096MB, Disk=20000MB) ...\n",
      "📶  \"minikube\" IP address is 192.168.99.100\n",
      "🐳  Configuring Docker as the container runtime ...\n",
      "✨  Preparing Kubernetes environment ...\n",
      "🚜  Pulling images required by Kubernetes v1.13.3 ...\n",
      "🚀  Launching Kubernetes v1.13.3 using kubeadm ... \n",
      "🔑  Configuring cluster permissions ...\n",
      "🤔  Verifying component health .....\n",
      "💗  kubectl is now configured to use \"minikube\"\n",
      "🏄  Done! Thank you for using minikube!\n"
     ]
    }
   ],
   "source": [
    "!minikube start --memory 4096"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clusterrolebinding.rbac.authorization.k8s.io/kube-system-cluster-admin created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!helm init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"tiller-deploy\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"tiller-deploy\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Tue Apr 16 13:52:58 2019\n",
      "NAMESPACE: seldon-system\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Service\n",
      "NAME                                        TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)  AGE\n",
      "seldon-operator-controller-manager-service  ClusterIP  10.100.187.38  <none>       443/TCP  0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME                        SECRETS  AGE\n",
      "seldon-spartakus-volunteer  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME                        AGE\n",
      "seldon-spartakus-volunteer  0s\n",
      "\n",
      "==> v1/Secret\n",
      "NAME                                   TYPE    DATA  AGE\n",
      "seldon-operator-webhook-server-secret  Opaque  0     0s\n",
      "\n",
      "==> v1/ClusterRole\n",
      "NAME                          AGE\n",
      "seldon-operator-manager-role  0s\n",
      "\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME                                 AGE\n",
      "seldon-operator-manager-rolebinding  0s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                        DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "seldon-spartakus-volunteer  1        0        0           0          0s\n",
      "\n",
      "==> v1/StatefulSet\n",
      "NAME                                DESIRED  CURRENT  AGE\n",
      "seldon-operator-controller-manager  1        1        0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME                        AGE\n",
      "seldon-spartakus-volunteer  0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                  READY  STATUS             RESTARTS  AGE\n",
      "seldon-operator-controller-manager-0  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/ConfigMap\n",
      "NAME                     DATA  AGE\n",
      "seldon-spartakus-config  3     0s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                         AGE\n",
      "seldondeployments.machinelearning.seldon.io  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usageMetrics.enabled=true --namespace seldon-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core-analytics\n",
      "LAST DEPLOYED: Tue Apr 16 13:53:06 2019\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Service\n",
      "NAME                      TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)       AGE\n",
      "alertmanager              ClusterIP  10.107.187.67  <none>       80/TCP        0s\n",
      "grafana-prom              NodePort   10.96.185.3    <none>       80:31405/TCP  0s\n",
      "prometheus-node-exporter  ClusterIP  None           <none>       9100/TCP      0s\n",
      "prometheus-seldon         ClusterIP  10.109.47.248  <none>       80/TCP        0s\n",
      "\n",
      "==> v1beta1/DaemonSet\n",
      "NAME                      DESIRED  CURRENT  READY  UP-TO-DATE  AVAILABLE  NODE SELECTOR  AGE\n",
      "prometheus-node-exporter  1        1        0      1           0          <none>         0s\n",
      "\n",
      "==> v1/Secret\n",
      "NAME                 TYPE    DATA  AGE\n",
      "grafana-prom-secret  Opaque  1     1s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "prometheus  1        0s\n",
      "\n",
      "==> v1/Job\n",
      "NAME                            DESIRED  SUCCESSFUL  AGE\n",
      "grafana-prom-import-dashboards  1        0           0s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                     DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "alertmanager-deployment  1        1        1           0          0s\n",
      "grafana-prom-deployment  1        1        1           0          0s\n",
      "prometheus-deployment    1        1        1           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                      READY  STATUS             RESTARTS  AGE\n",
      "grafana-prom-import-dashboards-k5zmm      0/1    ContainerCreating  0         0s\n",
      "alertmanager-deployment-7cd568f668-9fvhm  0/1    ContainerCreating  0         0s\n",
      "grafana-prom-deployment-899b4dd7b-n664s   0/1    ContainerCreating  0         0s\n",
      "prometheus-node-exporter-wzntd            0/1    Pending            0         0s\n",
      "prometheus-deployment-7554c97586-5dfl7    0/1    Pending            0         0s\n",
      "\n",
      "==> v1/ConfigMap\n",
      "NAME                       DATA  AGE\n",
      "alertmanager-server-conf   1     1s\n",
      "grafana-import-dashboards  11    1s\n",
      "prometheus-rules           0     0s\n",
      "prometheus-server-conf     1     0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "prometheus  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "prometheus  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install seldon-core-analytics --name seldon-core-analytics --set grafana_prom_admin_password=password --set persistence.enabled=false --repo https://storage.googleapis.com/seldon-charts "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "partitioned roll out complete: 1 new pods have been updated...\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status statefulset.apps/seldon-operator-controller-manager -n seldon-system"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Ingress\n",
    "Please note: There are reported gRPC issues with ambassador (see https://github.com/SeldonIO/seldon-core/issues/473)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   ambassador\n",
      "LAST DEPLOYED: Tue Apr 16 13:53:59 2019\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME               TYPE          CLUSTER-IP      EXTERNAL-IP  PORT(S)                     AGE\n",
      "ambassador-admins  ClusterIP     10.104.133.196  <none>       8877/TCP                    0s\n",
      "ambassador         LoadBalancer  10.103.248.191  <pending>    80:31100/TCP,443:31733/TCP  0s\n",
      "\n",
      "==> v1/Deployment\n",
      "NAME        DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "ambassador  3        3        3           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                         READY  STATUS             RESTARTS  AGE\n",
      "ambassador-5b89d44544-25vlr  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-dtnm6  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-j7km2  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "ambassador  1        0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Congratuations! You've successfully installed Ambassador.\n",
      "\n",
      "For help, visit our Slack at https://d6e.co/slack or view the documentation online at https://www.getambassador.io.\n",
      "\n",
      "To get the IP address of Ambassador, run the following commands:\n",
      "NOTE: It may take a few minutes for the LoadBalancer IP to be available.\n",
      "     You can watch the status of by running 'kubectl get svc -w  --namespace seldon ambassador'\n",
      "\n",
      "  On GKE/Azure:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace seldon ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n",
      "\n",
      "  On AWS:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace seldon ambassador -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n",
      "\n",
      "  echo http://$SERVICE_IP:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install stable/ambassador --name ambassador --set crds.keep=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"ambassador\" rollout to finish: 0 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 1 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 2 of 3 updated replicas are available...\n",
      "deployment \"ambassador\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deployment.apps/ambassador"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " * Port forward the dashboard when running\n",
    "  ```\n",
    "     kubectl port-forward $(kubectl get pods -n default -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n default 3000:3000\n",
    "  ```\n",
    "  * Visit http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1 and login using \"admin\" and the password you set above when launching with helm."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## REST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:0.12 model-with-metrics-rest:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/mymodel created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f deployment-rest.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"mymodel-mymodel-b79af31\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"mymodel-mymodel-b79af31\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/mymodel-mymodel-b79af31"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[7.564 2.469 4.288 1.109]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "  puid: \"mqi1cen7kiv8nlqs8pj443d9ns\"\n",
      "  requestPath {\n",
      "    key: \"complex-model\"\n",
      "    value: \"model-with-metrics-rest:0.1\"\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mycounter\"\n",
      "    value: 1.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mygauge\"\n",
      "    type: GAUGE\n",
      "    value: 100.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mytimer\"\n",
      "    type: TIMER\n",
      "    value: 20.200000762939453\n",
      "  }\n",
      "}\n",
      "data {\n",
      "  names: \"t:0\"\n",
      "  names: \"t:1\"\n",
      "  names: \"t:2\"\n",
      "  names: \"t:3\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 7.564\n",
      "        }\n",
      "        values {\n",
      "          number_value: 2.469\n",
      "        }\n",
      "        values {\n",
      "          number_value: 4.288\n",
      "        }\n",
      "        values {\n",
      "          number_value: 1.109\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-api-tester contract.json `minikube ip` `kubectl get svc ambassador -o jsonpath='{.spec.ports[0].nodePort}'` \\\n",
    "    mymodel --namespace seldon -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io \"mymodel\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f deployment-rest.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gRPC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build -E environment_grpc . seldonio/seldon-core-s2i-python3:0.12 model-with-metrics-grpc:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/mymodel created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f deployment-grpc.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"mymodel-mymodel-5818788\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"mymodel-mymodel-5818788\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/mymodel-mymodel-5818788"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validate on Grafana\n",
    "\n",
    "To check the metrics have appeared on Prometheus and are available in Grafana you could create a new graph in a dashboard and use the query:\n",
    "\n",
    "```\n",
    "mycounter_total\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[5.625 2.569 7.636 0.647]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "  puid: \"mu75mrgolbub9n24v11eflp7jr\"\n",
      "  requestPath {\n",
      "    key: \"complex-model\"\n",
      "    value: \"model-with-metrics-grpc:0.1\"\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mycounter\"\n",
      "    value: 1.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mygauge\"\n",
      "    type: GAUGE\n",
      "    value: 100.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mytimer\"\n",
      "    type: TIMER\n",
      "    value: 20.200000762939453\n",
      "  }\n",
      "}\n",
      "data {\n",
      "  names: \"t:0\"\n",
      "  names: \"t:1\"\n",
      "  names: \"t:2\"\n",
      "  names: \"t:3\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 5.625\n",
      "        }\n",
      "        values {\n",
      "          number_value: 2.569\n",
      "        }\n",
      "        values {\n",
      "          number_value: 7.636\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.647\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-api-tester contract.json `minikube ip` `kubectl get svc ambassador -o jsonpath='{.spec.ports[0].nodePort}'` \\\n",
    "    mymodel --namespace seldon -p --grpc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io \"mymodel\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f deployment-grpc.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🔥  Deleting \"minikube\" from virtualbox ...\n",
      "💔  The \"minikube\" cluster has been deleted.\n"
     ]
    }
   ],
   "source": [
    "!minikube delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}