notebooks/max_grpc_msg_size.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Increasing the Maximum Message Size for gRPC\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running this notebook\n",
    "\n",
    "You will need to start Jupyter with settings to allow for large payloads, for example:\n",
    "\n",
    "```\n",
    "jupyter notebook --NotebookApp.iopub_data_rate_limit=1000000000\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "33bfe200-7796-4d33-8d9e-e38c73b09c96"
   },
   "outputs": [],
   "source": [
    "from IPython.core.magic import register_line_cell_magic\n",
    "\n",
    "\n",
    "@register_line_cell_magic\n",
    "def writetemplate(line, cell):\n",
    "    with open(line, \"w\") as f:\n",
    "        f.write(cell.format(**globals()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Seldon Core\n",
    "\n",
    "Use the setup notebook to [Setup Cluster](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Setup-Cluster) with [Ambassador Ingress](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Ambassador) and [Install Seldon Core](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Install-Seldon-Core). Instructions [also online](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "52d0c099-0a33-42c5-b456-948eae966e4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "7824b2b0-afaf-4d4d-b2ad-95155cb78826"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context \"kind-kind\" modified.\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl config set-context $(kubectl config current-context) --namespace=seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "3ef294db-a2ff-4e18-a2dd-0a45d08ed74c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.5.0-dev'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VERSION = !cat ../version.txt\n",
    "VERSION = VERSION[0]\n",
    "VERSION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now add in our model config file the annotations `\"seldon.io/rest-timeout\":\"100000\"` and `\"seldon.io/grpc-timeout\":\"100000\"`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "bf9604be-072a-48e6-a349-b34cfd2bf0d9",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%writetemplate resources/model_long_timeouts.yaml\n",
    "apiVersion: machinelearning.seldon.io/v1alpha2\n",
    "kind: SeldonDeployment\n",
    "metadata:\n",
    "  labels:\n",
    "    app: seldon\n",
    "  name: model-long-timeout\n",
    "spec:\n",
    "  annotations:\n",
    "    deployment_version: v1\n",
    "    seldon.io/grpc-timeout: '100000'\n",
    "    seldon.io/rest-timeout: '100000'\n",
    "  name: long-to\n",
    "  predictors:\n",
    "  - annotations:\n",
    "      predictor_version: v1\n",
    "    componentSpecs:\n",
    "    - spec:\n",
    "        containers:\n",
    "        - image: seldonio/mock_classifier:{VERSION}\n",
    "          imagePullPolicy: IfNotPresent\n",
    "          name: classifier\n",
    "          resources:\n",
    "            requests:\n",
    "              memory: 1Mi\n",
    "        terminationGracePeriodSeconds: 20\n",
    "    graph:\n",
    "      children: []\n",
    "      name: classifier\n",
    "      type: MODEL\n",
    "    name: test\n",
    "    replicas: 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Seldon Deployment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deploy the runtime graph to kubernetes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "cfa1b0dc-11b2-4254-a60a-249d58571c3c",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/model-long-timeout created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl apply -f resources/model_long_timeouts.yaml -n seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "c7637004-278a-43d2-a5fc-c8695f3bca3e",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deployment \"model-long-timeout-test-0-classifier\" successfully rolled out\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=model-long-timeout -o jsonpath='{.items[0].metadata.name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "fcd4b922-5655-45be-be29-062c74008cec",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from seldon_core.seldon_client import SeldonClient\n",
    "\n",
    "sc = SeldonClient(\n",
    "    deployment_name=\"model-long-timeout\",\n",
    "    namespace=\"seldon\",\n",
    "    grpc_max_send_message_length=50 * 1024 * 1024,\n",
    "    grpc_max_receive_message_length=50 * 1024 * 1024,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send a small request which should succeed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "c9dfbeba-4421-4b66-84e7-35f20ca356e8",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success:True message:\n",
      "Request:\n",
      "{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.4806932754099743]}}}\n",
      "Response:\n",
      "{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.08047035772935462]}}}\n"
     ]
    }
   ],
   "source": [
    "r = sc.predict(gateway=\"ambassador\", transport=\"grpc\")\n",
    "assert r.success == True\n",
    "print(r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send a large request which will fail as the default for the model will be 4G."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "4fe5089e-77b3-4c8d-b684-d20748aecb5b",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False <_InactiveRpcError of RPC that terminated with:\n",
      "\tstatus = StatusCode.RESOURCE_EXHAUSTED\n",
      "\tdetails = \"Received message larger than max (8000023 vs. 4194304)\"\n",
      "\tdebug_error_string = \"{\"created\":\"@1603887710.710555595\",\"description\":\"Error received from peer ipv6:[::1]:8003\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1061,\"grpc_message\":\"Received message larger than max (8000023 vs. 4194304)\",\"grpc_status\":8}\"\n",
      ">\n"
     ]
    }
   ],
   "source": [
    "r = sc.predict(gateway=\"ambassador\", transport=\"grpc\", shape=(1000000, 1))\n",
    "print(r.success, r.msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "c56d7fa4-9edb-4eb3-b953-18612c69a02a",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io \"model-long-timeout\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f resources/model_long_timeouts.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Allowing larger gRPC messages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we change our SeldonDeployment to include a annotation for max grpx message size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "cdfcb802-15ae-4fe3-b3f9-1db1bab035f8"
   },
   "outputs": [],
   "source": [
    "%%writetemplate resources/model_grpc_size.yaml\n",
    "apiVersion: machinelearning.seldon.io/v1alpha2\n",
    "kind: SeldonDeployment\n",
    "metadata:\n",
    "  labels:\n",
    "    app: seldon\n",
    "  name: seldon-model\n",
    "spec:\n",
    "  annotations:\n",
    "    seldon.io/grpc-max-message-size: '10000000'\n",
    "    seldon.io/grpc-timeout: '100000'\n",
    "    seldon.io/rest-timeout: '100000'\n",
    "  name: test-deployment\n",
    "  predictors:\n",
    "  - annotations:\n",
    "      predictor_version: v1\n",
    "    componentSpecs:\n",
    "    - spec:\n",
    "        containers:\n",
    "        - image: seldonio/mock_classifier:{VERSION}\n",
    "          imagePullPolicy: IfNotPresent\n",
    "          name: classifier\n",
    "          resources:\n",
    "            requests:\n",
    "              memory: 1Mi\n",
    "        terminationGracePeriodSeconds: 20\n",
    "    graph:\n",
    "      children: []\n",
    "      endpoint:\n",
    "        type: GRPC\n",
    "      name: classifier\n",
    "      type: MODEL\n",
    "    name: grpc-size\n",
    "    replicas: 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "53b29b7a-0ffd-432d-a590-1bb104af667f",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/seldon-model created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f resources/model_grpc_size.yaml -n seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "6368d66c-e924-4cfa-8bb5-75ae15d238e3",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"seldon-model-grpc-size-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"seldon-model-grpc-size-0-classifier\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=seldon-model -o jsonpath='{.items[0].metadata.name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send a request via ambassador. This should succeed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "bcb34458-b62f-4fae-b3c3-f4ac6c0acad7",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "sc = SeldonClient(\n",
    "    deployment_name=\"seldon-model\",\n",
    "    namespace=\"seldon\",\n",
    "    grpc_max_send_message_length=50 * 1024 * 1024,\n",
    "    grpc_max_receive_message_length=50 * 1024 * 1024,\n",
    ")\n",
    "r = sc.predict(gateway=\"ambassador\", transport=\"grpc\", shape=(1000000, 1))\n",
    "assert r.success == True\n",
    "print(r.success)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "cdb5dac6-5670-46e4-bfb2-17c047a97b24",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io \"seldon-model\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f resources/model_grpc_size.json -n seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "77663b29-bdb1-4b58-87cd-3b7e5f4f47ba"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}