appwrapper-pytorchjob/mnist/mnist_pytorch_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3c39a92f-a2d8-4ff2-b960-4aacba121251",
   "metadata": {},
   "source": [
    "# Fetch the Mnist_784 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bd6b9443-c626-41a0-9686-44bdd217be01",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "import numpy as np\n",
    "import json\n",
    "import requests\n",
    "X, y = fetch_openml('mnist_784', return_X_y=True, parser='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "93ce3bdd-a925-45b0-af7b-e3657e771261",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed3fd588-8f58-417c-906e-c5a1e8e08fdb",
   "metadata": {},
   "source": [
    "Deploy the model in your Data Science project using RHODS UI under \"Models and model servers\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e697e7f6-86d6-4810-a0be-25f3bd9ad3cb",
   "metadata": {},
   "source": [
    "# Submit HTTP REST request to the ModelMesh for single sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "56387f1d-26b0-4b0c-b774-7a68c3123db4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "{\"model_name\":\"mnist123__isvc-8d6ab6dcea\",\"model_version\":\"1\",\"outputs\":[{\"name\":\"output_0\",\"datatype\":\"FP32\",\"shape\":[1,10],\"data\":[-814.69055,-957.3951,-849.8701,-50.25647,-1386.2316,0,-1192.0592,-917.001,-584.47845,-549.07214]}]}\n"
     ]
    }
   ],
   "source": [
    "model_name=\"mnist123\"\n",
    "\n",
    "import requests\n",
    "import json\n",
    "URL='http://modelmesh-serving.huggingface.svc.cluster.local:8008/v2/models/'+model_name+'/infer' # underscore characters are removed\n",
    "headers = {}\n",
    "payload = {\n",
    "        \"inputs\": [{ \"name\": \"input_0\", \"shape\": (1,1,28,28), \"datatype\": \"FP32\", \"data\": X.iloc[0].values.tolist()}]\n",
    "    }\n",
    "headers = {\"content-type\": \"application/json\"}\n",
    "res = requests.post(URL, json=payload, headers=headers)\n",
    "print(res)\n",
    "print(res.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23c9bfe1-5600-4571-82f8-3938159592a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected 5 Actual 5\n"
     ]
    }
   ],
   "source": [
    "print(\"Expected\",y.iloc[0],\"Actual\",np.argmax(res.json()['outputs'][0]['data']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37a497a8-f137-4f0b-9227-0e72a1f8a0fc",
   "metadata": {},
   "source": [
    "# Submit HTTP REST request to the ModelMesh for a batch of samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1988c792-d0f4-451d-8254-bbd7cd0846b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "{\"model_name\":\"mnist123__isvc-8d6ab6dcea\",\"model_version\":\"1\",\"outputs\":[{\"name\":\"output_0\",\"datatype\":\"FP32\",\"shape\":[5,10],\"data\":[-814.69055,-957.39514,-849.8701,-50.25641,-1386.2317,0,-1192.0592,-917.00104,-584.47845,-549.07214,0,-1738.2633,-1001.20197,-1479.0795,-1671.8254,-1337.5544,-1102.626,-1238.0874,-1310.7225,-1436.8561,-1826.0315,-1556.1779,-1231.7928,-1325.793,0,-1360.6669,-1473.7661,-1042.4226,-1417.6643,-456.1117,-1381.7842,0,-658.9868,-1014.41394,-828.6969,-1272.7472,-1026.6971,-779.3543,-694.4404,-1050.7561,-1771.2993,-1376.7881,-1287.1708,-1171.8777,-408.81616,-1147.683,-1725.4868,-665.1205,-822.9125,0]}]}\n"
     ]
    }
   ],
   "source": [
    "model_name=\"mnist123\"\n",
    "\n",
    "import requests\n",
    "import json\n",
    "URL='http://modelmesh-serving.huggingface.svc.cluster.local:8008/v2/models/'+model_name+'/infer' # underscore characters are removed\n",
    "headers = {}\n",
    "payload = {\n",
    "        \"inputs\": [{ \"name\": \"input_0\", \"shape\": (5,1,28,28), \"datatype\": \"FP32\", \"data\": X.loc[0:4].values.flatten().tolist()}]\n",
    "    }\n",
    "#print(payload)\n",
    "headers = {\"content-type\": \"application/json\"}\n",
    "res = requests.post(URL, json=payload, headers=headers)\n",
    "print(res)\n",
    "print(res.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2bfb8ec0-43b5-46a7-b6b3-d07908454528",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['5', '0', '4', '1', '9'] [5 0 4 1 9]\n"
     ]
    }
   ],
   "source": [
    "actual=y[0:5].values.tolist()\n",
    "expected=np.argmax(np.array(res.json()['outputs'][0]['data']).reshape(res.json()['outputs'][0]['shape']),axis=1)\n",
    "print(actual,expected)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9afd1cda-cd29-4ac9-9dae-54176017feaa",
   "metadata": {},
   "source": [
    "# Submit gRPC request to the ModelMesh for single sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f786bde3-5d60-45ca-b1a2-fec960fa8129",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: grpcio in /opt/app-root/lib/python3.8/site-packages (1.54.0)\n",
      "Requirement already satisfied: grpcio-tools==1.46.0 in /opt/app-root/lib/python3.8/site-packages (1.46.0)\n",
      "Requirement already satisfied: protobuf<4.0dev,>=3.12.0 in /opt/app-root/lib/python3.8/site-packages (from grpcio-tools==1.46.0) (3.20.3)\n",
      "Requirement already satisfied: setuptools in /opt/app-root/lib/python3.8/site-packages (from grpcio-tools==1.46.0) (67.3.1)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install grpcio grpcio-tools==1.46.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f4f8847d-96cd-447e-9ab3-1eb2e76cb139",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-06-09 15:56:52--  https://raw.githubusercontent.com/kserve/modelmesh-serving/main/fvt/proto/kfs_inference_v2.proto\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.111.133, 185.199.110.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 10394 (10K) [text/plain]\n",
      "Saving to: ‘kfs_inference_v2.proto’\n",
      "\n",
      "kfs_inference_v2.pr 100%[===================>]  10.15K  --.-KB/s    in 0s      \n",
      "\n",
      "2023-06-09 15:56:52 (87.6 MB/s) - ‘kfs_inference_v2.proto’ saved [10394/10394]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#!wget https://raw.githubusercontent.com/kserve/kserve/master/docs/predict-api/v2/grpc_predict_v2.proto\n",
    "!wget https://raw.githubusercontent.com/kserve/modelmesh-serving/main/fvt/proto/kfs_inference_v2.proto\n",
    "!python3 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. ./kfs_inference_v2.proto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "931779a8-4538-4150-acf4-b2e2a7a8dbd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name=\"mnist123\"\n",
    "payload = { \"model_name\": model_name,\n",
    "            \"inputs\": [{ \"name\": \"input_0\", \"shape\": (1,1,28,28), \"datatype\": \"FP32\", \"contents\": {\"fp32_contents\":X.iloc[0].values.tolist()}}]\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "44b108a8-a0b6-4f27-80b3-bede2e8aca72",
   "metadata": {},
   "outputs": [],
   "source": [
    "import grpc\n",
    "import kfs_inference_v2_pb2, kfs_inference_v2_pb2_grpc\n",
    "grpc_url=\"modelmesh-serving.huggingface.svc.cluster.local:8033\"\n",
    "request=kfs_inference_v2_pb2.ModelInferRequest(model_name=model_name,inputs=payload[\"inputs\"])\n",
    "grpc_channel = grpc.insecure_channel(grpc_url)\n",
    "grpc_stub = kfs_inference_v2_pb2_grpc.GRPCInferenceServiceStub(grpc_channel)\n",
    "response = grpc_stub.ModelInfer(request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c86e43d0-15ae-43e8-bd9f-80f1a0b14c70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'google.protobuf.pyext._message.RepeatedCompositeContainer'> <class 'google.protobuf.pyext._message.RepeatedScalarContainer'>\n",
      "{'name': 'output_0', 'datatype': 'FP32', 'shape': ['1', '10']}\n"
     ]
    }
   ],
   "source": [
    "print(type(response.outputs),type(response.raw_output_contents))\n",
    "from google.protobuf.json_format import MessageToDict\n",
    "d = MessageToDict(response.outputs[0])\n",
    "print(d)\n",
    "binary_data=bytes([x for x in response.raw_output_contents[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "41708894-c834-4cf1-ae5b-d8f4b7ed8e7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected 5 Actual 5\n"
     ]
    }
   ],
   "source": [
    "import struct\n",
    "import base64\n",
    "FLOAT = 'f'\n",
    "fmt = '<' + FLOAT * (len(binary_data) // struct.calcsize(FLOAT))\n",
    "print(\"Expected\",y.iloc[0],\"Actual\",np.argmax(np.array(struct.unpack(fmt, binary_data))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c33c02a-cd26-409a-be93-0a84b9c4c2e0",
   "metadata": {},
   "source": [
    "# Submit gRPC request to the ModelMesh for batch of samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "98cc5f15-6032-45fe-9d5a-486980a0f17d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name=\"mnist123\"\n",
    "payload = { \"model_name\": model_name,\n",
    "            \"inputs\": [{ \"name\": \"input_0\", \"shape\": (5,1,28,28), \"datatype\": \"FP32\", \"contents\": {\"fp32_contents\":X.loc[0:4].values.flatten().tolist()}}]\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "53ba68ed-2f24-4095-963b-687c04bfcc6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import grpc\n",
    "import kfs_inference_v2_pb2, kfs_inference_v2_pb2_grpc\n",
    "grpc_url=\"modelmesh-serving.huggingface.svc.cluster.local:8033\"\n",
    "request=kfs_inference_v2_pb2.ModelInferRequest(model_name=model_name,inputs=payload[\"inputs\"])\n",
    "grpc_channel = grpc.insecure_channel(grpc_url)\n",
    "grpc_stub = kfs_inference_v2_pb2_grpc.GRPCInferenceServiceStub(grpc_channel)\n",
    "response = grpc_stub.ModelInfer(request)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ea84757e-dc33-4552-858f-83d1d1b3ad52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'google.protobuf.pyext._message.RepeatedCompositeContainer'> <class 'google.protobuf.pyext._message.RepeatedScalarContainer'>\n",
      "{'name': 'output_0', 'datatype': 'FP32', 'shape': ['5', '10']}\n"
     ]
    }
   ],
   "source": [
    "print(type(response.outputs),type(response.raw_output_contents))\n",
    "from google.protobuf.json_format import MessageToDict\n",
    "d = MessageToDict(response.outputs[0])\n",
    "print(d)\n",
    "binary_data=bytes([x for x in response.raw_output_contents[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "36350206-bdd9-485f-9d8b-77246007fbd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected ['5', '0', '4', '1', '9'] Actual ['5', '0', '4', '1', '9']\n"
     ]
    }
   ],
   "source": [
    "import struct\n",
    "import base64\n",
    "FLOAT = 'f'\n",
    "fmt = '<' + FLOAT * (len(binary_data) // struct.calcsize(FLOAT))\n",
    "numbers = [str(n) for n in np.argmax(np.array(struct.unpack(fmt, binary_data)).reshape(*[int(shapeval) for shapeval in d['shape']]),axis=1)]\n",
    "print(\"Expected\",y[0:5].values.tolist(),\"Actual\",numbers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fad0c66-bc4c-4f3d-a1f6-76e821a6fc72",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "ae72ec8d55aeb4773d9bab14ab14ec6c410f2dd8be83850b7c2732f479ead773"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}