model-calling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9430e8a1-26f2-44ca-bda5-6bca0e88ab65",
   "metadata": {},
   "source": [
    "# Inferencing Deployed LLM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cc489f59-4c73-4c51-8207-642d115691f1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import string\n",
    "import cmlapi\n",
    "from src.api import ApiUtility\n",
    "import cdsw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f174c917-8049-4943-9207-e8f4d8e1e6de",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# You can use an APIV2-based utility to access the latest model's metadata. For more, explore the src folder\u001b[39;00m\n\u001b[1;32m      6\u001b[0m apiUtil \u001b[38;5;241m=\u001b[39m ApiUtility()\n\u001b[0;32m----> 7\u001b[0m Model_AccessKey \u001b[38;5;241m=\u001b[39m \u001b[43mapiUtil\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_latest_deployment_details\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mText Summarization LLM v1 Test2\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel_access_key\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m      8\u001b[0m Deployment_CRN \u001b[38;5;241m=\u001b[39m apiUtil\u001b[38;5;241m.\u001b[39mget_latest_deployment_details(model_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mText Summarization LLM v1 Test2\u001b[39m\u001b[38;5;124m\"\u001b[39m)[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlatest_deployment_crn\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
      "File \u001b[0;32m~/src/api.py:89\u001b[0m, in \u001b[0;36mApiUtility.get_latest_deployment_details\u001b[0;34m(self, model_name)\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[38;5;66;03m# gather model details\u001b[39;00m\n\u001b[1;32m     84\u001b[0m models \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m     85\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient\u001b[38;5;241m.\u001b[39mlist_models(project_id\u001b[38;5;241m=\u001b[39mproject_id, async_req\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m     86\u001b[0m     \u001b[38;5;241m.\u001b[39mget()\n\u001b[1;32m     87\u001b[0m     \u001b[38;5;241m.\u001b[39mto_dict()\n\u001b[1;32m     88\u001b[0m )\n\u001b[0;32m---> 89\u001b[0m model_info \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\n\u001b[1;32m     90\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mmodels\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmodels\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\n\u001b[1;32m     91\u001b[0m \u001b[43m\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m     93\u001b[0m model_id \u001b[38;5;241m=\u001b[39m model_info[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m     94\u001b[0m model_crn \u001b[38;5;241m=\u001b[39m model_info[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcrn\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "client = cmlapi.default_client()\n",
    "project_id = os.environ[\"CDSW_PROJECT_ID\"]\n",
    "client.list_models(project_id)\n",
    "\n",
    "# You can use an APIV2-based utility to access the latest model's metadata. For more, explore the src folder\n",
    "apiUtil = ApiUtility()\n",
    "Model_AccessKey = apiUtil.get_latest_deployment_details(model_name=\"Text Summarization LLM v1 Test2\")[\"model_access_key\"]\n",
    "Deployment_CRN = apiUtil.get_latest_deployment_details(model_name=\"Text Summarization LLM v1 Test2\")[\"latest_deployment_crn\"]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1f57f6b3-0317-4310-8a1d-bc7261052762",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'action': 'summarize', 'document': 'The full cost of damage in Newton Stewart, one of the areas worst affected, is still being assessed.\\nRepair work is ongoing in Hawick and many roads in Peeblesshire remain badly affected by standing water.\\nTrains on the west coast mainline face disruption due to damage at the Lamington Viaduct.\\nMany businesses and householders were affected by flooding in Newton Stewart after the River Cree overflowed into the town.\\nFirst Minister Nicola Sturgeon visited the area to inspect the damage.\\nThe waters breached a retaining wall, flooding many commercial properties on Victoria Street - the main shopping thoroughfare.\\nJeanette Tate, who owns the Cinnamon Cafe which was badly affected, said she could not fault the multi-agency response once the flood hit.\\nHowever, she said more preventative work could have been carried out to ensure the retaining wall did not fail.\\n\"It is difficult but I do think there is so much publicity for Dumfries and the Nith - and I totally appreciate that - but it is almost like we\\'re neglected or forgotten,\" she said.\\n\"That may not be true but it is perhaps my perspective over the last few days.\\n\"Why were you not ready to help us a bit more when the warning and the alarm alerts had gone out?\"\\nMeanwhile, a flood alert remains in place across the Borders because of the constant rain.\\nPeebles was badly hit by problems, sparking calls to introduce more defences in the area.\\nScottish Borders Council has put a list on its website of the roads worst affected and drivers have been urged not to ignore closure signs.\\nThe Labour Party\\'s deputy Scottish leader Alex Rowley was in Hawick on Monday to see the situation first hand.\\nHe said it was important to get the flood protection plan right but backed calls to speed up the process.\\n\"I was quite taken aback by the amount of damage that has been done,\" he said.\\n\"Obviously it is heart-breaking for people who have been forced out of their homes and the impact on businesses.\"\\nHe said it was important that \"immediate steps\" were taken to protect the areas most vulnerable and a clear timetable put in place for flood prevention plans.\\nHave you been affected by flooding in Dumfries and Galloway or the Borders? Tell us about your experience of the situation and how it was handled. Email us on selkirk.news@bbc.co.uk or dumfries@bbc.co.uk.'}\n"
     ]
    }
   ],
   "source": [
    "document = 'The full cost of damage in Newton Stewart, one of the areas worst affected, is still being assessed.\\nRepair work is ongoing in Hawick and many roads in Peeblesshire remain badly affected by standing water.\\nTrains on the west coast mainline face disruption due to damage at the Lamington Viaduct.\\nMany businesses and householders were affected by flooding in Newton Stewart after the River Cree overflowed into the town.\\nFirst Minister Nicola Sturgeon visited the area to inspect the damage.\\nThe waters breached a retaining wall, flooding many commercial properties on Victoria Street - the main shopping thoroughfare.\\nJeanette Tate, who owns the Cinnamon Cafe which was badly affected, said she could not fault the multi-agency response once the flood hit.\\nHowever, she said more preventative work could have been carried out to ensure the retaining wall did not fail.\\n\"It is difficult but I do think there is so much publicity for Dumfries and the Nith - and I totally appreciate that - but it is almost like we\\'re neglected or forgotten,\" she said.\\n\"That may not be true but it is perhaps my perspective over the last few days.\\n\"Why were you not ready to help us a bit more when the warning and the alarm alerts had gone out?\"\\nMeanwhile, a flood alert remains in place across the Borders because of the constant rain.\\nPeebles was badly hit by problems, sparking calls to introduce more defences in the area.\\nScottish Borders Council has put a list on its website of the roads worst affected and drivers have been urged not to ignore closure signs.\\nThe Labour Party\\'s deputy Scottish leader Alex Rowley was in Hawick on Monday to see the situation first hand.\\nHe said it was important to get the flood protection plan right but backed calls to speed up the process.\\n\"I was quite taken aback by the amount of damage that has been done,\" he said.\\n\"Obviously it is heart-breaking for people who have been forced out of their homes and the impact on businesses.\"\\nHe said it was important that \"immediate steps\" were taken to protect the areas most vulnerable and a clear timetable put in place for flood prevention plans.\\nHave you been affected by flooding in Dumfries and Galloway or the Borders? Tell us about your experience of the situation and how it was handled. Email us on selkirk.news@bbc.co.uk or dumfries@bbc.co.uk.'\n",
    "my_dict = {\"action\":\"summarize\", \"document\":document}\n",
    "print(my_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6d99048e-82d1-4907-a283-50c70434ac77",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#calling the model\n",
    "response = cdsw.call_model(Model_AccessKey, my_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c5b5359e-333a-4600-b92c-dafdecd5ddb8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'success': True, 'response': {'summary': '<pad> repairs are ongoing in the area and many roads in peeblesshire remain badly affected. the full cost of damage in Newton Stewart is still being assessed. the water breached a retaining wall, flooding many commercial properties.</s>'}, 'ReplicaID': 'text-summarization-llm-1-9-6bcd9f7c4d-dl8j2', 'Size': 278, 'StatusCode': 200}\n"
     ]
    }
   ],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b62ff4b-16d0-40ce-849f-13da72d2c810",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}