llamainfer_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "447c568b-b227-4437-8e0e-17746f393554",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://github.com/git-lfs/git-lfs/releases/download/v3.4.0/git-lfs-linux-amd64-v3.4.0.tar.gz\n",
    "!tar -xvzf git-lfs-linux-amd64-v3.4.0.tar.gz\n",
    "!PREFIX=/opt/app-root/src/.local ./git-lfs-3.4.0/install.sh "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8aa3a67b-ebf7-4737-9f41-0730ee009009",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://github.com/caikit/caikit-nlp.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "752d93c2-dca2-4d23-9143-ad789a8b61ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install ./caikit-nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "130f5630-fb45-4351-9d5f-359667cb47f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://github.com/opendatahub-io/caikit-tgis-serving.git\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6500fde-8d50-4608-b794-8644facf29f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp caikit-tgis-serving/utils/convert.py . \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2553340c-1d04-4428-8f98-10b9f84f526b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --upgrade huggingface_hub\n",
    "!huggingface-cli login --token $HUGGINGFACE_TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9d55e0a-71d0-4150-87d9-e2381a59d336",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://huggingface.co/avijra/Llama-2-7b-chat-hf-fine-tuned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e4d5cf2-22fd-422e-845f-7014b4a7ace9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!./convert.py --model-path ./Llama-2-7b-chat-hf-fine-tuned --model-save-path ./Llama-2-7b-chat-hf-fine-tuned-caikit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d998e41e-9b06-43f5-b7a0-532b1b21cf2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install grpcio grpcio-tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c982810c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://github.com/fullstorydev/grpcurl/releases/download/v1.8.8/grpcurl_1.8.8_linux_x86_64.tar.gz\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6803c303",
   "metadata": {},
   "outputs": [],
   "source": [
    "!tar -xvzf grpcurl_1.8.8_linux_x86_64.tar.gz\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64d5397c",
   "metadata": {},
   "outputs": [],
   "source": [
    "grpcurl -insecure -d '{\"text\": \"How to please Demo Gods before the Demo?\", \"max_new_tokens\": 1024}' -H \"mm-model-id: Llama-2-7b-chat-hf-fine-tuned-caikit\" caikit-example-isvc-predictor-kserve-demo.apps.xxxxx-xxxxxt.xxx.xx.openshiftapps.com:443 caikit.runtime.Nlp.NlpService/TextGenerationTaskPredict"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}