prompt/4_advanced_prompting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Feb  1 03:15:57 2024       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 525.116.04   Driver Version: 525.116.04   CUDA Version: 12.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA RTX A4000    Off  | 00000000:00:05.0 Off |                  Off |\n",
      "| 41%   37C    P0    35W / 140W |      1MiB / 16376MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: accelerate in /usr/local/lib/python3.9/dist-packages (0.21.0)\n",
      "Requirement already satisfied: bitsandbytes in /usr/local/lib/python3.9/dist-packages (0.42.0)\n",
      "Requirement already satisfied: langchain in /usr/local/lib/python3.9/dist-packages (0.1.4)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.9/dist-packages (from accelerate) (6.0.1)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.9/dist-packages (from accelerate) (5.9.4)\n",
      "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.9/dist-packages (from accelerate) (2.1.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from accelerate) (23.2)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from accelerate) (1.23.4)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.9/dist-packages (from bitsandbytes) (1.11.4)\n",
      "Requirement already satisfied: langsmith<0.1,>=0.0.83 in /usr/local/lib/python3.9/dist-packages (from langchain) (0.0.85)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.9/dist-packages (from langchain) (3.8.3)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.9/dist-packages (from langchain) (4.0.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.9/dist-packages (from langchain) (1.10.13)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.9/dist-packages (from langchain) (1.33)\n",
      "Requirement already satisfied: langchain-core<0.2,>=0.1.16 in /usr/local/lib/python3.9/dist-packages (from langchain) (0.1.17)\n",
      "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.9/dist-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.9/dist-packages (from langchain) (8.2.3)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.9/dist-packages (from langchain) (1.4.41)\n",
      "Requirement already satisfied: langchain-community<0.1,>=0.0.14 in /usr/local/lib/python3.9/dist-packages (from langchain) (0.0.16)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/lib/python3.9/dist-packages (from langchain) (0.6.3)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.8.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (18.2.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.1.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.3)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.9/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.9/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.20.2)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.9/dist-packages (from jsonpatch<2.0,>=1.33->langchain) (2.4)\n",
      "Requirement already satisfied: anyio<5,>=3 in /usr/local/lib/python3.9/dist-packages (from langchain-core<0.2,>=0.1.16->langchain) (3.7.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic<3,>=1->langchain) (4.9.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests<3,>=2->langchain) (2019.11.28)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests<3,>=2->langchain) (2.8)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2->langchain) (1.26.14)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.9/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.1)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (2.1.0)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (10.3.2.106)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (3.9.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (2023.12.2)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (8.9.2.26)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (3.0)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (11.0.2.54)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (2.18.1)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.9/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate) (12.3.101)\n",
      "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.9/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.16->langchain) (1.3.0)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.9/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.16->langchain) (1.1.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain) (1.0.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.9/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install accelerate bitsandbytes langchain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content\n",
      "Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease\n",
      "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease\n",
      "Hit:3 https://deb.nodesource.com/node_16.x focal InRelease                     \n",
      "Hit:4 http://archive.ubuntu.com/ubuntu focal InRelease                         \n",
      "Hit:5 http://archive.ubuntu.com/ubuntu focal-updates InRelease             \n",
      "Hit:6 http://archive.ubuntu.com/ubuntu focal-backports InRelease\n",
      "Hit:7 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal InRelease\n",
      "Reading package lists... Done\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "5fefb8|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/model-00001-of-00002.safetensors\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "3c7b18|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/model-00002-of-00002.safetensors\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "0e002c|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/model.safetensors.index.json\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "ec7a79|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/special_tokens_map.json\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "2c202c|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/tokenizer.model\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "84a8eb|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/tokenizer_config.json\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "4157de|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/config.json\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n",
      "\n",
      "Download Results:\n",
      "gid   |stat|avg speed  |path/URI\n",
      "======+====+===========+=======================================================\n",
      "bc09e2|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/models/Llama-2-7b-chat-hf/generation_config.json\n",
      "\n",
      "Status Legend:\n",
      "(OK):download completed.\n"
     ]
    }
   ],
   "source": [
    "%cd /content\n",
    "!apt-get update -y \n",
    "!apt-get -y install -qq aria2\n",
    "\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/resolve/main/model-00001-of-00002.safetensors -d /content/models/Llama-2-7b-chat-hf -o model-00001-of-00002.safetensors\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/resolve/main/model-00002-of-00002.safetensors -d /content/models/Llama-2-7b-chat-hf -o model-00002-of-00002.safetensors\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/raw/main/model.safetensors.index.json -d /content/models/Llama-2-7b-chat-hf -o model.safetensors.index.json\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/raw/main/special_tokens_map.json -d /content/models/Llama-2-7b-chat-hf -o special_tokens_map.json\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/resolve/main/tokenizer.model -d /content/models/Llama-2-7b-chat-hf -o tokenizer.model\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/raw/main/tokenizer_config.json -d /content/models/Llama-2-7b-chat-hf -o tokenizer_config.json\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/raw/main/config.json -d /content/models/Llama-2-7b-chat-hf -o config.json\n",
    "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/4bit/Llama-2-7b-chat-hf/raw/main/generation_config.json -d /content/models/Llama-2-7b-chat-hf -o generation_config.json\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff94a731919a43a1a8d8b764e018e8d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/dist-packages/transformers/generation/configuration_utils.py:389: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.9` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.9/dist-packages/transformers/generation/configuration_utils.py:394: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.6` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LlamaForCausalLM(\n",
       "  (model): LlamaModel(\n",
       "    (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n",
       "    (layers): ModuleList(\n",
       "      (0-31): 32 x LlamaDecoderLayer(\n",
       "        (self_attn): LlamaSdpaAttention(\n",
       "          (q_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
       "          (k_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
       "          (v_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
       "          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
       "          (rotary_emb): LlamaRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): LlamaMLP(\n",
       "          (gate_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
       "          (up_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
       "          (down_proj): Linear4bit(in_features=11008, out_features=4096, bias=False)\n",
       "          (act_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): LlamaRMSNorm()\n",
       "        (post_attention_layernorm): LlamaRMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): LlamaRMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig\n",
    "import torch\n",
    "\n",
    "from transformers import BitsAndBytesConfig\n",
    "from torch import bfloat16\n",
    "\n",
    "# Our 4-bit configuration to load the LLM with less GPU memory\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,  # 4-bit quantization\n",
    "    bnb_4bit_quant_type='nf4',  # Normalized float 4\n",
    "    bnb_4bit_use_double_quant=True,  # Second quantization after the first\n",
    "    bnb_4bit_compute_dtype=bfloat16  # Computation type\n",
    ")\n",
    "\n",
    "model_id = \"/content/models/Llama-2-7b-chat-hf\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id, revision='main', use_fast=True, trust_remote_code=True)\n",
    "model_config = AutoConfig.from_pretrained(\n",
    "    model_id\n",
    ")\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    trust_remote_code=True,\n",
    "    config=model_config,\n",
    "    quantization_config=bnb_config,\n",
    "    device_map='auto'\n",
    ")\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from transformers import TextIteratorStreamer\n",
    "from threading import Thread\n",
    "\n",
    "def chat(messages, model, tokenizer, **kwargs):\n",
    "    input_ids = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors=\"pt\")\n",
    "    input_ids = input_ids.to(\"cuda\")\n",
    "    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)\n",
    "    \n",
    "    gen_kwargs = {\"input_ids\": input_ids, \"max_length\": 3512, \"eos_token_id\": tokenizer.eos_token_id, \"streamer\": streamer, 'do_sample': True, 'temperature': None, 'top_k': 10}\n",
    "    thread = Thread(target=model.generate, kwargs=gen_kwargs)\n",
    "    thread.start()\n",
    "    return streamer\n",
    "\n",
    "def display_chat(prompt, system=None, ret=False, **kwargs):\n",
    "    messages = [{'role': 'user', 'content': prompt}]\n",
    "    if system is not None:\n",
    "        messages = [{'role': 'system', 'content': system}] + messages\n",
    "\n",
    "    message = \"\"\n",
    "    for word in chat(messages, model, tokenizer):\n",
    "        print(word, end='')\n",
    "        message += word\n",
    "    \n",
    "    if ret:\n",
    "        return message\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "No chat template is defined for this tokenizer - using the default template for the LlamaTokenizerFast class. If the default is not appropriate for your model, please set `tokenizer.chat_template` to an appropriate template. See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n",
      "\n",
      "/usr/local/lib/python3.9/dist-packages/transformers/generation/utils.py:1518: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Sure, I'm happy to help! Here's how I would approach this problem:\n",
      "\n",
      "Thought: Hmm, let's see... when we divide 2 by 2, we get 1. So, if we multiply 8 by 2, we get 16. Now, let's divide 16 by 2, which gives us 8.\n",
      "\n",
      "Action: Let's calculate the final answer.\n",
      "Action Input: 8 + 2 / 2 * 2 = ?\n",
      "\n",
      "OBSERVATION: Hmm, that gives us 16.\n",
      "\n",
      "Thought: Great, we have our final answer! The final answer is 16.\n",
      "Final Answer: 16"
     ]
    }
   ],
   "source": [
    "PROMPT_TEMPLATE = \"\"\"\"You are an math expert which solve questions best as you can.\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Question: the input question you must answer\n",
    "Thought: you should always think about what to do\n",
    "Action: the action to take\n",
    "Action Input: the input to the action\n",
    "OBSERVATION: the result of the action\n",
    "... (this Thought/Action/Action Input/OBSERVATION can be repeated zero or more times)\n",
    "Thought: I now know the final answer\n",
    "Final Answer: the final answer to the original input question.\n",
    "\n",
    "Question: {query}\"\"\"\n",
    "display_chat(PROMPT_TEMPLATE.format(query=\"how much is 8 + 2 / 2 * 2?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Sure, I'm happy to help you with that problem! Here's how I would approach it:\n",
      "\n",
      "Thought: Okay, let's break this problem down step by step. We have 8 + 2, which is equal to 10. Now, we have 2 / 2, which is equal to 1. So, the total is 10 + 1, which is equal to 11.\n",
      "\n",
      "Action: I will calculate 11 multiplied by 2.\n",
      "Action Input: 11 multiplied by 2 is equal to 22.\n",
      "\n",
      "OBSERVATION: The result of multiplying 11 by 2 is 22.\n",
      "\n",
      "Thought: Great, now we have a total of 22. Let's check our answer by adding 8 and 2.\n",
      "\n",
      "Action: I will calculate 8 + 2.\n",
      "Action Input: 8 + 2 is equal to 10.\n",
      "\n",
      "OBSERVATION: The result of adding 8 and 2 is 10.\n",
      "\n",
      "Thought: Perfect! Now we have confirmed that our answer is correct. The final answer to the original input question is 10.\n",
      "\n",
      "Final Answer: 10. Sure! I'm happy to help you with your math problem. Here's how I would approach this question:\n",
      "\n",
      "Thought: Let's break down the problem step by step. We have 8 + 2 = 10, and then we have 2 * 2 = 4. So far, we have 10 + 4 = 14.\n",
      "\n",
      "Action: I will calculate the final answer by adding the last two numbers: 14 + 8 = 22.\n",
      "\n",
      "Action Input: The input to the action is the previous result, which is 14.\n",
      "\n",
      "OBSERVATION: The result of the action is 22.\n",
      "\n",
      "Thought: Great, now we have our final answer! The result of 8 + 2 / 2 * 2 is 22.\n",
      "\n",
      "Final Answer: 22. Great, let's dive into the problem!\n",
      "\n",
      "Question: How much is 8 + 2 / 2 * 2?\n",
      "\n",
      "Thought: Hmm, let's break this down step by step. First, we need to calculate the expression inside the parentheses. 2 / 2 = 1.\n",
      "\n",
      "Action: Write down the calculation: 8 + 2 / 1 * 2.\n",
      "\n",
      "Action Input: 8 + 2 = 10.\n",
      "\n",
      "OBSERVATION: The calculation gives us 10.\n",
      "\n",
      "Thought: Great, now we need to calculate the final answer.\n",
      "\n",
      "Action: Write down the final calculation: 10 + 2 = 12.\n",
      "\n",
      "Action Input: 12 is the final answer to the original input question.\n",
      "\n",
      "OBSERVATION: The result of the calculation is 12.\n",
      "\n",
      "Final Answer: 12.\n",
      "\n",
      "Therefore, the final answer to the question \"how much is 8 + 2 / 2 * 2?\" is 12."
     ]
    },
    {
     "data": {
      "text/plain": [
       "[\" Sure, I'm happy to help you with that problem! Here's how I would approach it:\\n\\nThought: Okay, let's break this problem down step by step. We have 8 + 2, which is equal to 10. Now, we have 2 / 2, which is equal to 1. So, the total is 10 + 1, which is equal to 11.\\n\\nAction: I will calculate 11 multiplied by 2.\\nAction Input: 11 multiplied by 2 is equal to 22.\\n\\nOBSERVATION: The result of multiplying 11 by 2 is 22.\\n\\nThought: Great, now we have a total of 22. Let's check our answer by adding 8 and 2.\\n\\nAction: I will calculate 8 + 2.\\nAction Input: 8 + 2 is equal to 10.\\n\\nOBSERVATION: The result of adding 8 and 2 is 10.\\n\\nThought: Perfect! Now we have confirmed that our answer is correct. The final answer to the original input question is 10.\\n\\nFinal Answer: 10.\",\n",
       " \" Sure! I'm happy to help you with your math problem. Here's how I would approach this question:\\n\\nThought: Let's break down the problem step by step. We have 8 + 2 = 10, and then we have 2 * 2 = 4. So far, we have 10 + 4 = 14.\\n\\nAction: I will calculate the final answer by adding the last two numbers: 14 + 8 = 22.\\n\\nAction Input: The input to the action is the previous result, which is 14.\\n\\nOBSERVATION: The result of the action is 22.\\n\\nThought: Great, now we have our final answer! The result of 8 + 2 / 2 * 2 is 22.\\n\\nFinal Answer: 22.\",\n",
       " ' Great, let\\'s dive into the problem!\\n\\nQuestion: How much is 8 + 2 / 2 * 2?\\n\\nThought: Hmm, let\\'s break this down step by step. First, we need to calculate the expression inside the parentheses. 2 / 2 = 1.\\n\\nAction: Write down the calculation: 8 + 2 / 1 * 2.\\n\\nAction Input: 8 + 2 = 10.\\n\\nOBSERVATION: The calculation gives us 10.\\n\\nThought: Great, now we need to calculate the final answer.\\n\\nAction: Write down the final calculation: 10 + 2 = 12.\\n\\nAction Input: 12 is the final answer to the original input question.\\n\\nOBSERVATION: The result of the calculation is 12.\\n\\nFinal Answer: 12.\\n\\nTherefore, the final answer to the question \"how much is 8 + 2 / 2 * 2?\" is 12.']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iterations = 3\n",
    "results = []\n",
    "for _ in range(iterations):\n",
    "    ret = display_chat(PROMPT_TEMPLATE.format(query=\"how much is 8 + 2 / 2 * 2?\"), ret=True)\n",
    "    results.append(ret)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Sure, I'm happy to help you with that problem! The user input is \"how much is 8 + 2 / 2 * 2?\".\n",
      "\n",
      "To solve the problem, I will follow the steps below:\n",
      "\n",
      "Thought: Okay, let's break this problem down step by step. We have 8 + 2, which is equal to 10. Now, we have 2 / 2, which is equal to 1. So, the total is 10 + 1, which is equal to 11.\n",
      "\n",
      "Action: I will calculate 11 multiplied by 2.\n",
      "Action Input: 11 multiplied by 2 is equal to 22.\n",
      "\n",
      "OBSERVATION: The result of multiplying 11 by 2 is 22.\n",
      "\n",
      "Thought: Great, now we have a total of 22. Let's check our answer by adding 8 and 2.\n",
      "\n",
      "Action: I will calculate 8 + 2.\n",
      "Action Input: 8 + 2 is equal to 10.\n",
      "\n",
      "OBSERVATION: The result of adding 8 and 2 is 10.\n",
      "\n",
      "Thought: Perfect! Now we have confirmed that our answer is correct. The final answer to the original input question is 10.\n",
      "\n",
      "Final Answer: 10.\n",
      "\n",
      "Therefore, the final answer to the question \"how much is 8 + 2 / 2 * 2?\" is 10."
     ]
    }
   ],
   "source": [
    "SELF_REFLECT_PROMPT = \"\"\"\n",
    "### User input ###\n",
    "{initial_prompt}:\n",
    "### Result ###\n",
    "{results}\n",
    "\n",
    "Reflect about the result regarding the user input and give the final answer\n",
    "\"\"\"\n",
    "display_chat(\n",
    "    SELF_REFLECT_PROMPT.format(\n",
    "        initial_prompt=PROMPT_TEMPLATE.format(query=\"how much is 8 + 2 / 2 * 2?\"),\n",
    "        results=\"\\n\".join(results)\n",
    "    )\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "### User input ###\n",
      "\"You are an math expert which solve questions best as you can.\n",
      "\n",
      "Use the following format:\n",
      "\n",
      "Question: the input question you must answer\n",
      "Thought: you should always think about what to do\n",
      "Action: the action to take\n",
      "Action Input: the input to the action\n",
      "OBSERVATION: the result of the action\n",
      "... (this Thought/Action/Action Input/OBSERVATION can be repeated zero or more times)\n",
      "Thought: I now know the final answer\n",
      "Final Answer: the final answer to the original input question.\n",
      "\n",
      "Question: how much is 8 + 2 / 2 * 2?:\n",
      "### Result ###\n",
      " Sure, I'm happy to help you with that problem! Here's how I would approach it:\n",
      "\n",
      "Thought: Okay, let's break this problem down step by step. We have 8 + 2, which is equal to 10. Now, we have 2 / 2, which is equal to 1. So, the total is 10 + 1, which is equal to 11.\n",
      "\n",
      "Action: I will calculate 11 multiplied by 2.\n",
      "Action Input: 11 multiplied by 2 is equal to 22.\n",
      "\n",
      "OBSERVATION: The result of multiplying 11 by 2 is 22.\n",
      "\n",
      "Thought: Great, now we have a total of 22. Let's check our answer by adding 8 and 2.\n",
      "\n",
      "Action: I will calculate 8 + 2.\n",
      "Action Input: 8 + 2 is equal to 10.\n",
      "\n",
      "OBSERVATION: The result of adding 8 and 2 is 10.\n",
      "\n",
      "Thought: Perfect! Now we have confirmed that our answer is correct. The final answer to the original input question is 10.\n",
      "\n",
      "Final Answer: 10.\n",
      " Sure! I'm happy to help you with your math problem. Here's how I would approach this question:\n",
      "\n",
      "Thought: Let's break down the problem step by step. We have 8 + 2 = 10, and then we have 2 * 2 = 4. So far, we have 10 + 4 = 14.\n",
      "\n",
      "Action: I will calculate the final answer by adding the last two numbers: 14 + 8 = 22.\n",
      "\n",
      "Action Input: The input to the action is the previous result, which is 14.\n",
      "\n",
      "OBSERVATION: The result of the action is 22.\n",
      "\n",
      "Thought: Great, now we have our final answer! The result of 8 + 2 / 2 * 2 is 22.\n",
      "\n",
      "Final Answer: 22.\n",
      " Great, let's dive into the problem!\n",
      "\n",
      "Question: How much is 8 + 2 / 2 * 2?\n",
      "\n",
      "Thought: Hmm, let's break this down step by step. First, we need to calculate the expression inside the parentheses. 2 / 2 = 1.\n",
      "\n",
      "Action: Write down the calculation: 8 + 2 / 1 * 2.\n",
      "\n",
      "Action Input: 8 + 2 = 10.\n",
      "\n",
      "OBSERVATION: The calculation gives us 10.\n",
      "\n",
      "Thought: Great, now we need to calculate the final answer.\n",
      "\n",
      "Action: Write down the final calculation: 10 + 2 = 12.\n",
      "\n",
      "Action Input: 12 is the final answer to the original input question.\n",
      "\n",
      "OBSERVATION: The result of the calculation is 12.\n",
      "\n",
      "Final Answer: 12.\n",
      "\n",
      "Therefore, the final answer to the question \"how much is 8 + 2 / 2 * 2?\" is 12.\n",
      "\n",
      "Reflect about the result regarding the user input and give the final answer\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(SELF_REFLECT_PROMPT.format(\n",
    "        initial_prompt=PROMPT_TEMPLATE.format(query=\"how much is 8 + 2 / 2 * 2?\"),\n",
    "        results=\"\\n\".join(results)\n",
    "))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}