Simple LLM using  Llama_2  .ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1db4c26247604dd1ac7307bec9cd1256": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_46f4ade1cdf44a75af56525e2b698f62",
              "IPY_MODEL_dde7c98e0c054dfbafe273aef1e408cb",
              "IPY_MODEL_7b7c19e1da3e48829d69a7bc9af2c665"
            ],
            "layout": "IPY_MODEL_fd78bef0b1024e6286cbae7ae631c478"
          }
        },
        "46f4ade1cdf44a75af56525e2b698f62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_54d69d1fc04545f1a69728cf8a1fafa4",
            "placeholder": "​",
            "style": "IPY_MODEL_9b1c2a42ef224baba6886fb3119fd254",
            "value": "llama-2-13b-chat.ggmlv3.q5_1.bin: 100%"
          }
        },
        "dde7c98e0c054dfbafe273aef1e408cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_15b868a7a306469694c6778d92d0df76",
            "max": 9763701888,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6f1738a2f1474a8bb9989f26bc1653c2",
            "value": 9763701888
          }
        },
        "7b7c19e1da3e48829d69a7bc9af2c665": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e97793a7ca574781afaec8a137148c91",
            "placeholder": "​",
            "style": "IPY_MODEL_f7603e2b9c014c969d898310c01a232f",
            "value": " 9.76G/9.76G [01:13&lt;00:00, 208MB/s]"
          }
        },
        "fd78bef0b1024e6286cbae7ae631c478": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54d69d1fc04545f1a69728cf8a1fafa4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b1c2a42ef224baba6886fb3119fd254": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15b868a7a306469694c6778d92d0df76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f1738a2f1474a8bb9989f26bc1653c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e97793a7ca574781afaec8a137148c91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7603e2b9c014c969d898310c01a232f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_dliwERb6Uh",
        "outputId": "202214e7-5592-4b48-c88f-ff6b28a1b9b0"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sat Mar  2 08:05:32 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla T4                       Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P8               9W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L0avf7xx2lcj",
        "outputId": "db341884-7a2c-4629-be66-c6831f38b8f9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using pip 23.1.2 from /usr/local/lib/python3.10/dist-packages/pip (python 3.10)\n",
            "Collecting llama-cpp-python==0.1.78\n",
            "  Downloading llama_cpp_python-0.1.78.tar.gz (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Running command pip subprocess to install build dependencies\n",
            "  Using pip 23.1.2 from /usr/local/lib/python3.10/dist-packages/pip (python 3.10)\n",
            "  Collecting setuptools>=42\n",
            "    Downloading setuptools-69.1.1-py3-none-any.whl (819 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 819.3/819.3 kB 9.0 MB/s eta 0:00:00\n",
            "  Collecting scikit-build>=0.13\n",
            "    Downloading scikit_build-0.17.6-py3-none-any.whl (84 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 84.3/84.3 kB 11.6 MB/s eta 0:00:00\n",
            "  Collecting cmake>=3.18\n",
            "    Downloading cmake-3.28.3-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (26.3 MB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 26.3/26.3 MB 38.2 MB/s eta 0:00:00\n",
            "  Collecting ninja\n",
            "    Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl (307 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 307.2/307.2 kB 30.2 MB/s eta 0:00:00\n",
            "  Collecting distro (from scikit-build>=0.13)\n",
            "    Downloading distro-1.9.0-py3-none-any.whl (20 kB)\n",
            "  Collecting packaging (from scikit-build>=0.13)\n",
            "    Downloading packaging-23.2-py3-none-any.whl (53 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 53.0/53.0 kB 7.6 MB/s eta 0:00:00\n",
            "  Collecting tomli (from scikit-build>=0.13)\n",
            "    Downloading tomli-2.0.1-py3-none-any.whl (12 kB)\n",
            "  Collecting wheel>=0.32.0 (from scikit-build>=0.13)\n",
            "    Downloading wheel-0.42.0-py3-none-any.whl (65 kB)\n",
            "       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.4/65.4 kB 9.5 MB/s eta 0:00:00\n",
            "  Installing collected packages: ninja, cmake, wheel, tomli, setuptools, packaging, distro, scikit-build\n",
            "    Creating /tmp/pip-build-env-wlclvr85/overlay/local/bin\n",
            "    changing mode of /tmp/pip-build-env-wlclvr85/overlay/local/bin/ninja to 755\n",
            "    changing mode of /tmp/pip-build-env-wlclvr85/overlay/local/bin/cmake to 755\n",
            "    changing mode of /tmp/pip-build-env-wlclvr85/overlay/local/bin/cpack to 755\n",
            "    changing mode of /tmp/pip-build-env-wlclvr85/overlay/local/bin/ctest to 755\n",
            "    changing mode of /tmp/pip-build-env-wlclvr85/overlay/local/bin/wheel to 755\n",
            "    changing mode of /tmp/pip-build-env-wlclvr85/overlay/local/bin/distro to 755\n",
            "  ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "  ipython 7.34.0 requires jedi>=0.16, which is not installed.\n",
            "  Successfully installed cmake-3.28.3 distro-1.9.0 ninja-1.11.1.1 packaging-23.2 scikit-build-0.17.6 setuptools-69.1.1 tomli-2.0.1 wheel-0.42.0\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Running command Getting requirements to build wheel\n",
            "  running egg_info\n",
            "  writing llama_cpp_python.egg-info/PKG-INFO\n",
            "  writing dependency_links to llama_cpp_python.egg-info/dependency_links.txt\n",
            "  writing requirements to llama_cpp_python.egg-info/requires.txt\n",
            "  writing top-level names to llama_cpp_python.egg-info/top_level.txt\n",
            "  reading manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  adding license file 'LICENSE.md'\n",
            "  writing manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Running command Preparing metadata (pyproject.toml)\n",
            "  running dist_info\n",
            "  creating /tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info\n",
            "  writing /tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info/PKG-INFO\n",
            "  writing dependency_links to /tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info/dependency_links.txt\n",
            "  writing requirements to /tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info/requires.txt\n",
            "  writing top-level names to /tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info/top_level.txt\n",
            "  writing manifest file '/tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  reading manifest file '/tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  adding license file 'LICENSE.md'\n",
            "  writing manifest file '/tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  creating '/tmp/pip-modern-metadata-iz_jqt97/llama_cpp_python-0.1.78.dist-info'\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting numpy==1.23.4\n",
            "  Downloading numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.1/17.1 MB\u001b[0m \u001b[31m168.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions>=4.5.0 (from llama-cpp-python==0.1.78)\n",
            "  Downloading typing_extensions-4.10.0-py3-none-any.whl (33 kB)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python==0.1.78)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m227.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: llama-cpp-python\n",
            "  Running command Building wheel for llama-cpp-python (pyproject.toml)\n",
            "\n",
            "\n",
            "  --------------------------------------------------------------------------------\n",
            "  -- Trying 'Ninja' generator\n",
            "  --------------------------------\n",
            "  ---------------------------\n",
            "  ----------------------\n",
            "  -----------------\n",
            "  ------------\n",
            "  -------\n",
            "  --\n",
            "  CMake Deprecation Warning at CMakeLists.txt:1 (cmake_minimum_required):\n",
            "    Compatibility with CMake < 3.5 will be removed from a future version of\n",
            "    CMake.\n",
            "\n",
            "    Update the VERSION argument <min> value or use a ...<max> suffix to tell\n",
            "    CMake that the project does not need compatibility with older versions.\n",
            "\n",
            "  Not searching for unused variables given on the command line.\n",
            "\n",
            "  -- The C compiler identification is GNU 11.4.0\n",
            "  -- Detecting C compiler ABI info\n",
            "  -- Detecting C compiler ABI info - done\n",
            "  -- Check for working C compiler: /usr/bin/cc - skipped\n",
            "  -- Detecting C compile features\n",
            "  -- Detecting C compile features - done\n",
            "  -- The CXX compiler identification is GNU 11.4.0\n",
            "  -- Detecting CXX compiler ABI info\n",
            "  -- Detecting CXX compiler ABI info - done\n",
            "  -- Check for working CXX compiler: /usr/bin/c++ - skipped\n",
            "  -- Detecting CXX compile features\n",
            "  -- Detecting CXX compile features - done\n",
            "  -- Configuring done (0.9s)\n",
            "  -- Generating done (0.0s)\n",
            "  -- Build files have been written to: /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_cmake_test_compile/build\n",
            "  --\n",
            "  -------\n",
            "  ------------\n",
            "  -----------------\n",
            "  ----------------------\n",
            "  ---------------------------\n",
            "  --------------------------------\n",
            "  -- Trying 'Ninja' generator - success\n",
            "  --------------------------------------------------------------------------------\n",
            "\n",
            "  Configuring Project\n",
            "    Working directory:\n",
            "      /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-build\n",
            "    Command:\n",
            "      /tmp/pip-build-env-wlclvr85/overlay/local/lib/python3.10/dist-packages/cmake/data/bin/cmake /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0 -G Ninja -DCMAKE_MAKE_PROGRAM:FILEPATH=/tmp/pip-build-env-wlclvr85/overlay/local/lib/python3.10/dist-packages/ninja/data/bin/ninja --no-warn-unused-cli -DCMAKE_INSTALL_PREFIX:PATH=/tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install -DPYTHON_VERSION_STRING:STRING=3.10.12 -DSKBUILD:INTERNAL=TRUE -DCMAKE_MODULE_PATH:PATH=/tmp/pip-build-env-wlclvr85/overlay/local/lib/python3.10/dist-packages/skbuild/resources/cmake -DPYTHON_EXECUTABLE:PATH=/usr/bin/python3 -DPYTHON_INCLUDE_DIR:PATH=/usr/include/python3.10 -DPYTHON_LIBRARY:PATH=/usr/lib/x86_64-linux-gnu/libpython3.10.so -DPython_EXECUTABLE:PATH=/usr/bin/python3 -DPython_ROOT_DIR:PATH=/usr -DPython_FIND_REGISTRY:STRING=NEVER -DPython_INCLUDE_DIR:PATH=/usr/include/python3.10 -DPython3_EXECUTABLE:PATH=/usr/bin/python3 -DPython3_ROOT_DIR:PATH=/usr -DPython3_FIND_REGISTRY:STRING=NEVER -DPython3_INCLUDE_DIR:PATH=/usr/include/python3.10 -DCMAKE_MAKE_PROGRAM:FILEPATH=/tmp/pip-build-env-wlclvr85/overlay/local/lib/python3.10/dist-packages/ninja/data/bin/ninja -DLLAMA_CUBLAS=on -DCMAKE_BUILD_TYPE:STRING=Release -DLLAMA_CUBLAS=on\n",
            "\n",
            "  Not searching for unused variables given on the command line.\n",
            "  -- The C compiler identification is GNU 11.4.0\n",
            "  -- The CXX compiler identification is GNU 11.4.0\n",
            "  -- Detecting C compiler ABI info\n",
            "  -- Detecting C compiler ABI info - done\n",
            "  -- Check for working C compiler: /usr/bin/cc - skipped\n",
            "  -- Detecting C compile features\n",
            "  -- Detecting C compile features - done\n",
            "  -- Detecting CXX compiler ABI info\n",
            "  -- Detecting CXX compiler ABI info - done\n",
            "  -- Check for working CXX compiler: /usr/bin/c++ - skipped\n",
            "  -- Detecting CXX compile features\n",
            "  -- Detecting CXX compile features - done\n",
            "  -- Found Git: /usr/bin/git (found version \"2.34.1\")\n",
            "  fatal: not a git repository (or any of the parent directories): .git\n",
            "  fatal: not a git repository (or any of the parent directories): .git\n",
            "  CMake Warning at vendor/llama.cpp/CMakeLists.txt:117 (message):\n",
            "    Git repository not found; to enable automatic generation of build info,\n",
            "    make sure Git is installed and the project is a Git repository.\n",
            "\n",
            "\n",
            "  -- Performing Test CMAKE_HAVE_LIBC_PTHREAD\n",
            "  -- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success\n",
            "  -- Found Threads: TRUE\n",
            "  -- Found CUDAToolkit: /usr/local/cuda/targets/x86_64-linux/include (found version \"12.2.140\")\n",
            "  -- cuBLAS found\n",
            "  -- The CUDA compiler identification is NVIDIA 12.2.140\n",
            "  -- Detecting CUDA compiler ABI info\n",
            "  -- Detecting CUDA compiler ABI info - done\n",
            "  -- Check for working CUDA compiler: /usr/local/cuda/bin/nvcc - skipped\n",
            "  -- Detecting CUDA compile features\n",
            "  -- Detecting CUDA compile features - done\n",
            "  -- Using CUDA architectures: 52;61;70\n",
            "  -- CMAKE_SYSTEM_PROCESSOR: x86_64\n",
            "  -- x86 detected\n",
            "  -- Configuring done (4.2s)\n",
            "  -- Generating done (0.0s)\n",
            "  -- Build files have been written to: /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-build\n",
            "  [1/9] Building C object vendor/llama.cpp/CMakeFiles/ggml.dir/ggml-alloc.c.o\n",
            "  [2/9] Building C object vendor/llama.cpp/CMakeFiles/ggml.dir/k_quants.c.o\n",
            "  [3/9] Building C object vendor/llama.cpp/CMakeFiles/ggml.dir/ggml.c.o\n",
            "  [4/9] Building CXX object vendor/llama.cpp/CMakeFiles/llama.dir/llama.cpp.o\n",
            "  [5/9] Building CUDA object vendor/llama.cpp/CMakeFiles/ggml.dir/ggml-cuda.cu.o\n",
            "  [6/9] Linking CUDA shared library vendor/llama.cpp/libggml_shared.so\n",
            "  [7/9] Linking CXX shared library vendor/llama.cpp/libllama.so\n",
            "  [8/9] Linking CUDA static library vendor/llama.cpp/libggml_static.a\n",
            "  [8/9] Install the project...\n",
            "  -- Install configuration: \"Release\"\n",
            "  -- Installing: /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install/lib/libggml_shared.so\n",
            "  -- Installing: /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install/lib/libllama.so\n",
            "  -- Set non-toolchain portion of runtime path of \"/tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install/lib/libllama.so\" to \"\"\n",
            "  -- Installing: /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install/bin/convert.py\n",
            "  -- Installing: /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install/bin/convert-lora-to-ggml.py\n",
            "  -- Installing: /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/libllama.so\n",
            "  -- Set non-toolchain portion of runtime path of \"/tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/_skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/libllama.so\" to \"\"\n",
            "\n",
            "  copying llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/__init__.py\n",
            "  copying llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_grammar.py\n",
            "  copying llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_cpp.py\n",
            "  copying llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama.py\n",
            "  copying llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_types.py\n",
            "  copying llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/utils.py\n",
            "  creating directory _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server\n",
            "  copying llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__init__.py\n",
            "  copying llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/app.py\n",
            "  copying llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__main__.py\n",
            "  copying /tmp/pip-install-cdbgjhyk/llama-cpp-python_e1fdd3e4201c41618d1f994e6716f3e0/llama_cpp/py.typed -> _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/py.typed\n",
            "\n",
            "  running bdist_wheel\n",
            "  running build\n",
            "  running build_py\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/py.typed -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/libllama.so -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server\n",
            "  copied 9 files\n",
            "  running build_ext\n",
            "  installing to _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel\n",
            "  running install\n",
            "  running install_lib\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama_grammar.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama_cpp.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/py.typed -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/libllama.so -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/llama_types.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server/__init__.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server/app.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/server/__main__.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp/server\n",
            "  copying _skbuild/linux-x86_64-3.10/setuptools/lib.linux-x86_64-cpython-310/llama_cpp/utils.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp\n",
            "  copied 11 files\n",
            "  running install_data\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/lib\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/lib/libllama.so -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/lib\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/lib/libggml_shared.so -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/lib\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/bin\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/bin/convert-lora-to-ggml.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/bin\n",
            "  copying _skbuild/linux-x86_64-3.10/cmake-install/bin/convert.py -> _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.data/data/bin\n",
            "  running install_egg_info\n",
            "  running egg_info\n",
            "  writing llama_cpp_python.egg-info/PKG-INFO\n",
            "  writing dependency_links to llama_cpp_python.egg-info/dependency_links.txt\n",
            "  writing requirements to llama_cpp_python.egg-info/requires.txt\n",
            "  writing top-level names to llama_cpp_python.egg-info/top_level.txt\n",
            "  reading manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  adding license file 'LICENSE.md'\n",
            "  writing manifest file 'llama_cpp_python.egg-info/SOURCES.txt'\n",
            "  Copying llama_cpp_python.egg-info to _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78-py3.10.egg-info\n",
            "  running install_scripts\n",
            "  copied 0 files\n",
            "  creating _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel/llama_cpp_python-0.1.78.dist-info/WHEEL\n",
            "  creating '/tmp/pip-wheel-kythfw8i/.tmp-__ov4g8f/llama_cpp_python-0.1.78-cp310-cp310-linux_x86_64.whl' and adding '_skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel' to it\n",
            "  adding 'llama_cpp/__init__.py'\n",
            "  adding 'llama_cpp/libllama.so'\n",
            "  adding 'llama_cpp/llama.py'\n",
            "  adding 'llama_cpp/llama_cpp.py'\n",
            "  adding 'llama_cpp/llama_grammar.py'\n",
            "  adding 'llama_cpp/llama_types.py'\n",
            "  adding 'llama_cpp/py.typed'\n",
            "  adding 'llama_cpp/utils.py'\n",
            "  adding 'llama_cpp/server/__init__.py'\n",
            "  adding 'llama_cpp/server/__main__.py'\n",
            "  adding 'llama_cpp/server/app.py'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/bin/convert-lora-to-ggml.py'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/bin/convert.py'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/lib/libggml_shared.so'\n",
            "  adding 'llama_cpp_python-0.1.78.data/data/lib/libllama.so'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/LICENSE.md'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/METADATA'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/WHEEL'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/top_level.txt'\n",
            "  adding 'llama_cpp_python-0.1.78.dist-info/RECORD'\n",
            "  removing _skbuild/linux-x86_64-3.10/setuptools/bdist.linux-x86_64/wheel\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.1.78-cp310-cp310-linux_x86_64.whl size=5811062 sha256=0b4cb7638230c41a7816e8dbb547c8272f37c29e46ff4aef7a66a97020a3dada\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-867qjqhe/wheels/61/f9/20/9ca660a9d3f2a47e44217059409478865948b5c8a1cba70030\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: typing-extensions, numpy, diskcache, llama-cpp-python\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing_extensions 4.10.0\n",
            "    Uninstalling typing_extensions-4.10.0:\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/__pycache__/typing_extensions.cpython-310.pyc\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/typing_extensions-4.10.0.dist-info/\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/typing_extensions.py\n",
            "      Successfully uninstalled typing_extensions-4.10.0\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.25.2\n",
            "    Uninstalling numpy-1.25.2:\n",
            "      Removing file or directory /usr/local/bin/f2py\n",
            "      Removing file or directory /usr/local/bin/f2py3\n",
            "      Removing file or directory /usr/local/bin/f2py3.10\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/numpy-1.25.2.dist-info/\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/numpy.libs/\n",
            "      Removing file or directory /usr/local/lib/python3.10/dist-packages/numpy/\n",
            "      Successfully uninstalled numpy-1.25.2\n",
            "  changing mode of /usr/local/bin/f2py to 755\n",
            "  changing mode of /usr/local/bin/f2py3 to 755\n",
            "  changing mode of /usr/local/bin/f2py3.10 to 755\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "chex 0.1.85 requires numpy>=1.24.1, but you have numpy 1.23.4 which is incompatible.\n",
            "tensorflow 2.15.0 requires numpy<2.0.0,>=1.23.5, but you have numpy 1.23.4 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed diskcache-5.6.3 llama-cpp-python-0.1.78 numpy-1.23.4 typing-extensions-4.10.0\n",
            "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.20.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.66.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.10.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (23.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2024.2.2)\n",
            "Requirement already satisfied: llama-cpp-python==0.1.78 in /usr/local/lib/python3.10/dist-packages (0.1.78)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python==0.1.78) (4.10.0)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python==0.1.78) (1.23.4)\n",
            "Requirement already satisfied: diskcache>=5.6.1 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python==0.1.78) (5.6.3)\n",
            "Requirement already satisfied: numpy==1.23.4 in /usr/local/lib/python3.10/dist-packages (1.23.4)\n"
          ]
        }
      ],
      "source": [
        "# GPU llama-cpp-python\n",
        "!CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" FORCE_CMAKE=1 pip install llama-cpp-python==0.1.78 numpy==1.23.4 --force-reinstall --upgrade --no-cache-dir --verbose\n",
        "!pip install huggingface_hub\n",
        "!pip install llama-cpp-python==0.1.78\n",
        "!pip install numpy==1.23.4"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_name_or_path = \"TheBloke/Llama-2-13B-chat-GGML\"\n",
        "model_basename = \"llama-2-13b-chat.ggmlv3.q5_1.bin\" # the model is in bin format"
      ],
      "metadata": {
        "id": "qJ90LnMv54Y-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**2: Import All the Required Libraries**"
      ],
      "metadata": {
        "id": "6lOmpKB36RJh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import hf_hub_download"
      ],
      "metadata": {
        "id": "Ak3ZtGjM6Wdp"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_cpp import Llama"
      ],
      "metadata": {
        "id": "85XOzmui6rGN"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**3: Download the Model**"
      ],
      "metadata": {
        "id": "haAb9kNm6J9n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_path = hf_hub_download(repo_id=model_name_or_path, filename=model_basename)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176,
          "referenced_widgets": [
            "1db4c26247604dd1ac7307bec9cd1256",
            "46f4ade1cdf44a75af56525e2b698f62",
            "dde7c98e0c054dfbafe273aef1e408cb",
            "7b7c19e1da3e48829d69a7bc9af2c665",
            "fd78bef0b1024e6286cbae7ae631c478",
            "54d69d1fc04545f1a69728cf8a1fafa4",
            "9b1c2a42ef224baba6886fb3119fd254",
            "15b868a7a306469694c6778d92d0df76",
            "6f1738a2f1474a8bb9989f26bc1653c2",
            "e97793a7ca574781afaec8a137148c91",
            "f7603e2b9c014c969d898310c01a232f"
          ]
        },
        "id": "qBgdGV4b6MxG",
        "outputId": "57f06386-2faf-484e-b449-c9919d7a0541"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "llama-2-13b-chat.ggmlv3.q5_1.bin:   0%|          | 0.00/9.76G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1db4c26247604dd1ac7307bec9cd1256"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_path"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "kL3LF6pfE3HD",
        "outputId": "715c46a0-b69a-4398-af2c-47691a35d3c0"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/root/.cache/huggingface/hub/models--TheBloke--Llama-2-13B-chat-GGML/snapshots/3140827b4dfcb6b562cd87ee3d7f07109b014dd0/llama-2-13b-chat.ggmlv3.q5_1.bin'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**4: Loading the Model**"
      ],
      "metadata": {
        "id": "VQ6OYnI46kKq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# GPU\n",
        "lcpp_llm = None\n",
        "lcpp_llm = Llama(\n",
        "    model_path=model_path,\n",
        "    n_threads=2, # CPU cores\n",
        "    n_batch=512, # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.\n",
        "    n_gpu_layers=32 # Change this value based on your model and your GPU VRAM pool.\n",
        "    )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "irftToUj6aWt",
        "outputId": "807ae850-8ccc-4c32-e954-12d070460fda"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | VSX = 0 | \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# See the number of layers in GPU\n",
        "lcpp_llm.params.n_gpu_layers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YG4Pylz662At",
        "outputId": "3bd011ef-9c78-48b4-ffc7-7759753d5570"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**5: Create a Prompt Template**"
      ],
      "metadata": {
        "id": "iE-M307R6_pT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = \"Give me travel destination for Delhi\"\n",
        "prompt_template=f'''SYSTEM: You are a helpful, respectful and honest assistant. Always answer as helpfully.\n",
        "\n",
        "USER: {prompt}\n",
        "\n",
        "ASSISTANT:\n",
        "\n",
        "\n",
        "'''"
      ],
      "metadata": {
        "id": "RfzwELMC7Dyg"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**6: Generating the Response**"
      ],
      "metadata": {
        "id": "aT8pg6zt7QzA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response=lcpp_llm(prompt=prompt_template, max_tokens=256, temperature=0.5, top_p=0.95,\n",
        "                  repeat_penalty=1.2, top_k=150,\n",
        "                  echo=True)"
      ],
      "metadata": {
        "id": "0aF0qWUJ7OPK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6604feff-8d81-46f4-de43-0eb03666cd85"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jlJ1JgR68DDO",
        "outputId": "a560be61-849e-4a84-a4c9-ba149b484f97"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id': 'cmpl-23d254f0-87bd-4d04-bed9-c2fb2253c01c', 'object': 'text_completion', 'created': 1709367203, 'model': '/root/.cache/huggingface/hub/models--TheBloke--Llama-2-13B-chat-GGML/snapshots/3140827b4dfcb6b562cd87ee3d7f07109b014dd0/llama-2-13b-chat.ggmlv3.q5_1.bin', 'choices': [{'text': \"SYSTEM: You are a helpful, respectful and honest assistant. Always answer as helpfully.\\n\\nUSER: Give me travel destination for Delhi\\n\\nASSISTANT:\\nDelhi is an amazing city with so much to offer! Here are some top destinations you might consider visiting while you're there:\\n\\n1. The Red Fort - A stunning example of Mughal architecture and a must-see attraction in Delhi.\\n2. Humayun's Tomb - Another beautiful piece of Mughal architecture that is often overlooked by tourists, but definitely worth a visit.\\n3. Qutub Minar - The tallest brick minaret in India, with intricate carvings and inscriptions throughout.\\n4. Akshardham Temple - A modern Hindu temple made of white marble that is an architectural wonder and a great place to learn about Hinduism.\\n5. Raj Ghat - A simple but powerful memorial to Mahatma Gandhi, the father of India's independence movement.\\n6. Lodi Gardens - A peaceful oasis in the heart of Delhi with beautiful gardens and historic buildings.\\n7. Chandni Chowk - One of the oldest and busiest markets in Delhi, where you can find everything from spices to jewelry to souvenirs.\", 'index': 0, 'logprobs': None, 'finish_reason': 'length'}], 'usage': {'prompt_tokens': 40, 'completion_tokens': 256, 'total_tokens': 296}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response[\"choices\"][0][\"text\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qona58gX8oAn",
        "outputId": "ad9867e5-2a16-45eb-9db1-8ddf95f3550f"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "SYSTEM: You are a helpful, respectful and honest assistant. Always answer as helpfully.\n",
            "\n",
            "USER: Give me travel destination for Delhi\n",
            "\n",
            "ASSISTANT:\n",
            "Delhi is an amazing city with so much to offer! Here are some top destinations you might consider visiting while you're there:\n",
            "\n",
            "1. The Red Fort - A stunning example of Mughal architecture and a must-see attraction in Delhi.\n",
            "2. Humayun's Tomb - Another beautiful piece of Mughal architecture that is often overlooked by tourists, but definitely worth a visit.\n",
            "3. Qutub Minar - The tallest brick minaret in India, with intricate carvings and inscriptions throughout.\n",
            "4. Akshardham Temple - A modern Hindu temple made of white marble that is an architectural wonder and a great place to learn about Hinduism.\n",
            "5. Raj Ghat - A simple but powerful memorial to Mahatma Gandhi, the father of India's independence movement.\n",
            "6. Lodi Gardens - A peaceful oasis in the heart of Delhi with beautiful gardens and historic buildings.\n",
            "7. Chandni Chowk - One of the oldest and busiest markets in Delhi, where you can find everything from spices to jewelry to souvenirs.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "N4uMV0zF8pQt"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}