code/MediumT3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c8e2d96dbd67440cad96527c8a5cf951": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_818b0bdb1a0943caaef418ef2d06e6e7",
              "IPY_MODEL_ecf59b72add94befa6acbb9c6a7a41b3",
              "IPY_MODEL_af450586c0d1453cbfc68aba12990a93"
            ],
            "layout": "IPY_MODEL_6f4a988c58a047ce9ca69a8e624ebadf"
          }
        },
        "818b0bdb1a0943caaef418ef2d06e6e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d77d7a900dbe4f4cbb5e1c7b7dbd6db8",
            "placeholder": "​",
            "style": "IPY_MODEL_0712f157af3b4553961db3ef1c868a31",
            "value": "gemma-2b-it.gguf: 100%"
          }
        },
        "ecf59b72add94befa6acbb9c6a7a41b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ef52640e8234339a17f23ebc20e862f",
            "max": 10031780704,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a0ecbd14c5c7479793d6c8f8f27b5a2e",
            "value": 10031780704
          }
        },
        "af450586c0d1453cbfc68aba12990a93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2309cfdc2f0844fb9bacd3a77f4d4d9c",
            "placeholder": "​",
            "style": "IPY_MODEL_eed1ad922da6480180027c6536c36970",
            "value": " 10.0G/10.0G [01:07&lt;00:00, 245MB/s]"
          }
        },
        "6f4a988c58a047ce9ca69a8e624ebadf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d77d7a900dbe4f4cbb5e1c7b7dbd6db8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0712f157af3b4553961db3ef1c868a31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0ef52640e8234339a17f23ebc20e862f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0ecbd14c5c7479793d6c8f8f27b5a2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2309cfdc2f0844fb9bacd3a77f4d4d9c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eed1ad922da6480180027c6536c36970": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Run Google Gemma + llama.cpp GGUF Inference Open LLM 🦙"
      ],
      "metadata": {
        "id": "rhAucpX5Li7V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Mostrar Imagen\n",
        "from IPython.display import Image\n",
        "url = 'https://github.com/AleNunezArroyo/Medium/blob/main/img/BannerM3.png?raw=true'\n",
        "Image(url=url, width=800)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 471
        },
        "cellView": "form",
        "id": "JWB-9CNLL2FI",
        "outputId": "4300be53-e0b9-4123-d0a5-286153ce60d3"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<img src=\"https://github.com/AleNunezArroyo/Medium/blob/main/img/BannerM3.png?raw=true\" width=\"800\"/>"
            ],
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GyK68jfLe5gy",
        "outputId": "b80ebac8-e595-4fbe-b91a-e3bc98261f91"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sun Feb 25 00:48:35 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla T4                       Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P8               9W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Installation"
      ],
      "metadata": {
        "id": "yMg4peWHxY4n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" pip install llama-cpp-python"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Irc-vi4pQxXM",
        "outputId": "c6b60a51-b503-4283-b96c-9880d382a7a1"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting llama-cpp-python\n",
            "  Downloading llama_cpp_python-0.2.50.tar.gz (36.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m36.7/36.7 MB\u001b[0m \u001b[31m30.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (4.9.0)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (1.25.2)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2>=2.11.3 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (3.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.3->llama-cpp-python) (2.1.5)\n",
            "Building wheels for collected packages: llama-cpp-python\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.50-cp310-cp310-manylinux_2_35_x86_64.whl size=22744090 sha256=b3bf05eee77a8676bd4eda67d34318a67094b118f987d5c8ba139f2b8d16abd7\n",
            "  Stored in directory: /root/.cache/pip/wheels/aa/86/b5/60afef0265d0fd0a5a193909db85382833ed61cd2ecf3ce138\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: diskcache, llama-cpp-python\n",
            "Successfully installed diskcache-5.6.3 llama-cpp-python-0.2.50\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Libraries"
      ],
      "metadata": {
        "id": "HWRSLxUeybL3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "mF9wach2QdlY"
      },
      "outputs": [],
      "source": [
        "from huggingface_hub import hf_hub_download\n",
        "from llama_cpp import Llama"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download the GGUF model"
      ],
      "metadata": {
        "id": "M0bWcVZ1ef3g"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = \"google/gemma-2b-it\"\n",
        "model_file = \"gemma-2b-it.gguf\"\n",
        "HF_TOKEN = \"your_token\"\n",
        "model_path = hf_hub_download(model_name,\n",
        "                             filename=model_file,\n",
        "                             local_dir='/content',\n",
        "                             token=HF_TOKEN)\n",
        "\n",
        "print(\"My model path: \", model_path)"
      ],
      "metadata": {
        "id": "Op-HQj_rxEQ2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "c8e2d96dbd67440cad96527c8a5cf951",
            "818b0bdb1a0943caaef418ef2d06e6e7",
            "ecf59b72add94befa6acbb9c6a7a41b3",
            "af450586c0d1453cbfc68aba12990a93",
            "6f4a988c58a047ce9ca69a8e624ebadf",
            "d77d7a900dbe4f4cbb5e1c7b7dbd6db8",
            "0712f157af3b4553961db3ef1c868a31",
            "0ef52640e8234339a17f23ebc20e862f",
            "a0ecbd14c5c7479793d6c8f8f27b5a2e",
            "2309cfdc2f0844fb9bacd3a77f4d4d9c",
            "eed1ad922da6480180027c6536c36970"
          ]
        },
        "outputId": "0cb42d11-b2d8-4850-8a9a-c2670e6f5e66"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "gemma-2b-it.gguf:   0%|          | 0.00/10.0G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c8e2d96dbd67440cad96527c8a5cf951"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "My model path:  /content/gemma-2b-it.gguf\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import the model"
      ],
      "metadata": {
        "id": "DOQojvozyhEf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_cpp import Llama\n",
        "\n",
        "llm = Llama(model_path=model_path,\n",
        "            n_gpu_layers=-1)"
      ],
      "metadata": {
        "id": "70_mXTOZUQhN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bc799d9f-dd05-40f8-8d79-c83c889e8897"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "llama_model_loader: loaded meta data with 19 key-value pairs and 164 tensors from /content/gemma-2b-it.gguf (version GGUF V3 (latest))\n",
            "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
            "llama_model_loader: - kv   0:                       general.architecture str              = gemma\n",
            "llama_model_loader: - kv   1:                               general.name str              = gemma-2b-it\n",
            "llama_model_loader: - kv   2:                       gemma.context_length u32              = 8192\n",
            "llama_model_loader: - kv   3:                          gemma.block_count u32              = 18\n",
            "llama_model_loader: - kv   4:                     gemma.embedding_length u32              = 2048\n",
            "llama_model_loader: - kv   5:                  gemma.feed_forward_length u32              = 16384\n",
            "llama_model_loader: - kv   6:                 gemma.attention.head_count u32              = 8\n",
            "llama_model_loader: - kv   7:              gemma.attention.head_count_kv u32              = 1\n",
            "llama_model_loader: - kv   8:                 gemma.attention.key_length u32              = 256\n",
            "llama_model_loader: - kv   9:               gemma.attention.value_length u32              = 256\n",
            "llama_model_loader: - kv  10:     gemma.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
            "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
            "llama_model_loader: - kv  12:                tokenizer.ggml.bos_token_id u32              = 2\n",
            "llama_model_loader: - kv  13:                tokenizer.ggml.eos_token_id u32              = 1\n",
            "llama_model_loader: - kv  14:            tokenizer.ggml.padding_token_id u32              = 0\n",
            "llama_model_loader: - kv  15:            tokenizer.ggml.unknown_token_id u32              = 3\n",
            "llama_model_loader: - kv  16:                      tokenizer.ggml.tokens arr[str,256128]  = [\"<pad>\", \"<eos>\", \"<bos>\", \"<unk>\", ...\n",
            "llama_model_loader: - kv  17:                      tokenizer.ggml.scores arr[f32,256128]  = [0.000000, 0.000000, 0.000000, 0.0000...\n",
            "llama_model_loader: - kv  18:                  tokenizer.ggml.token_type arr[i32,256128]  = [3, 3, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
            "llama_model_loader: - type  f32:  164 tensors\n",
            "llm_load_vocab: mismatch in special tokens definition ( 544/256128 vs 388/256128 ).\n",
            "llm_load_print_meta: format           = GGUF V3 (latest)\n",
            "llm_load_print_meta: arch             = gemma\n",
            "llm_load_print_meta: vocab type       = SPM\n",
            "llm_load_print_meta: n_vocab          = 256128\n",
            "llm_load_print_meta: n_merges         = 0\n",
            "llm_load_print_meta: n_ctx_train      = 8192\n",
            "llm_load_print_meta: n_embd           = 2048\n",
            "llm_load_print_meta: n_head           = 8\n",
            "llm_load_print_meta: n_head_kv        = 1\n",
            "llm_load_print_meta: n_layer          = 18\n",
            "llm_load_print_meta: n_rot            = 256\n",
            "llm_load_print_meta: n_embd_head_k    = 256\n",
            "llm_load_print_meta: n_embd_head_v    = 256\n",
            "llm_load_print_meta: n_gqa            = 8\n",
            "llm_load_print_meta: n_embd_k_gqa     = 256\n",
            "llm_load_print_meta: n_embd_v_gqa     = 256\n",
            "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
            "llm_load_print_meta: f_norm_rms_eps   = 1.0e-06\n",
            "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
            "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
            "llm_load_print_meta: n_ff             = 16384\n",
            "llm_load_print_meta: n_expert         = 0\n",
            "llm_load_print_meta: n_expert_used    = 0\n",
            "llm_load_print_meta: rope scaling     = linear\n",
            "llm_load_print_meta: freq_base_train  = 10000.0\n",
            "llm_load_print_meta: freq_scale_train = 1\n",
            "llm_load_print_meta: n_yarn_orig_ctx  = 8192\n",
            "llm_load_print_meta: rope_finetuned   = unknown\n",
            "llm_load_print_meta: model type       = 2B\n",
            "llm_load_print_meta: model ftype      = all F32 (guessed)\n",
            "llm_load_print_meta: model params     = 2.51 B\n",
            "llm_load_print_meta: model size       = 9.34 GiB (32.00 BPW) \n",
            "llm_load_print_meta: general.name     = gemma-2b-it\n",
            "llm_load_print_meta: BOS token        = 2 '<bos>'\n",
            "llm_load_print_meta: EOS token        = 1 '<eos>'\n",
            "llm_load_print_meta: UNK token        = 3 '<unk>'\n",
            "llm_load_print_meta: PAD token        = 0 '<pad>'\n",
            "llm_load_print_meta: LF token         = 227 '<0x0A>'\n",
            "llm_load_tensors: ggml ctx size =    0.13 MiB\n",
            "llm_load_tensors: offloading 18 repeating layers to GPU\n",
            "llm_load_tensors: offloading non-repeating layers to GPU\n",
            "llm_load_tensors: offloaded 19/19 layers to GPU\n",
            "llm_load_tensors:        CPU buffer size =  2001.00 MiB\n",
            "llm_load_tensors:      CUDA0 buffer size =  9561.29 MiB\n",
            ".............................................................\n",
            "llama_new_context_with_model: n_ctx      = 512\n",
            "llama_new_context_with_model: freq_base  = 10000.0\n",
            "llama_new_context_with_model: freq_scale = 1\n",
            "llama_kv_cache_init:      CUDA0 KV buffer size =     9.00 MiB\n",
            "llama_new_context_with_model: KV self size  =    9.00 MiB, K (f16):    4.50 MiB, V (f16):    4.50 MiB\n",
            "llama_new_context_with_model:  CUDA_Host input buffer size   =     6.01 MiB\n",
            "llama_new_context_with_model:      CUDA0 compute buffer size =   508.25 MiB\n",
            "llama_new_context_with_model:  CUDA_Host compute buffer size =     4.00 MiB\n",
            "llama_new_context_with_model: graph splits (measure): 3\n",
            "AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | \n",
            "Model metadata: {'tokenizer.ggml.unknown_token_id': '3', 'tokenizer.ggml.padding_token_id': '0', 'tokenizer.ggml.eos_token_id': '1', 'general.architecture': 'gemma', 'gemma.feed_forward_length': '16384', 'gemma.attention.head_count': '8', 'general.name': 'gemma-2b-it', 'gemma.context_length': '8192', 'gemma.block_count': '18', 'gemma.embedding_length': '2048', 'gemma.attention.head_count_kv': '1', 'gemma.attention.key_length': '256', 'tokenizer.ggml.model': 'llama', 'gemma.attention.value_length': '256', 'gemma.attention.layer_norm_rms_epsilon': '0.000001', 'tokenizer.ggml.bos_token_id': '2'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Inference from a question"
      ],
      "metadata": {
        "id": "Qozcln5DyjGq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response = llm(\"What is an LLM?\", max_tokens=1000)\n",
        "response"
      ],
      "metadata": {
        "id": "4901EHafGkhK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f9b2b729-9bf8-4a50-c906-d0aaa105df7e"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =     398.83 ms\n",
            "llama_print_timings:      sample time =    1639.28 ms /   393 runs   (    4.17 ms per token,   239.74 tokens per second)\n",
            "llama_print_timings: prompt eval time =       0.00 ms /     1 tokens (    0.00 ms per token,      inf tokens per second)\n",
            "llama_print_timings:        eval time =   16307.11 ms /   393 runs   (   41.49 ms per token,    24.10 tokens per second)\n",
            "llama_print_timings:       total time =   26690.02 ms /   394 tokens\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': 'cmpl-27b2d267-fc5d-4cf2-b7f9-ae02b9efc2a4',\n",
              " 'object': 'text_completion',\n",
              " 'created': 1708822813,\n",
              " 'model': '/content/gemma-2b-it.gguf',\n",
              " 'choices': [{'text': '\\n\\nAn LLM (Large Language Model) is a type of artificial intelligence (AI) that can perform a wide range of natural language processing (NLP) tasks, such as language translation, text summarization, question answering, and sentiment analysis.\\n\\n**Here are some key characteristics of LLMs:**\\n\\n* **Superhuman language understanding:** They are trained on massive datasets of text and code, enabling them to understand the meaning of language in a much more comprehensive way than traditional AI models.\\n* **End-to-end learning:** LLMs can learn from raw text data without the need for additional supervision or pre-training.\\n* **Multimodal capabilities:** They can process and understand language in multiple modalities, including text, images, and audio.\\n* **Wide range of applications:** LLMs have numerous potential applications in various industries, including healthcare, finance, education, and entertainment.\\n\\n**Examples of LLMs:**\\n\\n* **ChatGPT:** A large language model that can generate human-quality text based on the input provided.\\n* **BART:** A large language model that can be used for language translation and question answering.\\n* **T5:** A large language model that is even larger than BART and has more parameters.\\n* **RoBERTa:** A large language model that is particularly good at understanding and generating long and complex text.\\n\\n**Benefits of using LLMs:**\\n\\n* **Increased efficiency:** LLMs can automate many NLP tasks, saving time and resources.\\n* **Improved accuracy:** They can provide more accurate and relevant results compared to traditional AI models.\\n* **New possibilities:** LLMs open up new possibilities for language processing and understanding.\\n\\n**Limitations of LLMs:**\\n\\n* **Bias:** LLMs can reflect biases present in the data they are trained on.\\n* **Ethical considerations:** The development and use of LLMs raise ethical concerns related to privacy, transparency, and accountability.\\n',\n",
              "   'index': 0,\n",
              "   'logprobs': None,\n",
              "   'finish_reason': 'stop'}],\n",
              " 'usage': {'prompt_tokens': 7, 'completion_tokens': 392, 'total_tokens': 399}}"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response['choices'][0]['text'])"
      ],
      "metadata": {
        "id": "k47fPageQPi4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ed73092e-93fd-4c51-fc42-7d88fa171adc"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "An LLM (Large Language Model) is a type of artificial intelligence (AI) that can perform a wide range of natural language processing (NLP) tasks, such as language translation, text summarization, question answering, and sentiment analysis.\n",
            "\n",
            "**Here are some key characteristics of LLMs:**\n",
            "\n",
            "* **Superhuman language understanding:** They are trained on massive datasets of text and code, enabling them to understand the meaning of language in a much more comprehensive way than traditional AI models.\n",
            "* **End-to-end learning:** LLMs can learn from raw text data without the need for additional supervision or pre-training.\n",
            "* **Multimodal capabilities:** They can process and understand language in multiple modalities, including text, images, and audio.\n",
            "* **Wide range of applications:** LLMs have numerous potential applications in various industries, including healthcare, finance, education, and entertainment.\n",
            "\n",
            "**Examples of LLMs:**\n",
            "\n",
            "* **ChatGPT:** A large language model that can generate human-quality text based on the input provided.\n",
            "* **BART:** A large language model that can be used for language translation and question answering.\n",
            "* **T5:** A large language model that is even larger than BART and has more parameters.\n",
            "* **RoBERTa:** A large language model that is particularly good at understanding and generating long and complex text.\n",
            "\n",
            "**Benefits of using LLMs:**\n",
            "\n",
            "* **Increased efficiency:** LLMs can automate many NLP tasks, saving time and resources.\n",
            "* **Improved accuracy:** They can provide more accurate and relevant results compared to traditional AI models.\n",
            "* **New possibilities:** LLMs open up new possibilities for language processing and understanding.\n",
            "\n",
            "**Limitations of LLMs:**\n",
            "\n",
            "* **Bias:** LLMs can reflect biases present in the data they are trained on.\n",
            "* **Ethical considerations:** The development and use of LLMs raise ethical concerns related to privacy, transparency, and accountability.\n",
            "\n"
          ]
        }
      ]
    }
  ]
}