Qwen1_5_MoE_A2_7B_Chat_GPTQ_Int4.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Troyanovsky/Local-LLM-Comparison-Colab-UI/blob/main/Qwen1_5_MoE_A2_7B_Chat_GPTQ_Int4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XNkpBtLvuTOp"
      },
      "source": [
        "## Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4 WebUI\n",
        "\n",
        "1. Run the following cell, takes ~5 min\n",
        "(You may need to confirm to proceed by typing \"Y\")\n",
        "2. Pick the version you need from one of the last two cells and only run that cell.\n",
        "3. Click the gradio link at the bottom\n",
        "4. In Chat settings - Instruction Template: ChatML\n",
        "\n",
        "\n",
        "Original model: https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D-MiHp_bveP6",
        "outputId": "bd047ccf-30a1-4a62-dabc-4e1b616fb327"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n",
            "fatal: destination path 'text-generation-webui' already exists and is not an empty directory.\n",
            "/content/text-generation-webui\n",
            "Ignoring llama-cpp-python: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting llama-cpp-python==0.2.56+cpuavx2 (from -r requirements.txt (line 37))\n",
            "  Downloading https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/cpu/llama_cpp_python-0.2.56+cpuavx2-cp310-cp310-manylinux_2_31_x86_64.whl (2.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.8/2.8 MB\u001b[0m \u001b[31m21.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring llama-cpp-python: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring llama-cpp-python: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting llama-cpp-python-cuda==0.2.56+cu121 (from -r requirements.txt (line 45))\n",
            "  Downloading https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda-0.2.56+cu121-cp310-cp310-manylinux_2_31_x86_64.whl (67.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.9/67.9 MB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring llama-cpp-python-cuda-tensorcores: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda-tensorcores: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring llama-cpp-python-cuda-tensorcores: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting llama-cpp-python-cuda-tensorcores==0.2.56+cu121 (from -r requirements.txt (line 51))\n",
            "  Downloading https://github.com/oobabooga/llama-cpp-python-cuBLAS-wheels/releases/download/textgen-webui/llama_cpp_python_cuda_tensorcores-0.2.56+cu121-cp310-cp310-manylinux_2_31_x86_64.whl (55.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.2/55.2 MB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring auto-gptq: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring auto-gptq: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring auto-gptq: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting auto-gptq==0.6.0+cu121 (from -r requirements.txt (line 57))\n",
            "  Downloading https://github.com/jllllll/AutoGPTQ/releases/download/v0.6.0/auto_gptq-0.6.0+cu121-cp310-cp310-linux_x86_64.whl (4.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m93.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring exllamav2: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring exllamav2: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring exllamav2: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting exllamav2==0.0.15+cu121 (from -r requirements.txt (line 61))\n",
            "  Downloading https://github.com/oobabooga/exllamav2/releases/download/v0.0.15/exllamav2-0.0.15+cu121-cp310-cp310-linux_x86_64.whl (72.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.2/72.2 MB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring exllamav2: markers 'platform_system == \"Linux\" and platform_machine != \"x86_64\"' don't match your environment\n",
            "Ignoring flash-attn: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring flash-attn: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring flash-attn: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting flash-attn==2.5.6+cu122torch2.2cxx11abiFALSE (from -r requirements.txt (line 66))\n",
            "  Downloading https://github.com/Dao-AILab/flash-attention/releases/download/v2.5.6/flash_attn-2.5.6+cu122torch2.2cxx11abiFALSE-cp310-cp310-linux_x86_64.whl (120.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.6/120.6 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hIgnoring gptq-for-llama: markers 'platform_system == \"Windows\" and python_version == \"3.11\"' don't match your environment\n",
            "Ignoring gptq-for-llama: markers 'platform_system == \"Windows\" and python_version == \"3.10\"' don't match your environment\n",
            "Ignoring gptq-for-llama: markers 'platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version == \"3.11\"' don't match your environment\n",
            "Collecting gptq-for-llama==0.1.1+cu121 (from -r requirements.txt (line 70))\n",
            "  Downloading https://github.com/jllllll/GPTQ-for-LLaMa-CUDA/releases/download/0.1.1/gptq_for_llama-0.1.1+cu121-cp310-cp310-linux_x86_64.whl (739 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m739.5/739.5 kB\u001b[0m \u001b[31m51.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ctransformers==0.2.27+cu121 (from -r requirements.txt (line 71))\n",
            "  Downloading https://github.com/jllllll/ctransformers-cuBLAS-wheels/releases/download/AVX2/ctransformers-0.2.27+cu121-py3-none-any.whl (15.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.5/15.5 MB\u001b[0m \u001b[31m86.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: accelerate==0.27.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 1)) (0.27.2)\n",
            "Requirement already satisfied: aqlm[cpu,gpu]==1.1.2 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (1.1.2)\n",
            "Requirement already satisfied: bitsandbytes==0.43.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (0.43.0)\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 4)) (0.4.6)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (2.18.0)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 6)) (0.7.0)\n",
            "Requirement already satisfied: gradio==4.23.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (4.23.0)\n",
            "Requirement already satisfied: hqq==0.1.5 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (0.1.5)\n",
            "Requirement already satisfied: jinja2==3.1.2 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (3.1.2)\n",
            "Requirement already satisfied: lm_eval==0.3.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 10)) (0.3.0)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 11)) (3.6)\n",
            "Requirement already satisfied: numba==0.59.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 12)) (0.59.1)\n",
            "Requirement already satisfied: numpy==1.26.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 13)) (1.26.4)\n",
            "Requirement already satisfied: optimum==1.17.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 14)) (1.17.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 15)) (1.5.3)\n",
            "Requirement already satisfied: peft==0.8.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 16)) (0.8.2)\n",
            "Requirement already satisfied: Pillow>=9.5.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 17)) (10.3.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 18)) (6.0.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 19)) (2.31.0)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 20)) (13.7.1)\n",
            "Requirement already satisfied: safetensors==0.4.* in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 21)) (0.4.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 22)) (1.11.4)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 23)) (0.1.99)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 24)) (2.15.2)\n",
            "Collecting transformers==4.39.* (from -r requirements.txt (line 25))\n",
            "  Using cached transformers-4.39.3-py3-none-any.whl (8.8 MB)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 26)) (4.66.2)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 27)) (0.16.5)\n",
            "Requirement already satisfied: SpeechRecognition==3.10.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 30)) (3.10.0)\n",
            "Requirement already satisfied: flask_cloudflared==0.0.14 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 31)) (0.0.14)\n",
            "Requirement already satisfied: sse-starlette==1.6.5 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 32)) (1.6.5)\n",
            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 33)) (0.6.0)\n",
            "Requirement already satisfied: autoawq==0.2.3 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 72)) (0.2.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.27.*->-r requirements.txt (line 1)) (24.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate==0.27.*->-r requirements.txt (line 1)) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.27.*->-r requirements.txt (line 1)) (2.2.1+cu121)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate==0.27.*->-r requirements.txt (line 1)) (0.20.3)\n",
            "Requirement already satisfied: triton>=2.1 in /usr/local/lib/python3.10/dist-packages (from aqlm[cpu,gpu]==1.1.2->-r requirements.txt (line 2)) (2.2.0)\n",
            "Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (from aqlm[cpu,gpu]==1.1.2->-r requirements.txt (line 2)) (1.11.1.1)\n",
            "Requirement already satisfied: aiofiles<24.0,>=22.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (23.2.1)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (4.2.2)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.110.1)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.3.2)\n",
            "Requirement already satisfied: gradio-client==0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.14.0)\n",
            "Requirement already satisfied: httpx>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.27.0)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (6.4.0)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (2.1.5)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (3.7.1)\n",
            "Requirement already satisfied: orjson~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (3.10.0)\n",
            "Requirement already satisfied: pydantic>=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (2.6.4)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.25.1)\n",
            "Requirement already satisfied: python-multipart>=0.0.9 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.0.9)\n",
            "Requirement already satisfied: ruff>=0.2.2 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.3.5)\n",
            "Requirement already satisfied: semantic-version~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (2.10.0)\n",
            "Requirement already satisfied: tomlkit==0.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.12.0)\n",
            "Requirement already satisfied: typer[all]<1.0,>=0.9 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.9.4)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (4.10.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.*->-r requirements.txt (line 7)) (0.29.0)\n",
            "Requirement already satisfied: timm in /usr/local/lib/python3.10/dist-packages (from hqq==0.1.5->-r requirements.txt (line 8)) (0.9.16)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from hqq==0.1.5->-r requirements.txt (line 8)) (2.4.0)\n",
            "Requirement already satisfied: jsonlines in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (4.0.0)\n",
            "Requirement already satisfied: numexpr in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (2.9.0)\n",
            "Requirement already satisfied: openai>=0.6.4 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (1.16.1)\n",
            "Requirement already satisfied: pybind11>=2.6.2 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (2.12.0)\n",
            "Requirement already satisfied: pycountry in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (23.12.11)\n",
            "Requirement already satisfied: pytablewriter in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (1.2.0)\n",
            "Requirement already satisfied: rouge-score>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (0.1.2)\n",
            "Requirement already satisfied: sacrebleu==1.5.0 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (1.5.0)\n",
            "Requirement already satisfied: scikit-learn>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (1.2.2)\n",
            "Requirement already satisfied: sqlitedict in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (2.1.0)\n",
            "Requirement already satisfied: tqdm-multiprocess in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (0.0.11)\n",
            "Requirement already satisfied: zstandard in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.3.0->-r requirements.txt (line 10)) (0.22.0)\n",
            "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba==0.59.*->-r requirements.txt (line 12)) (0.42.0)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from optimum==1.17.*->-r requirements.txt (line 14)) (15.0.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum==1.17.*->-r requirements.txt (line 14)) (1.12)\n",
            "Requirement already satisfied: transformers[sentencepiece]>=4.26.0 in /usr/local/lib/python3.10/dist-packages (from optimum==1.17.*->-r requirements.txt (line 14)) (4.40.0.dev0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.39.*->-r requirements.txt (line 25)) (3.13.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.39.*->-r requirements.txt (line 25)) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers==4.39.*->-r requirements.txt (line 25)) (0.15.2)\n",
            "Requirement already satisfied: Flask>=0.8 in /usr/local/lib/python3.10/dist-packages (from flask_cloudflared==0.0.14->-r requirements.txt (line 31)) (2.2.5)\n",
            "Requirement already satisfied: starlette in /usr/local/lib/python3.10/dist-packages (from sse-starlette==1.6.5->-r requirements.txt (line 32)) (0.37.2)\n",
            "Requirement already satisfied: autoawq-kernels in /usr/local/lib/python3.10/dist-packages (from autoawq==0.2.3->-r requirements.txt (line 72)) (0.0.6)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.14.0->gradio==4.23.*->-r requirements.txt (line 7)) (2023.6.0)\n",
            "Requirement already satisfied: websockets<12.0,>=10.0 in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.14.0->gradio==4.23.*->-r requirements.txt (line 7)) (11.0.3)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.10/dist-packages (from sacrebleu==1.5.0->lm_eval==0.3.0->-r requirements.txt (line 10)) (2.8.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 5)) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 5)) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 5)) (0.3.8)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 5)) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 5)) (0.70.16)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 5)) (3.9.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 15)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 15)) (2023.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 19)) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 19)) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 19)) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 19)) (2024.2.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->-r requirements.txt (line 20)) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->-r requirements.txt (line 20)) (2.16.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (1.62.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (2.27.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (1.2.0)\n",
            "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (3.20.3)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (67.7.2)\n",
            "Requirement already satisfied: six>1.9 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (1.16.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard->-r requirements.txt (line 24)) (3.0.1)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 27)) (8.1.7)\n",
            "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 27)) (3.1.43)\n",
            "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 27)) (1.44.0)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 27)) (0.4.0)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 27)) (1.3.3)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 27)) (1.4.4)\n",
            "Requirement already satisfied: diskcache>=5.6.1 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python==0.2.56+cpuavx2->-r requirements.txt (line 37)) (5.6.3)\n",
            "Requirement already satisfied: rouge in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.6.0+cu121->-r requirements.txt (line 57)) (1.0.1)\n",
            "Requirement already satisfied: gekko in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.6.0+cu121->-r requirements.txt (line 57)) (1.1.0)\n",
            "Requirement already satisfied: fastparquet in /usr/local/lib/python3.10/dist-packages (from exllamav2==0.0.15+cu121->-r requirements.txt (line 61)) (2024.2.0)\n",
            "Requirement already satisfied: py-cpuinfo<10.0.0,>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from ctransformers==0.2.27+cu121->-r requirements.txt (line 71)) (9.0.0)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.23.*->-r requirements.txt (line 7)) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.23.*->-r requirements.txt (line 7)) (4.19.2)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.23.*->-r requirements.txt (line 7)) (0.12.1)\n",
            "Requirement already satisfied: itsdangerous>=2.0 in /usr/local/lib/python3.10/dist-packages (from Flask>=0.8->flask_cloudflared==0.0.14->-r requirements.txt (line 31)) (2.1.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 5)) (4.0.3)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 27)) (4.0.11)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 24)) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 24)) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 24)) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard->-r requirements.txt (line 24)) (1.3.1)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.*->-r requirements.txt (line 7)) (3.7.1)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.*->-r requirements.txt (line 7)) (1.0.5)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.*->-r requirements.txt (line 7)) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.1->gradio==4.23.*->-r requirements.txt (line 7)) (0.14.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->-r requirements.txt (line 20)) (0.1.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.*->-r requirements.txt (line 7)) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.*->-r requirements.txt (line 7)) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.*->-r requirements.txt (line 7)) (4.50.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.*->-r requirements.txt (line 7)) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.*->-r requirements.txt (line 7)) (3.1.2)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=0.6.4->lm_eval==0.3.0->-r requirements.txt (line 10)) (1.7.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio==4.23.*->-r requirements.txt (line 7)) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio==4.23.*->-r requirements.txt (line 7)) (2.16.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm_eval==0.3.0->-r requirements.txt (line 10)) (3.8.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm_eval==0.3.0->-r requirements.txt (line 10)) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm_eval==0.3.0->-r requirements.txt (line 10)) (3.4.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (3.2.1)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (12.1.105)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate==0.27.*->-r requirements.txt (line 1)) (12.4.99)\n",
            "INFO: pip is looking at multiple versions of transformers[sentencepiece] to determine which version is compatible with other requirements. This could take a while.\n",
            "Requirement already satisfied: shellingham<2.0.0,>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio==4.23.*->-r requirements.txt (line 7)) (1.5.4)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->optimum==1.17.*->-r requirements.txt (line 14)) (10.0)\n",
            "Requirement already satisfied: cramjam>=2.3 in /usr/local/lib/python3.10/dist-packages (from fastparquet->exllamav2==0.0.15+cu121->-r requirements.txt (line 61)) (2.8.3)\n",
            "Requirement already satisfied: DataProperty<2,>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm_eval==0.3.0->-r requirements.txt (line 10)) (1.0.1)\n",
            "Requirement already satisfied: mbstrdecoder<2,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm_eval==0.3.0->-r requirements.txt (line 10)) (1.1.3)\n",
            "Requirement already satisfied: pathvalidate<4,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm_eval==0.3.0->-r requirements.txt (line 10)) (3.2.0)\n",
            "Requirement already satisfied: tabledata<2,>=1.3.1 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm_eval==0.3.0->-r requirements.txt (line 10)) (1.3.3)\n",
            "Requirement already satisfied: tcolorpy<1,>=0.0.5 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm_eval==0.3.0->-r requirements.txt (line 10)) (0.1.4)\n",
            "Requirement already satisfied: typepy[datetime]<2,>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm_eval==0.3.0->-r requirements.txt (line 10)) (1.3.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum==1.17.*->-r requirements.txt (line 14)) (1.3.0)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from timm->hqq==0.1.5->-r requirements.txt (line 8)) (0.17.1+cu121)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.24.1->gradio==4.23.*->-r requirements.txt (line 7)) (1.2.0)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 27)) (5.0.1)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.23.*->-r requirements.txt (line 7)) (2023.12.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.23.*->-r requirements.txt (line 7)) (0.34.0)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.23.*->-r requirements.txt (line 7)) (0.18.0)\n",
            "Requirement already satisfied: chardet<6,>=3.0.4 in /usr/local/lib/python3.10/dist-packages (from mbstrdecoder<2,>=1.0.0->pytablewriter->lm_eval==0.3.0->-r requirements.txt (line 10)) (5.2.0)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 24)) (0.6.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard->-r requirements.txt (line 24)) (3.2.2)\n",
            "Installing collected packages: llama-cpp-python, transformers\n",
            "  Attempting uninstall: llama-cpp-python\n",
            "    Found existing installation: llama_cpp_python 0.2.58\n",
            "    Uninstalling llama_cpp_python-0.2.58:\n",
            "      Successfully uninstalled llama_cpp_python-0.2.58\n",
            "  Attempting uninstall: transformers\n",
            "    Found existing installation: transformers 4.40.0.dev0\n",
            "    Uninstalling transformers-4.40.0.dev0:\n",
            "      Successfully uninstalled transformers-4.40.0.dev0\n",
            "Successfully installed llama-cpp-python-0.2.56+cpuavx2 transformers-4.39.3\n",
            "Requirement already satisfied: gradio==4.23.0 in /usr/local/lib/python3.10/dist-packages (4.23.0)\n",
            "Requirement already satisfied: aiofiles<24.0,>=22.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (23.2.1)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (4.2.2)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.110.1)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.3.2)\n",
            "Requirement already satisfied: gradio-client==0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.14.0)\n",
            "Requirement already satisfied: httpx>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.27.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.20.3)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (6.4.0)\n",
            "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (3.1.2)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (2.1.5)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (3.7.1)\n",
            "Requirement already satisfied: numpy~=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (1.26.4)\n",
            "Requirement already satisfied: orjson~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (3.10.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (24.0)\n",
            "Requirement already satisfied: pandas<3.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (1.5.3)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (10.3.0)\n",
            "Requirement already satisfied: pydantic>=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (2.6.4)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.25.1)\n",
            "Requirement already satisfied: python-multipart>=0.0.9 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.0.9)\n",
            "Requirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (6.0.1)\n",
            "Requirement already satisfied: ruff>=0.2.2 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.3.5)\n",
            "Requirement already satisfied: semantic-version~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (2.10.0)\n",
            "Requirement already satisfied: tomlkit==0.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.12.0)\n",
            "Requirement already satisfied: typer[all]<1.0,>=0.9 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.9.4)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (4.10.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==4.23.0) (0.29.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.14.0->gradio==4.23.0) (2023.6.0)\n",
            "Requirement already satisfied: websockets<12.0,>=10.0 in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.14.0->gradio==4.23.0) (11.0.3)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.23.0) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.23.0) (4.19.2)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==4.23.0) (0.12.1)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.0) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.0) (2024.2.2)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.0) (1.0.5)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.0) (3.6)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio==4.23.0) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.1->gradio==4.23.0) (0.14.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio==4.23.0) (3.13.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio==4.23.0) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio==4.23.0) (4.66.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.0) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.0) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.0) (4.50.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.0) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.0) (3.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==4.23.0) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio==4.23.0) (2023.4)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio==4.23.0) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio==4.23.0) (2.16.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio==4.23.0) (8.1.7)\n",
            "Requirement already satisfied: colorama<0.5.0,>=0.4.3 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio==4.23.0) (0.4.6)\n",
            "Requirement already satisfied: shellingham<2.0.0,>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio==4.23.0) (1.5.4)\n",
            "Requirement already satisfied: rich<14.0.0,>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio==4.23.0) (13.7.1)\n",
            "Requirement already satisfied: starlette<0.38.0,>=0.37.2 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio==4.23.0) (0.37.2)\n",
            "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.23.0) (23.2.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.23.0) (2023.12.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.23.0) (0.34.0)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==4.23.0) (0.18.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio==4.23.0) (1.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio==4.23.0) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio==4.23.0) (2.16.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.24.1->gradio==4.23.0) (1.2.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.3->gradio==4.23.0) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.3->gradio==4.23.0) (2.0.7)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio==4.23.0) (0.1.2)\n",
            "Collecting git+https://github.com/huggingface/transformers\n",
            "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-iqs4vbpv\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-iqs4vbpv\n",
            "  Resolved https://github.com/huggingface/transformers to commit b44df05bc0866f88f06c8c14b392afc197a8c8b6\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (3.13.3)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (0.20.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.40.0.dev0) (4.66.2)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.40.0.dev0) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.40.0.dev0) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.40.0.dev0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.40.0.dev0) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.40.0.dev0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.40.0.dev0) (2024.2.2)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.40.0.dev0-py3-none-any.whl size=8806756 sha256=1e0b66d97701c4085a7533cdd37420f4d6cee12f14b1d332dbebc511cc918196\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-m9fynl2g/wheels/c0/14/d6/6c9a5582d2ac191ec0a483be151a4495fe1eb2a6706ca49f1b\n",
            "Successfully built transformers\n",
            "Installing collected packages: transformers\n",
            "  Attempting uninstall: transformers\n",
            "    Found existing installation: transformers 4.39.3\n",
            "    Uninstalling transformers-4.39.3:\n",
            "      Successfully uninstalled transformers-4.39.3\n",
            "Successfully installed transformers-4.40.0.dev0\n",
            "Found existing installation: llama_cpp_python 0.2.56+cpuavx2\n",
            "Uninstalling llama_cpp_python-0.2.56+cpuavx2:\n",
            "  Successfully uninstalled llama_cpp_python-0.2.56+cpuavx2\n",
            "Collecting llama-cpp-python\n",
            "  Downloading llama_cpp_python-0.2.58.tar.gz (37.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m37.4/37.4 MB\u001b[0m \u001b[31m207.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (4.10.0)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (1.26.4)\n",
            "Requirement already satisfied: diskcache>=5.6.1 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (5.6.3)\n",
            "Requirement already satisfied: jinja2>=2.11.3 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (3.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.3->llama-cpp-python) (2.1.5)\n",
            "Building wheels for collected packages: llama-cpp-python\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.58-cp310-cp310-manylinux_2_35_x86_64.whl size=38815794 sha256=da51d9e1a3e9986bbb6636196dcec9e0a7bd2145d7a32f80705a9ca141e247d6\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-43dizeb0/wheels/36/63/c3/f4195c83bf1bd0fadafe9a7a7688ee8a271053674e4712f170\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: llama-cpp-python\n",
            "Successfully installed llama-cpp-python-0.2.58\n",
            "Found existing installation: flash-attn 2.5.6\n",
            "Uninstalling flash-attn-2.5.6:\n",
            "  Successfully uninstalled flash-attn-2.5.6\n",
            "Collecting flash-attn==2.5.6\n",
            "  Using cached flash_attn-2.5.6-cp310-cp310-linux_x86_64.whl\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from flash-attn==2.5.6) (2.2.1+cu121)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from flash-attn==2.5.6) (0.7.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from flash-attn==2.5.6) (24.0)\n",
            "Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (from flash-attn==2.5.6) (1.11.1.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (3.13.3)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (4.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (2.19.3)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (12.1.105)\n",
            "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn==2.5.6) (2.2.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->flash-attn==2.5.6) (12.4.99)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->flash-attn==2.5.6) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->flash-attn==2.5.6) (1.3.0)\n",
            "Installing collected packages: flash-attn\n",
            "Successfully installed flash-attn-2.5.6\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "02d212|\u001b[1;32mOK\u001b[0m  |   798KiB/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//config.2.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "e3289d|\u001b[1;32mOK\u001b[0m  |    26KiB/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//generation_config.2.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "5588b0|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//merges.txt\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "f24391|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//model-00001-of-00003.safetensors\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "e5ae7b|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//model-00002-of-00003.safetensors\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "e7f97e|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//model-00003-of-00003.safetensors\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "d4dde3|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//model.safetensors.index.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "c6acf1|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//tokenizer.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "2582e0|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//tokenizer_config.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "207201|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4//vocab.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "/content/text-generation-webui\n",
            "\u001b[2;36m08:15:29-406961\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Starting Text generation web UI                                            \n",
            "\u001b[2;36m08:15:29-418592\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Loading \u001b[32m\"Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4\"\u001b[0m                                 \n",
            "2024-04-03 08:15:30.554769: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-04-03 08:15:30.554834: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-04-03 08:15:30.562262: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-04-03 08:15:32.688596: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py:4365: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
            "  warnings.warn(\n",
            "Loading checkpoint shards: 100% 3/3 [00:24<00:00,  8.28s/it]\n",
            "\u001b[2;36m08:17:16-080137\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m LOADER: \u001b[32m\"Transformers\"\u001b[0m                                                     \n",
            "\u001b[2;36m08:17:16-081790\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m TRUNCATION LENGTH: \u001b[1;36m32768\u001b[0m                                                   \n",
            "\u001b[2;36m08:17:16-082844\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m INSTRUCTION TEMPLATE: \u001b[32m\"Custom \u001b[0m\u001b[32m(\u001b[0m\u001b[32mobtained from model metadata\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\"\u001b[0m              \n",
            "\u001b[2;36m08:17:16-083869\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Loaded the model in \u001b[1;36m106.66\u001b[0m seconds.                                        \n",
            "\u001b[2;36m08:17:16-084993\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Loading the extension \u001b[32m\"gallery\"\u001b[0m                                            \n",
            "\n",
            "Running on local URL:  http://127.0.0.1:7860\n",
            "\n",
            "Running on public URL: https://07583b41b26194aa75.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n",
            "Output generated in 156.87 seconds (2.45 tokens/s, 384 tokens, context 237, seed 1260143756)\n"
          ]
        }
      ],
      "source": [
        "%cd /content\n",
        "!apt-get -y install -qq aria2\n",
        "\n",
        "!git clone -b V20240402 https://github.com/Troyanovsky/text-generation-webui\n",
        "%cd /content/text-generation-webui\n",
        "!pip install -r requirements.txt\n",
        "!pip install -U gradio==4.23.0\n",
        "!pip install --upgrade git+https://github.com/huggingface/transformers\n",
        "\n",
        "!pip uninstall -y llama-cpp-python -y\n",
        "!CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" FORCE_CMAKE=1 pip install llama-cpp-python --no-cache-dir\n",
        "\n",
        "!pip uninstall flash-attn -y\n",
        "!pip install --no-build-isolation flash-attn==2.5.6\n",
        "\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/config.json?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o config.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/generation_config.json?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o generation_config.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/merges.txt?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o merges.txt\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/model-00001-of-00003.safetensors?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o model-00001-of-00003.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/model-00002-of-00003.safetensors?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o model-00002-of-00003.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/model-00003-of-00003.safetensors?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o model-00003-of-00003.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/model.safetensors.index.json?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o model.safetensors.index.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/tokenizer.json?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o tokenizer.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/tokenizer_config.json?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o tokenizer_config.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/resolve/main/vocab.json?download=true -d /content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/ -o vocab.json\n",
        "\n",
        "import json\n",
        "\n",
        "with open(\"/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/config.json\", \"r\") as f:\n",
        "    config = json.load(f)\n",
        "config['quantization_config'][\"use_exllama\"] = False\n",
        "with open(\"/content/text-generation-webui/models/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4/config.json\", \"w\") as f:\n",
        "    json.dump(config, f, indent=4)\n",
        "\n",
        "%cd /content/text-generation-webui\n",
        "!python server.py --share --model Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyM/ZKj3xQum6ulzylUuJ6xy",
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}