ddPn08's_tensor-rt.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/ddPn08's_tensor-rt.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n",
        "# https://github.com/ddPn08/tensorrt-diffusion-colab modified\n",
        "import os\n",
        "\n",
        "!apt update && apt install software-properties-common -y && add-apt-repository --yes ppa:deadsnakes/ppa\n",
        "!apt update && apt install tensorrt tensorrt-dev tensorrt-libs git-lfs -y\n",
        "!pip install tensorrt==8.5.3.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!git clone --single-branch https://github.com/camenduru/TensorRT && \\\n",
        "    cd TensorRT && \\\n",
        "    git submodule update --init --recursive\n",
        "\n",
        "!cd TensorRT && mkdir -p build && \\\n",
        "    cd build && \\\n",
        "    cmake /content/TensorRT -DTRT_OUT_DIR=$PWD/out && \\\n",
        "    cd plugin && \\\n",
        "    make -j$(nproc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from huggingface_hub import create_repo, upload_folder\n",
        "hf_token = 'token_here' # @param{type:\"string\"}\n",
        "repo_id = 'camenduru/libnvinfer_plugin.so.8.5.3_colab' # @param{type:\"string\"}\n",
        "create_repo(repo_id, private=True, token=hf_token)\n",
        "upload_folder(folder_path='/content/TensorRT/build/out', path_in_repo='', repo_id=repo_id, commit_message='content', token=hf_token)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "TRT_OSSPATH = os.path.abspath(\"./TensorRT\")\n",
        "script_filepath = os.path.join(TRT_OSSPATH, \"demo\", \"Diffusion\", \"build.py\")\n",
        "!curl -o {script_filepath} -L https://raw.githubusercontent.com/camenduru/tensorrt-diffusion-colab/main/scripts/build.py\n",
        "!pip install --upgrade pip\n",
        "!cd TensorRT/demo/Diffusion && pip install -r requirements.txt && pip install numpy==1.21.6"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OEp_KcG-_yeA"
      },
      "outputs": [],
      "source": [
        "!git clone https://huggingface.co/trrt/anything-v4.5-vae-swapped /content/model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SSE-qQcmw20m"
      },
      "outputs": [],
      "source": [
        "unet_pretrained_model_id = \"ckpt/anything-v4.5-vae-swapped\"  # @param{type:\"string\"}\n",
        "vae_pretrained_model_id = \"ckpt/anything-v4.5-vae-swapped\"  # @param{type:\"string\"}\n",
        "clip_pretrained_model_id = \"openai/clip-vit-large-patch14\"  # @param{type:\"string\"}\n",
        "\n",
        "os.environ[\"UNET_PRETRAINED_MODEL_ID\"] = unet_pretrained_model_id\n",
        "os.environ[\"VAE_PRETRAINED_MODEL_ID\"] = vae_pretrained_model_id\n",
        "os.environ[\"CLIP_PRETRAINED_MODEL_ID\"] = clip_pretrained_model_id\n",
        "\n",
        "engine_dir = \"/content/model/engine\"  # @param{type:\"string\"}\n",
        "onnx_dir = \"/content/model/onnx\"  # @param{type:\"string\"}\n",
        "\n",
        "os.makedirs(engine_dir, exist_ok=True)\n",
        "os.makedirs(onnx_dir, exist_ok=True)\n",
        "\n",
        "hf_token = \"\"  # @param{type:\"string\"}\n",
        "denoising_prec = \"fp32\"  # @param [\"fp32\", \"fp16\"]\n",
        "scheduler = \"LMSD\"  # @param [\"LMSD\", \"DPM\"]\n",
        "height = 512  # @param{type:\"slider\", min:256, max:1024, step:64}\n",
        "width = 512  # @param{type:\"slider\", min:256, max:1024, step:64}\n",
        "\n",
        "onnx_opset = 16  # @param{type:\"integer\"}\n",
        "force_onnx_export = False  # @param{type:\"boolean\"}\n",
        "force_onnx_optimize = False  # @param{type:\"boolean\"}\n",
        "onnx_minimal_optimization = False  # @param{type:\"boolean\"}\n",
        "\n",
        "force_engine_build = False  # @param{type:\"boolean\"}\n",
        "build_static_batch = False  # @param{type:\"boolean\"}\n",
        "build_dynamic_shape = False  # @param{type:\"boolean\"}\n",
        "build_preview_features = False  # @param{type:\"boolean\"}\n",
        "\n",
        "def make_args(d):\n",
        "    arguments = []\n",
        "    for k, v in d.items():\n",
        "        k = k.replace(\"_\", \"-\")\n",
        "        if type(v) == bool:\n",
        "            arguments.append(f\"--{k}\" if v else \"\")\n",
        "        elif type(v) == str and v:\n",
        "            arguments.extend([f\"--{k}\", f\"{v}\"])\n",
        "        elif v:\n",
        "            arguments.extend([f\"--{k}\", f\"{v}\"])\n",
        "    return \" \".join(arguments)\n",
        "\n",
        "args = make_args({\n",
        "    \"engine-dir\": engine_dir,\n",
        "    \"onnx-dir\": onnx_dir,\n",
        "    \"hf-token\": hf_token,\n",
        "    \"denoising-prec\": denoising_prec,\n",
        "    \"scheduler\": scheduler,\n",
        "    \"height\": height,\n",
        "    \"width\": width,\n",
        "    \"onnx-opset\": onnx_opset,\n",
        "    \"force-onnx-export\": force_onnx_export,\n",
        "    \"force-onnx-optimize\": force_onnx_optimize,\n",
        "    \"onnx-minimal-optimization\": onnx_minimal_optimization,\n",
        "    \"force-engine-build\": force_engine_build,\n",
        "    \"build-static-batch\": build_static_batch,\n",
        "    \"build-dynamic-shape\": build_dynamic_shape,\n",
        "    \"build-preview-features\": build_preview_features\n",
        "})\n",
        "\n",
        "! cd /content/TensorRT/demo/Diffusion && LD_PRELOAD=\"/content/TensorRT/build/out/libnvinfer_plugin.so.8.5.3\" python {script_filepath} {args}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from huggingface_hub import create_repo, upload_folder\n",
        "hf_token = 'token_here' # @param{type:\"string\"}\n",
        "repo_id = 'trrt/anything-v4.5-vae-swapped' # @param{type:\"string\"}\n",
        "create_repo(repo_id, private=True, token=hf_token)\n",
        "upload_folder(folder_path='/content/model', path_in_repo='', repo_id=repo_id, commit_message='content', token=hf_token)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RHR74sk6bszi"
      },
      "outputs": [],
      "source": [
        "engine_dir = \"/content/model/engine\"  # @param{type:\"string\"}\n",
        "output_dir = \"/content/outputs\"  # @param{type:\"string\"}\n",
        "\n",
        "os.makedirs(output_dir, exist_ok=True)\n",
        "\n",
        "prompt = \"masterpiece, best quality, 1girl\"  # @param{type:\"string\"}\n",
        "negative_prompt = \"\"  # @param{type:\"string\"}\n",
        "repeat_prompt = 1  # @param [1, 2, 4, 8, 16] {type:\"raw\"}\n",
        "height = 512  # @param{type:\"slider\", min:256, max:1024, step:64}\n",
        "width = 512  # @param{type:\"slider\", min:256, max:1024, step:64}\n",
        "denoising_steps = 50  # @param{type:\"integer\"}\n",
        "denoising_prec = \"fp32\"  # @param [\"fp32\", \"fp16\"]\n",
        "scheduler = \"LMSD\"  # @param [\"LMSD\", \"DPM\"]\n",
        "seed = \"1\"  # @param{type:\"string\"}\n",
        "\n",
        "num_warmup_runs = 0  # @param{type:\"integer\"}\n",
        "\n",
        "def make_args(d):\n",
        "    arguments = []\n",
        "    for k, v in d.items():\n",
        "        k = k.replace(\"_\", \"-\")\n",
        "        if type(v) == bool:\n",
        "            arguments.append(f\"--{k}\" if v else \"\")\n",
        "        elif type(v) == str and v:\n",
        "            arguments.extend([f\"--{k}\", f\"{v}\"])\n",
        "        elif v:\n",
        "            arguments.extend([f\"--{k}\", f\"{v}\"])\n",
        "    return \" \".join(arguments)\n",
        "\n",
        "args = make_args({\n",
        "    \"engine-dir\": engine_dir,\n",
        "    \"output-dir\": output_dir,\n",
        "    \"negative-prompt\": negative_prompt,\n",
        "    \"repeat-prompt\": repeat_prompt,\n",
        "    \"height\": height,\n",
        "    \"width\": height,\n",
        "    \"denoising-steps\": denoising_steps,\n",
        "    \"denoising-prec\": denoising_prec,\n",
        "    \"scheduler\": scheduler,\n",
        "    \"seed\": seed,\n",
        "    \"num-warmup-runs\": num_warmup_runs\n",
        "})\n",
        "\n",
        "!cd /content/TensorRT/demo/Diffusion && LD_PRELOAD=\"/content/TensorRT/build/out/libnvinfer_plugin.so.8.5.3\" python demo-diffusion.py \"{prompt}\" {args}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && echo \"deb https://ngrok-agent.s3.amazonaws.com buster main\" | sudo tee /etc/apt/sources.list.d/ngrok.list && sudo apt update && sudo apt install ngrok"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!ngrok config add-authtoken token_here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!pip install colab-xterm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "%load_ext colabxterm\n",
        "%xterm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# %env CUDA_MODULE_LOADING=LAZY"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!npm install -g pnpm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!git clone https://github.com/camenduru/Lsmith.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!cd /content/Lsmith/frontend && pnpm i && pnpm build --out-dir ../dist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!LD_PRELOAD=\"/content/TensorRT/build/out/libnvinfer_plugin.so.8.5.3\" python -m uvicorn modules.main:app --host 0.0.0.0 --port 7860 "
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}