singleImageCannyControlNetWithGFPGANpost.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "premium"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/stepbot/StableDiffusionNotebooks/blob/main/singleImageCannyControlNetWithGFPGANpost.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zHkHsdtnry57"
      },
      "outputs": [],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "DawvBH93xBYv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZwH2ifWEYEfJ"
      },
      "source": [
        "# Clone GFPGAN and enter the GFPGAN folder\n",
        "%cd /content\n",
        "!rm -rf GFPGAN\n",
        "!git clone https://github.com/TencentARC/GFPGAN.git\n",
        "%cd GFPGAN\n",
        "\n",
        "# Set up the environment\n",
        "# Install basicsr - https://github.com/xinntao/BasicSR\n",
        "# We use BasicSR for both training and inference\n",
        "!pip install basicsr\n",
        "# Install facexlib - https://github.com/xinntao/facexlib\n",
        "# We use face detection and face restoration helper in the facexlib package\n",
        "!pip install facexlib\n",
        "# Install other depencencies\n",
        "!pip install -r requirements.txt\n",
        "!python setup.py develop\n",
        "!pip install realesrgan  # used for enhancing the background (non-face) regions\n",
        "# Download the pre-trained model\n",
        "# !wget https://github.com/TencentARC/GFPGAN/releases/download/v0.2.0/GFPGANCleanv1-NoCE-C2.pth -P experiments/pretrained_models\n",
        "# Now we use the V1.4 model for the demo\n",
        "!wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth -P experiments/pretrained_models\n",
        "%cd /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aIrgth7sqFML"
      },
      "outputs": [],
      "source": [
        "!pip install diffusers[torch]==0.14.0 transformers scipy ftfy accelerate\n",
        "!pip install -q opencv-contrib-python\n",
        "!pip install -q controlnet_aux"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel\n",
        "from diffusers.utils import load_image\n",
        "import torch\n",
        "import cv2\n",
        "from PIL import Image\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "yAG1Ei_MWzb1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# select you model here\n",
        "model_id = \"dreamlike-art/dreamlike-photoreal-2.0\"\n",
        "\n",
        "controlnet = ControlNetModel.from_pretrained(\"lllyasviel/sd-controlnet-canny\", torch_dtype=torch.float16)\n",
        "pipe = StableDiffusionControlNetPipeline.from_pretrained(\n",
        "    model_id, controlnet=controlnet, torch_dtype=torch.float16\n",
        ")"
      ],
      "metadata": {
        "id": "FqqIFzJPZBh3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xSKWBKFPArKS"
      },
      "outputs": [],
      "source": [
        "# load input image\n",
        "try:\n",
        "  input_image=Image.open('input.png')\n",
        "except:\n",
        "  input_image = load_image(\n",
        "    \"https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png\"\n",
        ")\n",
        "input_image"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "image = np.array(input_image)\n",
        "\n",
        "low_threshold = 100\n",
        "high_threshold = 200\n",
        "\n",
        "image = cv2.Canny(image, low_threshold, high_threshold)\n",
        "image = image[:, :, None]\n",
        "image = np.concatenate([image, image, image], axis=2)\n",
        "canny_image = Image.fromarray(image)\n",
        "canny_image.save(\"canny.png\")\n",
        "canny_image"
      ],
      "metadata": {
        "id": "BZZp9d7uW6bu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipe.safety_checker=None\n",
        "pipe = pipe.to(\"cuda\")\n",
        "\n",
        "# set seed\n",
        "generator = torch.Generator(\"cuda\").manual_seed(42)\n",
        "\n",
        "# set prompts\n",
        "prompt = \"Nicolas Cage\"\n",
        "n_prompt = \"worst quality,monochrome, bad anatomy, low resolution, extra limbs, unnatural, messed up body, extra fingers\"\n",
        "\n",
        "# generate an image\n",
        "output = pipe(prompt,canny_image, negative_prompt=n_prompt, guidance_scale=8, num_inference_steps=50, height=768, width=768).images[0]\n",
        "output.save(\"output.png\")\n",
        "output\n",
        "\n"
      ],
      "metadata": {
        "id": "mH3hO-DFZxEC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Now we use the GFPGAN to restore the above low-quality images\n",
        "# We use [Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN) for enhancing the background (non-face) regions\n",
        "# You can find the different models in https://github.com/TencentARC/GFPGAN#european_castle-model-zoo\n",
        "!rm -rf postprocessed\n",
        "!python /content/GFPGAN/inference_gfpgan.py -i \"output.png\" -o postprocessed -v 1.4 -s 2 --bg_upsampler realesrgan\n",
        "post_image=Image.open('/content/postprocessed/restored_imgs/output.png')\n",
        "post_image\n",
        "# Usage: python inference_gfpgan.py -i inputs/whole_imgs -o results -v 1.3 -s 2 [options]...\n",
        "# \n",
        "#  -h                   show this help\n",
        "#  -i input             Input image or folder. Default: inputs/whole_imgs\n",
        "#  -o output            Output folder. Default: results\n",
        "#  -v version           GFPGAN model version. Option: 1 | 1.2 | 1.3. Default: 1.3\n",
        "#  -s upscale           The final upsampling scale of the image. Default: 2\n",
        "#  -bg_upsampler        background upsampler. Default: realesrgan\n",
        "#  -bg_tile             Tile size for background sampler, 0 for no tile during testing. Default: 400\n",
        "#  -suffix              Suffix of the restored faces\n",
        "#  -only_center_face    Only restore the center face\n",
        "#  -aligned             Input are aligned faces\n",
        "#  -ext                 Image extension. Options: auto | jpg | png, auto means using the same extension as inputs. Default: auto"
      ],
      "metadata": {
        "id": "CXEaygOrqB6t"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}