006_whisper_voice_recognition_stable_diffusion.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMSGuct6JIGUoN+tr77KP9v",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/karaage0703/stable-diffusion-colab-tools/blob/main/006_whisper_voice_recognition_stable_diffusion.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Whisper Voice Recognition Stable Diffusion\n",
        "\n",
        "Reference:\n",
        "- https://gist.github.com/tam17aki/8bfa2a42dab0061ee2641aed32dd1d30\n",
        "- https://zenn.dev/karaage0703/articles/d47bbb085fcb83"
      ],
      "metadata": {
        "id": "GqdXIRWRFqQE"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Record audio from microphone"
      ],
      "metadata": {
        "id": "8JY_bSVwHRFS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Javascript\n",
        "from google.colab import output\n",
        "from base64 import b64decode\n",
        "\n",
        "RECORD = \"\"\"\n",
        "const sleep = time => new Promise(resolve => setTimeout(resolve, time))\n",
        "const b2text = blob => new Promise(resolve => {\n",
        "  const reader = new FileReader()\n",
        "  reader.onloadend = e => resolve(e.srcElement.result)\n",
        "  reader.readAsDataURL(blob)\n",
        "})\n",
        "var record = time => new Promise(async resolve => {\n",
        "  stream = await navigator.mediaDevices.getUserMedia({ audio: true })\n",
        "  recorder = new MediaRecorder(stream)\n",
        "  chunks = []\n",
        "  recorder.ondataavailable = e => chunks.push(e.data)\n",
        "  recorder.start()\n",
        "  await sleep(time)\n",
        "  recorder.onstop = async ()=>{\n",
        "    blob = new Blob(chunks)\n",
        "    text = await b2text(blob)\n",
        "    resolve(text)\n",
        "  }\n",
        "  recorder.stop()\n",
        "})\n",
        "\"\"\"\n",
        "\n",
        "def record(sec, filename='audio.wav'):\n",
        "  display(Javascript(RECORD))\n",
        "  s = output.eval_js('record(%d)' % (sec * 1000))\n",
        "  b = b64decode(s.split(',')[1])\n",
        "  with open(filename, 'wb+') as f:\n",
        "    f.write(b)"
      ],
      "metadata": {
        "id": "jMPHkWeTFtwg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "audiofile = \"input.wav\"\n",
        "second = 3\n",
        "print(f\"Speak to your microphone {second} sec...\")\n",
        "record(second, audiofile)\n",
        "print(\"Done!\")"
      ],
      "metadata": {
        "id": "qrDeVjHFF3RY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Ignore \"UserWarning: PySoundFile failed. Trying audioread instead.\"\n",
        "import librosa\n",
        "import librosa.display\n",
        "speech, rate = librosa.load(audiofile, sr=16000)\n",
        "librosa.display.waveplot(speech, sr=rate)"
      ],
      "metadata": {
        "id": "gUs8ZQPpF5FM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Voice recognition Test\n",
        "\n",
        "Install Whisper"
      ],
      "metadata": {
        "id": "pRHByARuHKkX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -qq git+https://github.com/openai/whisper.git"
      ],
      "metadata": {
        "id": "UHv14rrUGeDB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Test Whisper"
      ],
      "metadata": {
        "id": "rOxy1YM_Msjg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import whisper\n",
        "whisper_model = whisper.load_model('small')\n",
        "#whisper_model = whisper.load_model('base')\n",
        "\n",
        "result = whisper_model.transcribe('input.wav', verbose=True, language='ja', task='translate')\n",
        "print(result['text'])"
      ],
      "metadata": {
        "id": "DvRvH0vLGnDR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Stable Diffusion Test"
      ],
      "metadata": {
        "id": "FACrHoBXIoyM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Hugging Face Login"
      ],
      "metadata": {
        "id": "B_bfXPh2I5TV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip -qq install diffusers==0.6.0\n",
        "!pip -qq install transformers\n",
        "!pip install -qq tqdm\n",
        "\n",
        "from huggingface_hub import notebook_login\n",
        "notebook_login()"
      ],
      "metadata": {
        "id": "vhvZkhOwG-HK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Setup"
      ],
      "metadata": {
        "id": "_twkhZFsJBe7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from diffusers import StableDiffusionPipeline\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "fig = plt.figure(figsize=(10,10))\n",
        "\n",
        "device = \"cuda\"\n",
        "model_id = \"CompVis/stable-diffusion-v1-4\"\n",
        "\n",
        "pipe = StableDiffusionPipeline.from_pretrained(\n",
        "    model_id,\n",
        "    revision=\"fp16\",\n",
        "    torch_dtype=torch.float16,\n",
        "    use_auth_token=True,\n",
        ").to(device)"
      ],
      "metadata": {
        "id": "AZcCCD8XI3m2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generate Image"
      ],
      "metadata": {
        "id": "QsEKBLpRJmYI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = result['text']\n",
        "seed_number = 42\n",
        "num_inference_steps  = 20\n",
        "guidance_scale_value = 7.5\n",
        "width_image = 512\n",
        "height_image = 512\n",
        "\n",
        "def infer(prompt, seed_number, num_inference_steps, guidance_scale_value, width_image, height_image):\n",
        "    generator = torch.Generator(device=device)\n",
        "    latents = None\n",
        "\n",
        "    # Get a new random seed, store it and use it as the generator state\n",
        "    if seed_number < 0:\n",
        "        seed = generator.seed()\n",
        "    else:\n",
        "        seed = seed_number\n",
        "\n",
        "    generator = generator.manual_seed(seed)\n",
        "\n",
        "    image_latent = torch.randn(\n",
        "        (1, pipe.unet.in_channels, height_image // 8, width_image // 8),\n",
        "        generator = generator,\n",
        "        device = device\n",
        "    )\n",
        "\n",
        "    with torch.autocast('cuda'):\n",
        "        image = pipe(\n",
        "            [prompt],\n",
        "            width=width_image,\n",
        "            height=height_image,\n",
        "            guidance_scale=guidance_scale_value,\n",
        "            num_inference_steps=num_inference_steps,\n",
        "            latents = image_latent\n",
        "        ).images[0]\n",
        "\n",
        "    return image\n",
        "\n",
        "def draw_image(image):\n",
        "    fig = plt.figure(figsize=(10,10))\n",
        "    plt.imshow(image)\n",
        "    plt.axis('off')\n",
        "    plt.show()\n",
        "\n",
        "image = infer(prompt, seed_number, num_inference_steps, guidance_scale_value, width_image, height_image)\n",
        "\n",
        "draw_image(image)"
      ],
      "metadata": {
        "id": "pwcVCyWbJrvm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Whisper Voice Recognition Stable Diffusion"
      ],
      "metadata": {
        "id": "DBbpugADKC3S"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Voice Recognition"
      ],
      "metadata": {
        "id": "58TeTmMsKJOY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "audiofile = \"input.wav\"\n",
        "second = 3\n",
        "print(f\"Speak to your microphone {second} sec...\")\n",
        "record(second, audiofile)\n",
        "print(\"Done!\")\n",
        "\n",
        "result = whisper_model.transcribe('input.wav', verbose=True, language='ja', task='translate')\n",
        "print(result['text'])"
      ],
      "metadata": {
        "id": "1vkTlBlRKKP4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title **Whisper Voice Recognition Stable Diffusion**\n",
        "#@markdown　Enter Parameter  (Attention: Seed=-1 is random)\n",
        "\n",
        "prompt = result['text']\n",
        "seed_number = 42 #@param\n",
        "num_inference_steps  = 20 #@param {type:\"slider\", min:1, max:200, step:1}\n",
        "guidance_scale_value = 7.5 #@param {type:\"slider\", min:1, max:20, step:0.1}\n",
        "width_image = 512 #@param {type:\"slider\", min:60, max:640, step:8}\n",
        "height_image = 512 #@param {type:\"slider\", min:60, max:640, step:8}\n",
        "\n",
        "image = infer(prompt, seed_number, num_inference_steps, guidance_scale_value, width_image, height_image)\n",
        "\n",
        "draw_image(image)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "rfrDEZ2kJMgr"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}