ganozli.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ganozli.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/eyaler/avatars4all/blob/master/ganozli.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gjODWEXeSm3i"
      },
      "source": [
        "# Demo for paper \"Liquid Warping GAN: A Unified Framework for Human Motion Imitation, Appearance Transfer and Novel View Synthesis\"\n",
        "\n",
        "### Made just a little bit more accessible by Eyal Gruss (https://eyalgruss.com, eyalgruss@gmail.com)\n",
        "\n",
        "##### Original project: https://svip-lab.github.io/project/impersonator\n",
        "\n",
        "##### Original notebook: https://colab.research.google.com/github/svip-lab/impersonator/blob/master/impersonator.ipynb\n",
        "\n",
        "##### Impersonator++: https://colab.research.google.com/drive/1bwUnj-9NnJA2EMr7eWO4I45UuBtKudg_\n",
        "\n",
        "#### **Stuff I made**:\n",
        "##### Avatars4all repository: https://github.com/eyaler/avatars4all\n",
        "##### Notebook for live webcam in the browser: https://colab.research.google.com/github/eyaler/avatars4all/blob/master/fomm_live.ipynb\n",
        "##### Notebook for talking head model: https://colab.research.google.com/github/eyaler/avatars4all/blob/master/fomm_bibi.ipynb\n",
        "##### Notebook for full body models (FOMM): https://colab.research.google.com/github/eyaler/avatars4all/blob/master/fomm_fufu.ipynb\n",
        "##### Notebook for full body models (impersonator): https://colab.research.google.com/github/eyaler/avatars4all/blob/master/ganozli.ipynb\n",
        "##### Notebook for full body models (impersonator++): https://colab.research.google.com/github/eyaler/avatars4all/blob/master/ganivut.ipynb\n",
        "##### Notebook for Wav2Lip audio based lip syncing: https://colab.research.google.com/github/eyaler/avatars4all/blob/master/melaflefon.ipynb\n",
        "##### List of more generative tools: https://j.mp/generativetools"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TAoK6jAHL1sT",
        "cellView": "form"
      },
      "source": [
        "#@title Setup\n",
        "%cd /content\n",
        "!git clone --depth 1 https://github.com/eyaler/impersonator\n",
        "!pip install DriveDownloader\n",
        "import os\n",
        "if not os.path.exists('/content/impersonator/assets/pretrains.zip'):\n",
        "  !ddl --filename /content/impersonator/assets/pretrains.zip 'https://1drv.ms/u/s!AjjUqiJZsj8whLNw4QyntCMsDKQjSg?e=L77Elv'\n",
        "if not os.path.exists('/content/impersonator/assets/samples.zip'):\n",
        "  !ddl --filename /content/impersonator/assets/samples.zip 'https://1drv.ms/u/s!AjjUqiJZsj8whLNz4BqnSgqrVwAXoQ?e=bC86db'\n",
        "if not os.path.exists('/content/impersonator/outputs/checkpoints.zip'):\n",
        "  !ddl --filename /content/impersonator/outputs/checkpoints.zip 'https://1drv.ms/u/s!AjjUqiJZsj8whLNyoEh67Uu0LlxquA?e=dkOnhQ'\n",
        "!pip install -U youtube-dl\n",
        "!pip install imageio==2.9.0\n",
        "!pip install imageio-ffmpeg==0.4.5\n",
        "!pip install torch==1.3.0\n",
        "!pip install torchvision==0.5.0\n",
        "!pip install visdom==0.1.8.9\n",
        "!pip install ipdb==0.12.2\n",
        "%cd /content/impersonator/thirdparty/neural_renderer\n",
        "!python setup.py install &> /dev/null\n",
        "!unzip -qq -u /content/impersonator/assets/pretrains.zip -d /content/impersonator/assets\n",
        "!unzip -qq -u /content/impersonator/assets/samples.zip -d /content/impersonator/assets\n",
        "!unzip -qq -u /content/impersonator/outputs/checkpoints.zip -d /content/impersonator/outputs\n",
        "%cd /content/impersonator"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "75NDsf50Rmfr",
        "cellView": "form"
      },
      "source": [
        "#@title Get the Driver video and Avatar image from the web\n",
        "#@markdown 1. You can change the URLs to your **own** stuff!\n",
        "#@markdown 2. Alternatively, you can upload **local** files in the next cells\n",
        " \n",
        "video_url = 'https://www.youtube.com/watch?v=NdEPmitJvaA' #@param {type:\"string\"}\n",
        "image_url = 'https://i.pinimg.com/564x/74/9e/96/749e96293399ebe1b8c99d89be522383.jpg' #@param {type:\"string\"}\n",
        " \n",
        "if video_url:\n",
        "  !rm -f /content/video.mp4\n",
        "  !youtube-dl --no-playlist -f \"bestvideo[ext=mp4][vcodec!*=av01][height<=360]+bestaudio[ext=m4a]/mp4[height<=360][vcodec!*=av01]/mp4[vcodec!*=av01]/mp4\" \"$video_url\" --merge-output-format mp4 -o /content/video\n",
        "  !mv /content/video.mp4 /content/video \n",
        " \n",
        "if image_url:\n",
        "  !wget \"$image_url\" -O /content/image"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UASpOYyDRpW-",
        "cellView": "form"
      },
      "source": [
        "#@title Optionally upload local Driver video { run: \"auto\" }\n",
        "manually_upload_video = False #@param {type:\"boolean\"}\n",
        "if manually_upload_video:\n",
        "  from google.colab import files\n",
        "  import shutil\n",
        "\n",
        "  %cd /content/sample_data\n",
        "  try:\n",
        "    uploaded = files.upload()\n",
        "  except Exception as e:\n",
        "    %cd /content\n",
        "    raise e\n",
        "\n",
        "  for fn in uploaded:\n",
        "    shutil.move('/content/sample_data/'+fn, '/content/video')\n",
        "    break\n",
        "  %cd /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qb8oP4hlRsHn",
        "cellView": "form"
      },
      "source": [
        "#@title Optionally upload local Avatar image { run: \"auto\" }\n",
        "manually_upload_image = False #@param {type:\"boolean\"}\n",
        "if manually_upload_image:\n",
        "  from google.colab import files\n",
        "  import shutil\n",
        " \n",
        "  %cd /content/sample_data\n",
        "  try:\n",
        "    uploaded = files.upload()\n",
        "  except Exception as e:\n",
        "    %cd /content\n",
        "    raise e\n",
        "\n",
        "  for fn in uploaded:\n",
        "    shutil.move('/content/sample_data/'+fn, '/content/image')\n",
        "    break\n",
        "  %cd /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gJQL5gh_Rvz3",
        "cellView": "form"
      },
      "source": [
        "#@title Optionally shorten Driver video\n",
        "start_seconds = 0 #@param {type:\"number\"}\n",
        "duration_seconds =  60#@param {type:\"number\"}\n",
        "start_seconds = max(start_seconds,0)\n",
        "duration_seconds = max(duration_seconds,0)\n",
        " \n",
        "if duration_seconds: \n",
        "  !mv /content/video /content/full_video\n",
        "  !ffmpeg -ss $start_seconds -t $duration_seconds -i /content/full_video -f mp4 /content/video -y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SuyGfkQ3R0GH",
        "cellView": "form"
      },
      "source": [
        "#@title Prepare assets\n",
        "#@markdown If you ran out of RAM this means that the video is too large. You can shorten it above.\n",
        " \n",
        "center_video_to_body = True #@param {type:\"boolean\"}\n",
        "crop_video_to_body = True #@param {type:\"boolean\"}\n",
        "video_crop_expansion_factor = 1.05 #@param {type:\"number\"}\n",
        "center_image_to_body = True #@param {type:\"boolean\"}\n",
        "crop_image_to_body = False #@param {type:\"boolean\"}\n",
        "image_crop_expansion_factor = 1.05 #@param {type:\"number\"}\n",
        "video_crop_expansion_factor = max(video_crop_expansion_factor, 1)\n",
        "image_crop_expansion_factor = max(image_crop_expansion_factor, 1)\n",
        " \n",
        "import imageio\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.animation as animation\n",
        "from skimage.transform import resize\n",
        "from IPython.display import HTML, clear_output\n",
        "import cv2\n",
        "import shutil\n",
        "import os\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        " \n",
        "hog = cv2.HOGDescriptor()\n",
        "hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())\n",
        " \n",
        "def fix_dims(im):\n",
        "    if im.ndim == 2:\n",
        "        im = np.tile(im[..., None], [1, 1, 3])\n",
        "    return im[...,:3]\n",
        " \n",
        "def get_crop(im, center_body=True, crop_body=True, expansion_factor=1, rects=None):\n",
        "    im = fix_dims(im)\n",
        "    if (center_body or crop_body) and rects is None:\n",
        "        rects, _ = hog.detectMultiScale(im, winStride=(4, 4),padding=(8,8), scale=expansion_factor)\n",
        "    if (center_body or crop_body) and rects is not None and len(rects):\n",
        "        x0,y0,w,h = sorted(rects, key=lambda x: x[2]*x[3])[-1]\n",
        "        if crop_body:\n",
        "            x0 += w//2-h//2\n",
        "            x1 = x0+h\n",
        "            y1 = y0+h\n",
        "        else:\n",
        "            img_h,img_w = im.shape[:2]\n",
        "            x0 += (w-img_h)//2\n",
        "            x1 = x0+img_h\n",
        "            y0 = 0\n",
        "            y1 = img_h\n",
        "    else:\n",
        "        h,w = im.shape[:2]\n",
        "        x0 = (w-h)//2\n",
        "        x1 = (w+h)//2\n",
        "        y0 = 0\n",
        "        y1 = h\n",
        "    return int(x0),int(x1),int(y0),int(y1)\n",
        " \n",
        "def pad_crop_resize(im, x0=None, x1=None, y0=None, y1=None, new_h=256, new_w=256):\n",
        "    im = fix_dims(im)\n",
        "    h,w = im.shape[:2]\n",
        "    if x0 is None:\n",
        "      x0 = 0\n",
        "    if x1 is None:\n",
        "      x1 = w\n",
        "    if y0 is None:\n",
        "      y0 = 0\n",
        "    if y1 is None:\n",
        "      y1 = h\n",
        "    if x0<0 or x1>w or y0<0 or y1>h:\n",
        "        im = np.pad(im, pad_width=[(max(-y0,0),max(y1-h,0)),(max(-x0,0),max(x1-w,0)),(0,0)], mode='edge')\n",
        "    return resize(im[max(y0,0):y1-min(y0,0),max(x0,0):x1-min(x0,0)], (new_h, new_w))\n",
        " \n",
        " \n",
        "source_image = imageio.imread('/content/image')\n",
        "source_image = pad_crop_resize(source_image, *get_crop(source_image, center_body=center_image_to_body, crop_body=crop_image_to_body, expansion_factor=image_crop_expansion_factor))\n",
        "imageio.imwrite('/content/crop.png', (source_image*255).astype(np.uint8))\n",
        "\n",
        "shutil.rmtree('/content/images', ignore_errors=True)\n",
        "os.makedirs('/content/images')\n",
        "with imageio.get_reader('/content/video', format='mp4') as reader:\n",
        "  fps = reader.get_meta_data()['fps']\n",
        " \n",
        "  driving_video = []\n",
        "  rects = None\n",
        "  try:\n",
        "      for i,im in enumerate(reader):\n",
        "          if not crop_video_to_body:\n",
        "              break\n",
        "          rects, _ = hog.detectMultiScale(im, winStride=(4, 4),padding=(8,8), scale=video_crop_expansion_factor)\n",
        "          if rects is not None and len(rects):\n",
        "              break\n",
        "      x0,x1,y0,y1 = get_crop(im, center_body=center_video_to_body, crop_body=crop_video_to_body, expansion_factor=video_crop_expansion_factor, rects=rects)\n",
        "      reader.set_image_index(0)\n",
        "      for j,im in enumerate(reader):\n",
        "          driving_video.append(pad_crop_resize(im,x0,x1,y0,y1))\n",
        "          imageio.imwrite(os.path.join('/content/images','%05d.jpg'%j), (driving_video[-1]*255).astype(np.uint8))\n",
        "  except RuntimeError:\n",
        "      pass\n",
        " \n",
        "def vid_display(source, driving, generated=None):\n",
        "    fig = plt.figure(figsize=(8 + 4 * (generated is not None), 6))\n",
        " \n",
        "    ims = []\n",
        "    for i in range(len(driving)):\n",
        "        cols = [source]\n",
        "        cols.append(driving[i])\n",
        "        if generated is not None:\n",
        "            cols.append(generated[i])\n",
        "        im = plt.imshow(np.concatenate(cols, axis=1), animated=True)\n",
        "        plt.axis('off')\n",
        "        ims.append([im])\n",
        " \n",
        "    ani = animation.ArtistAnimation(fig, ims, interval=50, repeat_delay=1000)\n",
        "    plt.close()\n",
        "    return ani\n",
        " \n",
        "clear_output()\n",
        "if rects is not None and len(rects):\n",
        "    print('first found body in frame %d'%i)\n",
        "print('number of frames: %d'%j)\n",
        "HTML(vid_display(source_image, driving_video).to_html5_video())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UzEEAV_7w3xZ",
        "cellView": "form"
      },
      "source": [
        "#@title Animate\n",
        "\n",
        "bg_ks = 13 #@param {type:\"slider\", min:1, max:25, step:2}\n",
        "ft_ks = 3 #@param {type:\"slider\", min:1, max:25, step:2}\n",
        "has_detector = True #@param {type:\"boolean\"}\n",
        "post_tune = True #@param {type:\"boolean\"}\n",
        "front_warp = True #@param {type:\"boolean\"}\n",
        "cam_strategy = 'smooth' #@param ['smooth', 'source', 'copy'] {type:\"raw\"}\n",
        "\n",
        "run = 'python run_imitator.py --gpu_ids 0 --model imitator --output_dir ./outputs/results --src_path /content/crop.png --tgt_path /content/images --save_res --bg_ks %d --ft_ks %d %s %s %s --cam_strategy %s'%(\n",
        "    bg_ks, ft_ks, '--has_detector' if has_detector else '', '--post_tune' if post_tune else '', '--front_warp' if front_warp else '', cam_strategy)\n",
        "\n",
        "%cd /content/impersonator\n",
        "shutil.rmtree('./outputs/results', ignore_errors=True)\n",
        "!{run}\n",
        "!ffmpeg -f image2 -i /content/impersonator/outputs/results/imitators/gt_%05d.jpg -c:v libx264 -c:a aac -pix_fmt yuv420p -profile:v baseline -movflags +faststart out2.mp4 -y\n",
        "!ffmpeg -f image2 -i /content/impersonator/outputs/results/imitators/pred_%05d.jpg -c:v libx264 -c:a aac -pix_fmt yuv420p -profile:v baseline -movflags +faststart /content/output.mp4 -y\n",
        "#!ffmpeg -i out2.mp4 -filter_complex \"[0:v] palettegen\" palette.png -y\n",
        "#!ffmpeg -i out2.mp4 -i palette.png -filter_complex \"[0:v][1:v] paletteuse\" endresult.gif -y\n",
        "#!ffmpeg -i /content/output.mp4 -filter_complex \"[0:v] palettegen\" palette2.png -y\n",
        "#!ffmpeg -i /content/output.mp4 -i palette2.png -filter_complex \"[0:v][1:v] paletteuse\" endresult2.gif -y\n",
        "#!ffmpeg -i endresult.gif -i endresult2.gif -filter_complex hstack outputstack2.gif -y\n",
        "#!ffmpeg -i outputstack2.gif -f mp4 -c:v libx264 -c:a aac -pix_fmt yuv420p -profile:v baseline -movflags +faststart /content/combined.mp4\n",
        "!ffmpeg -i out2.mp4 -i /content/output.mp4 -filter_complex hstack -pix_fmt yuv420p -profile:v baseline -movflags +faststart /content/combined.mp4 -y\n",
        "#!ffmpeg -i /content/combined.mp4 -i /content/video -c copy -map 0:v -map 1:a? -pix_fmt yuv420p -profile:v baseline -movflags +faststart /content/final.mp4 -y\n",
        "\n",
        "from IPython.display import HTML, clear_output\n",
        "from base64 import b64encode\n",
        "\n",
        "clear_output()\n",
        "with open('/content/combined.mp4', 'rb') as f:\n",
        "  data_url = \"data:video/mp4;base64,\" + b64encode(f.read()).decode()\n",
        "display(HTML(\"\"\"\n",
        "<video width=600 controls autoplay loop>\n",
        "      <source src=\"%s\" type=\"video/mp4\">\n",
        "</video>\"\"\" % data_url))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tkvBCCF9R7NA",
        "cellView": "form"
      },
      "source": [
        "#@title Download\n",
        "#@markdown 1. If it fails try running this cell again.\n",
        "#@markdown 2. Alternatively, you can manually download \"output.mp4\", \"combined.mp4\" from the folder on the left (click \"Refresh\" if missing).\n",
        "\n",
        "print() #see https://github.com/googlecolab/colabtools/issues/468\n",
        "from google.colab import files\n",
        "files.download('/content/combined.mp4') #fails for Firefox private window\n",
        "files.download('/content/output.mp4')\n",
        "\n",
        "'''\n",
        "palette=\"palette.png\"\n",
        "filters=\"fps=15,scale=256:-1:flags=lanczos\"\n",
        "!ffmpeg -v warning -i /content/output.mp4 -vf \"$filters,palettegen=stats_mode=diff\" -y $palette\n",
        "!ffmpeg -v warning -i /content/output.mp4 -i $palette -lavfi \"$filters [x]; [x][1:v] paletteuse=dither=bayer:bayer_scale=5:diff_mode=rectangle\" -y /content/output.gif\n",
        "filters=\"fps=15,scale=512:-1:flags=lanczos\"\n",
        "!ffmpeg -v warning -i /content/combined.mp4 -vf \"$filters,palettegen=stats_mode=diff\" -y $palette\n",
        "!ffmpeg -v warning -i /content/combined.mp4 -i $palette -lavfi \"$filters [x]; [x][1:v] paletteuse=dither=bayer:bayer_scale=5:diff_mode=rectangle\" -y /content/combined.gif\n",
        "files.download('/content/combined.gif')\n",
        "files.download('/content/output.gif')\n",
        "'''\n",
        "pass"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}