iPERCore_demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mJqF1PJ1as5T"
      },
      "source": [
        "論文  \n",
        "https://arxiv.org/abs/2011.09055  \n",
        "\n",
        "Dataset  \n",
        "https://svip-lab.github.io/dataset/iPER_dataset.html  \n",
        "\n",
        "GitHub  \n",
        "https://github.com/iPERDance/iPERCore  \n",
        "  \n",
        "<a href=\"https://colab.research.google.com/github/kaz12tech/ai_demos/blob/master/iPERCore_demo.ipynb\" target=\"_blank\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B6PSygeBas5a"
      },
      "source": [
        "# ランタイムの設定\n",
        "「ランタイム」→「ランタイムのタイプを変更」→「ハードウェアアクセラレータ」をGPUに変更"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I-JciHcTas5a"
      },
      "source": [
        "# 実行方法\n",
        "「ランタイム」→「すべてのセルを実行」を選択"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-eFOArQbbRdg"
      },
      "source": [
        "# 環境セットアップ"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E-2MNtZkaw6k"
      },
      "source": [
        "## GPUの確認"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F_bcY1Adayeh"
      },
      "outputs": [],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IWVVKJNpa1gd"
      },
      "source": [
        "## ライブラリのインストール"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lG-soU1wavvA"
      },
      "outputs": [],
      "source": [
        "!pip install moviepy==0.2.3.5 imageio==2.4.1\n",
        "!pip install yt-dlp"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nGzLzZrubpY2"
      },
      "source": [
        "## ライブラリのインポート"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ppuUJbo7bsst"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import os.path as osp\n",
        "import platform\n",
        "import argparse\n",
        "import time\n",
        "import sys\n",
        "import subprocess\n",
        "\n",
        "from yt_dlp import YoutubeDL\n",
        "from google.colab import files\n",
        "from moviepy.editor import *\n",
        "from moviepy.video.fx.resize import resize"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PYzRbTRXbWl-"
      },
      "source": [
        "## 環境変数設定"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oytRV4rDa_6W"
      },
      "outputs": [],
      "source": [
        "!ls /usr/local/ | grep cuda\n",
        "\n",
        "# CUDA_HOMEにcuda-11のパスを設定\n",
        "os.environ[\"CUDA_HOME\"] = \"/usr/local/cuda-11\"\n",
        "\n",
        "!echo $CUDA_HOME"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DrE8J4tvcO9H"
      },
      "source": [
        "# iPERCoreのセットアップ"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j4GF_TfCbvU0"
      },
      "source": [
        "## GitHubからコードを取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4BSjUmwIbx9y"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/iPERDance/iPERCore.git\n",
        "\n",
        "# Commits on Oct 31, 2022\n",
        "%cd /content/iPERCore\n",
        "!git checkout 61f7e086316d7f64779be75d3a771da4e1e4d441"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3bDg1VIsb7Pr"
      },
      "source": [
        "## ライブラリのインストール"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T6_AnjRab-0u"
      },
      "outputs": [],
      "source": [
        "%cd /content/iPERCore/\n",
        "\n",
        "# cuda11.2以上の場合cuda11.1の設定でセットアップ\n",
        "!sed -i '158i \\    version = version if version != 11.2 else 11.1' setup.py\n",
        "\n",
        "!python setup.py develop"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MQ1_x64JccbY"
      },
      "source": [
        "## 学習済みモデルのダウンロード"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8GnWPBekcejK"
      },
      "outputs": [],
      "source": [
        "# Download all checkpoints\n",
        "#!wget -O assets/checkpoints.zip \"https://1drv.ws/u/s!AjjUqiJZsj8whLkwQyrk3W9_H7MzNA?e=rRje0G\"\n",
        "!wget -O assets/checkpoints.zip \"https://download.impersonator.org/iper_plus_plus_latest_checkpoints.zip\"\n",
        "!unzip -o assets/checkpoints.zip -d assets/\n",
        "\n",
        "!rm assets/checkpoints.zip"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7SQ27jyrcjnR"
      },
      "source": [
        "## サンプルデータのダウンロード"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VednQhMmcmLJ"
      },
      "outputs": [],
      "source": [
        "%cd /content/iPERCore\n",
        "\n",
        "# download samples\n",
        "!wget -O assets/samples.zip  \"http://101.32.75.151:10086/samples.zip\"\n",
        "!unzip -o assets/samples.zip -d  assets\n",
        "!rm assets/samples.zip\n",
        "\n",
        "# download checkpoints\n",
        "!wget -O assets/checkpoints.zip \"http://101.32.75.151:10086/checkpoints.zip\"\n",
        "!unzip -o assets/checkpoints.zip -d assets/\n",
        "\n",
        "!rm assets/checkpoints.zip"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V7Hq0WX8cxK-"
      },
      "source": [
        "# Motion Imitation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-iRzZAehdF4v"
      },
      "source": [
        "## configの詳細\n",
        "  --gpu_ids（str）：gpu_ids、デフォルトは「0」です  \n",
        "  --image_size（int）：画像サイズ。デフォルトは512です  \n",
        "  --num_source（int）：Attentionのソースイメージの数。デフォルトは2です。Largeにはより多くのGPUメモリが必要です  \n",
        "  --assets_dir（str）：アセットディレクトリ。configurationとすべての事前トレーニング済みチェックポイントが必要  \n",
        "  --output_dir（str）：出力ディレクトリ  \n",
        "  --src_path（str）：ソース入力情報  \n",
        "\n",
        "[使用画像](https://livedoor.blogimg.jp/chargomannsei/imgs/6/6/66e10295-s.jpg)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u7xM-5lRmAZX"
      },
      "outputs": [],
      "source": [
        "%cd /content/iPERCore\n",
        "!rm -rf images\n",
        "!mkdir images\n",
        "\n",
        "\n",
        "#@markdown  front_only:正面画像のみ、front_back:正面、背面画像、fron_back_bg:正面、背面、背景画像\n",
        "model_type = \"front_only\"  #@param [\"front_only\", \"front_back\", \"front_back_bg\"] {allow-input: false}\n",
        "\n",
        "gpu_ids = \"0\"\n",
        "#@markdown 出力画像サイズ\n",
        "image_size = 512 #@param {type:\"integer\"}\n",
        "num_source = 2\n",
        "assets_dir = \"/content/iPERCore/assets\"\n",
        "output_dir = \"/content/iPERCore/results\"\n",
        "cfg_path = \"/content/iPERCore/assets/configs/deploy.toml\"\n",
        "\n",
        "# model設定\n",
        "model_id = \"donald_trump_2\"\n",
        "if model_type == \"front_back\":\n",
        "  model_id = \"axing_1\"\n",
        "elif model_type == \"front_back_bg\":\n",
        "  model_id = \"afan_6=ns=2\"\n",
        "\n",
        "%cd /content/iPERCore/images\n",
        "# src画像アップロード\n",
        "if model_type == \"front_only\":\n",
        "  print(\"正面画像をアップロードしてください\")\n",
        "  front_img = files.upload()\n",
        "  front_img = list(front_img.keys())\n",
        "\n",
        "  front_img_full_path = os.path.join(\"/content/iPERCore/images\", front_img[0])\n",
        "  src_path = \"\\\"path?=\" + front_img_full_path + \",name?=front_only\\\"\"\n",
        "elif model_type == \"front_back\":\n",
        "  print(\"正面画像をアップロードしてください\")\n",
        "  front_img = files.upload()\n",
        "  front_img = list(front_img.keys())\n",
        "  print(\"背面画像をアップロードしてください\")\n",
        "  back_img = files.upload()\n",
        "  back_img = list(back_img.keys())\n",
        "\n",
        "  src_path = \"\\\"path?=/content/iPERCore/images,name?=front_back\\\"\"\n",
        "elif model_type == \"front_back_bg\":\n",
        "  print(\"正面画像をアップロードしてください\")\n",
        "  front_img = files.upload()\n",
        "  front_img = list(front_img.keys())\n",
        "  print(\"背面画像をアップロードしてください\")\n",
        "  back_img = files.upload()\n",
        "  back_img = list(back_img.keys())\n",
        "  print(\"背景画像をアップロードしてください\")\n",
        "  bg_img = files.upload()\n",
        "  bg_img = list(bg_img.keys())\n",
        "\n",
        "  bg_img_full_path = os.path.join(\"/content/iPERCore/images\", bg_img[0])\n",
        "  src_path = \"\\\"path?=/content/iPERCore/images,\" \\\n",
        "             \"name?=front_back_bg,\" \\\n",
        "             \"bg_path?=\" + bg_img_full_path + \"\\\"\"\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gHJBIN8UD68k"
      },
      "outputs": [],
      "source": [
        "%cd /content/iPERCore\n",
        "\n",
        "!rm -rf videos\n",
        "!mkdir videos\n",
        "\n",
        "video_url = 'https://www.youtube.com/watch?v=VSSPwzSflr0' #@param {type:\"string\"}\n",
        "\n",
        "#@markdown 動画の切り抜き範囲(秒)を指定してください。\\\n",
        "#@markdown 30秒以上の場合OOM発生の可能性が高いため注意\n",
        "start_sec =  3#@param {type:\"integer\"}\n",
        "end_sec =  10#@param {type:\"integer\"}\n",
        "\n",
        "(start_pt, end_pt) = (start_sec, end_sec)\n",
        "\n",
        "download_resolution = 360\n",
        "full_video_path = '/content/iPERCore/videos/full_video.mp4'\n",
        "subclip_path = '/content/iPERCore/videos/subclip.mp4'\n",
        "video_name = 'subclip'\n",
        "\n",
        "# 動画ダウンロード\n",
        "ydl_opts = {'format': f'best[height<={download_resolution}]', 'overwrites': True, 'outtmpl': full_video_path}\n",
        "with YoutubeDL(ydl_opts) as ydl:\n",
        "    ydl.download([video_url])\n",
        "\n",
        "# 指定区間切り抜き\n",
        "with VideoFileClip(full_video_path) as video:\n",
        "    subclip = video.subclip(start_pt, end_pt)\n",
        "    subclip.write_videofile(subclip_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rJ60_TrX_TuD"
      },
      "outputs": [],
      "source": [
        "ref_path = \"\\\"path?=\" + subclip_path + \",\" \\\n",
        "             \"name?=\" + video_name + \",\" \\\n",
        "             \"pose_fc?=400\\\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u5kb2pJU8d3Z"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "!rm -rf {output_dir}\n",
        "\n",
        "%cd /content/iPERCore/\n",
        "\n",
        "!python -m iPERCore.services.run_imitator  \\\n",
        "  --gpu_ids     $gpu_ids       \\\n",
        "  --num_source  $num_source    \\\n",
        "  --image_size  $image_size    \\\n",
        "  --output_dir  $output_dir    \\\n",
        "  --model_id    $model_id      \\\n",
        "  --cfg_path    $cfg_path      \\\n",
        "  --src_path    $src_path      \\\n",
        "  --ref_path    $ref_path"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GpbAm97rB7X3"
      },
      "outputs": [],
      "source": [
        "%cd /content/iPERCore/\n",
        "results_video = \"/content/iPERCore/results/primitives/\" + model_type + \"/synthesis/imitations/\" + model_type + \"-\" + video_name + \".mp4\"\n",
        "\n",
        "clip = VideoFileClip(results_video)\n",
        "clip = resize(clip, height=420)\n",
        "clip.ipython_display()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "B6PSygeBas5a",
        "I-JciHcTas5a",
        "-eFOArQbbRdg",
        "E-2MNtZkaw6k",
        "IWVVKJNpa1gd",
        "nGzLzZrubpY2",
        "PYzRbTRXbWl-",
        "j4GF_TfCbvU0",
        "3bDg1VIsb7Pr",
        "MQ1_x64JccbY",
        "7SQ27jyrcjnR"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 0
}