MVIMP_DAIN_Colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "1_MVIMP_DAIN_ZOTIKUS_v1_3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "A-i-1UgMfqFI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Check your current GPU\n",
        "# If you are lucky, you get 16GB VRAM. If you are not lucky, you get less. VRAM is important. The more VRAM, the higher the maximum resolution will go.\n",
        "\n",
        "# P100: 16GB (Works)\n",
        "# T4: 16GB (Not tested)\n",
        "# P4: 8GB (Not tested)\n",
        "# K80: 8GB (Not tested)\n",
        "\n",
        "!nvidia-smi -L"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_tUWizBkyEd8",
        "colab_type": "text"
      },
      "source": [
        "# **DAIN**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f_7rgW8TCBEd",
        "colab_type": "text"
      },
      "source": [
        "## **`PREPARE VM`**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJFI55OEztwp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def audio_notification():\n",
        "  # Play an audio beep. Any audio URL will do.\n",
        "  from google.colab import output\n",
        "  for x in range(1, 4):\n",
        "    output.eval_js('new Audio(\"https://notificationsounds.com/notification-sounds/maybe-one-day-584/download/mp3\").play()')\n",
        "  print(\"Done!\")\n",
        "print(\"OK!\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pd8qYTOdmleG",
        "colab_type": "text"
      },
      "source": [
        "### Downgrade gcc"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2KqGGbiantll",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%writefile /etc/apt/sources.list\n",
        "# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to\n",
        "# newer versions of the distribution.\n",
        "deb http://archive.ubuntu.com/ubuntu/ bionic main restricted\n",
        "deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted\n",
        "deb http://archive.ubuntu.com/ubuntu/ bionic universe\n",
        "deb http://archive.ubuntu.com/ubuntu/ bionic-updates universe\n",
        "deb http://archive.ubuntu.com/ubuntu/ bionic multiverse\n",
        "deb http://archive.ubuntu.com/ubuntu/ bionic-updates multiverse\n",
        "deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted universe multiverse\n",
        "deb http://security.ubuntu.com/ubuntu/ bionic-security main restricted\n",
        "deb http://security.ubuntu.com/ubuntu/ bionic-security universe\n",
        "deb http://security.ubuntu.com/ubuntu/ bionic-security multiverse\n",
        "deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/\n",
        "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main\n",
        "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEIa1W9IXqfU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!gcc -dumpversion\n",
        "!g++ -dumpversion"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oq4gm1PDPByf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!sudo apt update\n",
        "!sudo apt install gcc-4.9\n",
        "!sudo apt install g++-4.9\n",
        "\n",
        "# Remove & overwrite any existing symbolic links to activate freshly installed gcc\n",
        "\n",
        "!rm /usr/bin/gcc\n",
        "!rm /usr/bin/g++\n",
        "\n",
        "!ln -s /usr/bin/gcc-4.9 /usr/bin/gcc\n",
        "!ln -s /usr/bin/g++-4.9 /usr/bin/g++\n",
        "\n",
        "!gcc -dumpversion\n",
        "!g++ -dumpversion"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "jlkMCuyMNrg8"
      },
      "source": [
        "### Downgrade Torch version"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "p-qcCNvFNrg8",
        "colab": {}
      },
      "source": [
        "!yes | pip uninstall torch torchvision\n",
        "\n",
        "!pip install torch==1.0.0 torchvision==0.2.1\n",
        "\n",
        "import torch\n",
        "print(torch.__version__)\n",
        "\n",
        "audio_notification()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L6O8GLO-Vnc4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "print(torch.__version__)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a5cLND7upZ8K",
        "colab_type": "text"
      },
      "source": [
        "### Downgrade to Cuda 9.0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N2H4FjjhpX7f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb\n",
        "!dpkg -i cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb\n",
        "!apt-key add /var/cuda-repo-9-0-local/7fa2af80.pub\n",
        "!apt-get update\n",
        "!apt-get install cuda=9.0.176-1\n",
        "!echo ****** Cuda reinstall completed. Restarting runtime now! *******\n",
        "\n",
        "audio_notification()\n",
        "\n",
        "import os\n",
        "os.kill(os.getpid(), 9)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2H_Bzn10pdP5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvcc --version"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_JM5tR2ACvRN",
        "colab_type": "text"
      },
      "source": [
        "## **`PREPARE DAIN`**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Chdta8Jl2hG",
        "colab_type": "text"
      },
      "source": [
        "### Download Repository\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4vf1LTvY9fry",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Download repository\n",
        "!git clone https://github.com/BrokenSilence/MVIMP.git"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zk4MBH9ToH3D",
        "colab_type": "text"
      },
      "source": [
        "### Adjust the default nvcc_args"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OHgae2YSoFVF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%writefile /content/MVIMP/third_party/DAIN/my_package/DepthFlowProjection/setup.py\n",
        "\n",
        "#!/usr/bin/env python3\n",
        "import os\n",
        "import torch\n",
        "\n",
        "from setuptools import setup, find_packages\n",
        "from torch.utils.cpp_extension import BuildExtension, CUDAExtension\n",
        "\n",
        "cxx_args = ['-std=c++11']\n",
        "nvcc_args = [\n",
        "    #'-gencode', 'arch=compute_30,code=sm_30', #K80\n",
        "    #'-gencode', 'arch=compute_37,code=sm_37', #K80\n",
        "    #'-gencode', 'arch=compute_50,code=sm_50',\n",
        "    #'-gencode', 'arch=compute_52,code=sm_52',\n",
        "    '-gencode', 'arch=compute_60,code=sm_60',  #P100\n",
        "    #'-gencode', 'arch=compute_61,code=sm_61'  #P4\n",
        "    # '-gencode', 'arch=compute_70,code=sm_70',\n",
        "    #'-gencode', 'arch=compute_70,code=compute_70'\n",
        "]\n",
        "\n",
        "setup(\n",
        "    name='depthflowprojection_cuda',\n",
        "    ext_modules=[\n",
        "        CUDAExtension('depthflowprojection_cuda', [\n",
        "            'depthflowprojection_cuda.cc',\n",
        "            'depthflowprojection_cuda_kernel.cu'\n",
        "        ], extra_compile_args={'cxx': cxx_args, 'nvcc': nvcc_args})\n",
        "    ],\n",
        "    cmdclass={\n",
        "        'build_ext': BuildExtension\n",
        "    })"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zQzOA_VFmWip",
        "colab_type": "text"
      },
      "source": [
        "### Build DAIN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SQ4XKrXuOFFY",
        "colab": {}
      },
      "source": [
        "# If you played around uninstall previously build/installed packages\n",
        "!pip uninstall mindepthflowprojection_cuda flowprojection_cuda separableconv_cuda depthflowprojection_cuda interpolationch_cuda interpolation_cuda separableconvflow-cuda filterinterpolation_cuda correlation_cuda -y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EF2BEdHck-CG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# This takes a while. Just wait. ~15 minutes.\n",
        "# Building DAIN.\n",
        "%cd /content/MVIMP/third_party/DAIN/my_package/\n",
        "!CUDA_VISIBLE_DEVICES=0\n",
        "!./build.sh\n",
        "print(\"Building #1 done.\")\n",
        "\n",
        "# Wait again. ~5 minutes.\n",
        "# Building DAIN PyTorch correlation package.\n",
        "%cd /content/MVIMP/third_party/DAIN/PWCNet/correlation_package_pytorch1_0\n",
        "!CUDA_VISIBLE_DEVICES=0\n",
        "!./build.sh\n",
        "print(\"Building #2 done.\")\n",
        "\n",
        "# Download pre-trained model\n",
        "%cd /content/MVIMP/third_party/DAIN\n",
        "!mkdir model_weights\n",
        "!wget -O model_weights/best.pth http://vllab1.ucmerced.edu/~wenbobao/DAIN/best.pth\n",
        "!CUDA_VISIBLE_DEVICES=0\n",
        "!sudo apt-get install imagemagick imagemagick-doc\n",
        "!sudo apt install zip unzip\n",
        "\n",
        "audio_notification()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EvSGQcofyBE6",
        "colab_type": "text"
      },
      "source": [
        "### Update OpenCV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6kL5S05IFVsg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import cv2\n",
        "cv2.__version__"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mpsgzQdZf4JC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Maybe not needed anymore, will have to check later.\n",
        "!pip3 install --upgrade opencv-python==4.2.0.34"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cfRrBzcT425W",
        "colab_type": "text"
      },
      "source": [
        "### Connect Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "V35cgSpEOFFj",
        "colab": {}
      },
      "source": [
        "# Connect Google Drive\n",
        "import os\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "directoryi = '/content/gdrive/My Drive/DAIN/Input'\n",
        "directoryo = '/content/gdrive/My Drive/DAIN/Output'\n",
        "\n",
        "if not os.path.exists(directoryi):\n",
        "    os.makedirs(directoryi)\n",
        "if not os.path.exists(directoryo):\n",
        "    os.makedirs(directoryo)\n",
        "\n",
        "print('Google Drive connected.')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DEW1h4nvW5Yu",
        "colab_type": "text"
      },
      "source": [
        "## **Using DAIN**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "GT7w5wKlaBMx"
      },
      "source": [
        "### Configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mUYJQ3s-W_zD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Configuration\n",
        "\n",
        "# >Input File\n",
        "VIDEO_FILE = 'test.mkv'\n",
        "\n",
        "# >Time Step \n",
        "# Set the frame multiplier, 0.5 corresponds to 2X, 0.25 corresponds to 4X, and 0.125 corresponds to 8X.\n",
        "TIMESTEP = 0.5\n",
        "\n",
        "\n",
        "# >High Resolution \n",
        "# 16GB VRAM: Can handle 720p with HIGH_RES turned off.\n",
        "# 8GB VRAM: Can handle 480p with HIGH_RES turned off.\n",
        "\n",
        "# Default is False.\n",
        "# Set to True to split a single frame into 4 blocks and process them separately in order to reduce GPU memory usage.\n",
        "HIGH_RES = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "042MdwLc0UL7",
        "colab_type": "text"
      },
      "source": [
        "### Load Input Files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QZLO1FrwXTMI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Copy input file from Google Drive into VM ( \"DAIN/Input\" folder needs to be created in your Google Drive )\n",
        "\n",
        "import shutil\n",
        "import os\n",
        "shutil.copy(os.path.join('/content/gdrive/My Drive/DAIN/Input', VIDEO_FILE), '/content/MVIMP/Data/Input')\n",
        "audio_notification()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X-_6osgA0QQa",
        "colab_type": "text"
      },
      "source": [
        "### Interpolate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ujv4QOzD8nzC",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "# Output video file will be created in your Google Drive folder: DAIN/Output\n",
        "\n",
        "import os\n",
        "os.chdir('/content/MVIMP')\n",
        "\n",
        "!python3 inference_dain.py --input_video {video_file} --time_step {TIMESTEP} --high_resolution {HIGH_RES}\n",
        "audio_notification()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}