syncnet-runner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Syncnet Implementation\n",
    "\n",
    "https://github.com/voletiv/syncnet-in-keras/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cv2, os, sys, numpy as np\n",
    "import scipy.io.wavfile as wav\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import speechpy\n",
    "import dlib\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mouth detection from https://github.com/voletiv/lipreading-in-the-wild-experiments/tree/master/process-lrw\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from process_lrw_functions import detect_mouth_in_frame, extract_audio_from_mp4\n",
    "from syncnet_functions import load_pretrained_syncnet_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_video_input(video):\n",
    "\n",
    "    detector = dlib.get_frontal_face_detector()\n",
    "    predictor = dlib.shape_predictor(\"shape_predictor_68_face_landmarks.dat\")\n",
    "\n",
    "    cap         = cv2.VideoCapture(video)\n",
    "    frameFPS    = int(cap.get(cv2.CAP_PROP_FPS))\n",
    "    frameCount  = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "    frameWidth  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "    frameHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "\n",
    "    print(\"FPS: {}\".format(frameFPS))\n",
    "    print(\"Frames: {}\".format(frameCount))\n",
    "    print(\"Width: {}\".format(frameWidth))\n",
    "    print(\"Height: {}\".format(frameHeight))\n",
    "\n",
    "    face = dlib.rectangle(30, 30, 220, 220)\n",
    "\n",
    "    lip_model_input = []\n",
    "\n",
    "    frame_count = 0\n",
    "\n",
    "    while(cap.isOpened()):\n",
    "\n",
    "        # If frames are extracted from video, all frames are read\n",
    "        frames = []\n",
    "        for i in range(5):\n",
    "        \n",
    "            # print(\"Frame\", frame_count+1, \"of\", frameCount, end=\"\\r\")\n",
    "            _, frame = cap.read()\n",
    "            frame_count += 1\n",
    "            if(frame is None):\n",
    "                break\n",
    "\n",
    "            mouth, face = detect_mouth_in_frame(\n",
    "                frame, detector, predictor,\n",
    "                prevFace=face,\n",
    "                verbose=False)\n",
    "\n",
    "            mouth = cv2.cvtColor(mouth, cv2.COLOR_BGR2GRAY) # convert to grayscale\n",
    "            mouth = cv2.resize( mouth, (112,112))\n",
    "            # mouth = mouth[:, :,0] \t# drop the RGB channel\n",
    "            frames.append(mouth)\n",
    "\n",
    "        if len(frames) == 5:\n",
    "            stacked = np.stack(frames, axis=-1)\t#syncnet requires (112,112,5)\n",
    "            # input(stacked.shape)\n",
    "            lip_model_input.append(stacked)\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    return np.array(lip_model_input)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MFCC code thanks to michiyosony \n",
    "\n",
    "https://github.com/voletiv/syncnet-in-keras/issues/1#issuecomment-380149724\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EACH_MFCC_OUTPUT_FRAME_SIZE = 20\n",
    "\n",
    "def extract_mfcc_series(wav_file, target_dir=None):\n",
    "    (rate, sig) = wav.read(wav_file)\n",
    "    print(\"Sig length: {}\".format(len(sig)))\n",
    "    try:\n",
    "        mfcc_feat = speechpy.feature.mfcc(sig, sampling_frequency=rate, frame_length=0.010, frame_stride=0.01)\n",
    "    except IndexError:\n",
    "        print(\"index error occurred while extracting mfcc\")\n",
    "        return\n",
    "    print('sample_rate: {}, mfcc_feat length: {}, mfcc_feat[0] length: {}'.format(rate, len(mfcc_feat), len(mfcc_feat[0])))\n",
    "    num_output = len(mfcc_feat) // EACH_MFCC_OUTPUT_FRAME_SIZE\n",
    "    \n",
    "    print(mfcc_feat.shape)\n",
    "    print(int(num_output))\n",
    "    images = []\n",
    "\n",
    "    for index in tqdm.tqdm(range(num_output)):\n",
    "        img = Image.new('RGB', (20, 13), \"black\")\n",
    "        pixels = img.load()\n",
    "        for i in range(img.size[0]):\n",
    "            for j in range(img.size[1]):\n",
    "                frame_index = index * EACH_MFCC_OUTPUT_FRAME_SIZE + i\n",
    "                # print(frame_index)\n",
    "                try:\n",
    "                    if mfcc_feat[frame_index][j] < 0:\n",
    "                        red_amount = min(255, 255 * (mfcc_feat[frame_index][j] / -20))\n",
    "                        pixels[i, j] = (int(red_amount), 0, 0)\n",
    "                    elif (mfcc_feat[frame_index][j] > 0):\n",
    "                        blue_amount = min(255, 255 * (mfcc_feat[frame_index][j] / 20))\n",
    "                        pixels[i, j] = (0, 0, int(blue_amount))\n",
    "                except IndexError:\n",
    "                    print(\"index error occurred while extracting mfcc @ \" + str(frame_index) + \",\" + str(j))\n",
    "                    break\n",
    "        # img.save(\"{}/mfcc_{:03d}.png\".format(target_dir, index), 'PNG')\n",
    "        img_to_np = np.array(img)\n",
    "\n",
    "        # Convert to grayscale\n",
    "        gray_image = cv2.cvtColor(img_to_np, cv2.COLOR_RGB2GRAY)\n",
    "        \n",
    "        # Expand an axis\n",
    "        gray_image_exp = np.expand_dims(gray_image, axis=-1)\n",
    "\n",
    "        images.append(gray_image_exp)\n",
    "\n",
    "    return np.asarray(images)\n",
    "\n",
    "\n",
    "def get_audio_input(video):\n",
    "    audio_out = \"{}.wav\".format(video)\n",
    "    cmd=\"ffmpeg -y -loglevel panic -i {} -acodec pcm_s16le -ac 1 -ar 16000 {}\".format(video, audio_out)\n",
    "    os.system(cmd)\n",
    "    return extract_mfcc_series(audio_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the inputs to the models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make sure video is of 25fps!**\n",
    "If not, use the following ffmpeg command to convert fps:\n",
    "\n",
    "```\n",
    "ffmpeg -i video.mp4 -r 25 -y video_at_25_fps\n",
    ".mp4\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_video_to_25_fps(video):\n",
    "    cmd=\"ffmpeg -i {} -r 25 -y tmp.mp4\".format(video)\n",
    "    os.system(cmd)\n",
    "    cmd=\"mv tmp.mp4 {}\".format(video)\n",
    "    os.system(cmd)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "video_to_test = \"test/unsynced.mp4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "convert_video_to_25_fps(video_to_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FPS: 25\n",
      "Frames: 184\n",
      "Width: 320\n",
      "Height: 240\n",
      "(36, 112, 112, 5)\n"
     ]
    }
   ],
   "source": [
    "lip_input = get_video_input(video_to_test)\n",
    "print(lip_input.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sig length: 117077\n",
      "sample_rate: 16000, mfcc_feat length: 731, mfcc_feat[0] length: 13\n",
      "(731, 13)\n",
      "36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 36/36 [00:00<00:00, 1284.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(36, 13, 20, 1)\n"
     ]
    }
   ],
   "source": [
    "audio_input = get_audio_input(video_to_test)\n",
    "print(audio_input.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "version = 'v4'\n",
    "mode = 'both'\n",
    "syncnet_audio_model, syncnet_lip_model = load_pretrained_syncnet_model(version=version, mode=mode, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1_audio (Conv2D)         (None, 13, 20, 64)        640       \n",
      "_________________________________________________________________\n",
      "bn1_audio (BatchNormalizatio (None, 13, 20, 64)        256       \n",
      "_________________________________________________________________\n",
      "relu1_audio (Activation)     (None, 13, 20, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2_audio (Conv2D)         (None, 13, 20, 128)       73856     \n",
      "_________________________________________________________________\n",
      "bn2_audio (BatchNormalizatio (None, 13, 20, 128)       512       \n",
      "_________________________________________________________________\n",
      "relu2_audio (Activation)     (None, 13, 20, 128)       0         \n",
      "_________________________________________________________________\n",
      "pool2_audio (MaxPooling2D)   (None, 11, 9, 128)        0         \n",
      "_________________________________________________________________\n",
      "conv3_audio (Conv2D)         (None, 11, 9, 256)        295168    \n",
      "_________________________________________________________________\n",
      "bn3_audio (BatchNormalizatio (None, 11, 9, 256)        1024      \n",
      "_________________________________________________________________\n",
      "relu3_audio (Activation)     (None, 11, 9, 256)        0         \n",
      "_________________________________________________________________\n",
      "conv4_audio (Conv2D)         (None, 11, 9, 256)        590080    \n",
      "_________________________________________________________________\n",
      "bn4_audio (BatchNormalizatio (None, 11, 9, 256)        1024      \n",
      "_________________________________________________________________\n",
      "relu4_audio (Activation)     (None, 11, 9, 256)        0         \n",
      "_________________________________________________________________\n",
      "conv5_audio (Conv2D)         (None, 11, 9, 256)        590080    \n",
      "_________________________________________________________________\n",
      "bn5_audio (BatchNormalizatio (None, 11, 9, 256)        1024      \n",
      "_________________________________________________________________\n",
      "relu5_audio (Activation)     (None, 11, 9, 256)        0         \n",
      "_________________________________________________________________\n",
      "pool5_audio (MaxPooling2D)   (None, 5, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten_audio (Flatten)      (None, 5120)              0         \n",
      "_________________________________________________________________\n",
      "fc6_audio (Dense)            (None, 256)               1310976   \n",
      "_________________________________________________________________\n",
      "bn6_audio (BatchNormalizatio (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "relu6_audio (Activation)     (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "fc7_audio (Dense)            (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "bn7_audio (BatchNormalizatio (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "relu7_audio (Activation)     (None, 128)               0         \n",
      "=================================================================\n",
      "Total params: 2,899,072\n",
      "Trainable params: 2,896,384\n",
      "Non-trainable params: 2,688\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(syncnet_audio_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1_lip (Conv2D)           (None, 110, 110, 96)      4416      \n",
      "_________________________________________________________________\n",
      "bn1_lip (BatchNormalization) (None, 110, 110, 96)      384       \n",
      "_________________________________________________________________\n",
      "relu1_lip (Activation)       (None, 110, 110, 96)      0         \n",
      "_________________________________________________________________\n",
      "pool1_lip (MaxPooling2D)     (None, 54, 54, 96)        0         \n",
      "_________________________________________________________________\n",
      "conv2_lip (Conv2D)           (None, 50, 50, 256)       614656    \n",
      "_________________________________________________________________\n",
      "bn2_lip (BatchNormalization) (None, 50, 50, 256)       1024      \n",
      "_________________________________________________________________\n",
      "relu2_lip (Activation)       (None, 50, 50, 256)       0         \n",
      "_________________________________________________________________\n",
      "pool2_lip (MaxPooling2D)     (None, 24, 24, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv3_lip (Conv2D)           (None, 22, 22, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "bn3_lip (BatchNormalization) (None, 22, 22, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu3_lip (Activation)       (None, 22, 22, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv4_lip (Conv2D)           (None, 20, 20, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "bn4_lip (BatchNormalization) (None, 20, 20, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu4_lip (Activation)       (None, 20, 20, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv5_lip (Conv2D)           (None, 18, 18, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "bn5_lip (BatchNormalization) (None, 18, 18, 512)       2048      \n",
      "_________________________________________________________________\n",
      "relu5_lip (Activation)       (None, 18, 18, 512)       0         \n",
      "_________________________________________________________________\n",
      "pool5_lip (MaxPooling2D)     (None, 6, 6, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten_lip (Flatten)        (None, 18432)             0         \n",
      "_________________________________________________________________\n",
      "fc6_lip (Dense)              (None, 256)               4718848   \n",
      "_________________________________________________________________\n",
      "bn6_lip (BatchNormalization) (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "relu6_lip (Activation)       (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "fc7_lip (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "bn7_lip (BatchNormalization) (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "relu7_lip (Activation)       (None, 128)               0         \n",
      "=================================================================\n",
      "Total params: 11,279,680\n",
      "Trainable params: 11,275,136\n",
      "Non-trainable params: 4,544\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(syncnet_lip_model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate embedding Euclidian distance to see if video / audio is synced\n",
    "\n",
    "1. Pass the audio frame through the audio model to get its encoding (a 128-dimensional feature), pass the video frame through the lip model to get its encoding (a 128-dimensional features)\n",
    "\n",
    "2. Check the euclidean distance between the audio encoding and the video encoding.\n",
    "\n",
    "3. If the distance is greater than a threshold (say, 0.6), then it is said the audio and video are not in sync."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(36, 128)\n"
     ]
    }
   ],
   "source": [
    "audio_embeddings = syncnet_audio_model.predict(audio_input)\n",
    "print(audio_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(36, 128)\n"
     ]
    }
   ],
   "source": [
    "lip_embeddings = syncnet_lip_model.predict(lip_input)\n",
    "print(lip_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def euclidian_distance(np_data_1, np_data_2): \n",
    "    \n",
    "    \n",
    "    if( np_data_1.shape != np_data_2.shape):\n",
    "        print(\"==> Dimensions don't match {} {}. Clipping\".format(np_data_1.shape, np_data_2.shape))\n",
    "        min_dim = min(np_data_1.shape[0],  np_data_2.shape[0])\n",
    "\n",
    "        np_data_1 = np_data_1[:min_dim,:]\n",
    "        np_data_2 = np_data_2[:min_dim,:]\n",
    "\n",
    "    dist = np.linalg.norm(np_data_1-np_data_2)\n",
    "    return dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "distance_float = euclidian_distance(audio_embeddings, lip_embeddings)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47.7346\n"
     ]
    }
   ],
   "source": [
    "print(distance_float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def euclidian_distance_N(np_data_1, np_data_2): \n",
    "    dist = np.sqrt( np.sum(np.square(np.subtract(np_data_1, np_data_2)), axis=-1) )\n",
    "    return dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 14.6083622    6.9940815    6.94163227   6.37897682   4.63438559\n",
      "   4.05189753   9.52804565   8.16981983   8.11581326   7.10538149\n",
      "   9.50084972   7.13821268   6.25484085   4.89169073   3.47015381\n",
      "   4.1924243    6.65998745  11.94016838   9.45641804  10.38359356\n",
      "  10.20247841   7.65069389   7.0541501    7.41728783   9.06472588\n",
      "   5.98884869   7.62462854   6.5692277    4.89220715  11.58408165\n",
      "   7.99294329  11.91477299   7.73058414   4.51798201   6.656991\n",
      "   4.72928667]\n"
     ]
    }
   ],
   "source": [
    "distance_np = euclidian_distance_N(audio_embeddings, lip_embeddings)\n",
    "\n",
    "print(distance_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******************************\n",
      "test/bad-dub-01.mp4\n",
      "FPS: 25\n",
      "Frames: 37\n",
      "Width: 320\n",
      "Height: 240\n",
      "(7, 112, 112, 5)\n",
      "Sig length: 22528\n",
      "sample_rate: 16000, mfcc_feat length: 140, mfcc_feat[0] length: 13\n",
      "(140, 13)\n",
      "7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:00<00:00, 999.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7, 13, 20, 1)\n",
      "(7, 128)\n",
      "(7, 128)\n",
      "Distance: 22.37862777709961\n",
      "******************************\n",
      "test/bad-dub-02.mp4\n",
      "FPS: 25\n",
      "Frames: 69\n",
      "Width: 320\n",
      "Height: 240\n",
      "(13, 112, 112, 5)\n",
      "Sig length: 43008\n",
      "sample_rate: 16000, mfcc_feat length: 268, mfcc_feat[0] length: 13\n",
      "(268, 13)\n",
      "13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13/13 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13, 13, 20, 1)\n",
      "(13, 128)\n",
      "(13, 128)\n",
      "Distance: 29.282217025756836\n",
      "******************************\n",
      "test/bad-dub-03.mp4\n",
      "FPS: 25\n",
      "Frames: 134\n",
      "Width: 320\n",
      "Height: 240\n",
      "(26, 112, 112, 5)\n",
      "Sig length: 84651\n",
      "sample_rate: 16000, mfcc_feat length: 529, mfcc_feat[0] length: 13\n",
      "(529, 13)\n",
      "26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 26/26 [00:00<00:00, 1299.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(26, 13, 20, 1)\n",
      "(26, 128)\n",
      "(26, 128)\n",
      "Distance: 43.84505844116211\n",
      "******************************\n",
      "test/bad-dub-04.mp4\n",
      "FPS: 25\n",
      "Frames: 89\n",
      "Width: 320\n",
      "Height: 240\n",
      "(17, 112, 112, 5)\n",
      "Sig length: 56320\n",
      "sample_rate: 16000, mfcc_feat length: 352, mfcc_feat[0] length: 13\n",
      "(352, 13)\n",
      "17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17/17 [00:00<00:00, 3342.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17, 13, 20, 1)\n",
      "(17, 128)\n",
      "(17, 128)\n",
      "Distance: 35.79926300048828\n",
      "******************************\n",
      "test/bad-dub-05.mp4\n",
      "FPS: 25\n",
      "Frames: 42\n",
      "Width: 320\n",
      "Height: 240\n",
      "(8, 112, 112, 5)\n",
      "Sig length: 25941\n",
      "sample_rate: 16000, mfcc_feat length: 162, mfcc_feat[0] length: 13\n",
      "(162, 13)\n",
      "8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8, 13, 20, 1)\n",
      "(8, 128)\n",
      "(8, 128)\n",
      "Distance: 25.937734603881836\n",
      "******************************\n",
      "test/bad-dub-06.mp4\n",
      "FPS: 25\n",
      "Frames: 107\n",
      "Width: 320\n",
      "Height: 240\n",
      "(21, 112, 112, 5)\n",
      "Sig length: 67925\n",
      "sample_rate: 16000, mfcc_feat length: 424, mfcc_feat[0] length: 13\n",
      "(424, 13)\n",
      "21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 21/21 [00:00<00:00, 671.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21, 13, 20, 1)\n",
      "(21, 128)\n",
      "(21, 128)\n",
      "Distance: 35.53982925415039\n",
      "******************************\n",
      "test/bad-dub-07.mp4\n",
      "FPS: 25\n",
      "Frames: 119\n",
      "Width: 320\n",
      "Height: 240\n",
      "(23, 112, 112, 5)\n",
      "Sig length: 75776\n",
      "sample_rate: 16000, mfcc_feat length: 473, mfcc_feat[0] length: 13\n",
      "(473, 13)\n",
      "23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 23/23 [00:00<00:00, 717.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(23, 13, 20, 1)\n",
      "(23, 128)\n",
      "(23, 128)\n",
      "Distance: 40.302101135253906\n",
      "******************************\n",
      "test/bad-dub-multispeaker.mp4\n",
      "FPS: 25\n",
      "Frames: 545\n",
      "Width: 320\n",
      "Height: 240\n",
      "(109, 112, 112, 5)\n",
      "Sig length: 347477\n",
      "sample_rate: 16000, mfcc_feat length: 2171, mfcc_feat[0] length: 13\n",
      "(2171, 13)\n",
      "108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [00:00<00:00, 1298.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(108, 13, 20, 1)\n",
      "(108, 128)\n",
      "(109, 128)\n",
      "==> Dimensions don't match (108, 128) (109, 128). Clipping\n",
      "Distance: 81.50373077392578\n"
     ]
    }
   ],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "test_path=\"test/\"\n",
    "for f in listdir(test_path):\n",
    "    tfile=join(test_path, f)\n",
    "    if(isfile(tfile) and f.startswith(\"bad-dub\") and f.endswith(\".mp4\")):\n",
    "\n",
    "        print(\"*\" * 30)\n",
    "        print(tfile)\n",
    "        \n",
    "        convert_video_to_25_fps(tfile)\n",
    "\n",
    "        lip_input = get_video_input(tfile)\n",
    "        print(lip_input.shape)\n",
    "\n",
    "        audio_input = get_audio_input(tfile)\n",
    "        print(audio_input.shape)\n",
    "\n",
    "        audio_embeddings = syncnet_audio_model.predict(audio_input)\n",
    "        print(audio_embeddings.shape)\n",
    "\n",
    "        lip_embeddings = syncnet_lip_model.predict(lip_input)\n",
    "        print(lip_embeddings.shape)\n",
    "\n",
    "        distance_float = euclidian_distance(audio_embeddings, lip_embeddings)\n",
    "\n",
    "        print(\"Distance: {}\".format(distance_float))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}