Lecture 7.2 - Merging Faces.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import dlib\n",
    "import numpy \n",
    "from time import sleep\n",
    "import sys\n",
    "\n",
    "PREDICTOR_PATH = \"shape_predictor_68_face_landmarks.dat\"\n",
    "SCALE_FACTOR = 1\n",
    "FEATHER_AMOUNT = 11\n",
    "\n",
    "FACE_POINTS = list(range(17, 68))\n",
    "MOUTH_POINTS = list(range(48, 61))\n",
    "RIGHT_BROWN_POINTS = list(range(17, 22))\n",
    "LEFT_BROWN_POINTS = list(range(22, 27))\n",
    "RIGHT_EYE_POINTS = list(range(36, 42))\n",
    "LEFT_EYE_POINTS = list(range(42, 48))\n",
    "NOSE_POINTS = list(range(27, 34))\n",
    "JAW_POINTS = list(range(0, 17))\n",
    "\n",
    "# Point used up to line up the images.\n",
    "ALIGN_POINTS = (LEFT_BROWN_POINTS + RIGHT_EYE_POINTS + LEFT_EYE_POINTS + RIGHT_BROWN_POINTS + NOSE_POINTS + MOUTH_POINTS)\n",
    "\n",
    "# Points from the second image to overlay on the first. The convex hull of each element will be overlaid\n",
    "OVERLAY_POINTS = [LEFT_BROWN_POINTS + RIGHT_EYE_POINTS + LEFT_EYE_POINTS + RIGHT_BROWN_POINTS, \n",
    "                  RIGHT_BROWN_POINTS + NOSE_POINTS + MOUTH_POINTS,]\n",
    "\n",
    "# Amount of blur to use during color correction, as a fraction of a pupillary distance\n",
    "COLOUR_CORRECT_BLUR_FRAC = 0.6\n",
    "\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "predictor = dlib.shape_predictor(PREDICTOR_PATH)\n",
    "\n",
    "\n",
    "class TooManyFaces(Exception):\n",
    "    pass\n",
    "class NoFaces(Exception):\n",
    "    pass\n",
    "def get_landmarks(im):\n",
    "    # Return facial landmarks as (x,y) coordinates\n",
    "    rects = detector(im, 1)\n",
    "    \n",
    "    if len(rects) > 1:\n",
    "        raise TooManyFaces\n",
    "    if len(rects) == 0:\n",
    "        raise NoFaces\n",
    "        \n",
    "    return numpy.matrix([[p.x, p.y] for p in predictor(im, rects[0]).parts()])\n",
    "\n",
    "def annotate_landmarks(im, landmarks):\n",
    "    # Overlay the landmark points on the image itself\n",
    "    \n",
    "    im = im.copy()\n",
    "    for idx, point in enumerate(landmarks):\n",
    "        pos = (point[0, 0], point[0, 1])\n",
    "        cv2.putText(im, str(idx), pos, \n",
    "                   fontFace = cv2.FONT_HERSHEY_SCRIPT_SIMPLEX,\n",
    "                   fontScale = 0.4, \n",
    "                   color = (0, 0, 255))\n",
    "        cv2.circle(im, pos, 3, color=(0, 255, 255))\n",
    "        \n",
    "    return im\n",
    "\n",
    "def draw_convex_hull(im, points, color):\n",
    "    points = cv2.convexHull(points)\n",
    "    cv2.fillConvexPoly(im, points, color = color)\n",
    "    \n",
    "    \n",
    "def get_face_mask(im, landmarks):\n",
    "    im = numpy.zeros(im.shape[:2], dtype=numpy.float64)\n",
    "\n",
    "    for group in OVERLAY_POINTS:\n",
    "        draw_convex_hull(im,\n",
    "                         landmarks[group],\n",
    "                         color=1)\n",
    "\n",
    "    im = numpy.array([im, im, im]).transpose((1, 2, 0))\n",
    "\n",
    "    im = (cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0) * 1.0\n",
    "    im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)\n",
    "\n",
    "    return im\n",
    "\n",
    "def transformation_from_points(points1, points2):\n",
    "    \"\"\"\n",
    "    Return an affine transformation [s * R | T] such that:\n",
    "        sum || s * R *p1 + T - p2, i || ^2\n",
    "        is minimized  \n",
    "    \"\"\"\n",
    "    \n",
    "    # Solve the procrustes problem by subtracting centroids, scaling by the\n",
    "    # standard deviation, and then using the SVD to calculate the rotation. See\n",
    "    # the following for more details:\n",
    "    # https://en.wikipedia.org/wiki/Orthogonal_Procrustes_problem\n",
    "    \n",
    "    points1 = points1.astype(numpy.float64)\n",
    "    points2 = points2.astype(numpy.float64)\n",
    "    \n",
    "    c1 = numpy.mean(points1, axis=0)\n",
    "    c2 = numpy.mean(points2, axis=0)\n",
    "    \n",
    "    points1 -= c1\n",
    "    points2 -= c2\n",
    "    \n",
    "    s1 = numpy.std(points1)\n",
    "    s2 = numpy.std(points2)\n",
    "    \n",
    "    points1 /= s1\n",
    "    points2 /= s2\n",
    "    \n",
    "    U, S, Vt = numpy.linalg.svd(points1.T * points2)\n",
    "    \n",
    "    # The R we seek is in fact the transpose of the one given by U * Vt. This\n",
    "    # is because the above formulation assumes the matrix goes on the right\n",
    "    # (with row vectors) where as our solution requires the matrix to be on the\n",
    "    # left (with column vectors). \n",
    "    \n",
    "    R = (U * Vt).T\n",
    "    \n",
    "    return numpy.vstack([numpy.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T)), \n",
    "                        numpy.matrix([0., 0., 1.])])\n",
    "\n",
    "\n",
    "def read_im_and_landmarks(image):\n",
    "    \n",
    "    im = image\n",
    "    im = cv2.resize(im, None, fx=1, fy=1, interpolation = cv2.INTER_LINEAR)\n",
    "    im = cv2.resize(im, (im.shape[1] * SCALE_FACTOR, im.shape[0] * SCALE_FACTOR))\n",
    "    \n",
    "    s = get_landmarks(im)\n",
    "    \n",
    "    return im, s\n",
    "\n",
    "def warp_im(im, M, dshape):\n",
    "    output_im = numpy.zeros(dshape, dtype=im.dtype)\n",
    "    cv2.warpAffine(im, M[:2], (dshape[1], dshape[0]), \n",
    "                  dst=output_im,\n",
    "                  borderMode= cv2.BORDER_TRANSPARENT,\n",
    "                  flags = cv2.WARP_INVERSE_MAP)\n",
    "    \n",
    "    return output_im\n",
    "\n",
    "def correct_colours(im1, im2, landmarks1):\n",
    "    blur_amount = COLOUR_CORRECT_BLUR_FRAC * numpy.linalg.norm(\n",
    "                                                numpy.mean(landmarks1[LEFT_EYE_POINTS], axis=0) -\n",
    "                                                numpy.mean(landmarks1[RIGHT_EYE_POINTS], axis=0))\n",
    "    blur_amount = int(blur_amount)\n",
    "    \n",
    "    if blur_amount % 2 == 0:\n",
    "        blur_amount += 1\n",
    "        \n",
    "    im1_blur = cv2.GaussianBlur(im1, (blur_amount, blur_amount), 0)\n",
    "    im2_blur = cv2.GaussianBlur(im2, (blur_amount, blur_amount), 0)\n",
    "    \n",
    "    # Avoid divide_by_zero_errors.\n",
    "    im2_blur += (128 * (im2_blur <= 1.0)).astype(im2_blur.dtype)\n",
    "    \n",
    "    return (im2.astype(numpy.float64) * im1_blur.astype(numpy.float64) / im2_blur.astype(numpy.float64))\n",
    "\n",
    "\n",
    "def swappy(image1, image2):\n",
    "    \n",
    "    im1, landmarks1 = read_im_and_landmarks(image1)\n",
    "    im2, landmarks2 = read_im_and_landmarks(image2)\n",
    "    \n",
    "    M = transformation_from_points(landmarks1[ALIGN_POINTS], landmarks2[ALIGN_POINTS])\n",
    "    \n",
    "    mask = get_face_mask(im2, landmarks2)\n",
    "    warped_mask = warp_im(mask, M, im1.shape)\n",
    "    combined_mask = numpy.max([get_face_mask(im1, landmarks1), warped_mask], axis=0)\n",
    "    \n",
    "    warped_im2 = warp_im(im2, M, im1.shape)\n",
    "    warped_corrected_im2 = correct_colours(im1, warped_im2, landmarks1)\n",
    "    \n",
    "    output_im = im1 * (1.0 - combined_mask) + warped_corrected_im2 * combined_mask\n",
    "    \n",
    "    cv2.imwrite('output.jpg', output_im)\n",
    "    image = cv2.imread('output.jpg')\n",
    "    return image\n",
    "\n",
    "# Enter the paths to your input images here\n",
    "image1 = cv2.imread('images/Hillary.jpg')\n",
    "image2 = cv2.imread('images/Trump.jpg')\n",
    "\n",
    "swapped = swappy(image1, image2)\n",
    "cv2.imshow('Face Swap 1', swapped)\n",
    "\n",
    "swapped = swappy(image2, image1)\n",
    "cv2.imshow('Face Swap 2', swapped)\n",
    "\n",
    "cv2.waitKey(0)\n",
    "\n",
    "cv2.destroyAllWindows()\n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[22, 23, 24, 25, 26]\n"
     ]
    }
   ],
   "source": [
    "print list(range(22, 27))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}