Skip to content

Commit

Permalink
fix the bug: when the part of face is out of image...roughly
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiangshaocheng
xiangshaocheng committed Feb 20, 2019
1 parent 09fcd64 commit 98e10b6
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 102 deletions.
4 changes: 2 additions & 2 deletions face_detect_and_track.py
View file
Expand Up @@ -15,11 +15,11 @@ def __init__(self):
def face_detection(self,frame)->np.ndarray:
face_rects = self.__get_face_rects(frame)
if not isinstance(face_rects, np.ndarray):
logging.info("No face")
logging.debug("No face")
return 0

if not len(face_rects) == 1:
logging.info("Too many face")
logging.error("Too many face")
return 0

return face_rects[0]
Expand Down
3 changes: 2 additions & 1 deletion face_swap.py
View file
Expand Up @@ -5,7 +5,7 @@
import argparse
import numpy as np
import scipy.spatial as spatial

import logging
## 3D Transform
def bilinear_interpolate(img, coords):
""" Interpolates over every image channel
Expand Down Expand Up @@ -59,6 +59,7 @@ def process_warp(src_img, result_img, tri_affines, dst_points, delaunay):
x, y = coords.T
result_img[y, x] = bilinear_interpolate(src_img, out_coords)


return None

def triangular_affine_matrices(vertices, src_points, dst_points):
Expand Down
136 changes: 37 additions & 99 deletions video_handler.py
View file
Expand Up @@ -8,7 +8,6 @@
from face_detect_and_track import *
from face_points_detection import *
from face_swap import *
logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(lineno)d:%(message)s")


class VideoHandler(object):
Expand All @@ -33,8 +32,7 @@ def set_src_img(self,img_path):
def process_src_img(self):
src_face = self.detector.face_detection(self.src_img)
if isinstance(src_face, int):
logging.error("No face in src img")
exit(0)
raise Exception("No face detected in src image!")
src_face_rect=self.box_to_dlib_rect(src_face)
self.src_points=face_points_detection(self.src_img,src_face_rect)
src_mask = mask_from_points(self.src_img.shape[:2],self. src_points)
Expand All @@ -56,41 +54,45 @@ def fast_face_swap(self,dst_img,dst_face_rect:dlib.rectangle):
warped_src_img = warp_image_3d(self.src_img, self.src_points[:48], dst_points[:48],(h,w) )
src_only_face = correct_colours(dst_only_face, warped_src_img, dst_points)

r = cv2.boundingRect(dst_mask)
center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)))
center=tuple(dst_points[33])

output = cv2.seamlessClone(warped_src_img, dst_img, dst_mask, center, cv2.NORMAL_CLONE)
return output


def slow_face_swap(self,dst_img,dst_face_rect:dlib.rectangle):
dst_points=face_points_detection(dst_img,dst_face_rect) #4ms
w,h = dst_img.shape[:2]
t0=time.time()
warped_dst_img = warp_image_3d(dst_img, dst_points[:48], self.src_points[:48], self.src_only_face.shape[:2]) #140ms
t1=time.time()
logging.info(t1-t0)
self.src_only_face = correct_colours(warped_dst_img, self.src_only_face, self.src_points)
warped_src_img = warp_image_3d(self.src_only_face, self.src_points[:48], dst_points[:48], (w, h))
dst_mask = mask_from_points((w, h), dst_points)
r = cv2.boundingRect(dst_mask)
center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)))
output = cv2.seamlessClone(warped_src_img, dst_img, dst_mask, center, cv2.NORMAL_CLONE)
return output
# For TEST
# def run_face_swap(self,dst_img,dst_face_bbox):
# dst_points=face_points_detection(dst_img,dst_face_bbox)
# for (point_index,point) in enumerate(dst_points):
# # pt_pos=(point.x,point.y)
#
# cv2.circle(dst_img, (point[0],point[1]), 2, (0,0,255), -1)
# cv2.putText(dst_img,str(point_index),(point[0],point[1]),cv2.FONT_HERSHEY_SIMPLEX,0.35,(0,0,255),1)
# return dst_img

# For DEBUG
def draw_landmarks(self,dst_img,dst_face_bbox):
dst_points=face_points_detection(dst_img,dst_face_bbox)
for (point_index,point) in enumerate(dst_points):
# pt_pos=(point.x,point.y)

cv2.circle(dst_img, (point[0],point[1]), 2, (0,0,255), -1)
cv2.putText(dst_img,str(point_index),(point[0],point[1]),cv2.FONT_HERSHEY_SIMPLEX,0.35,(0,0,255),1)


def box_to_dlib_rect(self,bbox):
(x,y,w,h)=[int(v) for v in bbox]
dlib_rect=dlib.rectangle(x,y,x+w,y+h)
return dlib_rect

def _check_face_rect(self,face_rect,h,w):
if face_rect.left() > 0 and face_rect.top()>0 and face_rect.right()<w and face_rect.height()<h:
return True
else :
return False

def cascade_vh(self):
face_flag = 0
target_lose_cnt = 0
Expand All @@ -117,7 +119,7 @@ def cascade_vh(self):
if target_lose_cnt < self.lose_threshold:
(success, box_predict) = self.tracker.update_track(frame)
if not success:
logging.info("update failed")
logging.debug("update failed")
target_lose_cnt += 1
cv2.imshow("output", frame)
continue
Expand Down Expand Up @@ -154,93 +156,29 @@ def cascade_vh(self):
continue

face_rect:dlib.rectangle = self.box_to_dlib_rect(face_bbox)
frame = self.run_face_swap(frame, face_rect)
# frame_roi=frame[y:y+h,x:x+w]
# face_rect = self.box_to_dlib_rect(face_bbox)
# frame_roi = self.run_face_swap(frame_roi, face_rect)

end_tc = cv2.getTickCount()
fps = cv2.getTickFrequency() / (end_tc - start_tc)
logging.info("fps {}".format(fps))
cv2.imshow("frame", frame)
# Notice that dlib could extract the face's landmarks even we did not provide full face.
# It's a pretty rough solution. Maybe there is a method with more accurate
# if not self._check_face_rect(face_rect,*frame.shape[:2]):
# logging.info("face out frame")
# continue
# TODO: I still can't solve the problem when part of the face is out of picture, even # I add function "_check_face_rect(self,face_rect,h,w)".
try:
frame=self.run_face_swap(frame, face_rect)
except :
continue
else:
end_tc = cv2.getTickCount()
fps = cv2.getTickFrequency() / (end_tc - start_tc)
logging.info("fps {}".format(fps))
cv2.imshow("frame", frame)

cv2.destroyAllWindows()
self.cap.release()


def dlib_vh(self):
face_flag = 0
target_lose_cnt = 0
while (cv2.waitKey(1) != 27):
start_tc = cv2.getTickCount()
grabbed, frame = self.cap.read()
if not face_flag:
face_bbox = self.detector.face_detection(frame)
# No face has been detected
if isinstance(face_bbox, int):
cv2.imshow("frame", frame)
continue
# detect successfully
else:
logging.info("tracker init")
face_flag = 1
face_bbox = self.expand_bbox(*face_bbox)
self.tracker.start_track(frame, face_bbox)
target_lose_cnt = 0
(x, y, w, h) = face_bbox
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)

else:
# track
if target_lose_cnt < self.lose_threshold:
(success, box_predict) = self.tracker.update_track(frame)
if not success:
logging.info("update failed")
target_lose_cnt += 1
cv2.imshow("frame", frame)
continue
else:
(old_x, old_y, old_w, old_h) = (int(v) for v in box_predict)

# logging.debug("box_predict shape ({},{})".format(box_predict[2],box_predict[3]))

# maxY=np.min([old_y+old_h,frame.shape[0]])
# maxX=np.min([old_x+old_w,frame.shape[1]])
# bbox = d.cascade_method(frame[old_y:maxY,old_x:maxX])

# draw predict rect
cv2.rectangle(frame, (old_x, old_y), (old_x + old_w, old_y + old_h), (0, 0, 255), 2)
face_bbox = self.detector.face_detection(frame[old_y:old_y + old_h, old_x:old_x + old_w])
if isinstance(face_bbox, int):
target_lose_cnt += 1
cv2.imshow("frame", frame)
continue
target_lose_cnt = 0
(x, y, w, h) = self.expand_bbox(*face_bbox)
x = int(old_x + x)
y = int(old_y + y)
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)

face_bbox[0] = x
face_bbox[1] = y
face_rect = self.box_to_dlib_rect(face_bbox)
# lose target
else:
logging.info("losing target")
# reset tracker
self.tracker = Tracker()
face_flag = 0
continue
face_rect = self.box_to_dlib_rect(face_bbox)
frame = self.run_face_swap(frame, face_rect)
cv2.imshow("frame", frame)

end_tc = cv2.getTickCount()
fps = cv2.getTickFrequency() / (end_tc - start_tc)
logging.info("fps {}".format(fps))
cv2.destroyAllWindows()
self.cap.release()
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(lineno)d:%(message)s")

video_path = "D:/1120/Git/Other/PracticalPythonAndOpenCV_CaseStudies-master/Chapter03/video/adrian_face.mov"
test=VideoHandler()
test.set_src_img("D:/1120/Git/Mywork/FaceSwap/imgs/test7.jpg")
Expand Down

0 comments on commit 98e10b6

Please sign in to comment.