Skip to content
Permalink
Browse files

changed help message for pixel loss:

Pixel loss may help to enhance fine details and stabilize face color. Use it only if quality does not improve over time.

SAE:
previous SAE model will not work with this update.
Greatly decreased chance of model collapse.
Increased model accuracy.
Residual blocks now default and this option has been removed.
Improved 'learn mask'.
Added masked preview (switch by space key)

Converter:
fixed rct/lct in seamless mode
added mask mode (6) learned*FAN-prd*FAN-dst

added mask editor, its created for refining dataset for FANSeg model, and not for production, but you can spend your time and test it in regular fakes with face obstructions
  • Loading branch information...
iperov committed Apr 4, 2019
1 parent 01e98cd commit 5ac7e5d7f1af12ad2c0d5f7039614629a274cf83
@@ -77,11 +77,11 @@ def __init__(self, predictor_func,
self.hist_match_threshold = np.clip ( io.input_int("Hist match threshold [0..255] (skip:255) : ", 255), 0, 255)

if face_type == FaceType.FULL:
self.mask_mode = np.clip ( io.input_int ("Mask mode: (1) learned, (2) dst, (3) FAN-prd, (4) FAN-dst , (5) FAN-prd&dst (?) help. Default - %d : " % (1) , 1, help_message="If you learned mask, then option 1 should be choosed. 'dst' mask is raw shaky mask from dst aligned images. 'FAN-prd' - using super smooth mask by pretrained FAN-model from predicted face. 'FAN-dst' - using super smooth mask by pretrained FAN-model from dst face. 'FAN-prd&dst' - using multiplied FAN prd and dst mask. "), 1, 5 )
self.mask_mode = np.clip ( io.input_int ("Mask mode: (1) learned, (2) dst, (3) FAN-prd, (4) FAN-dst , (5) FAN-prd*FAN-dst (6) learned*FAN-prd*FAN-dst (?) help. Default - %d : " % (1) , 1, help_message="If you learned mask, then option 1 should be choosed. 'dst' mask is raw shaky mask from dst aligned images. 'FAN-prd' - using super smooth mask by pretrained FAN-model from predicted face. 'FAN-dst' - using super smooth mask by pretrained FAN-model from dst face. 'FAN-prd*FAN-dst' or 'learned*FAN-prd*FAN-dst' - using multiplied masks."), 1, 6 )
else:
self.mask_mode = np.clip ( io.input_int ("Mask mode: (1) learned, (2) dst . Default - %d : " % (1) , 1), 1, 2 )

if self.mask_mode == 3 or self.mask_mode == 4 or self.mask_mode == 5:
if self.mask_mode >= 3 or self.mask_mode <= 6:
self.fan_seg = None

if self.mode != 'raw':
@@ -117,7 +117,7 @@ def on_host_tick(self):

#overridable
def on_cli_initialize(self):
if (self.mask_mode == 3 or self.mask_mode == 4 or self.mask_mode == 5) and self.fan_seg == None:
if (self.mask_mode >= 3 and self.mask_mode <= 6) and self.fan_seg == None:
self.fan_seg = FANSegmentator(256, FaceType.toString(FaceType.FULL) )

#override
@@ -167,26 +167,28 @@ def cli_convert_face (self, img_bgr, img_face_landmarks, debug):

if self.mask_mode == 2: #dst
prd_face_mask_a_0 = cv2.resize (dst_face_mask_a_0, (output_size,output_size), cv2.INTER_CUBIC)
elif self.mask_mode >= 3 and self.mask_mode <= 5:
elif self.mask_mode >= 3 and self.mask_mode <= 6:

if self.mask_mode == 3 or self.mask_mode == 5: #FAN-prd
if self.mask_mode == 3 or self.mask_mode == 5 or self.mask_mode == 6:
prd_face_bgr_256 = cv2.resize (prd_face_bgr, (256,256) )
prd_face_bgr_256_mask = self.fan_seg.extract_from_bgr( prd_face_bgr_256[np.newaxis,...] ) [0]
FAN_prd_face_mask_a_0 = cv2.resize (prd_face_bgr_256_mask, (output_size,output_size), cv2.INTER_CUBIC)

if self.mask_mode == 4 or self.mask_mode == 5: #FAN-dst
if self.mask_mode == 4 or self.mask_mode == 5 or self.mask_mode == 6:
face_256_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, 256, face_type=FaceType.FULL)
dst_face_256_bgr = cv2.warpAffine(img_bgr, face_256_mat, (256, 256), flags=cv2.INTER_LANCZOS4 )
dst_face_256_mask = self.fan_seg.extract_from_bgr( dst_face_256_bgr[np.newaxis,...] ) [0]
FAN_dst_face_mask_a_0 = cv2.resize (dst_face_256_mask, (output_size,output_size), cv2.INTER_CUBIC)

if self.mask_mode == 3:
if self.mask_mode == 3: #FAN-prd
prd_face_mask_a_0 = FAN_prd_face_mask_a_0
elif self.mask_mode == 4:
elif self.mask_mode == 4: #FAN-dst
prd_face_mask_a_0 = FAN_dst_face_mask_a_0
elif self.mask_mode == 5:
prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0

elif self.mask_mode == 6:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0

prd_face_mask_a_0[ prd_face_mask_a_0 < 0.001 ] = 0.0

prd_face_mask_a = prd_face_mask_a_0[...,np.newaxis]
@@ -269,11 +271,12 @@ def cli_convert_face (self, img_bgr, img_face_landmarks, debug):
img_mask_blurry_aaa = cv2.blur(img_mask_blurry_aaa, (blur, blur) )

img_mask_blurry_aaa = np.clip( img_mask_blurry_aaa, 0, 1.0 )
face_mask_blurry_aaa = cv2.warpAffine( img_mask_blurry_aaa, face_mat, (output_size, output_size) )

if debug:
debugs += [img_mask_blurry_aaa.copy()]

if self.color_transfer_mode is not None:
if 'seamless' not in self.mode and self.color_transfer_mode is not None:
if self.color_transfer_mode == 'rct':
if debug:
debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
@@ -309,15 +312,20 @@ def cli_convert_face (self, img_bgr, img_face_landmarks, debug):

if self.masked_hist_match:
hist_mask_a *= prd_face_mask_a

white = (1.0-hist_mask_a)* np.ones ( prd_face_bgr.shape[:2] + (1,) , dtype=np.float32)

hist_match_1 = prd_face_bgr*hist_mask_a + (1.0-hist_mask_a)* np.ones ( prd_face_bgr.shape[:2] + (1,) , dtype=np.float32)
hist_match_1 = prd_face_bgr*hist_mask_a + white
hist_match_1[ hist_match_1 > 1.0 ] = 1.0

hist_match_2 = dst_face_bgr*hist_mask_a + (1.0-hist_mask_a)* np.ones ( prd_face_bgr.shape[:2] + (1,) , dtype=np.float32)
hist_match_2 = dst_face_bgr*hist_mask_a + white
hist_match_2[ hist_match_1 > 1.0 ] = 1.0

prd_face_bgr = imagelib.color_hist_match(hist_match_1, hist_match_2, self.hist_match_threshold )


#if self.masked_hist_match:
# prd_face_bgr -= white

if self.mode == 'hist-match-bw':
prd_face_bgr = prd_face_bgr.astype(dtype=np.float32)

@@ -364,6 +372,35 @@ def cli_convert_face (self, img_bgr, img_face_landmarks, debug):

out_img = np.clip( img_bgr*(1-img_mask_blurry_aaa) + (out_img*img_mask_blurry_aaa) , 0, 1.0 )

if 'seamless' in self.mode and self.color_transfer_mode is not None:
out_face_bgr = cv2.warpAffine( out_img, face_mat, (output_size, output_size) )

if self.color_transfer_mode == 'rct':
if debug:
debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]

new_out_face_bgr = imagelib.reinhard_color_transfer ( np.clip( (out_face_bgr*255).astype(np.uint8), 0, 255),
np.clip( (dst_face_bgr*255).astype(np.uint8), 0, 255),
source_mask=face_mask_blurry_aaa, target_mask=face_mask_blurry_aaa)
new_out_face_bgr = np.clip( new_out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)

if debug:
debugs += [ np.clip( cv2.warpAffine( new_out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]


elif self.color_transfer_mode == 'lct':
if debug:
debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]

new_out_face_bgr = imagelib.linear_color_transfer (out_face_bgr, dst_face_bgr)
new_out_face_bgr = np.clip( new_out_face_bgr, 0.0, 1.0)

if debug:
debugs += [ np.clip( cv2.warpAffine( new_out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]

new_out = cv2.warpAffine( new_out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT )
out_img = np.clip( img_bgr*(1-img_mask_blurry_aaa) + (new_out*img_mask_blurry_aaa) , 0, 1.0 )

if self.mode == 'seamless-hist-match':
out_face_bgr = cv2.warpAffine( out_img, face_mat, (output_size, output_size) )
new_out_face_bgr = imagelib.color_hist_match(out_face_bgr, dst_face_bgr, self.hist_match_threshold)
@@ -4,6 +4,7 @@
from enum import IntEnum
import mathlib
import imagelib
from imagelib import IEPolys
from mathlib.umeyama import umeyama
from facelib import FaceType
import math
@@ -153,7 +154,7 @@ def transform_points(points, mat, invert=False):
return points


def get_image_hull_mask (image_shape, image_landmarks):
def get_image_hull_mask (image_shape, image_landmarks, ie_polys=None):
if len(image_landmarks) != 68:
raise Exception('get_image_hull_mask works only with 68 landmarks')
int_lmrks = np.array(image_landmarks, dtype=np.int)
@@ -198,6 +199,9 @@ def get_image_hull_mask (image_shape, image_landmarks):
#nose
cv2.fillConvexPoly( hull_mask, cv2.convexHull(int_lmrks[27:36]), (1,) )

if ie_polys is not None:
ie_polys.overlay_mask(hull_mask)

return hull_mask

def get_image_eye_mask (image_shape, image_landmarks):
@@ -211,11 +215,6 @@ def get_image_eye_mask (image_shape, image_landmarks):

return hull_mask

def get_image_hull_mask_3D (image_shape, image_landmarks):
result = get_image_hull_mask(image_shape, image_landmarks)

return np.repeat ( result, (3,), -1 )

def blur_image_hull_mask (hull_mask):

maxregion = np.argwhere(hull_mask==1.0)
@@ -235,9 +234,6 @@ def blur_image_hull_mask (hull_mask):

return hull_mask

def get_blurred_image_hull_mask(image_shape, image_landmarks):
return blur_image_hull_mask ( get_image_hull_mask(image_shape, image_landmarks) )

mirror_idxs = [
[0,16],
[1,15],
@@ -282,7 +278,7 @@ def mirror_landmarks (landmarks, val):
result[:,0] = val - result[:,0] - 1
return result

def draw_landmarks (image, image_landmarks, color=(0,255,0), transparent_mask=False):
def draw_landmarks (image, image_landmarks, color=(0,255,0), transparent_mask=False, ie_polys=None):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')

@@ -310,11 +306,11 @@ def draw_landmarks (image, image_landmarks, color=(0,255,0), transparent_mask=Fa
cv2.circle(image, (x, y), 2, color, lineType=cv2.LINE_AA)

if transparent_mask:
mask = get_image_hull_mask (image.shape, image_landmarks)
mask = get_image_hull_mask (image.shape, image_landmarks, ie_polys)
image[...] = ( image * (1-mask) + image * mask / 2 )[...]

def draw_rect_landmarks (image, rect, image_landmarks, face_size, face_type, transparent_mask=False, landmarks_color=(0,255,0) ):
draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask)
def draw_rect_landmarks (image, rect, image_landmarks, face_size, face_type, transparent_mask=False, ie_polys=None, landmarks_color=(0,255,0) ):
draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask, ie_polys=ie_polys)
imagelib.draw_rect (image, rect, (255,0,0), 2 )

image_to_face_mat = get_transform_mat (image_landmarks, face_size, face_type)
@@ -0,0 +1,104 @@
import numpy as np
import cv2

class IEPolysPoints:
def __init__(self, IEPolys_parent, type):
self.parent = IEPolys_parent
self.type = type
self.points = np.empty( (0,2), dtype=np.int32 )
self.n_max = self.n = 0

def add(self,x,y):
self.points = np.append(self.points[0:self.n], [ (x,y) ], axis=0)
self.n_max = self.n = self.n + 1
self.parent.dirty = True

def n_dec(self):
self.n = max(0, self.n-1)
self.parent.dirty = True
return self.n

def n_inc(self):
self.n = min(len(self.points), self.n+1)
self.parent.dirty = True
return self.n

def n_clip(self):
self.points = self.points[0:self.n]
self.n_max = self.n

def cur_point(self):
return self.points[self.n-1]

def points_to_n(self):
return self.points[0:self.n]

def set_points(self, points):
self.points = np.array(points)
self.n_max = self.n = len(points)
self.parent.dirty = True

class IEPolys:
def __init__(self):
self.list = []
self.n_max = self.n = 0
self.dirty = True

def add(self, type):
self.list = self.list[0:self.n]
self.list.append ( IEPolysPoints(self, type) )
self.n_max = self.n = self.n + 1
self.dirty = True

def n_dec(self):
self.n = max(0, self.n-1)
self.dirty = True
return self.n

def n_inc(self):
self.n = min(len(self.list), self.n+1)
self.dirty = True
return self.n

def n_list(self):
return self.list[self.n-1]

def n_clip(self):
self.list = self.list[0:self.n]
self.n_max = self.n
if self.n > 0:
self.list[-1].n_clip()

def __iter__(self):
for n in range(self.n):
yield self.list[n]

def switch_dirty(self):
d = self.dirty
self.dirty = False
return d

def overlay_mask(self, mask):
h,w,c = mask.shape
white = (1,)*c
black = (0,)*c
for n in range(self.n):
poly = self.list[n]
if poly.n > 0:
cv2.fillPoly(mask, [poly.points_to_n()], white if poly.type == 1 else black )

def dump(self):
result = []
for n in range(self.n):
l = self.list[n]
result += [ (l.type, l.points_to_n().tolist() ) ]
return result

@staticmethod
def load(ie_polys=None):
obj = IEPolys()
if ie_polys is not None:
for (type, points) in ie_polys:
obj.add(type)
obj.n_list().set_points(points)
return obj
@@ -20,4 +20,6 @@

from .DCSCN import DCSCN

from .common import normalize_channels
from .common import normalize_channels

from .IEPolys import IEPolys
@@ -15,23 +15,24 @@ def _get_pil_font (font, size):
return ImageFont.load_default()

def get_text_image( shape, text, color=(1,1,1), border=0.2, font=None):
try:
size = shape[1]
pil_font = _get_pil_font( localization.get_default_ttf_font_name() , size)
text_width, text_height = pil_font.getsize(text)
h,w,c = shape
try:
pil_font = _get_pil_font( localization.get_default_ttf_font_name() , h-2)

canvas = Image.new('RGB', shape[0:2], (0,0,0) )
canvas = Image.new('RGB', (w,h) , (0,0,0) )
draw = ImageDraw.Draw(canvas)
offset = ( 0, 0)
draw.text(offset, text, font=pil_font, fill=tuple((np.array(color)*255).astype(np.int)) )

result = np.asarray(canvas) / 255
if shape[2] != 3:
result = np.concatenate ( (result, np.ones ( (shape[1],) + (shape[0],) + (shape[2]-3,)) ), axis=2 )


if c > 3:
result = np.concatenate ( (result, np.ones ((h,w,c-3)) ), axis=-1 )
elif c < 3:
result = result[...,0:c]
return result
except:
return np.zeros ( (shape[1], shape[0], shape[2]), dtype=np.float32 )
return np.zeros ( (h,w,c) )

def draw_text( image, rect, text, color=(1,1,1), border=0.2, font=None):
h,w,c = image.shape
@@ -18,8 +18,10 @@
class InteractBase(object):
EVENT_LBUTTONDOWN = 1
EVENT_LBUTTONUP = 2
EVENT_MBUTTONDOWN = 3
EVENT_MBUTTONUP = 4
EVENT_RBUTTONDOWN = 5
EVENT_RBUTTONUP = 6
EVENT_RBUTTONUP = 6
EVENT_MOUSEWHEEL = 10

def __init__(self):
@@ -263,6 +265,8 @@ def onMouse(event, x, y, flags, param):
elif event == cv2.EVENT_LBUTTONUP: ev = InteractBase.EVENT_LBUTTONUP
elif event == cv2.EVENT_RBUTTONDOWN: ev = InteractBase.EVENT_RBUTTONDOWN
elif event == cv2.EVENT_RBUTTONUP: ev = InteractBase.EVENT_RBUTTONUP
elif event == cv2.EVENT_MBUTTONDOWN: ev = InteractBase.EVENT_MBUTTONDOWN
elif event == cv2.EVENT_MBUTTONUP: ev = InteractBase.EVENT_MBUTTONUP
elif event == cv2.EVENT_MOUSEWHEEL: ev = InteractBase.EVENT_MOUSEWHEEL

else: ev = 0
Oops, something went wrong.

0 comments on commit 5ac7e5d

Please sign in to comment.
You can’t perform that action at this time.