diff --git a/.travis.yml b/.travis.yml index fcd733f..01c3f9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ before_install: install: - conda install boost scipy ; + - conda install -c menpo opencv ; # Install dlib from git (with pip) as there was an issue with libjpeg # Which was solved and not yet released. - pip install git+https://github.com/davisking/dlib.git@02f6da285149a61bc58728d9c5e77932151118b5#egg=dlib ; diff --git a/file_metadata/image/image_file.py b/file_metadata/image/image_file.py index 9256c3d..ff65583 100644 --- a/file_metadata/image/image_file.py +++ b/file_metadata/image/image_file.py @@ -15,6 +15,7 @@ import skimage import skimage.io import skimage.color +import skimage.transform import zbar from PIL import Image from pycolorname.pantone.pantonepaint import PantonePaint @@ -81,6 +82,11 @@ def fetch(self, key=''): .format(self.fetch('filename'))) # Use empty array as the file cannot be read. return numpy.ndarray(0) + elif key == 'ndarray_grey': + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return skimage.img_as_ubyte( + skimage.color.rgb2grey(self.fetch('ndarray'))) return super(ImageFile, self).fetch(key) def analyze_softwares(self): @@ -183,6 +189,173 @@ def analyze_color_average(self): 'Color:ClosestLabeledColor': closest_label, 'Color:AverageRGB': tuple(round(i, 3) for i in mean_color)} + @staticmethod + def _haarcascade(image, filename, directory=None, **kwargs): + """ + Use OpenCV's haarcascade classifiers to detect certain features. + + :param image: Image to use when detecting with the haarcascade. + :param filename: The file to create the CascadeClassifier with. + :param directory: The directory of the haarcascade file. + :param kwagrs: Keyword args to pass to cascade's detectMultiScale(). + :return: List of rectangles of the detected objects. A rect + is defined by an array with 4 values i the order: + left, top, width, height. + """ + try: + import cv2 + except ImportError: + logging.warn('HAAR Cascade analysis requires the optional ' + 'dependency OpenCV to be installed.') + return [] + + directory = (directory if directory is not None + else os.path.abspath(os.path.join( + os.path.realpath(cv2.__file__), + *([os.pardir] * 4 + ['share', 'OpenCV', 'haarcascades'])))) + cascade = cv2.CascadeClassifier(os.path.join(directory, filename),) + features = cascade.detectMultiScale(image, **kwargs) + return features + + def analyze_face_haarcascades(self): + """ + Use opencv's haar cascade filters to identify faces, right eye, left + eye, upper body, etc.. + """ + try: + import cv2 + from cv2 import cv + except ImportError: + logging.warn('HAAR Cascade analysis requires the optional ' + 'dependency OpenCV 2.x to be installed.') + return {} + + image_array = self.fetch('ndarray_grey') + if image_array.ndim == 3: + logging.warn('Faces cannot be detected in animated images ' + 'using haarcascades yet.') + return {} + + # The "scale" given here is relevant for the detection rate. + scale = max(1.0, numpy.average(image_array.shape) / 500.0) + + # Equalize the histogram and make the size smaller + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + img_shape = map(lambda x: int(x / scale), image_array.shape) + img = skimage.img_as_ubyte( + skimage.exposure.equalize_hist( + skimage.transform.resize(image_array, + output_shape=img_shape, + preserve_range=True))) + + def haar(im, key, single=False, **kwargs): + cascades = { + 'frontal_face': 'haarcascade_frontalface_alt.xml', + 'profile_face': 'haarcascade_profileface.xml', + 'nested': 'haarcascade_eye_tree_eyeglasses.xml', + 'mouth': 'haarcascade_mcs_mouth.xml', + 'nose': 'haarcascade_mcs_nose.xml', + 'right_eye': 'haarcascade_righteye_2splits.xml', + 'left_eye': 'haarcascade_lefteye_2splits.xml', + 'left_ear': 'haarcascade_mcs_leftear.xml', + 'right_ear': 'haarcascade_mcs_rightear.xml', + 'upper_body': 'haarcascade_upperbody.xml', + 'lower_body': 'haarcascade_lowerbody.xml'} + # Set some default kwargs + kwargs['scaleFactor'] = kwargs.get('scaleFactor', 1.1) + kwargs['minNeighbors'] = kwargs.get('minNeighbors', 2) + kwargs['minSize'] = kwargs.get('minSize', (30, 30)) + flags = cv.CV_HAAR_SCALE_IMAGE + if single: + flags = (flags | cv.CV_HAAR_FIND_BIGGEST_OBJECT | + cv.CV_HAAR_DO_ROUGH_SEARCH) + kwargs['flags'] = kwargs.get('flags', flags) + return list(self._haarcascade(im, cascades[key], **kwargs)) + + def drop_overlapping_regions(regions): + drop = set() + # Sort regions by area (leftmost is smallest and dropped first) + regions = sorted(regions, key=lambda x: x[-1] * x[-2]) + # overlap: Neither range is completely greater than the other + overlap = (lambda x_min, x_width, y_min, y_width: + x_min <= y_min + y_width and y_min <= x_min + x_width) + for i1, reg1 in enumerate(regions): + for i2, reg2 in enumerate(regions[:i1]): + if (i2 not in drop and + overlap(reg1[0], reg1[2], reg2[0], reg2[2]) and + overlap(reg1[1], reg1[3], reg2[1], reg2[3])): + drop.add(i2) + for i, reg in enumerate(regions): + if i not in drop: + yield reg + + frontal = haar(img, 'frontal_face') + profile = haar(img, 'profile_face') + faces = list(drop_overlapping_regions(frontal + profile)) + + if len(faces) == 0: + return {} + + data = [] + for face in faces: + scaled_face = list(map(lambda x: int(x * scale), face)) + fdata = {'position': { + 'left': scaled_face[0], 'top': scaled_face[1], + 'width': scaled_face[2], 'height': scaled_face[3]}} + roi = list(map(int, [ + max(0, face[0] - (face[2] / 8)), + max(0, face[1] - (face[3] / 8)), + min(img.shape[0], face[2] + (2 * face[2] / 8)), + min(img.shape[1], face[3] + (2 * face[3] / 8))])) + face_img = img[roi[1]:roi[1] + roi[3] - 1, + roi[0]:roi[0] + roi[2] - 1] + + def feat_mid(rect, offx, offy): + return (int(scale * (roi[0] + rect[0] + offx + rect[2] / 2)), + int(scale * (roi[1] + rect[1] + offy + rect[3] // 2))) + + eye_img = face_img[:roi[3] // 2, :] + nested = list(drop_overlapping_regions(haar(eye_img, 'nested'))) + if len(nested) == 2: + nested = sorted(nested, key=lambda x: x[0]) + fdata['eyes'] = (feat_mid(nested[0], 0, 0), + feat_mid(nested[1], 0, 0)) + else: + eyes_found = [] + for eye in ['left_eye', 'right_eye']: + eye_feats = haar(eye_img, eye, single=True) + if len(eye_feats) == 1: + eyes_found.append(feat_mid(eye_feats[0], 0, 0)) + if len(eyes_found) > 0: + fdata['eyes'] = tuple(eyes_found) + + ear_offy = roi[3] // 8 + ear_img = face_img[ear_offy:roi[3] * 7 // 8, :] + ears_found = [] + for ear in ['left_ear', 'right_ear']: + ear_feats = haar(ear_img, ear, single=True) + if len(ear_feats) == 1: + ears_found.append(feat_mid(ear_feats[0], 0, ear_offy)) + if len(ears_found) > 0: + fdata['ears'] = tuple(ears_found) + + nose_offx, nose_offy = roi[2] // 4, roi[3] // 4 + nose_img = face_img[nose_offy:roi[3] * 3 // 4, + nose_offx:roi[2] * 3 // 4] + nose_feats = haar(nose_img, 'nose', single=True) + if len(nose_feats) == 1: + fdata['nose'] = feat_mid(nose_feats[0], nose_offx, nose_offy) + + mouth_offy = roi[3] // 2 + mouth_img = face_img[mouth_offy:, :] + mouth_feats = haar(mouth_img, 'mouth', single=True) + if len(mouth_feats) == 1: + fdata['mouth'] = feat_mid(mouth_feats[0], 0, mouth_offy) + + data.append(fdata) + return {'OpenCV:Faces': data} + def analyze_facial_landmarks(self, with_landmarks=True, detector_upsample_num_times=0): @@ -272,9 +445,9 @@ def tup2(pt1, pt2): # Point 34 is the tip of the nose fdata['nose'] = tup(shape.part(34)) # Point 40 and 37 are the two corners of the left eye - fdata['left_eye'] = tup2(shape.part(40), shape.part(37)) # Point 46 and 43 are the two corners of the right eye - fdata['right_eye'] = tup2(shape.part(46), shape.part(43)) + fdata['eyes'] = (tup2(shape.part(40), shape.part(37)), + tup2(shape.part(46), shape.part(43))) # Point 49 and 55 are the two outer corners of the mouth fdata['mouth'] = tup2(shape.part(49), shape.part(55)) data.append(fdata) @@ -380,11 +553,11 @@ def analyze_barcode_zbar(self): - confidence - The quality of the barcode. The higher it is the more accurate the detection is. """ - with warnings.catch_warnings(): - # Supress warning about precision lost in float -> ubyte - warnings.simplefilter("ignore") - image_array = skimage.img_as_ubyte( - skimage.color.rgb2grey(self.fetch('ndarray'))) + image_array = self.fetch('ndarray_grey') + if image_array.ndim == 3: + logging.warn('Barcodes cannot be detected in animated images ' + 'using zbar.') + return {} height, width = image_array.shape zbar_img = zbar.Image(width, height, 'Y800', image_array.tobytes()) scanner = zbar.ImageScanner() diff --git a/tests/image/image_file_test.py b/tests/image/image_file_test.py index dece79a..2619156 100644 --- a/tests/image/image_file_test.py +++ b/tests/image/image_file_test.py @@ -136,6 +136,51 @@ def test_color_average_animated_image(self): self.assertEqual(data['Color:ClosestLabeledColorRGB'], (223, 223, 227)) +class ImageFileFaceHAARCascadesTest(unittest.TestCase): + + def test_face_haarcascade_charlie_chaplin(self): + with ImageFile(fetch_file('charlie_chaplin.jpg')) as uut: + data = uut.analyze_face_haarcascades() + self.assertIn('OpenCV:Faces', data) + self.assertEqual(len(data['OpenCV:Faces']), 1) + + face = data['OpenCV:Faces'][0] + print(face) + self.assertIn((662, 558), face['eyes']) + self.assertEqual(face['nose'], (776, 688)) + self.assertEqual(face['mouth'], (735, 794)) + + def test_face_haarcascade_mona_lisa(self): + with ImageFile(fetch_file('mona_lisa.jpg')) as uut: + data = uut.analyze_face_haarcascades() + self.assertIn('OpenCV:Faces', data) + self.assertEqual(len(data['OpenCV:Faces']), 1) + + face = data['OpenCV:Faces'][0] + self.assertEqual(face['nose'], (318, 310)) + self.assertEqual(face['mouth'], (325, 341)) + + def test_face_haarcascade_monkey_face(self): + _file = ImageFile(fetch_file('monkey_face.jpg')) + data = _file.analyze_face_haarcascades() + self.assertEqual(data, {}) + + def test_face_haarcascade_baby_face(self): + _file = ImageFile(fetch_file('baby_face.jpg')) + data = _file.analyze_face_haarcascades() + self.assertIn('OpenCV:Faces', data) + self.assertEqual(len(data['OpenCV:Faces']), 1) + + face = data['OpenCV:Faces'][0] + self.assertEqual(face['mouth'], (851, 1381)) + self.assertIn('position', face) + + def test_face_haarcascade_animated_image(self): + _file = ImageFile(fetch_file('animated.gif')) + data = _file.analyze_face_haarcascades() + self.assertEqual(data, {}) + + # Increase the timeout as the first time it will need to download the # shape predictor data ~60MB @pytest.mark.timeout(300) @@ -153,8 +198,9 @@ def test_facial_landmarks_mona_lisa(self): self.assertEqual(len(data['dlib:Faces']), 1) face = data['dlib:Faces'][0] - self.assertEqual(face['left_eye'], (288, 252)) - self.assertEqual(face['right_eye'], (361, 251)) + self.assertEqual((face['eyes']), 2) + self.assertIn((288, 252), face['eyes']) + self.assertIn((361, 251), face['eyes']) self.assertEqual(face['nose'], (325, 318)) self.assertEqual(face['mouth'], (321, 338)) @@ -165,8 +211,7 @@ def test_facial_landmarks_baby_face(self): self.assertEqual(len(data['dlib:Faces']), 1) face = data['dlib:Faces'][0] - self.assertNotIn('left_eye', face) - self.assertNotIn('right_eye', face) + self.assertNotIn('eyes', face) self.assertNotIn('nose', face) self.assertNotIn('mouth', face) self.assertIn('position', face)