Merge branch 'staging'

lib/align/aligned_face.py

@@ -234,6 +234,11 @@ class _FaceCache:  # pylint:disable=too-many-instance-attributes
     average_distance: float, optional
         The average distance of the core landmarks (18-67) from the mean face that was used for
         aligning the image.  Default: `0.0`
+    relative_eye_mouth_position: float, optional
+        A float value representing the relative position of the lowest eye/eye-brow point to the
+        highest mouth point. Positive values indicate that eyes/eyebrows are aligned above the
+        mouth, negative values indicate that eyes/eyebrows are misaligned below the mouth.
+        Default: `0.0`
     adjusted_matrix: :class:`numpy.ndarray`, optional
         The 3x2 transformation matrix for extracting and aligning the core face area out of the
         original frame with padding and sizing applied. Default: ``None``
@@ -251,6 +256,7 @@ class _FaceCache:  # pylint:disable=too-many-instance-attributes
     landmarks: Optional[np.ndarray] = None
     landmarks_normalized: Optional[np.ndarray] = None
     average_distance: float = 0.0
+    relative_eye_mouth_position: float = 0.0
     adjusted_matrix: Optional[np.ndarray] = None
     interpolators: Tuple[int, int] = (0, 0)
     cropped_roi: Dict[CenteringType, np.ndarray] = field(default_factory=dict)
@@ -464,6 +470,22 @@ def average_distance(self) -> float:
                 self._cache.average_distance = average_distance
         return self._cache.average_distance
 
+    @property
+    def relative_eye_mouth_position(self) -> float:
+        """ float: Value representing the relative position of the lowest eye/eye-brow point to the
+        highest mouth point. Positive values indicate that eyes/eyebrows are aligned above the
+        mouth, negative values indicate that eyes/eyebrows are misaligned below the mouth. """
+        with self._cache.lock("relative_eye_mouth_position"):
+            if not self._cache.relative_eye_mouth_position:
+                lowest_eyes = np.max(self.normalized_landmarks[np.r_[17:27, 36:48], 1])
+                highest_mouth = np.min(self.normalized_landmarks[48:68, 1])
+                position = highest_mouth - lowest_eyes
+                logger.trace("lowest_eyes: %s, highest_mouth: %s, "  # type: ignore
+                             "relative_eye_mouth_position: %s", lowest_eyes, highest_mouth,
+                             position)
+                self._cache.relative_eye_mouth_position = position
+        return self._cache.relative_eye_mouth_position
+
     @classmethod
     def _padding_from_coverage(cls, size: int, coverage_ratio: float) -> Dict[CenteringType, int]:
         """ Return the image padding for a face from coverage_ratio set against a

plugins/extract/_config.py

@@ -86,6 +86,15 @@ def set_globals(self):
             "degrees. Aligned faces should have a roll value close to zero. Values that are a "
             "significant distance from 0 degrees tend to be misaligned images. These can usually "
             "be safely disgarded.")
+        self.add_item(
+            section=section,
+            title="aligner_features",
+            datatype=bool,
+            default=True,
+            group="filters",
+            info="Filters out faces where the lowest point of the aligned face's eye or eyebrow "
+            "is lower than the highest point of the aligned face's mouth. Any faces where this "
+            "occurs are misaligned and can be safely disgarded.")
         self.add_item(
             section=section,
             title="filter_refeed",

plugins/extract/align/_base.py

@@ -111,7 +111,8 @@ def __init__(self,
         self.set_normalize_method(normalize_method)
 
         self._plugin_type = "align"
-        self._filter = AlignedFilter(min_scale=self.config["aligner_min_scale"],
+        self._filter = AlignedFilter(feature_filter=self.config["aligner_features"],
+                                     min_scale=self.config["aligner_min_scale"],
                                      max_scale=self.config["aligner_max_scale"],
                                      distance=self.config["aligner_distance"],
                                      roll=self.config["aligner_roll"],
@@ -537,6 +538,9 @@ class AlignedFilter():
 
     Parameters
     ----------
+    feature_filter: bool
+        ``True`` to enable filter to check relative position of eyes/eyebrows and mouth. ``False``
+        to disable.
     min_scale: float
         Filters out faces that have been aligned at below this value as a multiplier of the
         minimum frame dimension. Set to ``0`` for off.
@@ -556,22 +560,33 @@ class AlignedFilter():
         ``True`` to disable the filter regardless of config options. Default: ``False``
     """
     def __init__(self,
+                 feature_filter: bool,
                  min_scale: float,
                  max_scale: float,
                  distance: float,
                  roll: float,
                  save_output: bool,
                  disable: bool = False) -> None:
-        logger.debug("Initializing %s: (min_scale: %s, max_scale: %s, distance: %s, roll, %s"
-                     "save_output: %s, disable: %s)", self.__class__.__name__, min_scale,
-                     max_scale, distance, roll, save_output, disable)
+        logger.debug("Initializing %s: (feature_filter: %s, min_scale: %s, max_scale: %s, "
+                     "distance: %s, roll, %s, save_output: %s, disable: %s)",
+                     self.__class__.__name__, feature_filter, min_scale, max_scale, distance, roll,
+                     save_output, disable)
+        self._features = feature_filter
         self._min_scale = min_scale
         self._max_scale = max_scale
         self._distance = distance / 100.
         self._roll = roll
         self._save_output = save_output
-        self._active = not disable and (max_scale > 0.0 or min_scale > 0.0 or distance > 0.0)
-        self._counts: Dict[str, int] = dict(min_scale=0, max_scale=0, distance=0, roll=0)
+        self._active = not disable and (feature_filter or
+                                        max_scale > 0.0 or
+                                        min_scale > 0.0 or
+                                        distance > 0.0 or
+                                        roll > 0.0)
+        self._counts: Dict[str, int] = dict(features=0,
+                                            min_scale=0,
+                                            max_scale=0,
+                                            distance=0,
+                                            roll=0)
         logger.debug("Initialized %s: ", self.__class__.__name__)
 
     def __call__(self, faces: List[DetectedFace], minimum_dimension: int
@@ -602,6 +617,13 @@ def __call__(self, faces: List[DetectedFace], minimum_dimension: int
         for idx, face in enumerate(faces):
             aligned = AlignedFace(landmarks=face.landmarks_xy, centering="face")
 
+            if self._features and aligned.relative_eye_mouth_position < 0.0:
+                self._counts["features"] += 1
+                if self._save_output:
+                    retval.append(face)
+                    sub_folders[idx] = "_align_filt_features"
+                continue
+
             min_max = self._scale_test(aligned, minimum_dimension)
             if min_max in ("min", "max"):
                 self._counts[f"{min_max}_scale"] += 1
@@ -617,7 +639,7 @@ def __call__(self, faces: List[DetectedFace], minimum_dimension: int
                     sub_folders[idx] = "_align_filt_distance"
                 continue
 
-            if not 0.0 < abs(aligned.pose.roll) < self._roll:
+            if self._roll != 0.0 and not 0.0 < abs(aligned.pose.roll) < self._roll:
                 self._counts["roll"] += 1
                 if self._save_output:
                     retval.append(face)
@@ -682,11 +704,13 @@ def filtered_mask(self, faces: List[DetectedFace], minimum_dimension: List[int])
         retval = [True for _ in range(len(faces))]
         for idx, (face, dim) in enumerate(zip(faces, minimum_dimension)):
             aligned = AlignedFace(landmarks=face.landmarks_xy)
+            if self._features and aligned.relative_eye_mouth_position < 0.0:
+                continue
             if self._scale_test(aligned, dim) is not None:
                 continue
             if 0.0 < self._distance < aligned.average_distance:
                 continue
-            if not 0.0 < abs(aligned.pose.roll) < self._roll:
+            if self._roll != 0.0 and not 0.0 < abs(aligned.pose.roll) < self._roll:
                 continue
             retval[idx] = False