Merge pull request #487 from aleju/ooi_removal

Improve CBA removal and clipping

changelogs/master/added/20191106_ooi_removal.md

@@ -0,0 +1,28 @@
+# Removal of Coordinate-Based Augmentables Outside of the Image Plane
+
+* Added `Keypoint.is_out_of_image()`.
+
+* Added `BoundingBox.compute_out_of_image_area()`.
+* Added `Polygon.compute_out_of_image_area()`.
+
+* Added `Keypoint.compute_out_of_image_fraction()`
+* Added `BoundingBox.compute_out_of_image_fraction()`.
+* Added `Polygon.compute_out_of_image_fraction()`.
+* Added `LineString.compute_out_of_image_fraction()`.
+
+* Added `KeypointsOnImage.remove_out_of_image_fraction()`.
+* Added `BoundingBoxesOnImage.remove_out_of_image_fraction()`.
+* Added `PolygonsOnImage.remove_out_of_image_fraction()`.
+* Added `LineStringsOnImage.remove_out_of_image_fraction()`.
+
+* Added `KeypointsOnImage.clip_out_of_image()`.
+
+* Added `imgaug.augmenters.meta.RemoveCBAsByOutOfImageFraction`.
+  Removes coordinate-based augmentables (e.g. BBs) that have at least a
+  specified fraction of their area outside of the image plane.
+* Added `imgaug.augmenters.meta.ClipCBAsToImagePlanes`.
+  Clips off all parts from coordinate-based augmentables (e.g. BBs) that are
+  outside of the corresponding image.
+
+* Changed `Polygon.area` to return `0.0` if the polygon contains less than
+  three points (previously: exception).

imgaug/augmentables/bbs.py

@@ -7,7 +7,8 @@
 import skimage.measure
 
 from .. import imgaug as ia
-from .utils import normalize_shape, project_coords
+from .utils import (normalize_shape, project_coords,
+                    _remove_out_of_image_fraction)
 
 
 # TODO functions: square(), to_aspect_ratio(), contains_point()
@@ -368,6 +369,66 @@ def iou(self, other):
         area_union = self.area + other.area - inters.area
         return inters.area / area_union if area_union > 0 else 0.0
 
+    def compute_out_of_image_area(self, image):
+        """Compute the area of the BB that is outside of the image plane.
+
+        Parameters
+        ----------
+        image : (H,W,...) ndarray or tuple of int
+            Image dimensions to use.
+            If an ``ndarray``, its shape will be used.
+            If a ``tuple``, it is assumed to represent the image shape
+            and must contain at least two integers.
+
+        Returns
+        -------
+        float
+            Total area of the bounding box that is outside of the image plane.
+            Can be ``0.0``.
+
+        """
+        shape = normalize_shape(image)
+        height, width = shape[0:2]
+        bb_image = BoundingBox(x1=0, y1=0, x2=width, y2=height)
+        inter = self.intersection(bb_image, default=None)
+        area = self.area
+        return area if inter is None else area - inter.area
+
+    def compute_out_of_image_fraction(self, image):
+        """Compute fraction of BB area outside of the image plane.
+
+        This estimates ``f = A_ooi / A``, where ``A_ooi`` is the area of the
+        bounding box that is outside of the image plane, while ``A`` is the
+        total area of the bounding box.
+
+        Parameters
+        ----------
+        image : (H,W,...) ndarray or tuple of int
+            Image dimensions to use.
+            If an ``ndarray``, its shape will be used.
+            If a ``tuple``, it is assumed to represent the image shape
+            and must contain at least two integers.
+
+        Returns
+        -------
+        float
+            Fraction of the bounding box area that is outside of the image
+            plane. Returns ``0.0`` if the bounding box is fully inside of
+            the image plane. If the bounding box has an area of zero, the
+            result is ``1.0`` if its coordinates are outside of the image
+            plane, otherwise ``0.0``.
+
+        """
+        area = self.area
+        if area == 0:
+            shape = normalize_shape(image)
+            height, width = shape[0:2]
+            y1_outside = self.y1 < 0 or self.y1 >= height
+            x1_outside = self.x1 < 0 or self.x1 >= width
+            is_outside = (y1_outside or x1_outside)
+            return 1.0 if is_outside else 0.0
+        return self.compute_out_of_image_area(image) / area
+
     def is_fully_within_image(self, image):
         """Estimate whether the bounding box is fully inside the image area.
 
@@ -1359,6 +1420,27 @@ def remove_out_of_image(self, fully=True, partly=False):
             if not bb.is_out_of_image(self.shape, fully=fully, partly=partly)]
         return BoundingBoxesOnImage(bbs_clean, shape=self.shape)
 
+    def remove_out_of_image_fraction(self, fraction):
+        """Remove all BBs with an out of image fraction of at least `fraction`.
+
+        Parameters
+        ----------
+        fraction : number
+            Minimum out of image fraction that a bounding box has to have in
+            order to be removed. A fraction of ``1.0`` removes only bounding
+            boxes that are ``100%`` outside of the image. A fraction of ``0.0``
+            removes all bounding boxes.
+
+        Returns
+        -------
+        imgaug.augmentables.bbs.BoundingBoxesOnImage
+            Reduced set of bounding boxes, with those that had an out of image
+            fraction greater or equal the given one removed.
+
+        """
+        return _remove_out_of_image_fraction(self, fraction,
+                                             BoundingBoxesOnImage)
+
     @ia.deprecated(alt_func="BoundingBoxesOnImage.clip_out_of_image()",
                    comment="clip_out_of_image() has the exactly same "
                            "interface.")

imgaug/augmentables/kps.py

@@ -7,7 +7,8 @@
 import six.moves as sm
 
 from .. import imgaug as ia
-from .utils import normalize_shape, project_coords
+from .utils import (normalize_shape, project_coords,
+                    _remove_out_of_image_fraction)
 
 
 def compute_geometric_median(points=None, eps=1e-5, X=None):
@@ -149,6 +150,54 @@ def project(self, from_shape, to_shape):
         xy_proj = project_coords([(self.x, self.y)], from_shape, to_shape)
         return self.deepcopy(x=xy_proj[0][0], y=xy_proj[0][1])
 
+    def is_out_of_image(self, image):
+        """Estimate whether this point is outside of the given image plane.
+
+        Parameters
+        ----------
+        image : (H,W,...) ndarray or tuple of int
+            Image dimensions to use.
+            If an ``ndarray``, its shape will be used.
+            If a ``tuple``, it is assumed to represent the image shape
+            and must contain at least two integers.
+
+        Returns
+        -------
+        bool
+            ``True`` is the point is inside the image plane, ``False``
+            otherwise.
+
+        """
+        shape = normalize_shape(image)
+        height, width = shape[0:2]
+        y_inside = (0 <= self.y < height)
+        x_inside = (0 <= self.x < width)
+        return not y_inside or not x_inside
+
+    def compute_out_of_image_fraction(self, image):
+        """Compute fraction of the keypoint that is out of the image plane.
+
+        The fraction is always either ``1.0`` (point is outside of the image
+        plane) or ``0.0`` (point is inside the image plane). This method
+        exists for consistency with other augmentables, e.g. bounding boxes.
+
+        Parameters
+        ----------
+        image : (H,W,...) ndarray or tuple of int
+            Image dimensions to use.
+            If an ``ndarray``, its shape will be used.
+            If a ``tuple``, it is assumed to represent the image shape
+            and must contain at least two integers.
+
+        Returns
+        -------
+        float
+            Either ``1.0`` (point is outside of the image plane) or
+            ``0.0`` (point is inside of it).
+
+        """
+        return float(self.is_out_of_image(image))
+
     def shift(self, x=0, y=0):
         """Move the keypoint around on an image.
 
@@ -586,6 +635,45 @@ def draw_on_image(self, image, color=(0, 255, 0), alpha=1.0, size=3,
                 raise_if_out_of_image=raise_if_out_of_image)
         return image
 
+    def remove_out_of_image_fraction(self, fraction):
+        """Remove all KPs with an out of image fraction of at least `fraction`.
+
+        This method exists for consistency with other augmentables, e.g.
+        bounding boxes.
+
+        Parameters
+        ----------
+        fraction : number
+            Minimum out of image fraction that a keypoint has to have in
+            order to be removed. Note that any keypoint can only have a
+            fraction of either ``1.0`` (is outside) or ``0.0`` (is inside).
+            Set this to ``0.0+eps`` to remove all points that are outside of
+            the image. Setting this to ``0.0`` will remove all points.
+
+        Returns
+        -------
+        imgaug.augmentables.kps.KeypointsOnImage
+            Reduced set of keypoints, with those thathad an out of image
+            fraction greater or equal the given one removed.
+
+        """
+        return _remove_out_of_image_fraction(self, fraction, KeypointsOnImage)
+
+    def clip_out_of_image(self):
+        """Remove all KPs that are outside of the image plane.
+
+        This method exists for consistency with other augmentables, e.g.
+        bounding boxes.
+
+        Returns
+        -------
+        imgaug.augmentables.kps.KeypointsOnImage
+            Keypoints that are inside the image plane.
+
+        """
+        # we could use anything >0 here as the fraction
+        return self.remove_out_of_image_fraction(0.5)
+
     def shift(self, x=0, y=0):
         """Move the keypoints on the x/y-axis.
 

imgaug/augmentables/lines.py

@@ -8,7 +8,8 @@
 import cv2
 
 from .. import imgaug as ia
-from .utils import normalize_shape, project_coords, interpolate_points
+from .utils import (normalize_shape, project_coords, interpolate_points,
+                    _remove_out_of_image_fraction)
 
 
 # TODO Add Line class and make LineString a list of Line elements
@@ -333,6 +334,42 @@ def project(self, from_shape, to_shape):
         coords_proj = project_coords(self.coords, from_shape, to_shape)
         return self.copy(coords=coords_proj)
 
+    def compute_out_of_image_fraction(self, image):
+        """Compute fraction of polygon area outside of the image plane.
+
+        This estimates ``f = A_ooi / A``, where ``A_ooi`` is the area of the
+        polygon that is outside of the image plane, while ``A`` is the
+        total area of the bounding box.
+
+        Parameters
+        ----------
+        image : (H,W,...) ndarray or tuple of int
+            Image dimensions to use.
+            If an ``ndarray``, its shape will be used.
+            If a ``tuple``, it is assumed to represent the image shape
+            and must contain at least two integers.
+
+        Returns
+        -------
+        float
+            Fraction of the polygon area that is outside of the image
+            plane. Returns ``0.0`` if the polygon is fully inside of
+            the image plane. If the polygon has an area of zero, the polygon
+            is treated similarly to a :class:`LineString`, i.e. the fraction
+            of the line that is inside the image plane is returned.
+
+        """
+        length = self.length
+        if length == 0:
+            if len(self.coords) == 0:
+                return 0.0
+            points_ooi = ~self.get_pointwise_inside_image_mask(image)
+            return 1.0 if np.all(points_ooi) else 0.0
+        lss_clipped = self.clip_out_of_image(image)
+        length_after_clip = sum([ls.length for ls in lss_clipped])
+        inside_image_factor = length_after_clip / length
+        return 1.0 - inside_image_factor
+
     def is_fully_within_image(self, image, default=False):
         """Estimate whether the line string is fully inside an image plane.
 
@@ -1762,6 +1799,26 @@ def remove_out_of_image(self, fully=True, partly=False):
                          self.shape, fully=fully, partly=partly)]
         return LineStringsOnImage(lss_clean, shape=self.shape)
 
+    def remove_out_of_image_fraction(self, fraction):
+        """Remove all LS with an out of image fraction of at least `fraction`.
+
+        Parameters
+        ----------
+        fraction : number
+            Minimum out of image fraction that a line string has to have in
+            order to be removed. A fraction of ``1.0`` removes only line
+            strings that are ``100%`` outside of the image. A fraction of
+            ``0.0`` removes all line strings.
+
+        Returns
+        -------
+        imgaug.augmentables.lines.LineStringsOnImage
+            Reduced set of line strings, with those that had an out of image
+            fraction greater or equal the given one removed.
+
+        """
+        return _remove_out_of_image_fraction(self, fraction, LineStringsOnImage)
+
     def clip_out_of_image(self):
         """
         Clip off all parts of the line strings that are outside of an image.