transform_data.py

import argparse
import cv2
import imgaug as ia
from imgaug import augmenters as iaa
from imgaug import parameters as iap
import imutils
import math
import numpy as np
import os
import pandas as pd
import random
from shapely import geometry

import fetch_data
import generate_data
from config import Config


def key_pts_to_yolo(key_pts, w_img, h_img):
    """
    Convert a list of keypoints into a yolo training format
    :param key_pts: list of keypoints
    :param w_img: width of the entire image
    :param h_img: height of the entire image
    :return: <x> <y> <width> <height>
    """
    x1 = max(0, min([pt[0] for pt in key_pts]))
    x2 = min(w_img, max([pt[0] for pt in key_pts]))
    y1 = max(0, min([pt[1] for pt in key_pts]))
    y2 = min(h_img, max([pt[1] for pt in key_pts]))
    x = (x2 + x1) / 2 / w_img
    y = (y2 + y1) / 2 / h_img
    width = (x2 - x1) / w_img
    height = (y2 - y1) / h_img
    return x, y, width, height


class ImageGenerator:
    """
    A template for generating a training image
    An ImageGenerator contains a background image, list of cards, and other environmental parameters to
    set up a training image for YOLO network
    """
    def __init__(self, img_bg, class_ids, width, height, skew=None, cards=None):
        """
        :param img_bg: background (textile) image
        :param width: width of the training image
        :param height: height of the training image
        :param skew: 4 coordinates that indicates the corners (in normalized form) for perspective transform
        :param cards: list of Card objects
        """
        self.img_bg = img_bg
        self.class_ids = class_ids
        self.img_result = None
        self.width = width
        self.height = height
        if cards is None:
            self.cards = []
        else:
            self.cards = cards

        # Compute transform matrix for perspective transform (used for skewing the final result)
        if skew is not None:
            orig_corner = np.array([[0, 0], [0, height], [width, height], [width, 0]], dtype=np.float32)
            new_corner = np.array([[width * s[0], height * s[1]] for s in skew], dtype=np.float32)
            self.M = cv2.getPerspectiveTransform(orig_corner, new_corner)
            pass
        else:
            self.M = None
        pass

    def add_card(self, card, x=None, y=None, theta=0.0, scale=1.0):
        """
        Add a card to this generator scenario.
        :param card: card to be added
        :param x: new X-coordinate for the centre of the card
        :param y: new Y-coordinate for the centre of the card
        :param theta: new angle for the card
        :param scale: new scale for the card
        :return: none
        """
        # If the position isn't given, push it out of the image so that it won't be visible during rendering
        if x is None:
            x = -len(card.img[0]) / 2
        if y is None:
            y = -len(card.img) / 2
        self.cards.append(card)
        card.x = x
        card.y = y
        card.theta = theta
        card.scale = scale
        pass

    def render(self, visibility=0.5, aug=None, display=False, debug=False):
        """
        Display the current state of the generator.
        :param visibility: portion of the card's image that must not be overlapped by other cards for the card to be
                           considered as visible
        :param aug: image augmentator to apply during rendering
        :param display: flag for displaying the rendering result
        :param debug: flag for debug
        :return: none
        """
        self.check_visibility(visibility=visibility)
        img_result = np.zeros((self.height, self.width, 3), dtype=np.uint8)
        card_mask = cv2.imread(Config.card_mask_path)

        for card in self.cards:
            card_x = int(card.x + 0.5)
            card_y = int(card.y + 0.5)

            # Scale & rotate card image
            img_card = cv2.resize(card.img, (int(len(card.img[0]) * card.scale), int(len(card.img) * card.scale)))
            # Add a random glaring on individual card - it happens frequently in real life as MTG cards can reflect
            # the lights very well.
            if aug is not None:
                seq = iaa.Sequential([
                    iaa.SimplexNoiseAlpha(first=iaa.Add(random.randrange(128)), size_px_max=[1, 3],
                                          upscale_method="cubic"),  # Lighting
                ])
                img_card = seq.augment_image(img_card)
            mask_scale = cv2.resize(card_mask, (int(len(card_mask[0]) * card.scale), int(len(card_mask) * card.scale)))
            img_mask = cv2.bitwise_and(img_card, mask_scale)
            img_rotate = imutils.rotate_bound(img_mask, card.theta / math.pi * 180)
            
            # Calculate the position of the card image in relation to the background
            # Crop the card image if it's out of boundary
            card_w = len(img_rotate[0])
            card_h = len(img_rotate)
            card_crop_x1 = max(0, card_w // 2 - card_x)
            card_crop_x2 = min(card_w, card_w // 2 + len(img_result[0]) - card_x)
            card_crop_y1 = max(0, card_h // 2 - card_y)
            card_crop_y2 = min(card_h, card_h // 2 + len(img_result) - card_y)
            img_card_crop = img_rotate[card_crop_y1:card_crop_y2, card_crop_x1:card_crop_x2]

            # Calculate the position of the corresponding area in the background
            bg_crop_x1 = max(0, card_x - (card_w // 2))
            bg_crop_x2 = min(len(img_result[0]), int(card_x + (card_w / 2) + 0.5))
            bg_crop_y1 = max(0, card_y - (card_h // 2))
            bg_crop_y2 = min(len(img_result), int(card_y + (card_h / 2) + 0.5))
            img_result_crop = img_result[bg_crop_y1:bg_crop_y2, bg_crop_x1:bg_crop_x2]

            # Override the background with the current card
            img_result_crop = np.where(img_card_crop, img_card_crop, img_result_crop)
            img_result[bg_crop_y1:bg_crop_y2, bg_crop_x1:bg_crop_x2] = img_result_crop
            
            if debug:
                for ext_obj in card.objects:
                    if ext_obj.visible:
                        for pt in ext_obj.key_pts:
                            cv2.circle(img_result, card.coordinate_in_generator(pt[0], pt[1]), 2, (1, 1, 255), 10)
                        bounding_box = card.bb_in_generator(ext_obj.key_pts)
                        cv2.rectangle(img_result, bounding_box[0], bounding_box[2], (1, 255, 1), 5)

        img_result = cv2.GaussianBlur(img_result, (5, 5), 0)

        # Skew the cards if it's provided
        if self.M is not None:
            img_result = cv2.warpPerspective(img_result, self.M, (self.width, self.height))
            if debug:
                for card in self.cards:
                    for ext_obj in card.objects:
                        if ext_obj.visible:
                            new_pts = np.array([[list(card.coordinate_in_generator(pt[0], pt[1]))]
                                                for pt in ext_obj.key_pts], dtype=np.float32)
                            new_pts = cv2.perspectiveTransform(new_pts, self.M)
                            for pt in new_pts:
                                cv2.circle(img_result, (pt[0][0], pt[0][1]), 2, (255, 1, 1), 10)

        img_bg = cv2.resize(self.img_bg, (self.width, self.height))
        img_result = np.where(img_result, img_result, img_bg)

        # Apply image augmentation
        if aug is not None:
            img_result = aug.augment_image(img_result)

        if display or debug:
            cv2.imshow('Result', img_result)
            cv2.waitKey(0)

        self.img_result = img_result
        pass

    def generate_horizontal_span(self, gap=None, scale=None, theta=0, shift=None, jitter=None):
        """
        Generating the first scenario where the cards are laid out in a straight horizontal line
        :param gap: horizontal offset between each adjacent cards
        :param scale: scale of each cards in the generator
        :param theta: rotation of the entire span in radian
        :param shift: range of arbitrary offset for each card
        :param jitter: range of in-place rotation for each card in radian
        :return: True if successfully generated, otherwise False
        """
        # Set scale of the cards, variance of shift & jitter to be applied if they're not given
        card_size = (len(self.cards[0].img[0]), len(self.cards[0].img))
        if scale is None:
            # Scale the cards so that card takes about 50% of the image's height
            coverage_ratio = 0.5
            scale = self.height * coverage_ratio / card_size[1]
        if shift is None:
            # Plus minus 5% of the card's height
            shift = [-card_size[1] * scale * 0.05, card_size[1] * scale * 0.05]
            pass
        if jitter is None:
            # Plus minus 10 degrees
            jitter = [-math.pi / 18, math.pi / 18]
        if gap is None:
            # 25% of the card's width - set symbol and 1-2 mana symbols will be visible on each card
            gap = card_size[0] * scale * 0.4

        # Determine the location of the first card
        # The cards will cover (width of a card + (# of cards - 1) * gap) pixels wide and (height of a card) pixels high
        x_anchor = int(self.width / 2 + (len(self.cards) - 1) * gap / 2)
        y_anchor = self.height // 2
        for card in self.cards:
            card.scale = scale
            card.x = x_anchor
            card.y = y_anchor
            card.theta = 0
            card.shift(shift, shift)
            card.rotate(jitter)
            card.rotate(theta, centre=(self.width // 2 - x_anchor, self.height // 2 - y_anchor))
            x_anchor -= gap

        return True

    def generate_vertical_span(self, gap=None, scale=None, theta=0, shift=None, jitter=None):
        """
        Generating the second scenario where the cards are laid out in a straight vertical line
        :param gap: horizontal offset between each adjacent cards
        :param scale: scale of each cards in the generator
        :param theta: rotation of the entire span in radian
        :param shift: range of arbitrary offset for each card
        :param jitter: range of in-place rotation for each card in radian
        :return: True if successfully generated, otherwise False
        :return: True if successfully generated, otherwise False
        """
        # Set scale of the cards, variance of shift & jitter to be applied if they're not given
        card_size = (len(self.cards[0].img[0]), len(self.cards[0].img))
        if scale is None:
            # Scale the cards so that card takes about 50% of the image's height
            coverage_ratio = 0.5
            scale = self.height * coverage_ratio / card_size[1]
        if shift is None:
            # Plus minus 5% of the card's height
            shift = [-card_size[1] * scale * 0.05, card_size[1] * scale * 0.05]
            pass
        if jitter is None:
            # Plus minus 5 degrees
            jitter = [-math.pi / 36, math.pi / 36]
        if gap is None:
            # 15% of the card's height - the title bar (with mana symbols) will be visible
            gap = card_size[1] * scale * 0.25

        # Determine the location of the first card
        # The cards will cover (width of a card) pixels wide and (height of a card + (# of cards - 1) * gap) pixels high
        x_anchor = self.width // 2
        y_anchor = int(self.height / 2 - (len(self.cards) - 1) * gap / 2)
        for card in self.cards:
            card.scale = scale
            card.x = x_anchor
            card.y = y_anchor
            card.theta = 0
            card.shift(shift, shift)
            card.rotate(jitter)
            card.rotate(theta, centre=(self.width // 2 - x_anchor, self.height // 2 - y_anchor))
            y_anchor += gap
        return True

    def generate_fan_out(self, centre, theta_between_cards=None, scale=None, shift=None, jitter=None):
        """
        Generating the third scenario where the cards are laid out in a fan shape
        :return: True if successfully generated, otherwise False
        """
        # TODO
        return False

    def generate_non_obstructive(self, tolerance=0.90, scale=None):
        """
        Generating the fourth scenario where the cards are laid in arbitrary position that doesn't obstruct other cards
        :param tolerance: minimum level of visibility for each cards
        :param scale: scale of each cards in generator
        :return: True if successfully generated, otherwise False
        """
        card_size = (len(self.cards[0].img[0]), len(self.cards[0].img))
        if scale is None:
            # Total area of the cards should cover about 25-40% of the entire image, depending on the number of cards
            scale = math.sqrt(self.width * self.height * min(0.25 + 0.02 * len(self.cards), 0.4)
                              / (card_size[0] * card_size[1] * len(self.cards)))
        # Position each card at random location that doesn't obstruct other cards
        i = 0
        while i < len(self.cards):
            card = self.cards[i]
            card.scale = scale
            rep = 0
            while True:
                card.x = random.uniform(card_size[1] * scale / 2, self.width - card_size[1] * scale)
                card.y = random.uniform(card_size[1] * scale / 2, self.height - card_size[1] * scale)
                card.theta = random.uniform(-math.pi, math.pi)
                self.check_visibility(self.cards[:i + 1], visibility=tolerance)
                # This position is not obstructive if all of the cards are visible
                is_visible = [other_card.objects[0].visible for other_card in self.cards[:i + 1]]
                non_obstructive = all(is_visible)
                if non_obstructive:
                    i += 1
                    break
                rep += 1
                if rep >= 1000:
                    # Reassign previous card's position
                    i -= 1
                    break
        return True

    def check_visibility(self, cards=None, i_check=None, visibility=0.5):
        """
        Check whether if extracted objects in a card is visible in the current scenario, and update their status
        :param cards: list of cards (in a correct Z-order). All cards in this Generator are checked by default.
        :param i_check: indices of cards that needs to be checked. Cards that aren't in this list will only be used
        to check visibility of other cards. All cards are checked by default.
        :param visibility: minimum ratio of the object's area that aren't covered by another card to be visible
        :return: none
        """
        if cards is None:
            cards = self.cards
        if i_check is None:
            i_check = range(len(cards))

        # Create a polygon of each card
        card_poly_list = [geometry.Polygon([card.coordinate_in_generator(0, 0),
                                            card.coordinate_in_generator(0, len(card.img)),
                                            card.coordinate_in_generator(len(card.img[0]), len(card.img)),
                                            card.coordinate_in_generator(len(card.img[0]), 0)]) for card in self.cards]
        template_poly = geometry.Polygon([(0, 0), (self.width, 0), (self.width, self.height), (0, self.height)])

        # First card in the list is overlaid on the bottom of the card pile
        for i in i_check:
            card = cards[i]
            for ext_obj in card.objects:
                obj_poly = geometry.Polygon([card.coordinate_in_generator(pt[0], pt[1]) for pt in ext_obj.key_pts])
                obj_area = obj_poly.area
                # Check if the other cards are blocking this object or if it's out of the template
                # If there are other polygons with higher indices in the list, that card is overlapping this object
                # We assume that no objects from the same card is on top of each other
                for card_poly in card_poly_list[i + 1:]:
                    obj_poly = obj_poly.difference(card_poly)
                obj_poly = obj_poly.intersection(template_poly)
                visible_area = obj_poly.area
                ext_obj.visible = obj_area * visibility <= visible_area

    def export_training_data(self, out_name, visibility=0.5, aug=None):
        """
        Export the generated training image along with the txt file for all bounding boxes
        :param out_name: path of the output file (without extension)
        :param visibility: portion of the card's image that must not be overlapped by other cards for the card to be
                           considered as visible
        :param aug: image augmentator to be applied
        :return: none
        """
        self.render(visibility, aug=aug)
        cv2.imwrite(out_name + '.jpg', self.img_result)
        out_txt = open(out_name + '.txt', 'w')
        for card in self.cards:
            for ext_obj in card.objects:
                if not ext_obj.visible:
                    continue
                coords_in_gen = [card.coordinate_in_generator(key_pt[0], key_pt[1]) for key_pt in ext_obj.key_pts]
                obj_yolo_info = key_pts_to_yolo(coords_in_gen, self.width, self.height)
                if ext_obj.label == 'card':
                    #class_id = self.class_ids[card.info['name']]
                    class_id = 0  # since only the entire card is used
                    out_txt.write(str(class_id) + ' %.6f %.6f %.6f %.6f\n' % obj_yolo_info)
        out_txt.close()


class Card:
    """
    A class for storing required information about a card in relation to the ImageGenerator
    """
    def __init__(self, img, card_info, objects, x=None, y=None, theta=None, scale=None):
        """
        :param img: image of the card
        :param card_info: details like name, mana cost, type, set, etc
        :param objects: list of ExtractedObjects like mana & set symbol, etc
        :param x: X-coordinate of the card's centre in relation to the generator
        :param y: Y-coordinate of the card's centre in relation to the generator
        :param theta: angle of rotation of the card in relation to the generator
        :param scale: scale of the card in the generator in relation to the original image
        """
        self.img = img
        self.info = card_info
        self.objects = objects
        self.x = x
        self.y = y
        self.theta = theta
        self.scale = scale
        pass

    def shift(self, x, y):
        """
        Apply a X/Y translation on this image
        :param x: amount of X-translation. If range is given, translate by a random amount within that range
        :param y: amount of Y-translation. If range is given, translate by a random amount within that range
        :return: none
        """
        if isinstance(x, tuple) or (isinstance(x, list) and len(x) == 2):
            self.x += random.uniform(x[0], x[1])
        else:
            self.x += x
        if isinstance(y, tuple) or (isinstance(y, list) and len(y) == 2):
            self.y += random.uniform(y[0], y[1])
        else:
            self.y += y
        pass

    def rotate(self, theta, centre=(0, 0)):
        """
        Apply a rotation on this image with a centre
        :param theta: amount of rotation in radian (clockwise). If a range is given, rotate by a random amount within
                      that range
        :param centre: coordinate of the centre of the rotation in relation to the centre of this card
        :return: none
        """
        if isinstance(theta, tuple) or (isinstance(theta, list) and len(theta) == 2):
            theta = random.uniform(theta[0], theta[1])

        # If the centre given is the centre of this card, the whole math simplifies a bit
        # (This still works without the if statement, but let's not do useless trigs if we know the answer already)
        if centre is not (0, 0):
            # Rotation math
            self.x -= -centre[1] * math.sin(theta) + centre[0] * math.cos(theta)
            self.y -= centre[1] * math.cos(theta) + centre[0] * math.sin(theta)

            # Offset for the coordinate translation
            self.x += centre[0]
            self.y += centre[1]

        self.theta += theta
        pass

    def coordinate_in_generator(self, x, y):
        """
        Converting coordinate within the card into the coordinate in the generator it is associated with
        :param x: x coordinate within the card
        :param y: y coordinate within the card
        :return: (x, y) coordinate in the generator
        """
        # Relative distance in X & Y axis, if the centre of the card is at the origin (0, 0)
        rel_x = x - len(self.img[0]) // 2
        rel_y = y - len(self.img) // 2

        # Scaling
        rel_x *= self.scale
        rel_y *= self.scale

        # Rotation
        rot_x = rel_x - rel_y * math.sin(self.theta) + rel_x * math.cos(self.theta)
        rot_y = rel_y + rel_y * math.cos(self.theta) + rel_x * math.sin(self.theta)

        # Negate offset
        rot_x -= rel_x
        rot_y -= rel_y

        # Shift
        gen_x = rot_x + self.x
        gen_y = rot_y + self.y

        return int(gen_x), int(gen_y)

    def bb_in_generator(self, key_pts):
        """
        Convert a keypoints of bounding box in card into the coordinate in the generator
        :param key_pts: keypoints of the bounding box
        :return: bounding box represented by 4 points in the generator
        """
        coords_in_gen = [self.coordinate_in_generator(key_pt[0], key_pt[1]) for key_pt in key_pts]
        x1 = min([pt[0] for pt in coords_in_gen])
        x2 = max([pt[0] for pt in coords_in_gen])
        y1 = min([pt[1] for pt in coords_in_gen])
        y2 = max([pt[1] for pt in coords_in_gen])
        return [(x1, y1), (x2, y1), (x2, y2), (x1, y2)]


class ExtractedObject:
    """
    Simple struct to hold information about an extracted object
    """
    def __init__(self, label, key_pts):
        self.label = label
        self.key_pts = key_pts
        self.visible = False


def main(args):
    random.seed()
    ia.seed(random.randrange(10000))

    bg_images = generate_data.load_dtd(dtd_dir='%s/dtd/images' % Config.data_dir, dump_it=False)
    background = generate_data.Backgrounds(images=bg_images)

    card_pool = pd.DataFrame()
    for set_name in Config.all_set_list:
        df = fetch_data.load_all_cards_text('%s/csv/%s.csv' % (Config.data_dir, set_name))
        card_pool = card_pool.append(df)
    class_ids = {}
    with open('%s/obj.names' % Config.data_dir) as names_file:
        class_name_list = names_file.read().splitlines()
        for i in range(len(class_name_list)):
            class_ids[class_name_list[i]] = i

    for i in range(args.num_gen):
        # Arbitrarily select top left and right corners for perspective transformation
        # Since the training image are generated with random rotation, don't need to skew all four sides
        skew = [[random.uniform(0, 0.25), 0], [0, 1], [1, 1],
                [random.uniform(0.75, 1), 0]]
        generator = ImageGenerator(background.get_random(), class_ids, args.width, args.height, skew=skew)
        out_name = ''

        # Use 2 to 5 cards per generator
        for _, card_info in card_pool.sample(random.randint(2, 5)).iterrows():
            img_name = '%s/card_img/png/%s/%s_%s.png' % (Config.data_dir, card_info['set'],
                                                         card_info['collector_number'],
                                                         fetch_data.get_valid_filename(card_info['name']))
            out_name += '%s%s_' % (card_info['set'], card_info['collector_number'])
            card_img = cv2.imread(img_name)
            if card_img is None:
                fetch_data.fetch_card_image(card_info, out_dir='%s/card_img/png/%s' % (Config.data_dir,
                                                                                       card_info['set']))
                card_img = cv2.imread(img_name)
            if card_img is None:
                print('WARNING: card %s is not found!' % img_name)
            detected_object_list = generate_data.apply_bounding_box(card_img, card_info)
            card = Card(card_img, card_info, detected_object_list)
            generator.add_card(card)

        for j in range(args.num_iter):
            seq = iaa.Sequential([
                iaa.Multiply((0.8, 1.2)),  # darken / brighten the whole image
                iaa.SimplexNoiseAlpha(first=iaa.Add(random.randrange(64)), per_channel=0.1, size_px_max=[3, 6],
                                      upscale_method="cubic"),  # Lighting
                iaa.AdditiveGaussianNoise(scale=random.uniform(0, 0.05) * 255, per_channel=0.1),  # Noises
                iaa.Dropout(p=[0, 0.05], per_channel=0.1)  # Dropout
            ])

            if i % 3 == 0:
                generator.generate_non_obstructive()
                generator.export_training_data(visibility=0.0, out_name='%s/train/non_obstructive_update/%s%d'
                                                                        % (Config.data_dir, out_name, j), aug=seq)
            elif i % 3 == 1:
                generator.generate_horizontal_span(theta=random.uniform(-math.pi, math.pi))
                generator.export_training_data(visibility=0.0, out_name='%s/train/horizontal_span_update/%s%d'
                                                                        % (Config.data_dir, out_name, j), aug=seq)
            else:
                generator.generate_vertical_span(theta=random.uniform(-math.pi, math.pi))
                generator.export_training_data(visibility=0.0, out_name='%s/train/vertical_span_update/%s%d'
                                                                        % (Config.data_dir, out_name, j), aug=seq)

            #generator.generate_horizontal_span(theta=random.uniform(-math.pi, math.pi))
            #generator.render(display=True, aug=seq, debug=True)
            print('Generated %s%d' % (out_name, j))
            generator.img_bg = background.get_random()
    pass


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--num_gen', dest='num_gen', help='Number of training images to generate',
                        type=int, required=True)
    parser.add_argument('-ni', '--num_iter', dest='num_iter', help='Number of iterations to generate each config',
                        type=int, default=1)
    parser.add_argument('-w', '--width', dest='width', help='Width of the training image', type=int, default=1440)
    parser.add_argument('-ht', '--height', dest='height', help='Height of the training image', type=int, default=960)
    args = parser.parse_args()
    main(args)