CelebAMask_HQ_Dataset_Processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CelebAMask HQ Dataset Processing",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMy8G3IL+8++WTZ/K7SQBqo"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3opGFbBLdzGi"
      },
      "source": [
        "#### <b>Load Dataset</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y2x8aEoX1i0V",
        "outputId": "7cf90906-2c83-4712-8dad-c37cc3e1e6d9"
      },
      "source": [
        "!wget https://postechackr-my.sharepoint.com/:u:/g/personal/dongbinna_postech_ac_kr/Eb37jNPPA7hHl0fmktYqcV8B-qmPLx-ZKYQ1eFk4UPBV_A?download=1 -O CelebAMask-HQ.zip\r\n",
        "!wget https://postechackr-my.sharepoint.com/:t:/g/personal/dongbinna_postech_ac_kr/EVRoUY8_txRFv56-KWvZrksBDWbD6adkjBxwwRN7qAC6bg?download=1 -O CelebA-HQ-identity.txt\r\n",
        "!wget https://postechackr-my.sharepoint.com/:t:/g/personal/dongbinna_postech_ac_kr/EVrdIrPOkR1OlEWBVK8lE3AB9bFh741GnKBkNgPa8trNuA?download=1 -O CelebA-HQ-attribute.txt"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-03-12 07:55:29--  https://postechackr-my.sharepoint.com/:u:/g/personal/dongbinna_postech_ac_kr/Eb37jNPPA7hHl0fmktYqcV8B-qmPLx-ZKYQ1eFk4UPBV_A?download=1\n",
            "Resolving postechackr-my.sharepoint.com (postechackr-my.sharepoint.com)... 13.107.136.9\n",
            "Connecting to postechackr-my.sharepoint.com (postechackr-my.sharepoint.com)|13.107.136.9|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /personal/dongbinna_postech_ac_kr/Documents/Research/datasets/CelebAMask-HQ/CelebAMask-HQ.zip?originalPath=aHR0cHM6Ly9wb3N0ZWNoYWNrci1teS5zaGFyZXBvaW50LmNvbS86dTovZy9wZXJzb25hbC9kb25nYmlubmFfcG9zdGVjaF9hY19rci9FYjM3ak5QUEE3aEhsMGZta3RZcWNWOEItcW1QTHgtWktZUTFlRms0VVBCVl9BP3J0aW1lPTBhRWROU3psMkVn [following]\n",
            "--2021-03-12 07:55:30--  https://postechackr-my.sharepoint.com/personal/dongbinna_postech_ac_kr/Documents/Research/datasets/CelebAMask-HQ/CelebAMask-HQ.zip?originalPath=aHR0cHM6Ly9wb3N0ZWNoYWNrci1teS5zaGFyZXBvaW50LmNvbS86dTovZy9wZXJzb25hbC9kb25nYmlubmFfcG9zdGVjaF9hY19rci9FYjM3ak5QUEE3aEhsMGZta3RZcWNWOEItcW1QTHgtWktZUTFlRms0VVBCVl9BP3J0aW1lPTBhRWROU3psMkVn\n",
            "Reusing existing connection to postechackr-my.sharepoint.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3153930546 (2.9G) [application/x-zip-compressed]\n",
            "Saving to: ‘CelebAMask-HQ.zip’\n",
            "\n",
            "CelebAMask-HQ.zip   100%[===================>]   2.94G  22.4MB/s    in 1m 50s  \n",
            "\n",
            "2021-03-12 07:57:21 (27.4 MB/s) - ‘CelebAMask-HQ.zip’ saved [3153930546/3153930546]\n",
            "\n",
            "--2021-03-12 07:57:21--  https://postechackr-my.sharepoint.com/:t:/g/personal/dongbinna_postech_ac_kr/EVRoUY8_txRFv56-KWvZrksBDWbD6adkjBxwwRN7qAC6bg?download=1\n",
            "Resolving postechackr-my.sharepoint.com (postechackr-my.sharepoint.com)... 13.107.136.9\n",
            "Connecting to postechackr-my.sharepoint.com (postechackr-my.sharepoint.com)|13.107.136.9|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /personal/dongbinna_postech_ac_kr/Documents/Research/datasets/CelebAMask-HQ/CelebA-HQ-identity.txt?originalPath=aHR0cHM6Ly9wb3N0ZWNoYWNrci1teS5zaGFyZXBvaW50LmNvbS86dDovZy9wZXJzb25hbC9kb25nYmlubmFfcG9zdGVjaF9hY19rci9FVlJvVVk4X3R4UkZ2NTYtS1d2WnJrc0JEV2JENmFka2pCeHd3Uk43cUFDNmJnP3J0aW1lPWxJVDBkeXpsMkVn [following]\n",
            "--2021-03-12 07:57:22--  https://postechackr-my.sharepoint.com/personal/dongbinna_postech_ac_kr/Documents/Research/datasets/CelebAMask-HQ/CelebA-HQ-identity.txt?originalPath=aHR0cHM6Ly9wb3N0ZWNoYWNrci1teS5zaGFyZXBvaW50LmNvbS86dDovZy9wZXJzb25hbC9kb25nYmlubmFfcG9zdGVjaF9hY19rci9FVlJvVVk4X3R4UkZ2NTYtS1d2WnJrc0JEV2JENmFka2pCeHd3Uk43cUFDNmJnP3J0aW1lPWxJVDBkeXpsMkVn\n",
            "Reusing existing connection to postechackr-my.sharepoint.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 427557 (418K) [text/plain]\n",
            "Saving to: ‘CelebA-HQ-identity.txt’\n",
            "\n",
            "CelebA-HQ-identity. 100%[===================>] 417.54K   443KB/s    in 0.9s    \n",
            "\n",
            "2021-03-12 07:57:23 (443 KB/s) - ‘CelebA-HQ-identity.txt’ saved [427557/427557]\n",
            "\n",
            "--2021-03-12 07:57:23--  https://postechackr-my.sharepoint.com/:t:/g/personal/dongbinna_postech_ac_kr/EVrdIrPOkR1OlEWBVK8lE3AB9bFh741GnKBkNgPa8trNuA?download=1\n",
            "Resolving postechackr-my.sharepoint.com (postechackr-my.sharepoint.com)... 13.107.136.9\n",
            "Connecting to postechackr-my.sharepoint.com (postechackr-my.sharepoint.com)|13.107.136.9|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /personal/dongbinna_postech_ac_kr/Documents/Research/datasets/CelebAMask-HQ/CelebA-HQ-attribute.txt?originalPath=aHR0cHM6Ly9wb3N0ZWNoYWNrci1teS5zaGFyZXBvaW50LmNvbS86dDovZy9wZXJzb25hbC9kb25nYmlubmFfcG9zdGVjaF9hY19rci9FVnJkSXJQT2tSMU9sRVdCVks4bEUzQUI5YkZoNzQxR25LQmtOZ1BhOHRyTnVBP3J0aW1lPV9sVDBlQ3psMkVn [following]\n",
            "--2021-03-12 07:57:24--  https://postechackr-my.sharepoint.com/personal/dongbinna_postech_ac_kr/Documents/Research/datasets/CelebAMask-HQ/CelebA-HQ-attribute.txt?originalPath=aHR0cHM6Ly9wb3N0ZWNoYWNrci1teS5zaGFyZXBvaW50LmNvbS86dDovZy9wZXJzb25hbC9kb25nYmlubmFfcG9zdGVjaF9hY19rci9FVnJkSXJQT2tSMU9sRVdCVks4bEUzQUI5YkZoNzQxR25LQmtOZ1BhOHRyTnVBP3J0aW1lPV9sVDBlQ3psMkVn\n",
            "Reusing existing connection to postechackr-my.sharepoint.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3622218 (3.5M) [text/plain]\n",
            "Saving to: ‘CelebA-HQ-attribute.txt’\n",
            "\n",
            "CelebA-HQ-attribute 100%[===================>]   3.45M  3.59MB/s    in 1.0s    \n",
            "\n",
            "2021-03-12 07:57:25 (3.59 MB/s) - ‘CelebA-HQ-attribute.txt’ saved [3622218/3622218]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNDdBv8z1teG"
      },
      "source": [
        "%%capture\r\n",
        "!rm -rf ./CelebAMask-HQ\r\n",
        "!unzip CelebAMask-HQ.zip"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p_4UTUJAL_t7"
      },
      "source": [
        "#### <b>Generate Facial Identity Recognition Dataset</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U17_Cso7RB0D",
        "outputId": "7f825776-30e4-4cdf-92fe-1c3e65d6b8ab"
      },
      "source": [
        "identities = {}\r\n",
        "\r\n",
        "with open('./CelebA-HQ-identity.txt') as f:\r\n",
        "    lines = f.readlines()\r\n",
        "    for line in lines:\r\n",
        "        file_name, identity = line.strip().split()\r\n",
        "        identities[file_name] = identity\r\n",
        "\r\n",
        "print(f'There are {len(set(identities.values()))} identities.')\r\n",
        "print(f'There are {len(identities.keys())} images.')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 6217 identities.\n",
            "There are 30000 images.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Ejc5bR7MACf"
      },
      "source": [
        "import os\r\n",
        "from shutil import copyfile\r\n",
        "\r\n",
        "\r\n",
        "source_root = './CelebAMask-HQ/CelebA-HQ-img/'\r\n",
        "target_root = './identity_dataset/'\r\n",
        "file_list = os.listdir(source_root)\r\n",
        "\r\n",
        "for file in file_list:\r\n",
        "    identity = identities[file]\r\n",
        "    source = os.path.join(source_root, file)\r\n",
        "    target = os.path.join(target_root, str(identity), file)\r\n",
        "    if not os.path.exists(os.path.join(target_root, str(identity))):\r\n",
        "        os.makedirs(os.path.join(target_root, str(identity)))\r\n",
        "    copyfile(source, target)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ulW7t-MsMm-C",
        "outputId": "377502dc-36cc-4b29-c06b-b6ab9ad4739f"
      },
      "source": [
        "folder_root = './identity_dataset/'\r\n",
        "folder_list = os.listdir(folder_root)\r\n",
        "\r\n",
        "threshold = 15\r\n",
        "identity_cnt = 0\r\n",
        "\r\n",
        "train_images = 0\r\n",
        "test_images = 0\r\n",
        "train_ratio = 0.8\r\n",
        "\r\n",
        "for folder in folder_list:\r\n",
        "    file_list = os.path.join(folder_root, folder)\r\n",
        "    file_list = os.listdir(file_list)\r\n",
        "    if len(file_list) >= threshold:\r\n",
        "        identity_cnt += 1\r\n",
        "        num_train = int(train_ratio * len(file_list))\r\n",
        "        for file in file_list[:num_train]:\r\n",
        "            train_images += 1\r\n",
        "            source = os.path.join(folder_root, folder, file)\r\n",
        "            target = os.path.join(folder_root, 'train', folder, file)\r\n",
        "            if not os.path.exists(os.path.join(folder_root, 'train', folder)):\r\n",
        "                os.makedirs(os.path.join(folder_root, 'train', folder))\r\n",
        "            os.rename(source, target)\r\n",
        "        for file in file_list[num_train:]:\r\n",
        "            test_images += 1\r\n",
        "            source = os.path.join(folder_root, folder, file)\r\n",
        "            target = os.path.join(folder_root, 'test', folder, file)\r\n",
        "            if not os.path.exists(os.path.join(folder_root, 'test', folder)):\r\n",
        "                os.makedirs(os.path.join(folder_root, 'test', folder))\r\n",
        "            os.rename(source, target)\r\n",
        "\r\n",
        "print(f'There are {identity_cnt} identities that have more than {threshold} images.')\r\n",
        "print(f'There are {train_images} train images.')\r\n",
        "print(f'There are {test_images} test images.')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 307 identities that have more than 15 images.\n",
            "There are 4263 train images.\n",
            "There are 1215 test images.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-2w3kSzYEcq"
      },
      "source": [
        "!mkdir -p ./facial_identity_dataset/train\r\n",
        "!mkdir -p ./facial_identity_dataset/test\r\n",
        "os.rename('./identity_dataset/train', './facial_identity_dataset/train')\r\n",
        "os.rename('./identity_dataset/test', './facial_identity_dataset/test')"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Fx5hBw-YCLG"
      },
      "source": [
        "#### <b>Generate Face Gender Recognition Dataset</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S3eB3332YLUu",
        "outputId": "9fa82f13-9892-43f1-edb5-6089254657ad"
      },
      "source": [
        "gender_map = {}\r\n",
        "\r\n",
        "with open('./CelebA-HQ-attribute.txt') as f:\r\n",
        "    lines = f.readlines()\r\n",
        "    for line in lines[2:]:\r\n",
        "        splited = line.strip().split()\r\n",
        "        file_name, male = splited[0], splited[21]\r\n",
        "        gender_map[file_name] = male\r\n",
        "\r\n",
        "print(f'There are {len(set(gender_map.values()))} classes.')\r\n",
        "print(f'There are {len(gender_map.keys())} images.')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 2 classes.\n",
            "There are 30000 images.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlaOVHCtZBVk"
      },
      "source": [
        "import os\r\n",
        "from shutil import copyfile\r\n",
        "\r\n",
        "\r\n",
        "source_root = './CelebAMask-HQ/CelebA-HQ-img/'\r\n",
        "target_root = './gender_dataset/'\r\n",
        "file_list = os.listdir(source_root)\r\n",
        "\r\n",
        "for file in file_list:\r\n",
        "    gender = gender_map[file]\r\n",
        "    if gender == '1':\r\n",
        "        gender = 'male'\r\n",
        "    else:\r\n",
        "        gender = 'female'\r\n",
        "    source = os.path.join(source_root, file)\r\n",
        "    target = os.path.join(target_root, gender, file)\r\n",
        "    if not os.path.exists(os.path.join(target_root, gender)):\r\n",
        "        os.makedirs(os.path.join(target_root, gender))\r\n",
        "    copyfile(source, target)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8ws6CVhUZCr3",
        "outputId": "76e3e207-9bf5-4585-9e19-0b9b2d168bae"
      },
      "source": [
        "folder_root = './gender_dataset/'\r\n",
        "folder_list = os.listdir(folder_root)\r\n",
        "\r\n",
        "male_cnt = 0\r\n",
        "female_cnt = 0\r\n",
        "\r\n",
        "train_images = 0\r\n",
        "test_images = 0\r\n",
        "train_ratio = 0.8\r\n",
        "\r\n",
        "for folder in folder_list:\r\n",
        "    file_list = os.path.join(folder_root, folder)\r\n",
        "    file_list = os.listdir(file_list)\r\n",
        "    if folder == 'male':\r\n",
        "        male_cnt += len(file_list)\r\n",
        "    else:\r\n",
        "        female_cnt += len(file_list)\r\n",
        "    num_train = int(train_ratio * len(file_list))\r\n",
        "    for file in file_list[:num_train]:\r\n",
        "        train_images += 1\r\n",
        "        source = os.path.join(folder_root, folder, file)\r\n",
        "        target = os.path.join(folder_root, 'train', folder, file)\r\n",
        "        if not os.path.exists(os.path.join(folder_root, 'train', folder)):\r\n",
        "            os.makedirs(os.path.join(folder_root, 'train', folder))\r\n",
        "        os.rename(source, target)\r\n",
        "    for file in file_list[num_train:]:\r\n",
        "        test_images += 1\r\n",
        "        source = os.path.join(folder_root, folder, file)\r\n",
        "        target = os.path.join(folder_root, 'test', folder, file)\r\n",
        "        if not os.path.exists(os.path.join(folder_root, 'test', folder)):\r\n",
        "            os.makedirs(os.path.join(folder_root, 'test', folder))\r\n",
        "        os.rename(source, target)\r\n",
        "\r\n",
        "print(f'There are {male_cnt} male images.')\r\n",
        "print(f'There are {female_cnt} female images.')\r\n",
        "print(f'There are {train_images} train images.')\r\n",
        "print(f'There are {test_images} test images.')"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 11057 male images.\n",
            "There are 18943 female images.\n",
            "There are 23999 train images.\n",
            "There are 6001 test images.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SV9ULnfebDV_"
      },
      "source": [
        "!mkdir -p ./face_gender_dataset/train\r\n",
        "!mkdir -p ./face_gender_dataset/test\r\n",
        "os.rename('./gender_dataset/train', './face_gender_dataset/train')\r\n",
        "os.rename('./gender_dataset/test', './face_gender_dataset/test')"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EEF-bLZieN3k"
      },
      "source": [
        "#### <b>Save Processed Dataset</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4AJbOdjZd4Qq"
      },
      "source": [
        "%%capture\r\n",
        "!zip -r facial_identity_dataset.zip ./facial_identity_dataset/\r\n",
        "!zip -r face_gender_dataset.zip ./face_gender_dataset/"
      ],
      "execution_count": 19,
      "outputs": []
    }
  ]
}