generate_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "OMNIGLOT_DATA = os.path.join(os.getcwd(), 'omniglot/')\n",
    "DATASET_DIR = os.path.join(os.getcwd(), 'cluttered_omniglot/')\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['figure.figsize'] = (12.0, 12.0)\n",
    "\n",
    "import dataset_utils\n",
    "\n",
    "from joblib import Parallel, delayed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create config file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = dataset_utils.DatasetGeneratorConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Omniglot Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reorder_chars(chars):\n",
    "    reordered_chars=[]\n",
    "    for alph in range(len(chars)):\n",
    "        for char in range(len(chars[alph])):\n",
    "            reordered_chars.append(chars[alph][char])\n",
    "    \n",
    "    return reordered_chars\n",
    "\n",
    "#Load chars from pickle file\n",
    "path = OMNIGLOT_DATA\n",
    "\n",
    "#Train split\n",
    "with open(path + 'chars_train.pickle', 'rb') as fp:\n",
    "    chars_train = pickle.load(fp)\n",
    "    chars_train = reorder_chars(chars_train)\n",
    "    \n",
    "#Evaluation split\n",
    "with open(path + 'chars_eval.pickle', 'rb') as fp:\n",
    "    chars_eval = pickle.load(fp)\n",
    "    chars_eval = reorder_chars(chars_eval)\n",
    "    \n",
    "#Test split\n",
    "with open(path + 'chars_test.pickle', 'rb') as fp:\n",
    "    chars_test = pickle.load(fp)\n",
    "    chars_test = reorder_chars(chars_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset Generation Script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checksums = {4: [b'\\x8c\\x99\\x9e\\\\J\\x9a\\x811g.eB\\xa0\\xb3\\xf2f', \n",
    "              b';\\xf8\\x05\\xb2vk#!\\xf9\\xad\\xb9\\x88\\xd1\\n\\x0f\\xba', \n",
    "              b'\\xc6\\x84\\xf4\\x99\\x18<cx\\x82\\xeb\\xed*\\xbb\\xca\\x12\\x8b', \n",
    "              b'\\xd3\\x8c\\x88\\rM\\xdfSpK\\xa7\\xf2f\\n?\\x0e\\xad'], \n",
    "             8: [b'\\x0cY\\x0f\\xf4#\\x872:2\\xe1\\x12R\\xcf\\x95rh', \n",
    "              b'\\x92\\x7f\\xa3>@\\x0cl\\xaa\\x96\\xd7\\xad\\xbaOj\\xdd\\xac', \n",
    "              b'\\xb9m\\xc0\\xbdN8\\xf2\\x0fkk\\x8c&N\\x88\\xe2\\xb0', \n",
    "              b'M\\xf1\\x8d\\r\\xe6\\xf5\\xba\\tR\\xe3\\xcc\\xd2\\x91\\xf2\\x96\\x81'], \n",
    "             16: [b'2}\\x08\\xf3\\xe1\\xb1\\xb1\\xb5\\xed\\x89\\x05\\xf1c\\xedy^', \n",
    "              b'\\x8bs>\\x95\\xa8n9Nw\\xf0E\\x83\\xf8~+\\xf2', \n",
    "              b'\\xde8\\xa6\\x04\\xdfq<\\xf0\\x05&a\\x00EB\\xc9B', \n",
    "              b'\\xbf\\x87uy\\xec\\x90\\xbf \\x9a\\x96x\\x9e\\xd9w\\xd3\\x94'], \n",
    "             32: [b'\\x813\\x89v\\xc8\\xbd\\xc16\\x8dN\\xc5Z\\x9eS=y', \n",
    "              b'o\\xd4\\xe0a\\x8a\\x80\\x0f\\xba2_\\x06\\xffE\\xed\\xf9n', \n",
    "              b':\\x05\\xe2\\x03\\xfb\\x85\\x84\\x18\\xc5\\x18\\x81d\\xd9V\\xb9[', \n",
    "              b'\\xcc\\x96\\xe1D\\x90\\xba\\xed\\xa3\\xff\\x1c\\xd7\\xd7\\xdf\\xe4\\xd0A'], \n",
    "             64: [b'\\x11\\x19HZ\\x05-\\xf3A\\xe8\\xe6\\x0f\\x9f\\x14\\n\\xcb\\x17', \n",
    "              b'VP\\x0c\\xcf\\xa0\\xef\\x07_\\x85[\\xa7\\x0cL*|\\xa2', \n",
    "              b'(\\x0f\\xba,\\x0c\\x98+\\x85\\xa9n\\xd0a\\x1b\\xe0p\\x03', \n",
    "              b'W\\xf4\\xc7\\xed\\x19f\\xc50f\\xb3\\xb3\\xc8\\x80\\xf0\\xda\\x90'], \n",
    "             128: [b'\\x8a\\x11O\\x805\\xb6\\xcf\\xf0L\\xaf\\xad\\x93\\xbf&/\\xb2', \n",
    "              b'\\xc8\\xd16\\xfa\\xb2\\x16h\\xd8\\xa3h\\xf5\\x8e\\xa7\\x00\\x90v', \n",
    "              b'\\x05\\xd2\\xd94T\\xe1\\xbb\\xb63\\x90\\x9eX\\xd8\\t\\xfc\\xb9', \n",
    "              b'el\\x85\\xda\\xfcw\\xce\\xe0\\xf5\\x9c\\xb7 v\\xc6J='], \n",
    "             256: [b's\\xffZO\\xc1j\\x87\\x7f\\xbf\\x1fj\\xf2]\\xc9\\x9f\\x14', \n",
    "              b'D\\xbe\\x0eZ%\\xb2\\x16Z\\xea\\x18S\\x1d!6\\xd0w', \n",
    "              b'\\xfbZ\\xe4$\\x06\\xd88J\\x99\\xfa\\x0b*\\x92\\xdd\\xa3\\xa8', \n",
    "              b'z\\xee@d\\x0f}\\xe8\\xd3\\xf3\\xad\\xf7\\xe8\\xc9]\\xfbb']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate indicidual datasets for each clutter level\n",
    "nname =  [4, 8, 16, 32, 64, 128, 256]\n",
    "nchars = [3, 7, 15, 31, 63, 127, 255]\n",
    "\n",
    "for i in range(len(nname)):\n",
    "    # Set and print saving directory\n",
    "    dset_dir = DATASET_DIR + '{}_characters/'.format(nname[i])\n",
    "    print('')\n",
    "    print(dset_dir)\n",
    "\n",
    "    # Set number of images per parallel job\n",
    "    config.JOBLENGTH = 2000\n",
    "    # Set number of distractors\n",
    "    config.DISTRACTORS = nchars[i]\n",
    "    \n",
    "    ### Generate training set ###\n",
    "    \n",
    "    # Set dataset split\n",
    "    config.DRAWER_SPLIT = 'train'\n",
    "    config.set_drawer_split()\n",
    "    # Define number of train images\n",
    "    dataset_size = 2000000\n",
    "    # Choose training alphabets\n",
    "    chars = chars_train\n",
    "    # Set path\n",
    "    path = dset_dir + 'train/'\n",
    "    # Set a fixed seed\n",
    "    seed_train = 2209944264\n",
    "\n",
    "    # Generate dataset\n",
    "    print('Generating dataset train/')\n",
    "    dataset_utils.generate_dataset(path, dataset_size, chars, config, seed=seed_train, save=True)\n",
    "    print('')\n",
    "\n",
    "\n",
    "    ### Generate evaluation and test sets ###\n",
    "\n",
    "    # Set dataset split\n",
    "    config.DRAWER_SPLIT = 'val'\n",
    "    config.set_drawer_split()\n",
    "    # Define number of val/test images\n",
    "    dataset_size = 10000\n",
    "\n",
    "    #Generate evaluation set on train characters\n",
    "    seed_val_train = 4020197800\n",
    "    chars = chars_train\n",
    "    path = dset_dir + 'val-train/'\n",
    "    print('Generating dataset val-train/')\n",
    "    dataset_utils.generate_dataset(path, dataset_size, chars, config, seed=seed_val_train, save=True, checksum=checksums[nname[i]][0])\n",
    "    print('')\n",
    "    \n",
    "    #Generate test set on train characters\n",
    "    seed_test_train = 1665765955\n",
    "    chars = chars_train\n",
    "    path = dset_dir + 'test-train/'\n",
    "    print('Generating dataset test-train/')\n",
    "    dataset_utils.generate_dataset(path, dataset_size, chars, config, seed=seed_test_train, save=True, checksum=checksums[nname[i]][1])\n",
    "    print('')\n",
    "\n",
    "    #Generate evaluation set on one-shot characters\n",
    "    seed_val_one_shot = 3755213170\n",
    "    chars = chars_eval\n",
    "    path = dset_dir + 'val-one-shot/'\n",
    "    print('Generating dataset val-one-shot/')\n",
    "    dataset_utils.generate_dataset(path, dataset_size, chars, config, seed=seed_val_one_shot, save=True, checksum=checksums[nname[i]][2])\n",
    "    print('')\n",
    "\n",
    "    #Generate test set on one-shot characters\n",
    "    seed_test_one_shot = 2301871561\n",
    "    chars = chars_test\n",
    "    path = dset_dir + 'test-one-shot/'\n",
    "    print('Generating dataset test-one-shot/')\n",
    "    dataset_utils.generate_dataset(path, dataset_size, chars, config, seed=seed_test_one_shot, save=True, checksum=checksums[nname[i]][3])\n",
    "    print('')\n",
    "\n",
    "print('')\n",
    "print('All Done')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset Visualization Script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Show some randomly generated example images\n",
    "\n",
    "# Define number of images\n",
    "dataset_size = 5\n",
    "\n",
    "# Choose training alphabets\n",
    "chars = chars_train\n",
    "# Set number of images per parallel job\n",
    "config.JOBLENGTH = 1\n",
    "# Set number of distractors\n",
    "config.DISTRACTORS = 7\n",
    "# Set dataset split\n",
    "config.DRAWER_SPLIT = 'train'\n",
    "config.set_drawer_split()\n",
    "\n",
    "# set path if images should be saved\n",
    "dset_dir = DATASET_DIR + '{}_characters/'.format(config.DISTRACTORS + 1)\n",
    "path = DATASET_DIR + 'visualize/'\n",
    "\n",
    "# Run visualization\n",
    "dataset_utils.generate_dataset(path, dataset_size, chars, config, seed=None, save=False, show=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}