1_notmnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5hIbr52I7Z7U"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 1\n",
    "------------\n",
    "\n",
    "The objective of this assignment is to learn about simple data curation practices, and familiarize you with some of the data we'll be reusing later.\n",
    "\n",
    "This notebook uses the [notMNIST](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) dataset to be used with python experiments. This dataset is designed to look like the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset, while looking a little more like real data: it's a harder task, and the data is a lot less 'clean' than MNIST."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "apJbCsBHl-2A"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import tarfile\n",
    "import urllib\n",
    "from IPython.display import display, Image\n",
    "from scipy import ndimage\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from collections import defaultdict\n",
    "import cPickle as pickle\n",
    "\n",
    "# http://stackoverflow.com/questions/29772158/make-ipython-notebook-print-in-real-time\n",
    "oldsysstdout = sys.stdout\n",
    "class flushfile():\n",
    "    def __init__(self, f):\n",
    "        self.f = f\n",
    "    def __getattr__(self,name): \n",
    "        return object.__getattribute__(self.f, name)\n",
    "    def write(self, x):\n",
    "        self.f.write(x)\n",
    "        self.f.flush()\n",
    "    def flush(self):\n",
    "        self.f.flush()\n",
    "sys.stdout = flushfile(sys.stdout)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jNWGtZaXn-5j"
   },
   "source": [
    "First, we'll download the dataset to our local machine. The data consists of characters rendered in a variety of fonts on a 28x28 image. The labels are limited to 'A' through 'J' (10 classes). The training set has about 500k and the testset 19000 labelled examples. Given these sizes, it should be possible to train models quickly on any machine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 186058,
     "status": "ok",
     "timestamp": 1444485672507,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "2a0a5e044bb03b66",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "EYRJ4ICW6-da",
    "outputId": "0d0f85df-155f-4a89-8e7e-ee32df36ec8d"
   },
   "outputs": [],
   "source": [
    "url = 'http://yaroslavvb.com/upload/notMNIST/'\n",
    "\n",
    "def maybe_download(filename, expected_bytes):\n",
    "  \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "  if not os.path.exists(filename):\n",
    "    filename, _ = urllib.urlretrieve(url + filename, filename)\n",
    "  statinfo = os.stat(filename)\n",
    "  if statinfo.st_size == expected_bytes:\n",
    "    print 'Found and verified', filename\n",
    "  else:\n",
    "    raise Exception(\n",
    "      'Failed to verify' + filename + '. Can you get to it with a browser?')\n",
    "  return filename\n",
    "\n",
    "train_filename = maybe_download('notMNIST_large.tar.gz', 247336696)\n",
    "test_filename = maybe_download('notMNIST_small.tar.gz', 8458043)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cC3p0oEyF8QT"
   },
   "source": [
    "Extract the dataset from the compressed .tar.gz file.\n",
    "This should give you a set of directories, labelled A through J."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_folders = [\n",
    "    'notMNIST_large/A',\n",
    "    'notMNIST_large/B',\n",
    "    'notMNIST_large/C',\n",
    "    'notMNIST_large/D',\n",
    "    'notMNIST_large/E',\n",
    "    'notMNIST_large/F',\n",
    "    'notMNIST_large/G',\n",
    "    'notMNIST_large/H',\n",
    "    'notMNIST_large/I',\n",
    "    'notMNIST_large/J'\n",
    "]\n",
    "test_folders = [\n",
    "    'notMNIST_small/A',\n",
    "    'notMNIST_small/B',\n",
    "    'notMNIST_small/C',\n",
    "    'notMNIST_small/D',\n",
    "    'notMNIST_small/E',\n",
    "    'notMNIST_small/F',\n",
    "    'notMNIST_small/G',\n",
    "    'notMNIST_small/H',\n",
    "    'notMNIST_small/I',\n",
    "    'notMNIST_small/J'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 186055,
     "status": "ok",
     "timestamp": 1444485672525,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "2a0a5e044bb03b66",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "H8CBE-WZ8nmj",
    "outputId": "ef6c790c-2513-4b09-962e-27c79390c762"
   },
   "outputs": [],
   "source": [
    "num_classes = 10\n",
    "\n",
    "def extract(filename):\n",
    "  tar = tarfile.open(filename)\n",
    "  tar.extractall()\n",
    "  tar.close()\n",
    "  root = os.path.splitext(os.path.splitext(filename)[0])[0]  # remove .tar.gz\n",
    "  data_folders = [os.path.join(root, d) for d in sorted(os.listdir(root))]\n",
    "  if len(data_folders) != num_classes:\n",
    "    raise Exception(\n",
    "      'Expected %d folders, one per class. Found %d instead.' % (\n",
    "        num_folders, len(data_folders)))\n",
    "  print data_folders\n",
    "  return data_folders\n",
    "  \n",
    "train_folders = extract(train_filename)\n",
    "test_folders = extract(test_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4riXK3IoHgx6"
   },
   "source": [
    "---\n",
    "Problem 1\n",
    "---------\n",
    "\n",
    "Let's take a peek at some of the data to make sure it looks sensible. Each exemplar should be an image of a character A through J rendered in a different font. Display a sample of the images that we just downloaded. Hint: you can use the package IPython.display.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Image(filename=\"notMNIST_small/A/MDEtMDEtMDAudHRm.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Image(filename=\"notMNIST_large/A/a2F6b28udHRm.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Image(filename=\"notMNIST_large/C/ZXVyb2Z1cmVuY2UgaXRhbGljLnR0Zg==.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Image(filename=\"notMNIST_small/I/SVRDIEZyYW5rbGluIEdvdGhpYyBEZW1pLnBmYg==.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PBdkjESPK8tw"
   },
   "source": [
    "Now let's load the data in a more manageable format.\n",
    "\n",
    "We'll convert the entire dataset into a 3D array (image index, x, y) of floating point values, normalized to have approximately zero mean and standard deviation ~0.5 to make training easier down the road. The labels will be stored into a separate array of integers 0 through 9.\n",
    "\n",
    "A few images might not be readable, we'll just skip them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "powers = np.power(np.full(50, 2, int), np.arange(0,50))\n",
    "\n",
    "def compute_hash(pixels):\n",
    "    # http://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html\n",
    "    rows = np.stack(np.split(pixels.astype(int), 7, 1))\n",
    "    blocks = np.stack(np.split(rows, 7, 1))\n",
    "    sums = np.ravel(blocks.sum(2).sum(2))\n",
    "    subs = sums - 128 * 16\n",
    "    signs = np.sign(subs)\n",
    "    bits = (signs + 1) / 2.0\n",
    "    if bits.sum() == 0:\n",
    "        subs = sums - 32\n",
    "        signs = np.sign(subs)\n",
    "        bits = (signs + 1) / 2.0\n",
    "        bits = np.append(bits, 0)\n",
    "    else:\n",
    "        bits = np.append(bits, 1)\n",
    "    return int(np.multiply(bits, powers).sum())\n",
    "compute_hash(np.stack(np.split(np.floor(np.linspace(0, 255, num = 28 * 28)),28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 30
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 399874,
     "status": "ok",
     "timestamp": 1444485886378,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "2a0a5e044bb03b66",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "h7q0XhG3MJdf",
    "outputId": "92c391bb-86ff-431d-9ada-315568a19e59"
   },
   "outputs": [],
   "source": [
    "image_size = 28  # Pixel width and height.\n",
    "pixel_depth = 255.0  # Number of levels per pixel.\n",
    "skip_list = [\n",
    " 'notMNIST_large/A/SG90IE11c3RhcmQgQlROIFBvc3Rlci50dGY=.png',\n",
    " 'notMNIST_large/A/RnJlaWdodERpc3BCb29rSXRhbGljLnR0Zg==.png',\n",
    " 'notMNIST_large/A/Um9tYW5hIEJvbGQucGZi.png',\n",
    " 'notMNIST_large/B/TmlraXNFRi1TZW1pQm9sZEl0YWxpYy5vdGY=.png',\n",
    " 'notMNIST_large/D/VHJhbnNpdCBCb2xkLnR0Zg==.png',\n",
    " 'notMNIST_small/A/RGVtb2NyYXRpY2FCb2xkT2xkc3R5bGUgQm9sZC50dGY=.png',\n",
    " 'notMNIST_small/F/Q3Jvc3NvdmVyIEJvbGRPYmxpcXVlLnR0Zg==.png'\n",
    "]\n",
    "\n",
    "# Map of has to list of (dataset id, index) pairs.\n",
    "image_hashes = defaultdict(list)\n",
    "\n",
    "def load(data_folders, set_id, min_num_images, max_num_images):\n",
    "  dataset = np.ndarray(\n",
    "    shape=(max_num_images, image_size, image_size), dtype=np.float32)\n",
    "  labels = np.ndarray(shape=(max_num_images), dtype=np.int32)\n",
    "  label_index = 0\n",
    "  image_index = 0\n",
    "  for folder in data_folders:\n",
    "    print folder\n",
    "    for image in os.listdir(folder):\n",
    "      if image_index >= max_num_images:\n",
    "        raise Exception('More images than expected: %d >= %d' % (\n",
    "          num_images, max_num_images))\n",
    "      image_file = os.path.join(folder, image)\n",
    "      if image_file in skip_list:\n",
    "        continue\n",
    "      try:\n",
    "        raw_data = ndimage.imread(image_file)\n",
    "        image_data = (raw_data.astype(float) - pixel_depth / 2) / pixel_depth\n",
    "       \n",
    "        if image_data.shape != (image_size, image_size):\n",
    "          raise Exception('Unexpected image shape: %s' % str(image_data.shape))\n",
    "\n",
    "        hash = compute_hash(raw_data)\n",
    "        \n",
    "        dataset[image_index, :, :] = image_data\n",
    "        labels[image_index] = label_index\n",
    "        image_hashes[hash].append((set_id, image_index))\n",
    "        image_index += 1\n",
    "      except IOError as e:\n",
    "        skip_list.append(image_file)\n",
    "        print 'Could not read:', image_file, ':', e, '- it\\'s ok, skipping.'\n",
    "    label_index += 1\n",
    "  num_images = image_index\n",
    "  dataset = dataset[0:num_images, :, :]\n",
    "  labels = labels[0:num_images]\n",
    "  if num_images < min_num_images:\n",
    "    raise Exception('Many fewer images than expected: %d < %d' % (\n",
    "        num_images, min_num_images))\n",
    "  print 'Full dataset tensor:', dataset.shape\n",
    "  print 'Mean:', np.mean(dataset)\n",
    "  print 'Standard deviation:', np.std(dataset)\n",
    "  print 'Labels:', labels.shape\n",
    "  return dataset, labels\n",
    "\n",
    "image_hashes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_dataset, train_labels = load(train_folders, 'train', 450000, 550000)\n",
    "test_dataset, test_labels = load(test_folders, 'test', 18000, 20000)\n",
    "\n",
    "skip_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "counts = defaultdict(lambda: 0)\n",
    "max_hash = 0\n",
    "print len(image_hashes)\n",
    "\n",
    "for (hash, indicies) in image_hashes.iteritems():\n",
    "    image_hashes[int(hash)] = indicies\n",
    "    counts[len(indicies)] += 1\n",
    "    if len(indicies) > max_hash:\n",
    "        max_hash = len(indicies)\n",
    "        print hash, max_hash\n",
    "\n",
    "print image_hashes[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "datasets = {\n",
    "    'test' : test_dataset,\n",
    "    \"train\": train_dataset\n",
    "}\n",
    "\n",
    "whites = []\n",
    "blacks = []\n",
    "collisions = [whites, blacks]\n",
    "\n",
    "white_image = np.stack(np.split(np.full(28*28, 0.5),28))\n",
    "white_hash = compute_hash(np.stack(np.split(np.full(28*28, 255, int),28)))\n",
    "black_image = compute_hash(np.stack(np.split(np.full(28*28, -0.5),28)))\n",
    "black_hash = compute_hash(np.stack(np.split(np.full(28*28, 0, int),28)))\n",
    "\n",
    "compare_count = 0\n",
    "max_match = 0\n",
    "\n",
    "def find_collision(collisions, index):\n",
    "    for match in collisions:\n",
    "        if index in match:\n",
    "            return match\n",
    "    return None\n",
    "\n",
    "def in_collisions(collisions, index_a, index_b):\n",
    "    for match in collisions:\n",
    "        if index_a in match and index_b in match:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "print \"Checking for collisions:\"\n",
    "\n",
    "percent = len(image_hashes) / 100\n",
    "for (i, (hash, indicies)) in enumerate(image_hashes.iteritems()):\n",
    "    if i % percent == 0:\n",
    "        print (i / percent) - 1, \"% complete.\"\n",
    "    if len(indicies) > 1:\n",
    "        if len(indicies) > 100:\n",
    "            print \"Large: \", len(indicies)\n",
    "        if len(indicies) > max_match:\n",
    "            max_match = len(indicies)\n",
    "            print \"Max match: \", max_match\n",
    "        if compare_count % 5000 == 0:\n",
    "            print \"Compared: \", compare_count\n",
    "        local_collisions = []\n",
    "        for i in range(0, len(indicies) - 1):\n",
    "            index_a = indicies[i]\n",
    "            dataset_a = datasets[index_a[0]]\n",
    "            image_a = dataset_a[index_a[1]]\n",
    "            if hash == white_hash:\n",
    "                if np.allclose(image_a, white_image):\n",
    "                    whites.append(index_a)\n",
    "                    continue\n",
    "            if hash == black_hash:\n",
    "                if np.allclose(image_a, black_image):\n",
    "                    blacks.append(index_a)\n",
    "                    continue\n",
    "            for index_b in indicies[i + 1:]:\n",
    "                if in_collisions(local_collisions, index_a, index_b):\n",
    "                    continue\n",
    "                dataset_b = datasets[index_b[0]]\n",
    "                compare_count += 1\n",
    "                if np.allclose(image_a, dataset_b[index_b[1]]):\n",
    "                    hit = find_collision(local_collisions, index_a)\n",
    "                    if hit == None:\n",
    "                        local_collisions.append([index_a, index_b])\n",
    "                    else:\n",
    "                        hit.append(index_b)\n",
    "                    \n",
    "        collisions.extend(local_collisions)\n",
    "print \"100 % complete - all done.\"\n",
    "print \"Total collisions:\", len(collisions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST_collisions.pickle'\n",
    "\n",
    "try:\n",
    "  f = open(pickle_file, 'wb')\n",
    "  save = {\n",
    "    'collisions': collisions\n",
    "    }\n",
    "  pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
    "  f.close()\n",
    "except Exception as e:\n",
    "  print 'Unable to save data to', pickle_file, ':', e\n",
    "  raise\n",
    "    \n",
    "statinfo = os.stat(pickle_file)\n",
    "print 'Compressed pickle size:', statinfo.st_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST_collisions.pickle'\n",
    "\n",
    "collisions = []\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  collisions = save['collisions']\n",
    "  del save  # hint to help gc free up memory\n",
    "  print 'Collisions:', len(collisions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print \"Total collisions:\", len(collisions)\n",
    "print \"Whites: \", len(collisions[0])\n",
    "print \"Blacks: \", len(collisions[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for c in collisions:\n",
    "    if len(c) > 20:\n",
    "        print len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "exemplar = plt.imshow(test_dataset[16829])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "exemplar = plt.imshow(train_dataset[513353])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vUdbskYE2d87"
   },
   "source": [
    "---\n",
    "Problem 2\n",
    "---------\n",
    "\n",
    "Let's verify that the data still looks good. Displaying a sample of the labels and images from the ndarray. Hint: you can use matplotlib.pyplot.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "exemplar = plt.imshow(train_dataset[0])\n",
    "train_labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "exemplar = plt.imshow(train_dataset[373])\n",
    "train_labels[373]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "exemplar = plt.imshow(test_dataset[18169])\n",
    "test_labels[18169]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "exemplar = plt.imshow(train_dataset[9])\n",
    "train_labels[9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GPTCnjIcyuKN"
   },
   "source": [
    "Next, we'll randomize the data. It's important to have the labels well shuffled for the training and test distributions to match."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST_full.pickle'\n",
    "\n",
    "try:\n",
    "  f = open(pickle_file, 'wb')\n",
    "  save = {\n",
    "    'train_dataset': train_dataset,\n",
    "    'train_labels': train_labels,\n",
    "    'test_dataset': test_dataset,\n",
    "    'test_labels': test_labels,\n",
    "    'image_hashes': image_hashes\n",
    "    }\n",
    "  pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
    "  f.close()\n",
    "except Exception as e:\n",
    "  print 'Unable to save data to', pickle_file, ':', e\n",
    "  raise\n",
    "\n",
    "statinfo = os.stat(pickle_file)\n",
    "print 'Compressed pickle size:', statinfo.st_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST_full.pickle'\n",
    "\n",
    "train_dataset = []\n",
    "train_labels = []\n",
    "test_dataset = []\n",
    "test_labels = []\n",
    "image_hashes = []\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  train_dataset = save['train_dataset']\n",
    "  train_labels = save['train_labels']\n",
    "  test_dataset = save['test_dataset']\n",
    "  test_labels = save['test_labels']\n",
    "  image_hashes = save['image_hashes']\n",
    "  del save  # hint to help gc free up memory\n",
    "  print 'Training set', train_dataset.shape, train_labels.shape\n",
    "  print 'Test set', test_dataset.shape, test_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "6WZ2l2tN2zOL"
   },
   "outputs": [],
   "source": [
    "np.random.seed(133)\n",
    "def randomize(dataset, labels):\n",
    "  permutation = np.random.permutation(labels.shape[0])\n",
    "  shuffled_dataset = dataset[permutation,:,:]\n",
    "  shuffled_labels = labels[permutation]\n",
    "  return shuffled_dataset, shuffled_labels\n",
    "train_dataset, train_labels = randomize(train_dataset, train_labels)\n",
    "test_dataset, test_labels = randomize(test_dataset, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "puDUTe6t6USl"
   },
   "source": [
    "---\n",
    "Problem 3\n",
    "---------\n",
    "Convince yourself that the data is still good after shuffling!\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "exemplar = plt.imshow(train_dataset[-1])\n",
    "train_labels[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cYznx5jUwzoO"
   },
   "source": [
    "---\n",
    "Problem 4\n",
    "---------\n",
    "Another check: we expect the data to be balanced across classes. Verify that.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.bincount(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.bincount(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LA7M7K22ynCt"
   },
   "source": [
    "Prune the training data as needed. Depending on your computer setup, you might not be able to fit it all in memory, and you can tune train_size as needed.\n",
    "\n",
    "Also create a validation dataset for hyperparameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 411281,
     "status": "ok",
     "timestamp": 1444485897869,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "2a0a5e044bb03b66",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "s3mWgZLpyuzq",
    "outputId": "8af66da6-902d-4719-bedc-7c9fb7ae7948"
   },
   "outputs": [],
   "source": [
    "train_size = 200000\n",
    "valid_size = 10000\n",
    "\n",
    "valid_dataset = train_dataset[:valid_size,:,:]\n",
    "valid_labels = train_labels[:valid_size]\n",
    "train_dataset = train_dataset[valid_size:valid_size+train_size,:,:]\n",
    "train_labels = train_labels[valid_size:valid_size+train_size]\n",
    "print 'Training', train_dataset.shape, train_labels.shape\n",
    "print 'Validation', valid_dataset.shape, valid_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.bincount(valid_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tIQJaJuwg5Hw"
   },
   "source": [
    "Finally, let's save the data for later reuse:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "QiR_rETzem6C"
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "try:\n",
    "  f = open(pickle_file, 'wb')\n",
    "  save = {\n",
    "    'train_dataset': train_dataset,\n",
    "    'train_labels': train_labels,\n",
    "    'valid_dataset': valid_dataset,\n",
    "    'valid_labels': valid_labels,\n",
    "    'test_dataset': test_dataset,\n",
    "    'test_labels': test_labels,\n",
    "    }\n",
    "  pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n",
    "  f.close()\n",
    "except Exception as e:\n",
    "  print 'Unable to save data to', pickle_file, ':', e\n",
    "  raise\n",
    "    \n",
    "print 'Training', train_dataset.shape, train_labels.shape\n",
    "print 'Validation', valid_dataset.shape, valid_labels.shape\n",
    "print 'Test', test_dataset.shape, test_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 413065,
     "status": "ok",
     "timestamp": 1444485899688,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "2a0a5e044bb03b66",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "hQbLjrW_iT39",
    "outputId": "b440efc6-5ee1-4cbc-d02d-93db44ebd956"
   },
   "outputs": [],
   "source": [
    "statinfo = os.stat(pickle_file)\n",
    "print 'Compressed pickle size:', statinfo.st_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "train_dataset = []\n",
    "train_labels = []\n",
    "valid_dataset = []\n",
    "valid_labels = []\n",
    "test_dataset = []\n",
    "test_labels = []\n",
    "\n",
    "print \"Opening\", pickle_file\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "  save = pickle.load(f)\n",
    "  train_dataset = save['train_dataset']\n",
    "  train_labels = save['train_labels']\n",
    "  valid_dataset = save['valid_dataset']\n",
    "  valid_labels = save['valid_labels']\n",
    "  test_dataset = save['test_dataset']\n",
    "  test_labels = save['test_labels']\n",
    "  del save  # hint to help gc free up memory\n",
    "  print 'Training set', train_dataset.shape, train_labels.shape\n",
    "  print 'Validation set', valid_dataset.shape, valid_labels.shape\n",
    "  print 'Test set', test_dataset.shape, test_labels.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gE_cRAQB33lk"
   },
   "source": [
    "---\n",
    "Problem 5\n",
    "---------\n",
    "\n",
    "By construction, this dataset might contain a lot of overlapping samples, including training data that's also contained in the validation and test set! Overlap between training and test can skew the results if you expect to use your model in an environment where there is never an overlap, but are actually ok if you expect to see training samples recur when you use it.\n",
    "Measure how much overlap there is between training, validation and test samples.\n",
    "Optional questions:\n",
    "- What about near duplicates between datasets? (images that are almost identical)\n",
    "- Create a sanitized validation and test set, and compare your accuracy on those in subsequent assignments.\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L8oww1s4JMQx"
   },
   "source": [
    "---\n",
    "Problem 6\n",
    "---------\n",
    "\n",
    "Let's get an idea of what an off-the-shelf classifier can give you on this data. It's always good to check that there is something to learn, and that it's a problem that is not so trivial that a canned solution solves it.\n",
    "\n",
    "Train a simple model on this data using 50, 100, 1000 and 5000 training samples. Hint: you can use the LogisticRegression model from sklearn.linear_model.\n",
    "\n",
    "Optional question: train an off-the-shelf model on all the data!\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(test_set, test_labels, validation_set, validation_labels, count):\n",
    "    reg = LogisticRegression()\n",
    "    test_flattened = np.reshape(test_set[:count], (count, test_set.shape[1] * test_set.shape[2]))\n",
    "    reg.fit(test_flattened, test_labels[:count])\n",
    "    validation_flattened = np.reshape(validation_set, (validation_set.shape[0], validation_set.shape[1] * validation_set.shape[2]))\n",
    "    prediction = reg.predict(validation_flattened)\n",
    "    return accuracy_score(validation_labels, prediction)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train(test_dataset, test_labels, valid_dataset, valid_labels, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train(test_dataset, test_labels, valid_dataset, valid_labels, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train(test_dataset, test_labels, valid_dataset, valid_labels, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train(test_dataset, test_labels, valid_dataset, valid_labels, 5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train(test_dataset, test_labels, valid_dataset, valid_labels, test_dataset.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colabVersion": "0.3.2",
  "colab_default_view": {},
  "colab_views": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}