nips2016/mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys, os\n",
    "sys.path.insert(0, '..')\n",
    "from lib import models, graph, coarsening, utils\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "flags = tf.app.flags\n",
    "FLAGS = flags.FLAGS\n",
    "\n",
    "# Graphs.\n",
    "flags.DEFINE_integer('number_edges', 8, 'Graph: minimum number of edges per vertex.')\n",
    "flags.DEFINE_string('metric', 'euclidean', 'Graph: similarity measure (between features).')\n",
    "# TODO: change cgcnn for combinatorial Laplacians.\n",
    "flags.DEFINE_bool('normalized_laplacian', True, 'Graph Laplacian: normalized.')\n",
    "flags.DEFINE_integer('coarsening_levels', 4, 'Number of coarsened graphs.')\n",
    "\n",
    "# Directories.\n",
    "flags.DEFINE_string('dir_data', os.path.join('..', 'data', 'mnist'), 'Directory to store data.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def grid_graph(m, corners=False):\n",
    "    z = graph.grid(m)\n",
    "    dist, idx = graph.distance_sklearn_metrics(z, k=FLAGS.number_edges, metric=FLAGS.metric)\n",
    "    A = graph.adjacency(dist, idx)\n",
    "\n",
    "    # Connections are only vertical or horizontal on the grid.\n",
    "    # Corner vertices are connected to 2 neightbors only.\n",
    "    if corners:\n",
    "        import scipy.sparse\n",
    "        A = A.toarray()\n",
    "        A[A < A.max()/1.5] = 0\n",
    "        A = scipy.sparse.csr_matrix(A)\n",
    "        print('{} edges'.format(A.nnz))\n",
    "\n",
    "    print(\"{} > {} edges\".format(A.nnz//2, FLAGS.number_edges*m**2//2))\n",
    "    return A\n",
    "\n",
    "t_start = time.process_time()\n",
    "A = grid_graph(28, corners=False)\n",
    "A = graph.replace_random_edges(A, 0)\n",
    "graphs, perm = coarsening.coarsen(A, levels=FLAGS.coarsening_levels, self_connections=False)\n",
    "L = [graph.laplacian(A, normalized=True) for A in graphs]\n",
    "print('Execution time: {:.2f}s'.format(time.process_time() - t_start))\n",
    "graph.plot_spectrum(L)\n",
    "del A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(FLAGS.dir_data, one_hot=False)\n",
    "\n",
    "train_data = mnist.train.images.astype(np.float32)\n",
    "val_data = mnist.validation.images.astype(np.float32)\n",
    "test_data = mnist.test.images.astype(np.float32)\n",
    "train_labels = mnist.train.labels\n",
    "val_labels = mnist.validation.labels\n",
    "test_labels = mnist.test.labels\n",
    "\n",
    "t_start = time.process_time()\n",
    "train_data = coarsening.perm_data(train_data, perm)\n",
    "val_data = coarsening.perm_data(val_data, perm)\n",
    "test_data = coarsening.perm_data(test_data, perm)\n",
    "print('Execution time: {:.2f}s'.format(time.process_time() - t_start))\n",
    "del perm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#model = fc1()\n",
    "#model = fc2(nhiddens=100)\n",
    "#model = cnn2(K=5, F=10)  # K=28 is equivalent to filtering with fgcnn.\n",
    "#model = fcnn2(F=10)\n",
    "#model = fgcnn2(L[0], F=10)\n",
    "#model = lgcnn2_2(L[0], F=10, K=10)\n",
    "#model = cgcnn2_3(L[0], F=10, K=5)\n",
    "#model = cgcnn2_4(L[0], F=10, K=5)\n",
    "#model = cgcnn2_5(L[0], F=10, K=5)\n",
    "\n",
    "if False:\n",
    "    K = 5  # 5 or 5^2\n",
    "    t_start = time.process_time()\n",
    "    mnist.test._images = graph.lanczos(L, mnist.test._images.T, K).T\n",
    "    mnist.train._images = graph.lanczos(L, mnist.train._images.T, K).T\n",
    "    model = lgcnn2_1(L, F=10, K=K)\n",
    "    print('Execution time: {:.2f}s'.format(time.process_time() - t_start))\n",
    "    ph_data = tf.placeholder(tf.float32, (FLAGS.batch_size, mnist.train.images.shape[1], K), 'data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "common = {}\n",
    "common['dir_name']       = 'mnist/'\n",
    "common['num_epochs']     = 20\n",
    "common['batch_size']     = 100\n",
    "common['decay_steps']    = mnist.train.num_examples / common['batch_size']\n",
    "common['eval_frequency'] = 30 * common['num_epochs']\n",
    "common['brelu']          = 'b1relu'\n",
    "common['pool']           = 'mpool1'\n",
    "C = max(mnist.train.labels) + 1  # number of classes\n",
    "\n",
    "model_perf = utils.model_perf()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "    name = 'softmax'\n",
    "    params = common.copy()\n",
    "    params['dir_name'] += name\n",
    "    params['regularization'] = 5e-4\n",
    "    params['dropout']        = 1\n",
    "    params['learning_rate']  = 0.02\n",
    "    params['decay_rate']     = 0.95\n",
    "    params['momentum']       = 0.9\n",
    "    params['F']              = []\n",
    "    params['K']              = []\n",
    "    params['p']              = []\n",
    "    params['M']              = [C]\n",
    "    model_perf.test(models.cgcnn(L, **params), name, params,\n",
    "                    train_data, train_labels, val_data, val_labels, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Common hyper-parameters for networks with one convolutional layer.\n",
    "common['regularization'] = 0\n",
    "common['dropout']        = 1\n",
    "common['learning_rate']  = 0.02\n",
    "common['decay_rate']     = 0.95\n",
    "common['momentum']       = 0.9\n",
    "common['F']              = [10]\n",
    "common['K']              = [20]\n",
    "common['p']              = [1]\n",
    "common['M']              = [C]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "    name = 'fgconv_softmax'\n",
    "    params = common.copy()\n",
    "    params['dir_name'] += name\n",
    "    params['filter'] = 'fourier'\n",
    "    params['K'] = [L[0].shape[0]]\n",
    "    model_perf.test(models.cgcnn(L, **params), name, params,\n",
    "                    train_data, train_labels, val_data, val_labels, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "    name = 'sgconv_softmax'\n",
    "    params = common.copy()\n",
    "    params['dir_name'] += name\n",
    "    params['filter'] = 'spline'\n",
    "    model_perf.test(models.cgcnn(L, **params), name, params,\n",
    "                    train_data, train_labels, val_data, val_labels, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# With 'chebyshev2' and 'b2relu', it corresponds to cgcnn2_2(L[0], F=10, K=20).\n",
    "if True:\n",
    "    name = 'cgconv_softmax'\n",
    "    params = common.copy()\n",
    "    params['dir_name'] += name\n",
    "    params['filter'] = 'chebyshev5'\n",
    "#    params['filter'] = 'chebyshev2'\n",
    "#    params['brelu'] = 'b2relu'\n",
    "    model_perf.test(models.cgcnn(L, **params), name, params,\n",
    "                    train_data, train_labels, val_data, val_labels, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Common hyper-parameters for LeNet5-like networks.\n",
    "common['regularization'] = 5e-4\n",
    "common['dropout']        = 0.5\n",
    "common['learning_rate']  = 0.02  # 0.03 in the paper but sgconv_sgconv_fc_softmax has difficulty to converge\n",
    "common['decay_rate']     = 0.95\n",
    "common['momentum']       = 0.9\n",
    "common['F']              = [32, 64]\n",
    "common['K']              = [25, 25]\n",
    "common['p']              = [4, 4]\n",
    "common['M']              = [512, C]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Architecture of TF MNIST conv model (LeNet-5-like).\n",
    "# Changes: regularization, dropout, decaying learning rate, momentum optimizer, stopping condition, size of biases.\n",
    "# Differences: training data randomization, init conv1 biases at 0.\n",
    "if True:\n",
    "    name = 'fgconv_fgconv_fc_softmax' #  'Non-Param'\n",
    "    params = common.copy()\n",
    "    params['dir_name'] += name\n",
    "    params['filter'] = 'fourier'\n",
    "    params['K'] = [L[0].shape[0], L[2].shape[0]]\n",
    "    model_perf.test(models.cgcnn(L, **params), name, params,\n",
    "                    train_data, train_labels, val_data, val_labels, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "    name = 'sgconv_sgconv_fc_softmax'  # 'Spline'\n",
    "    params = common.copy()\n",
    "    params['dir_name'] += name\n",
    "    params['filter'] = 'spline'\n",
    "    model_perf.test(models.cgcnn(L, **params), name, params,\n",
    "                    train_data, train_labels, val_data, val_labels, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "    name = 'cgconv_cgconv_fc_softmax'  # 'Chebyshev'\n",
    "    params = common.copy()\n",
    "    params['dir_name'] += name\n",
    "    params['filter'] = 'chebyshev5'\n",
    "    model_perf.test(models.cgcnn(L, **params), name, params,\n",
    "                    train_data, train_labels, val_data, val_labels, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model_perf.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if False:\n",
    "    grid_params = {}\n",
    "    data = (train_data, train_labels, val_data, val_labels, test_data, test_labels)\n",
    "    utils.grid_search(params, grid_params, *data, model=lambda x: models.cgcnn(L,**x))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}