Add files via upload

keras.json.for_TensorFlow

@@ -0,0 +1,6 @@
+{
+    "epsilon": 1e-07,
+    "backend": "tensorflow",
+    "floatx": "float32",
+    "image_data_format": "channels_last"
+}

keras.json.for_Theano

@@ -0,0 +1,7 @@
+{
+    "epsilon": 1e-07,
+    "backend": "theano",
+    "floatx": "float32",
+    "image_dim_ordering": "th",
+    "image_data_format": "channels_first"
+}

nbs2/Keras-Tensorflow-Tutorial.ipynb

@@ -0,0 +1,179 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-09-20T09:46:27.105499Z",
+     "start_time": "2017-09-20T09:46:17.701636Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using TensorFlow backend.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import tensorflow as tf\n",
+    "sess = tf.Session()\n",
+    "\n",
+    "from keras import backend as K\n",
+    "K.set_session(sess)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-09-20T09:46:29.009523Z",
+     "start_time": "2017-09-20T09:46:29.003665Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# this placeholder will contain our input digits, as flat vectors\n",
+    "img = tf.placeholder(tf.float32, shape=(None, 784))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-09-20T09:46:29.653563Z",
+     "start_time": "2017-09-20T09:46:29.595156Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from keras.layers import Dense\n",
+    "\n",
+    "# Keras layers can be called on TensorFlow tensors:\n",
+    "x = Dense(128, activation='relu')(img)  # fully-connected layer with 128 units and ReLU activation\n",
+    "x = Dense(128, activation='relu')(x)\n",
+    "preds = Dense(10, activation='softmax')(x)  # output layer with 10 units and a softmax activation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-09-20T09:46:30.315748Z",
+     "start_time": "2017-09-20T09:46:30.291351Z"
+    },
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "labels = tf.placeholder(tf.float32, shape=(None, 10))\n",
+    "\n",
+    "from keras.objectives import categorical_crossentropy\n",
+    "loss = tf.reduce_mean(categorical_crossentropy(labels, preds))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2017-09-20T09:46:39.950492Z",
+     "start_time": "2017-09-20T09:46:30.955584Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
+      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
+      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
+      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
+      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
+      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
+      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
+      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tensorflow.examples.tutorials.mnist import input_data\n",
+    "mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
+    "\n",
+    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
+    "\n",
+    "sess.run(tf.global_variables_initializer())  # - required for Tensorflow variable initialization\n",
+    "\n",
+    "with sess.as_default():\n",
+    "    for i in range(100):\n",
+    "        batch = mnist_data.train.next_batch(50)\n",
+    "        train_step.run(feed_dict={img: batch[0],\n",
+    "                                  labels: batch[1]})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "p3",
+   "language": "python",
+   "name": "p3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.2"
+  },
+  "toc": {
+   "colors": {
+    "hover_highlight": "#DAA520",
+    "navigate_num": "#000000",
+    "navigate_text": "#333333",
+    "running_highlight": "#FF0000",
+    "selected_highlight": "#FFD700",
+    "sidebar_border": "#EEEEEE",
+    "wrapper_background": "#FFFFFF"
+   },
+   "moveMenuLeft": true,
+   "nav_menu": {
+    "height": "12px",
+    "width": "252px"
+   },
+   "navigate_menu": true,
+   "number_sections": true,
+   "sideBar": true,
+   "threshold": 4,
+   "toc_cell": false,
+   "toc_section_display": "block",
+   "toc_window_display": false,
+   "widenNotebook": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

nbs2/attention_wrapper.py

@@ -0,0 +1,100 @@
+# - Introduced a few changes for Keras 2
+from keras import backend as K
+# from keras.initializations import zero  # - Not used with Keras 2
+from keras.engine import InputSpec
+from keras.models import Sequential
+from keras.layers import LSTM, activations, Wrapper, Recurrent, Layer
+
+class Attention(Layer):
+    def __init__(self, fn_rnn, nlayers=1, **kwargs):
+        self.supports_masking = True
+        self.fn_rnn = fn_rnn
+        self.nlayers = nlayers
+        self.input_spec = [InputSpec(ndim=3), InputSpec(ndim=3)]
+        super(Attention, self).__init__(**kwargs)
+
+
+    def all_attrs(self, name):
+        return sum([getattr(layer, name, []) for layer in self.layers], [])
+
+
+    def w(self, dims, init, name):
+        # - Keras 2 different parameter order
+        return self.add_weight(dims, name=name.format(self.name), initializer=init) 
+
+
+    def build(self, input_shape):
+        self.enc_shape, self.dec_shape = input_shape
+        assert len(self.enc_shape) >= 3
+        self.layers = [self.fn_rnn() for i in range(self.nlayers)]
+        nb_samples, nb_time, nb_dims = self.dec_shape
+        l0 = self.layers[0]
+
+        out_shape = self.compute_output_shape(input_shape)  # - changed from self.get_output_shape_for
+        for layer in self.layers:
+            if not layer.built: layer.build(out_shape)
+
+        init = l0.kernel_initializer  # - changed from l0.init
+        out_dim = l0.units  # - changed from l0.output_dim
+        self.W1 = self.w((self.enc_shape[-1], nb_dims), init, '{}_W1')
+        self.W2 = self.w((out_dim, nb_dims), init, '{}_W2')
+        self.b2 = self.w((nb_dims,), 'zeros', '{}_b2') # - changed from zero
+        self.V =  self.w((nb_dims,), init, '{}_V')
+        self.W3 = self.w((nb_dims+out_dim, out_dim), init, '{}_W3')
+        self.b3 = self.w((out_dim,), 'zeros', '{}_b3') # - changed from zero
+
+        self.trainable_weights += self.all_attrs( 'trainable_weights')
+        self.non_trainable_weights += self.all_attrs( 'non_trainable_weights')
+        #self.losses += self.all_attrs( 'losses')  # - seems not available for layer in Keras 2
+        #self.updates = self.all_attrs( 'updates') # - seems not available for layer in Keras 2
+        self.constraints = getattr(self.layers[0], 'constraints', {}) # FIXME
+        super(Attention, self).build(input_shape)
+
+
+    def compute_output_shape(self, input_shape): # - changed from self.get_output_shape_for
+        return self.layers[0].compute_output_shape(input_shape[1])  # - changed from self.get_output_shape_for
+
+    def step(self, x, states):
+        h = states[0]
+        enc_output = states[-1]
+        xW1 = states[-2]
+
+        hW2 = K.expand_dims(K.dot(h,self.W2)+self.b2, 1)
+        u = K.tanh(xW1+hW2)
+        a = K.expand_dims(K.softmax(K.sum(self.V*u,2)), -1)
+        Xa = K.sum(a*enc_output,1)
+        h = K.dot(K.concatenate([x,Xa],1),self.W3)+self.b3
+
+        for layer in self.layers: h, new_states = layer.step(h, states)
+        return h, new_states
+
+
+    def get_constants(self, enc_output, constants):
+        constants.append(K.dot(enc_output,self.W1))
+        constants.append(enc_output)
+        return constants
+
+
+    def compute_mask(self, input, mask):
+        return self.layers[0].compute_mask(input, mask[1])
+
+
+    def call(self, x, mask=None):
+        l0 = self.layers[0]
+        enc_output, dec_input = x
+
+        if l0.stateful: initial_states = l0.states
+        else: initial_states = l0.get_initial_state(dec_input)  # - changed from get_initial_states
+        constants = l0.get_constants(dec_input)
+        constants = self.get_constants(enc_output, constants)
+        preprocessed_input = l0.preprocess_input(dec_input)
+        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
+             initial_states, go_backwards=l0.go_backwards,  mask=mask, # - changed from mask=mask[1]
+             constants=constants, unroll=l0.unroll, input_length=self.dec_shape[1])
+        if l0.stateful:
+            self.updates = []
+            for i in range(len(states)):
+                self.updates.append((l0.states[i], states[i]))
+
+        return outputs if l0.return_sequences else last_output
+