Create custom layer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create your own layer\n",
    "\n",
    "We are going to create a cusom layer. The goal of this layer is to create a multi layer perceptron. By doing so, we can also learn how to use some low level operation with tensorflow 2.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Be sure to used Tensorflow 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert hasattr(tf, \"function\") # Be sure to use tensorflow 2.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi Layer Perceptron\n",
    "\n",
    "This layer has no real purpose and should never be used in production. This is just an example to show how to create a custom layer. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.5, 0.5],\n",
       "       [0.5, 0.5],\n",
       "       [0.5, 0.5],\n",
       "       [0.5, 0.5],\n",
       "       [0.5, 0.5]], dtype=float32)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MlpLayer(tf.keras.layers.Layer):\n",
    "\n",
    "    def __init__(self, units, activations, **kwargs):\n",
    "        super(MlpLayer, self).__init__(**kwargs)\n",
    "        # Set the property to the layer\n",
    "        self.units = units\n",
    "        self.activations_list = activations\n",
    "        self.weights_list = []\n",
    "\n",
    "    # The build method will be called once\n",
    "    # we know the shape of the previous Layer: input_shape\n",
    "    def build(self, input_shape):\n",
    "        # Create trainable weights variables for this layer.\n",
    "        # We create matrix of weights for each layer\n",
    "        # Each weight have this shape: (previous_layer_size, layer_size)\n",
    "        i = 0\n",
    "        for units in self.units:\n",
    "            weights = self.add_weight(\n",
    "                        name='weights-%s' % i,\n",
    "                        shape=(input_shape[1], units),\n",
    "                        initializer='uniform',\n",
    "                        trainable=True\n",
    "            )\n",
    "            i += 1\n",
    "            self.weights_list.append(weights)\n",
    "            input_shape = (None, units)\n",
    "        super(MlpLayer, self).build(input_shape)\n",
    "        \n",
    "    def call(self, x):\n",
    "        output = x\n",
    "\n",
    "        # We go through each weight to compute the dot product between the previous\n",
    "        # activation and the weight of the layer.\n",
    "        # At the first pass, the previous activation is just the variable \"x\": The input vector\n",
    "        for weights, activation in zip(self.weights_list, self.activations_list):\n",
    "            # We can still used low level operations as tf.matmul, tf.nn.relu... \n",
    "            output = tf.matmul(output, weights)\n",
    "\n",
    "            if activation == \"relu\":\n",
    "                output = tf.nn.relu(output)\n",
    "            elif activation == \"sigmoid\":\n",
    "                output = tf.nn.sigmoid(output)\n",
    "            elif activation == \"softmax\":\n",
    "                output = tf.nn.softmax(output)\n",
    "        \n",
    "        return output\n",
    "\n",
    "    # By adding the get_config method you can then save your model with the custom layer\n",
    "    # and retrieve the model with the same parameters\n",
    "    def get_config(self):\n",
    "        config = {\n",
    "            'units': self.units,\n",
    "            \"activations\": self.activations_list\n",
    "        }\n",
    "        # Retrieve the config from the parent layer\n",
    "        base_config = super(MlpLayer, self).get_config()\n",
    "        # Return the final config\n",
    "        return dict(list(base_config.items()) + list(config.items()))\n",
    "\n",
    "# Flatten\n",
    "model = tf.keras.models.Sequential()\n",
    "\n",
    "# Add the layers\n",
    "model.add(MlpLayer([4 , 2], [\"relu\", \"softmax\"]))\n",
    "model.predict(np.zeros((5, 10)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"custom_layer_in_model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0302 09:41:52.265264 139932001146688 hdf5_format.py:224] No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
     ]
    }
   ],
   "source": [
    "custom_objects={'MlpLayer': MlpLayer}\n",
    "loaded_model = tf.keras.models.load_model(\"custom_layer_in_model.h5\", custom_objects=custom_objects)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}