chapter10_dl-for-timeseries.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThis notebook was generated for TensorFlow 2.6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# Deep learning for timeseries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Different kinds of timeseries tasks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## A temperature-forecasting example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!wget https://s3.amazonaws.com/keras-datasets/jena_climate_2009_2016.csv.zip\n",
    "!unzip jena_climate_2009_2016.csv.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Inspecting the data of the Jena weather dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "fname = os.path.join(\"jena_climate_2009_2016.csv\")\n",
    "\n",
    "with open(fname) as f:\n",
    "    data = f.read()\n",
    "\n",
    "lines = data.split(\"\\n\")\n",
    "header = lines[0].split(\",\")\n",
    "lines = lines[1:]\n",
    "print(header)\n",
    "print(len(lines))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Parsing the data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "temperature = np.zeros((len(lines),))\n",
    "raw_data = np.zeros((len(lines), len(header) - 1))\n",
    "for i, line in enumerate(lines):\n",
    "    values = [float(x) for x in line.split(\",\")[1:]]\n",
    "    temperature[i] = values[1]\n",
    "    raw_data[i, :] = values[:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Plotting the temperature timeseries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "plt.plot(range(len(temperature)), temperature)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Plotting the first 10 days of the temperature timeseries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "plt.plot(range(1440), temperature[:1440])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Computing the number of samples we'll use for each data split**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_train_samples = int(0.5 * len(raw_data))\n",
    "num_val_samples = int(0.25 * len(raw_data))\n",
    "num_test_samples = len(raw_data) - num_train_samples - num_val_samples\n",
    "print(\"num_train_samples:\", num_train_samples)\n",
    "print(\"num_val_samples:\", num_val_samples)\n",
    "print(\"num_test_samples:\", num_test_samples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Preparing the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Normalizing the data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "mean = raw_data[:num_train_samples].mean(axis=0)\n",
    "raw_data -= mean\n",
    "std = raw_data[:num_train_samples].std(axis=0)\n",
    "raw_data /= std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow import keras\n",
    "int_sequence = np.arange(10)\n",
    "dummy_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    data=int_sequence[:-3],\n",
    "    targets=int_sequence[3:],\n",
    "    sequence_length=3,\n",
    "    batch_size=2,\n",
    ")\n",
    "\n",
    "for inputs, targets in dummy_dataset:\n",
    "    for i in range(inputs.shape[0]):\n",
    "        print([int(x) for x in inputs[i]], int(targets[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Instantiating datasets for training, validation, and testing**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "sampling_rate = 6\n",
    "sequence_length = 120\n",
    "delay = sampling_rate * (sequence_length + 24 - 1)\n",
    "batch_size = 256\n",
    "\n",
    "train_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    raw_data[:-delay],\n",
    "    targets=temperature[delay:],\n",
    "    sampling_rate=sampling_rate,\n",
    "    sequence_length=sequence_length,\n",
    "    shuffle=True,\n",
    "    batch_size=batch_size,\n",
    "    start_index=0,\n",
    "    end_index=num_train_samples)\n",
    "\n",
    "val_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    raw_data[:-delay],\n",
    "    targets=temperature[delay:],\n",
    "    sampling_rate=sampling_rate,\n",
    "    sequence_length=sequence_length,\n",
    "    shuffle=True,\n",
    "    batch_size=batch_size,\n",
    "    start_index=num_train_samples,\n",
    "    end_index=num_train_samples + num_val_samples)\n",
    "\n",
    "test_dataset = keras.utils.timeseries_dataset_from_array(\n",
    "    raw_data[:-delay],\n",
    "    targets=temperature[delay:],\n",
    "    sampling_rate=sampling_rate,\n",
    "    sequence_length=sequence_length,\n",
    "    shuffle=True,\n",
    "    batch_size=batch_size,\n",
    "    start_index=num_train_samples + num_val_samples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Inspecting the output of one of our datasets**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "for samples, targets in train_dataset:\n",
    "    print(\"samples shape:\", samples.shape)\n",
    "    print(\"targets shape:\", targets.shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### A common-sense, non-machine-learning baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Computing the common-sense baseline MAE**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def evaluate_naive_method(dataset):\n",
    "    total_abs_err = 0.\n",
    "    samples_seen = 0\n",
    "    for samples, targets in dataset:\n",
    "        preds = samples[:, -1, 1] * std[1] + mean[1]\n",
    "        total_abs_err += np.sum(np.abs(preds - targets))\n",
    "        samples_seen += samples.shape[0]\n",
    "    return total_abs_err / samples_seen\n",
    "\n",
    "print(f\"Validation MAE: {evaluate_naive_method(val_dataset):.2f}\")\n",
    "print(f\"Test MAE: {evaluate_naive_method(test_dataset):.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Let's try a basic machine-learning model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Training and evaluating a densely connected model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.Flatten()(inputs)\n",
    "x = layers.Dense(16, activation=\"relu\")(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_dense.keras\",\n",
    "                                    save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(train_dataset,\n",
    "                    epochs=10,\n",
    "                    validation_data=val_dataset,\n",
    "                    callbacks=callbacks)\n",
    "\n",
    "model = keras.models.load_model(\"jena_dense.keras\")\n",
    "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Plotting results**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "loss = history.history[\"mae\"]\n",
    "val_loss = history.history[\"val_mae\"]\n",
    "epochs = range(1, len(loss) + 1)\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, \"bo\", label=\"Training MAE\")\n",
    "plt.plot(epochs, val_loss, \"b\", label=\"Validation MAE\")\n",
    "plt.title(\"Training and validation MAE\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Let's try a 1D convolutional model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.Conv1D(8, 24, activation=\"relu\")(inputs)\n",
    "x = layers.MaxPooling1D(2)(x)\n",
    "x = layers.Conv1D(8, 12, activation=\"relu\")(x)\n",
    "x = layers.MaxPooling1D(2)(x)\n",
    "x = layers.Conv1D(8, 6, activation=\"relu\")(x)\n",
    "x = layers.GlobalAveragePooling1D()(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_conv.keras\",\n",
    "                                    save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(train_dataset,\n",
    "                    epochs=10,\n",
    "                    validation_data=val_dataset,\n",
    "                    callbacks=callbacks)\n",
    "\n",
    "model = keras.models.load_model(\"jena_conv.keras\")\n",
    "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### A first recurrent baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**A simple LSTM-based model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.LSTM(16)(inputs)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_lstm.keras\",\n",
    "                                    save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(train_dataset,\n",
    "                    epochs=10,\n",
    "                    validation_data=val_dataset,\n",
    "                    callbacks=callbacks)\n",
    "\n",
    "model = keras.models.load_model(\"jena_lstm.keras\")\n",
    "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Understanding recurrent neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**NumPy implementation of a simple RNN**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "timesteps = 100\n",
    "input_features = 32\n",
    "output_features = 64\n",
    "inputs = np.random.random((timesteps, input_features))\n",
    "state_t = np.zeros((output_features,))\n",
    "W = np.random.random((output_features, input_features))\n",
    "U = np.random.random((output_features, output_features))\n",
    "b = np.random.random((output_features,))\n",
    "successive_outputs = []\n",
    "for input_t in inputs:\n",
    "    output_t = np.tanh(np.dot(W, input_t) + np.dot(U, state_t) + b)\n",
    "    successive_outputs.append(output_t)\n",
    "    state_t = output_t\n",
    "final_output_sequence = np.stack(successive_outputs, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### A recurrent layer in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**An RNN layer that can process sequences of any length**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_features = 14\n",
    "inputs = keras.Input(shape=(None, num_features))\n",
    "outputs = layers.SimpleRNN(16)(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**An RNN layer that returns only its last output step**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_features = 14\n",
    "steps = 120\n",
    "inputs = keras.Input(shape=(steps, num_features))\n",
    "outputs = layers.SimpleRNN(16, return_sequences=False)(inputs)\n",
    "print(outputs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**An RNN layer that returns its full output sequence**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "num_features = 14\n",
    "steps = 120\n",
    "inputs = keras.Input(shape=(steps, num_features))\n",
    "outputs = layers.SimpleRNN(16, return_sequences=True)(inputs)\n",
    "print(outputs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Stacking RNN layers**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(steps, num_features))\n",
    "x = layers.SimpleRNN(16, return_sequences=True)(inputs)\n",
    "x = layers.SimpleRNN(16, return_sequences=True)(x)\n",
    "outputs = layers.SimpleRNN(16)(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Advanced use of recurrent neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Using recurrent dropout to fight overfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Training and evaluating a dropout-regularized LSTM**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.LSTM(32, recurrent_dropout=0.25)(inputs)\n",
    "x = layers.Dropout(0.5)(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_lstm_dropout.keras\",\n",
    "                                    save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(train_dataset,\n",
    "                    epochs=50,\n",
    "                    validation_data=val_dataset,\n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, num_features))\n",
    "x = layers.LSTM(32, recurrent_dropout=0.2, unroll=True)(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Stacking recurrent layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Training and evaluating a dropout-regularized, stacked GRU model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.GRU(32, recurrent_dropout=0.5, return_sequences=True)(inputs)\n",
    "x = layers.GRU(32, recurrent_dropout=0.5)(x)\n",
    "x = layers.Dropout(0.5)(x)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"jena_stacked_gru_dropout.keras\",\n",
    "                                    save_best_only=True)\n",
    "]\n",
    "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(train_dataset,\n",
    "                    epochs=50,\n",
    "                    validation_data=val_dataset,\n",
    "                    callbacks=callbacks)\n",
    "model = keras.models.load_model(\"jena_stacked_gru_dropout.keras\")\n",
    "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Using bidirectional RNNs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "**Training and evaluating a bidirectional LSTM**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n",
    "x = layers.Bidirectional(layers.LSTM(16))(inputs)\n",
    "outputs = layers.Dense(1)(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "\n",
    "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n",
    "history = model.fit(train_dataset,\n",
    "                    epochs=10,\n",
    "                    validation_data=val_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Going even further"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Summary"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "chapter10_dl-for-timeseries.i",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}