WW-MLP3-BatchSizes.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "WW-MLP3-BatchSizes.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CalculatedContent/WeightWatcher-Examples/blob/main/WW-MLP3-BatchSizes.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This notebook shows that\n",
        "# Alpha decreases with increasing Test Accuracy\n",
        "\n",
        "## But if alpha < 2, the model may be overtrained\n",
        "\n",
        "### This is the simplest possible modern MLP, 3 layers trained on MNIST\n",
        "\n",
        "- Using Keras, following this sample code:\n",
        "    - https://www.kaggle.com/sathianpong/3-ways-to-implement-mlp-with-keras\n",
        "\n",
        "- trained  with a early stopping on the training loss,\n",
        "    - upto 1000 epochs (but usually < 100)\n",
        "    - delta_min = 0.001\n",
        "- all layers are frozen, except for layer 1 (which is 300 x 100)\n",
        "- the batch size is varied to induce the effect at very small batch_sizes\n",
        "\n",
        "\n",
        "Note: To apply the theory in this way, it is necessary to:\n",
        "- to train the all models  for as long / to as high of accuracy as possible\n",
        "- with the exact same initial conditions\n",
        "\n",
        "We don't necessarily need to freeze the other layers, but this is done for clarity here.  \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "VhMwt7M7cEZV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Keras Sequential API"
      ],
      "metadata": {
        "id": "2IcMPtALnDnT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import tensorflow as tf\n",
        "from IPython.display import Image, clear_output\n",
        "\n",
        "sns.set()\n",
        "tf.__version__"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "3b0V4XmCdshz",
        "outputId": "ec7787af-7567-4c81-e0fb-e6c14018e1ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2.15.0'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_e9GgbKxJgSZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://stackoverflow.com/questions/32419510/how-to-get-reproducible-results-in-keras\n",
        "\n",
        "# https://tensorflow.google.cn/api_docs/python/tf/config/experimental/enable_op_determinism\n",
        "\n",
        "%env CUBLAS_WORKSPACE_CONFIG=:4096:8\n",
        "\n",
        "import random\n",
        "def reset_random_seeds(seed_value=1):\n",
        "   os.environ['PYTHONHASHSEED']=str(seed_value)\n",
        "   tf.random.set_seed(seed_value)\n",
        "   tf.keras.utils.set_random_seed(seed_value)\n",
        "   np.random.seed(seed_value)\n",
        "   random.seed(seed_value)\n",
        "   tf.config.experimental.enable_op_determinism()\n",
        "\n",
        "reset_random_seeds()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bd9maNqsfZ2e",
        "outputId": "acafd37a-5376-4ac6-c3e5-a73e9088cd94"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "env: CUBLAS_WORKSPACE_CONFIG=:4096:8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cpOecISC8nX7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install weightwatcher"
      ],
      "metadata": {
        "id": "TSR7E0neyeaH",
        "outputId": "c5bbef2c-0804-4a9b-f151-49ff796ed6af",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting weightwatcher\n",
            "  Downloading weightwatcher-0.7.5-py3-none-any.whl (79 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/79.8 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m71.7/79.8 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.8/79.8 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from weightwatcher) (1.25.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from weightwatcher) (1.5.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from weightwatcher) (3.7.1)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from weightwatcher) (0.1.6)\n",
            "Collecting powerlaw (from weightwatcher)\n",
            "  Downloading powerlaw-1.5-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from weightwatcher) (1.2.2)\n",
            "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from weightwatcher) (0.4.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from weightwatcher) (4.66.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (4.49.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (1.4.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (23.2)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (9.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (3.1.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->weightwatcher) (2.8.2)\n",
            "Requirement already satisfied: traitlets in /usr/local/lib/python3.10/dist-packages (from matplotlib-inline->weightwatcher) (5.7.1)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->weightwatcher) (2023.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from powerlaw->weightwatcher) (1.11.4)\n",
            "Requirement already satisfied: mpmath in /usr/local/lib/python3.10/dist-packages (from powerlaw->weightwatcher) (1.3.0)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->weightwatcher) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->weightwatcher) (3.3.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->weightwatcher) (1.16.0)\n",
            "Installing collected packages: powerlaw, weightwatcher\n",
            "Successfully installed powerlaw-1.5 weightwatcher-0.7.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Download the Fashion MNIST dataset"
      ],
      "metadata": {
        "id": "ZFEgVvqsrzJY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = tf.keras.datasets.mnist\n",
        "\n",
        "(X_train, y_train), (X_test, y_test) = data.load_data()\n",
        "class_names = ['T-shirt/top','Trouser','Pullover','Dress','Coat','Sandal','Shirt','Sneaker','Bag','Ankle boot']\n",
        "\n",
        "clear_output()\n",
        "print(\"X_train shape :\",X_train.shape)\n",
        "print(\"X_test shape :\", X_test.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kXZ-3WMNnDFd",
        "outputId": "79575ea7-0518-472a-839c-f66f3cbe10c4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "X_train shape : (60000, 28, 28)\n",
            "X_test shape : (10000, 28, 28)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Recale the data"
      ],
      "metadata": {
        "id": "EbTiCnR1s23b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X_train = X_train/255.0\n",
        "X_test = X_test/255.0"
      ],
      "metadata": {
        "id": "4_OaMj00s2Ak"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Build the simplest MLP, need 3 layers\n",
        "\n",
        "- input layer\n",
        "- hidden layer\n",
        "- output layer\n",
        "\n",
        "Initialize all models with the same weight matrices"
      ],
      "metadata": {
        "id": "7Ng0L5O7s45O"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def build_model():\n",
        "\n",
        "  reset_random_seeds()\n",
        "\n",
        "  initializer = tf.keras.initializers.GlorotNormal(seed=1)\n",
        "\n",
        "  # Specify the loss fuction, optimizer, metrics\n",
        "  model = tf.keras.models.Sequential([\n",
        "      tf.keras.layers.Flatten(input_shape = [28,28]),\n",
        "      tf.keras.layers.Dense(300, activation='relu', kernel_initializer=initializer),\n",
        "      tf.keras.layers.Dense(100, activation='relu', kernel_initializer=initializer),\n",
        "      tf.keras.layers.Dense(10, activation='softmax', kernel_initializer=initializer),\n",
        "  ])\n",
        "\n",
        "\n",
        "  model.compile(\n",
        "  loss = tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "  optimizer = tf.keras.optimizers.SGD(),\n",
        "  metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n",
        "  )\n",
        "\n",
        "  model.layers[0].trainable = True\n",
        "  model.layers[1].trainable = True\n",
        "  model.layers[2].trainable = True\n",
        "  model.layers[3].trainable = True\n",
        "\n",
        "  return model\n",
        "\n"
      ],
      "metadata": {
        "id": "BiJWotzhsqch"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "bBtp_838C992"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = build_model()\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AjtwYJJEurxW",
        "outputId": "0c2a088c-1194-479c-bfc1-2476825dfe98"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " flatten (Flatten)           (None, 784)               0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 300)               235500    \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 100)               30100     \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 10)                1010      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 266610 (1.02 MB)\n",
            "Trainable params: 235500 (919.92 KB)\n",
            "Non-trainable params: 31110 (121.52 KB)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "model = build_model()\n",
        "\n",
        "history = model.fit(\n",
        "  X_train, y_train, epochs=1, batch_size=16, verbose=2\n",
        ")"
      ],
      "metadata": {
        "id": "qRxqeYcvgU0l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Should be:\n",
        "\n",
        "3750/3750 - 9s - loss: 0.6044 - sparse_categorical_accuracy: 0.7953 - 9s/epoch - 2ms/step\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "QuTYuRHagqJ0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = build_model()\n",
        "\n",
        "history = model.fit(\n",
        "  X_train, y_train, epochs=1, batch_size=16, verbose=2\n",
        ")"
      ],
      "metadata": {
        "id": "LrmAjytchBR0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train the model\n",
        "def train_model(bs):\n",
        "\n",
        "    model = build_model()\n",
        "\n",
        "    early_stopping_callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3, verbose=2, min_delta=0.0001, restore_best_weights=True)\n",
        "    history = model.fit(\n",
        "        X_train, y_train, epochs=1000, batch_size=bs,\n",
        "        validation_split=0.1,\n",
        "        verbose=2, callbacks=[early_stopping_callback]\n",
        "    )\n",
        "    return model, history"
      ],
      "metadata": {
        "id": "DZ0VhgAoyaJ0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "model, H = train_model(bs=16)"
      ],
      "metadata": {
        "id": "W0aMuO9XykYX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "H.history.keys()"
      ],
      "metadata": {
        "id": "6DilM7OjtO_R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pd.DataFrame(H.history).plot()\n",
        "plt.xlabel(\"epoch\")\n",
        "plt.legend(bbox_to_anchor=(1.05, 1), loc=2)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "qleFyBgX4RW0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import weightwatcher as ww\n",
        "watcher = ww.WeightWatcher(model=model)\n",
        "watcher.describe()"
      ],
      "metadata": {
        "id": "AIQwGkrCvxMY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "watcher.analyze(plot=True)"
      ],
      "metadata": {
        "id": "awNvVrG-v4NE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### We can now vary the hyperparameters and see how the test accuracy changes\n",
        " > defaults: https://keras.io/api/optimizers/sgd/'\n",
        "\n",
        "The goal is to get some significant variation in the test (validation) accuraies, and see how our weightwatcher metrics compare"
      ],
      "metadata": {
        "id": "MsIYeYBk6QJd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Changes just the learning rate first\n",
        "all_reaults = []\n",
        "#all_bs = [256,128,64,13,16,8,4,2, 1]\n",
        "all_bs = [1,2,4,8,16,32]\n",
        "\n",
        "\n",
        "mlp3_results = None\n",
        "models = {}\n",
        "histories = {}\n",
        "for bs in all_bs:\n",
        "    layer_ids = [1]\n",
        "    modelname = 'MLP3'\n",
        "    print(f\"Analyzing {modelname} bs={bs}\")\n",
        "    model, H = train_model(bs)\n",
        "    watcher = ww.WeightWatcher(model=model)\n",
        "    results = watcher.analyze(layers=layer_ids)\n",
        "\n",
        "    val_acc = H.history['val_sparse_categorical_accuracy'][-1]\n",
        "\n",
        "    histories[bs] = H\n",
        "    models[bs]= model\n",
        "\n",
        "    results['modelname'] = modelname\n",
        "    results['batch_size'] = bs\n",
        "    results['H_val_acc'] = H.history['val_sparse_categorical_accuracy'][-1]\n",
        "\n",
        "\n",
        "    score = model.evaluate(X_train, y_train, verbose = 0)\n",
        "    results['train_accuracy'] = score[1]\n",
        "    score = model.evaluate(X_test, y_test, verbose = 0)\n",
        "    results['test_accuracy'] = score[1]\n",
        "\n",
        "\n",
        "    pd.DataFrame(H.history).plot()\n",
        "    plt.xlabel(\"epoch\")\n",
        "    plt.legend(bbox_to_anchor=(1.05, 1), loc=2)\n",
        "    plt.title(f\"batch size {bs}\")\n",
        "    plt.show()\n",
        "\n",
        "    #filename = f\"/content/drive/My Drive/MLP3/model_bs{bs}\"\n",
        "    #model.save(filename)\n",
        "\n",
        "    if mlp3_results is None:\n",
        "        mlp3_results = results\n",
        "    else:\n",
        "        mlp3_results = pd.concat((mlp3_results, results))\n",
        "\n",
        "#filename = f\"/content/drive/My Drive/MLP3/mlp3_results.feather\"\n",
        "#mlp3_results.to_feather(filename)"
      ],
      "metadata": {
        "id": "s7PXIKTa6_yN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mlp3_results.plot.scatter(x='batch_size',y='test_accuracy')\n",
        "plt.title(r\"Batch Size  vs Test Accuracy\")\n",
        "plt.xlabel(r\"Batch Size\")\n",
        "plt.ylabel(\"Test Accuracy\")"
      ],
      "metadata": {
        "id": "K1iOuK5B8acI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mlp3_results.plot.scatter(x='alpha',y='train_accuracy')\n",
        "mlp3_results.plot.scatter(x='alpha',y='test_accuracy')"
      ],
      "metadata": {
        "id": "pmXt5xdEp2vg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mlp3_results.plot.scatter(x='alpha',y='test_accuracy')\n",
        "plt.title(r\"WeightWatcher PL Alpha ($\\alpha$) metric vs Test Accuracy\")\n",
        "plt.xlabel(r\"Alpha ($\\alpha$) metric\")\n",
        "plt.ylabel(\"Test Accuracy\")"
      ],
      "metadata": {
        "id": "E-7XvkG77wgX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "we7XugRTFjgu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "qYm9PdPiRsOk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "3a-m8jIaNYit"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}