week_06/Week #06 Task #02 Hyperparameter Tuning using Weights and Biases.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Week #06 Task #02 Hyperparameter Tuning using Weights and Biases.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNepLrtPIfGc4cSKw0HDKrn"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"RIf7eFgwyZ06"},"source":["# 1 Import libraries"]},{"cell_type":"code","metadata":{"id":"L4OeKOISEobo"},"source":["import tensorflow as tf\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import h5py\n","import time\n","import datetime\n","import pytz\n","import IPython"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aRdkrAFXxYcb"},"source":["print('TF version:', tf.__version__)\n","print('GPU devices:', tf.config.list_physical_devices('GPU'))"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"MSK_tRj1yipL"},"source":["# 2 Data load and preprocessing"]},{"cell_type":"code","metadata":{"id":"z7Rhm9scwVx1"},"source":["# download train_catvnoncat.h5\n","!gdown https://drive.google.com/uc?id=1ZPWKlEATuDjFtZJPgHCc5SURrcKaVP9Z\n","\n","# download test_catvnoncat.h5\n","!gdown https://drive.google.com/uc?id=1ndRNAwidOqEgqDHBurA0PGyXqHBlvzz-"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"evhmAkbKxYy_"},"source":["def load_dataset():\n","    # load the train data\n","    train_dataset = h5py.File('train_catvnoncat.h5', \"r\")\n","\n","    # your train set features\n","    train_set_x_orig = np.array(train_dataset[\"train_set_x\"][:]) \n","\n","    # your train set labels\n","    train_set_y_orig = np.array(train_dataset[\"train_set_y\"][:]) \n","\n","    # load the test data\n","    test_dataset = h5py.File('test_catvnoncat.h5', \"r\")\n","\n","    # your test set features\n","    test_set_x_orig = np.array(test_dataset[\"test_set_x\"][:]) \n","\n","    # your test set labels  \n","    test_set_y_orig = np.array(test_dataset[\"test_set_y\"][:]) \n","\n","    # the list of classes\n","    classes = np.array(test_dataset[\"list_classes\"][:]) \n","\n","    # reshape the test data\n","    train_set_y_orig = train_set_y_orig.reshape((train_set_y_orig.shape[0],1))\n","    test_set_y_orig = test_set_y_orig.reshape((test_set_y_orig.shape[0],1))\n","\n","    return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"mjQYsrdPHSyh"},"source":["# Loading the data (cat/non-cat)\n","train_set_x_orig, train_y, test_set_x_orig, test_y, classes = load_dataset()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6ihL83MbHlhc"},"source":["# Reshape the training and test examples\n","train_set_x_flatten = train_set_x_orig.reshape(train_set_x_orig.shape[0],-1)\n","test_set_x_flatten = test_set_x_orig.reshape(test_set_x_orig.shape[0],-1)\n","\n","# Standardize the dataset\n","train_x = train_set_x_flatten/255\n","test_x = test_set_x_flatten/255"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"sQn-Brt-IhK_"},"source":["print (\"train_x shape: \" + str(train_x.shape))\n","print (\"train_y shape: \" + str(train_y.shape))\n","print (\"test_x  shape: \" + str(test_x.shape))\n","print (\"test_y  shape: \" + str(test_y.shape))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ATILeNhddrcC"},"source":["# visualize a sample modified data\n","index = 13\n","plt.imshow(train_x[index].reshape(64,64,3))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Xa9dcSRx1__x"},"source":["# visualize a sample raw data\n","index = 13\n","plt.imshow(train_set_x_orig[index])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DvACru73ysxG"},"source":["class MyCustomCallback(tf.keras.callbacks.Callback):\n","\n","  def on_train_begin(self, batch, logs=None):\n","    self.begins = time.time()\n","    print('Training: begins at {}'.format(datetime.datetime.now(pytz.timezone('America/Fortaleza')).strftime(\"%a, %d %b %Y %H:%M:%S\")))\n","\n","  def on_train_end(self, logs=None):\n","    print('Training: ends at {}'.format(datetime.datetime.now(pytz.timezone('America/Fortaleza')).strftime(\"%a, %d %b %Y %H:%M:%S\")))\n","    print('Duration: {:.2f} seconds'.format(time.time() - self.begins))    "],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"iyHu7kEeyrh-"},"source":["# 3 Base Model"]},{"cell_type":"code","metadata":{"id":"HPkKk-ZayvfU"},"source":["# Instantiate a simple classification model\n","model = tf.keras.Sequential([\n","  tf.keras.layers.Dense(8, activation=tf.nn.relu, dtype='float64'),\n","  tf.keras.layers.Dense(8, activation=tf.nn.relu, dtype='float64'),\n","  tf.keras.layers.Dense(1, activation=tf.nn.sigmoid, dtype='float64')\n","])\n","\n","# Instantiate a logistic loss function that expects integer targets.\n","loss = tf.keras.losses.BinaryCrossentropy()\n","\n","# Instantiate an accuracy metric.\n","accuracy = tf.keras.metrics.BinaryAccuracy()\n","\n","# Instantiate an optimizer.\n","optimizer = tf.keras.optimizers.SGD(learning_rate=0.001)\n","\n","# configure the optimizer, loss, and metrics to monitor.\n","model.compile(optimizer=optimizer, loss=loss, metrics=[accuracy])\n","\n","# training \n","history = model.fit(x=train_x,\n","                    y=train_y,\n","                    batch_size=32,\n","                    epochs=500,\n","                    validation_data=(test_x,test_y),\n","                    callbacks=[MyCustomCallback()],\n","                    verbose=1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nLdVTFjGyyDe"},"source":["loss, acc = model.evaluate(x=train_x,y=train_y, batch_size=32)\n","print('Train loss: %.4f - acc: %.4f' % (loss, acc))\n","\n","loss_, acc_ = model.evaluate(x=test_x,y=test_y, batch_size=32)\n","print('Test loss: %.4f - acc: %.4f' % (loss_, acc_))"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"RooPTo2ezBBh"},"source":["# 4 Hyperparameter Tuning "]},{"cell_type":"code","metadata":{"id":"ESSHH5_UzQ3o"},"source":["%%capture\n","!pip install wandb==0.10.17"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Y_Tjh1Sbz1tJ"},"source":["!wandb login --relogin"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kKxNLU83GV4q"},"source":["## 4.1 Monitoring a neural network"]},{"cell_type":"code","metadata":{"id":"Hi7lXapOz50x"},"source":["import wandb\n","from wandb.keras import WandbCallback\n","from tensorflow.keras.callbacks import EarlyStopping\n","\n","# Default values for hyperparameters\n","defaults = dict(layer_1 = 8,\n","                layer_2 = 8,\n","                learn_rate = 0.001,\n","                batch_size = 32,\n","                epoch = 500)\n","\n","wandb.init(project=\"week06\", config= defaults, name=\"week06_run_01\")\n","config = wandb.config"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-bKBiY1q1QdJ"},"source":["# Instantiate a simple classification model\n","model = tf.keras.Sequential([\n","  tf.keras.layers.Dense(config.layer_1, activation=tf.nn.relu, dtype='float64'),\n","  tf.keras.layers.Dense(config.layer_2, activation=tf.nn.relu, dtype='float64'),\n","  tf.keras.layers.Dense(1, activation=tf.nn.sigmoid, dtype='float64')\n","])\n","\n","# Instantiate a logistic loss function that expects integer targets.\n","loss = tf.keras.losses.BinaryCrossentropy()\n","\n","# Instantiate an accuracy metric.\n","accuracy = tf.keras.metrics.BinaryAccuracy()\n","\n","# Instantiate an optimizer.\n","optimizer = tf.keras.optimizers.SGD(learning_rate=config.learn_rate)\n","\n","# configure the optimizer, loss, and metrics to monitor.\n","model.compile(optimizer=optimizer, loss=loss, metrics=[accuracy])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TuzJG3XG2jjw"},"source":["%%wandb\n","# Add WandbCallback() to the fit function\n","model.fit(x=train_x,\n","          y=train_y,\n","          batch_size=config.batch_size,\n","          epochs=config.epoch,\n","          validation_data=(test_x,test_y),\n","          callbacks=[WandbCallback(log_weights=True)],\n","          verbose=1)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"vh562csI3WFO"},"source":["## 4.2 Sweeps"]},{"cell_type":"code","metadata":{"id":"mo_cE96gG8Tq"},"source":[" # The sweep calls this function with each set of hyperparameters\n","def train():\n","    # Default values for hyper-parameters we're going to sweep over\n","    defaults = dict(layer_1 = 8,\n","                layer_2 = 8,\n","                learn_rate = 0.001,\n","                batch_size = 32,\n","                epoch = 500)\n","    \n","    # Initialize a new wandb run\n","    wandb.init(project=\"week06\", config= defaults)\n","\n","    # Config is a variable that holds and saves hyperparameters and inputs\n","    config = wandb.config\n","    \n","    # Instantiate a simple classification model\n","    model = tf.keras.Sequential([\n","                                 tf.keras.layers.Dense(config.layer_1, activation=tf.nn.relu, dtype='float64'),\n","                                 tf.keras.layers.Dense(config.layer_2, activation=tf.nn.relu, dtype='float64'),\n","                                 tf.keras.layers.Dense(1, activation=tf.nn.sigmoid, dtype='float64')\n","                                 ])\n","\n","    # Instantiate a logistic loss function that expects integer targets.\n","    loss = tf.keras.losses.BinaryCrossentropy()\n","\n","    # Instantiate an accuracy metric.\n","    accuracy = tf.keras.metrics.BinaryAccuracy()\n","\n","    # Instantiate an optimizer.\n","    optimizer = tf.keras.optimizers.SGD(learning_rate=config.learn_rate)\n","\n","    # configure the optimizer, loss, and metrics to monitor.\n","    model.compile(optimizer=optimizer, loss=loss, metrics=[accuracy])  \n","\n","    model.fit(train_x, train_y, batch_size=config.batch_size,\n","              epochs=config.epoch,\n","              validation_data=(test_x, test_y),\n","              callbacks=[WandbCallback(),\n","                          EarlyStopping(patience=100)]\n","              )   "],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"S6whtzZ1eior"},"source":["# See the source code in order to see other parameters\n","# https://github.com/wandb/client/tree/master/wandb/sweeps"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1Ibov1wrLgS9"},"source":["# Configure the sweep – specify the parameters to search through, the search strategy, the optimization metric et all.\n","sweep_config = {\n","    'method': 'random', #grid, random\n","    'metric': {\n","      'name': 'binary_accuracy',\n","      'goal': 'maximize'   \n","    },\n","    'parameters': {\n","        'layer_1': {\n","            'max': 32,\n","            'min': 8,\n","            'distribution': 'int_uniform',\n","        },\n","        'layer_2': {\n","            'max': 32,\n","            'min': 8,\n","            'distribution': 'int_uniform',\n","        },\n","        'learn_rate': {\n","            'min': -4,\n","            'max': -2,\n","            'distribution': 'log_uniform',  \n","        },\n","        'epoch': {\n","            'values': [300,400,600]\n","        },\n","        'batch_size': {\n","            'values': [32,64]\n","        }\n","    }\n","}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1rCRIA2HMG1Y"},"source":["# Initialize a new sweep\n","# Arguments:\n","#     – sweep_config: the sweep config dictionary defined above\n","#     – entity: Set the username for the sweep\n","#     – project: Set the project name for the sweep\n","sweep_id = wandb.sweep(sweep_config, entity=\"ivanovitchm\", project=\"week06\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1V0Cobv4MahM"},"source":["# Initialize a new sweep\n","# Arguments:\n","#     – sweep_id: the sweep_id to run - this was returned above by wandb.sweep()\n","#     – function: function that defines your model architecture and trains it\n","wandb.agent(sweep_id = sweep_id, function=train,count=20)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"P_dpK2SHjIlu"},"source":["### 4.2.1 Restore a model\n","\n","Restore a file, such as a model checkpoint, into your local run folder to access in your script.\n","\n","See [the restore docs](https://docs.wandb.com/library/restore) for more details."]},{"cell_type":"code","metadata":{"id":"-ikTvdN61mm0"},"source":["%%capture\n","!pip install wandb==0.10.17"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"V8dR38tFEpby"},"source":["!pip install wandb"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"xi-Fv1nfAU6q","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1628246847147,"user_tz":180,"elapsed":799,"user":{"displayName":"Ivanovitch Silva","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Git9r91cROvzBPiAlvwQtPMEFxLz44uDidMPM-PrQ=s64","userId":"06428777505436195303"}},"outputId":"cff675dc-f568-495d-d032-92d48ea5d47a"},"source":[" import wandb\n"," wandb.__version__"],"execution_count":2,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'0.11.2'"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"id":"5zhf0Gix1nYD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1628246856152,"user_tz":180,"elapsed":5630,"user":{"displayName":"Ivanovitch Silva","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Git9r91cROvzBPiAlvwQtPMEFxLz44uDidMPM-PrQ=s64","userId":"06428777505436195303"}},"outputId":"7e4ab300-d488-4807-a689-15b628270563"},"source":["!wandb login"],"execution_count":3,"outputs":[{"output_type":"stream","text":["\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n","\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter: \n","\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"0LB6j3O-jIsd"},"source":["# restore the raw model file \"model-best.h5\" from a specific run by user \"ivanovitchm\"\n","# in project \"lesson04\" from run \"sqdv5ccj\"\n","best_model = wandb.restore('model-best.h5', run_path=\"ivanovitchm/week06/cbwfq70j\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wo_JI5RPHzKu"},"source":["# restore the model for tf.keras\n","model = tf.keras.models.load_model(best_model.name)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aeM9gLcrDiz7"},"source":["# execute the loss and accuracy using the test dataset\n","loss_, acc_ = model.evaluate(x=test_x,y=test_y, batch_size=64)\n","print('Test loss: %.3f - acc: %.3f' % (loss_, acc_))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a55_JCKuR4kJ"},"source":["# source: https://github.com/wandb/awesome-dl-projects/blob/master/ml-tutorial/EMNIST_Dense_Classification.ipynb\n","import seaborn as sns\n","from sklearn.metrics import confusion_matrix\n","\n","predictions = np.greater_equal(model.predict(test_x),0.5).astype(int)\n","cm = confusion_matrix(y_true = test_y, y_pred = predictions)\n","\n","plt.figure(figsize=(6,6));\n","sns.heatmap(cm, annot=True)\n","plt.savefig('confusion_matrix.png', bbox_inches='tight')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wTu0f6DiR7oW"},"source":["wandb.init(project=\"week06\")\n","wandb.log({\"image_confusion_matrix\": [wandb.Image('confusion_matrix.png')]})"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XM5q_a_zqMY0"},"source":["# visualize the images and instances with error\n","# ground-truth\n","print(\"Ground-truth\\n\",test_y[~np.equal(predictions,test_y)])\n","\n","# predictions\n","print(\"Predictions\\n\",predictions[~np.equal(predictions,test_y)])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"9iJtJ-wWvvl5"},"source":["# Images predicted as non-cat\n","fig, ax = plt.subplots(2,6,figsize=(10,6))\n","wrong_images = (~np.equal(predictions,test_y)).astype(int)\n","index = np.where(wrong_images == 1)[0]\n","\n","for i,value in enumerate(index):\n","  ax[i//6,i%6].imshow(test_x[value].reshape(64,64,3))\n","plt.savefig('wrong_predictions.png', bbox_inches='tight')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"LnywcEUHxvL_"},"source":["wandb.log({\"wrong_predictions\": [wandb.Image('wrong_predictions.png')]})"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"jhBR6ePBvHy7"},"source":["# 5 References"]},{"cell_type":"markdown","metadata":{"id":"Hb3mCmDDvJjw"},"source":["1. https://github.com/wandb/awesome-dl-projects\n","2. https://docs.wandb.ai/app/features/panels/parameter-importance\n","3. https://wandb.ai/wandb/DistHyperOpt/reports/Modern-Scalable-Hyperparameter-Tuning-Methods--VmlldzoyMTQxODM"]}]}