tensor_core_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tensorflow.keras as keras\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.mixed_precision.experimental import Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Mixed precision setting\n",
    "## 이곳을 실행하지 않으면 Tensor core를 사용하지 않음\n",
    "\n",
    "policy = Policy('mixed_float16')\n",
    "tf.keras.mixed_precision.experimental.set_policy(policy)\n",
    "print('Compute dtype: %s' % policy.compute_dtype)\n",
    "print('Variable dtype: %s' % policy.variable_dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(784,), name='digits')\n",
    "if tf.config.list_physical_devices('GPU'):\n",
    "  print('The model will run with 4096 units on a GPU')\n",
    "  num_units = 4096\n",
    "else:\n",
    "  # Use fewer units on CPUs so the model finishes in a reasonable amount of time\n",
    "  print('The model will run with 64 units on a CPU')\n",
    "  num_units = 64\n",
    "dense1 = layers.Dense(num_units, activation='relu', name='dense_1')\n",
    "x = dense1(inputs)\n",
    "dense2 = layers.Dense(num_units, activation='relu', name='dense_2')\n",
    "x = dense2(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('x.dtype: %s' % x.dtype.name)\n",
    "# 'kernel' is dense1's variable\n",
    "print('dense1.kernel.dtype: %s' % dense1.kernel.dtype.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# INCORRECT: softmax and model output will be float16, when it should be float32\n",
    "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
    "print('Outputs dtype: %s' % outputs.dtype.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CORRECT: softmax and model output are float32\n",
    "x = layers.Dense(10, name='dense_logits')(x)\n",
    "outputs = layers.Activation('softmax', dtype='float32', name='predictions')(x)\n",
    "print('Outputs dtype: %s' % outputs.dtype.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The linear activation is an identity function. So this simply casts 'outputs'\n",
    "# to float32. In this particular case, 'outputs' is already float32 so this is a\n",
    "# no-op.\n",
    "outputs = layers.Activation('linear', dtype='float32')(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.Model(inputs=inputs, outputs=outputs)\n",
    "model.compile(loss='sparse_categorical_crossentropy',\n",
    "              optimizer=keras.optimizers.RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n",
    "x_test = x_test.reshape(10000, 784).astype('float32') / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=8192,\n",
    "                    epochs=5,\n",
    "                    validation_split=0.2)\n",
    "test_scores = model.evaluate(x_test, y_test, verbose=2)\n",
    "print('Test loss:', test_scores[0])\n",
    "print('Test accuracy:', test_scores[1])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mixed precision setting\n",
    "### Test gpu : RTX 2080ti\n",
    "\n",
    "Epoch 1/5  \n",
    "6/6 [==============================] - 0s 64ms/step - loss: 5.1665 - accuracy: 0.4094 - val_loss: 0.7263 - val_accuracy: 0.8307  \n",
    "Epoch 2/5  \n",
    "6/6 [==============================] - 0s 37ms/step - loss: 0.6760 - accuracy: 0.7938 - val_loss: 0.4567 - val_accuracy: 0.8476  \n",
    "Epoch 3/5  \n",
    "6/6 [==============================] - 0s 38ms/step - loss: 0.3648 - accuracy: 0.8856 - val_loss: 0.3276 - val_accuracy: 0.9015  \n",
    "Epoch 4/5  \n",
    "6/6 [==============================] - 0s 37ms/step - loss: 0.3125 - accuracy: 0.8989 - val_loss: 0.5661 - val_accuracy: 0.8050  \n",
    "Epoch 5/5  \n",
    "6/6 [==============================] - 0s 37ms/step - loss: 0.3052 - accuracy: 0.9035 - val_loss: 0.1625 - val_accuracy: 0.9500  \n",
    "313/313 - 1s - loss: 0.1676 - accuracy: 0.9485  \n",
    "Test loss: 0.16764894127845764  \n",
    "Test accuracy: 0.9484999775886536  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No mixed precision setting\n",
    "### Test gpu : RTX 2080ti\n",
    "Epoch 1/5  \n",
    "6/6 [==============================] - 1s 106ms/step - loss: 4.2295 - accuracy: 0.4273 - val_loss: 0.7785 - val_accuracy: 0.7990  \n",
    "Epoch 2/5  \n",
    "6/6 [==============================] - 1s 86ms/step - loss: 0.6895 - accuracy: 0.7926 - val_loss: 0.3278 - val_accuracy: 0.9089  \n",
    "Epoch 3/5  \n",
    "6/6 [==============================] - 1s 86ms/step - loss: 0.3661 - accuracy: 0.8807 - val_loss: 0.2844 - val_accuracy: 0.9078  \n",
    "Epoch 4/5  \n",
    "6/6 [==============================] - 1s 86ms/step - loss: 0.3186 - accuracy: 0.9001 - val_loss: 0.1877 - val_accuracy: 0.9470  \n",
    "Epoch 5/5  \n",
    "6/6 [==============================] - 1s 87ms/step - loss: 0.1981 - accuracy: 0.9417 - val_loss: 0.5932 - val_accuracy: 0.8552  \n",
    "313/313 - 1s - loss: 0.6030 - accuracy: 0.8537  \n",
    "Test loss: 0.6030229926109314  \n",
    "Test accuracy: 0.8536999821662903  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}