05_pet_breeds_DataParallel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook \"Pet Breeds\" using Data Parallel (DP)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Author: [Pierre Guillou](https://www.linkedin.com/in/pierreguillou)\n",
    "- Date: June 2020\n",
    "- Sources:\n",
    "  - original notebook from fastai v2: https://github.com/fastai/fastbook/blob/master/05_pet_breeds.ipynb\n",
    "  - example of code with DP and DDP: https://github.com/fastai/fastai2/blob/master/nbs/examples/train_imagenette.py\n",
    "  - doc about Data Parallel with fastai v2: https://dev.fast.ai/distributed#ParallelTrainer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Intitialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *\n",
    "from fastai2.vision.all import *\n",
    "from fastai2.distributed import *\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get data and dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.PETS)\n",
    "\n",
    "Path.BASE_PATH = path\n",
    "\n",
    "pets = DataBlock(blocks = (ImageBlock, CategoryBlock),\n",
    "                 get_items=get_image_files, \n",
    "                 splitter=RandomSplitter(seed=42),\n",
    "                 get_y=using_attr(RegexLabeller(r'(.+)_\\d+.jpg$'), 'name'),\n",
    "                 item_tfms=Resize(460),\n",
    "                 batch_tfms=aug_transforms(size=224, min_scale=0.75))\n",
    "dls = pets.dataloaders(path/\"images\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training without Data Parallel "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.539825</td>\n",
       "      <td>0.330290</td>\n",
       "      <td>0.103518</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.513095</td>\n",
       "      <td>0.318171</td>\n",
       "      <td>0.103518</td>\n",
       "      <td>00:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.324813</td>\n",
       "      <td>0.236592</td>\n",
       "      <td>0.069689</td>\n",
       "      <td>00:15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = cnn_learner(dls, resnet34, metrics=error_rate).to_fp16()\n",
    "learn.fine_tune(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Parallel training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### with PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.702696</td>\n",
       "      <td>0.319656</td>\n",
       "      <td>0.109608</td>\n",
       "      <td>00:19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.573543</td>\n",
       "      <td>0.277454</td>\n",
       "      <td>0.079838</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.398447</td>\n",
       "      <td>0.239030</td>\n",
       "      <td>0.072395</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get learner\n",
    "learn = cnn_learner(dls, resnet34, metrics=error_rate).to_fp16()\n",
    "\n",
    "if torch.cuda.device_count() > 1:\n",
    "    # Use all GPUs\n",
    "    learn.model = nn.DataParallel(learn.model)\n",
    "    # Choose GPUs\n",
    "    # learn.model = nn.DataParallel(learn.model, device_ids=[0,1])\n",
    "    \n",
    "learn.fine_tune(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### with fastai v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training in parallel_ctx context on GPU [0, 1]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.621625</td>\n",
       "      <td>0.344931</td>\n",
       "      <td>0.104195</td>\n",
       "      <td>00:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.572673</td>\n",
       "      <td>0.317181</td>\n",
       "      <td>0.099459</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.394333</td>\n",
       "      <td>0.234659</td>\n",
       "      <td>0.073748</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get number of GPUs\n",
    "gpu = None\n",
    "if torch.cuda.is_available():\n",
    "    if gpu is not None: torch.cuda.set_device(gpu)\n",
    "    n_gpu = torch.cuda.device_count()\n",
    "else:\n",
    "    n_gpu = None\n",
    "    \n",
    "# Get learner\n",
    "learn = cnn_learner(dls, resnet34, metrics=error_rate).to_fp16()\n",
    "\n",
    "# The context manager way of dp/ddp, both can handle single GPU base case.\n",
    "if gpu is None and n_gpu is not None:\n",
    "    ctx = learn.parallel_ctx\n",
    "    with partial(ctx, gpu)():\n",
    "        print(f\"Training in {ctx.__name__} context on GPU {list(range(n_gpu))}\")\n",
    "        learn.fine_tune(2)\n",
    "else:\n",
    "    learn.fine_tune(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## END"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "split_at_heading": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}