debug/pytorch/mem-frag-map.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Memory Fragmentation Causing and Mapping \n",
    "\n",
    "This notebook investigates how to emulate memory fragmentation and how to print the fragmentation map.\n",
    "\n",
    "Currently I'm stuck at emulating fragmentation: https://discuss.pytorch.org/t/gpu-ram-fragmentation-diagnostics/34073/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pynvml, torch, gc\n",
    "from ipyexperiments import IPyExperimentsPytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# light weight humanize from https://stackoverflow.com/a/1094933/9201239 w/ tweaks\n",
    "def hs(num, suffix='B'):\n",
    "    for unit in ['','K','M','G','T','P','E','Z']:\n",
    "        if abs(num) < 1024.0: return \"%3.1f %s%s\" % (num, unit, suffix)\n",
    "        num /= 1024.0\n",
    "    return \"%.1f %s%s\" % (num, 'Y', suffix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def globals_unset(var_names):\n",
    "    \" this is useful for re-running the cell, so that it resets the initial state or cleanup at the end of the cell\"\n",
    "    for x in var_names: \n",
    "        if x in globals(): \n",
    "            del globals()[x]   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pynvml.nvmlInit()\n",
    "id = torch.cuda.current_device()\n",
    "def mem_free():\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    handle = pynvml.nvmlDeviceGetHandleByIndex(id)\n",
    "    info = pynvml.nvmlDeviceGetMemoryInfo(handle)\n",
    "    return int( info.free / 2**20 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "run_control": {
     "marked": false
    }
   },
   "outputs": [],
   "source": [
    "def mem_report(): print(f\"free mem={mem_free()}\")\n",
    "\n",
    "def mem_allocate_mbs_last(n, fatal=False): \n",
    "    \" allocate n MBs, return the var holding it on success, None on failure \"\n",
    "    if n < 6: return None # don't try to allocate less than 6MB\n",
    "    try:\n",
    "        d = int(2**9*n**0.5)\n",
    "        return torch.ones((d, d)).cuda().contiguous()\n",
    "    except Exception as e:\n",
    "        if not fatal: return None\n",
    "        print(f\"allocated={hs(torch.cuda.memory_allocated())}, max allocated={hs(torch.cuda.max_memory_allocated())}, cached={hs(torch.cuda.memory_cached())}, max cached={hs(torch.cuda.max_memory_cached())} \")\n",
    "        raise e "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def leave_free_mbs(n):\n",
    "    \" consume whatever memory is needed so that n MBs are left free \"\n",
    "    avail = mem_free()\n",
    "    assert avail > n, f\"already have less available mem than desired {n}MBs\"\n",
    "    consume = avail - n\n",
    "    print(f\"consuming {consume}MB to bring free mem to {n}MBs\")\n",
    "    return mem_allocate_mbs_last(consume, fatal=True)         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mem_get(n):\n",
    "    print(f\"have {mem_free():4d}, allocating {n}\")\n",
    "    return mem_allocate_mbs(n, fatal=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "*** Experiment started with the Pytorch backend\n",
      "Device: ID 0, GeForce GTX 1070 Ti (8119 RAM)\n",
      "\n",
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.000\n",
      "･ CPU:         0       0     2151 MB |\n",
      "･ GPU:         0       0     6334 MB |\n"
     ]
    }
   ],
   "source": [
    "exp = IPyExperimentsPytorch(exp_enable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.001\n",
      "･ CPU:         0       0     2151 MB |\n",
      "･ GPU:         0       0     6334 MB |\n"
     ]
    }
   ],
   "source": [
    "def mem_allocate_mbs(n, fatal=False): \n",
    "    \" allocate n MBs, return the var holding it on success, None on failure \"\n",
    "    if n < 6: return None # don't try to allocate less than 6MB\n",
    "    try:\n",
    "        return torch.ByteTensor([1]*n*2**20).cuda().contiguous()\n",
    "    except Exception as e:\n",
    "        if not fatal: return None\n",
    "        print(f\"allocated={hs(torch.cuda.memory_allocated())}, max allocated={hs(torch.cuda.max_memory_allocated())}, cached={hs(torch.cuda.memory_cached())}, max cached={hs(torch.cuda.max_memory_cached())} \")\n",
    "        raise e "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yay\n",
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:08.124\n",
      "･ CPU:         0    5280     2149 MB |\n",
      "･ GPU:         0     660      503 MB |\n"
     ]
    }
   ],
   "source": [
    "### test mem_allocate_mbs ###\n",
    "globals_unset(['x1'])\n",
    "\n",
    "x1 = mem_allocate_mbs(660)\n",
    "if x1 is None:\n",
    "    print(\"failed to allocate\")\n",
    "else:\n",
    "    print(\"yay\")\n",
    "globals_unset(['x1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "660.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.005\n",
      "･ CPU:         0       0     2149 MB |\n",
      "･ GPU:         0       0      503 MB |\n"
     ]
    }
   ],
   "source": [
    "exp.cl.data[1].peaked_delta / 2**20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avail: 7616\n",
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.019\n",
      "･ CPU:         0       0     2149 MB |\n",
      "･ GPU:         0       0      503 MB |\n"
     ]
    }
   ],
   "source": [
    "avail = mem_free()\n",
    "print(f\"Avail: {avail}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mem fragmentation mapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.009\n",
      "･ CPU:         0       0     2151 MB |\n",
      "･ GPU:         0       0     6334 MB |\n"
     ]
    }
   ],
   "source": [
    "threshold = 10\n",
    "\n",
    "def get(goal, want=0, delta=0, depth=0):\n",
    "    \"\"\" measure what size of a contiguous memory chunk it's possible to allocate up to a `goal`\n",
    "      return the max size if possible, 0 otherwise.\n",
    "      this is a recursive implementation.\n",
    "    \"\"\"\n",
    "    if want  == 0: want  = goal        \n",
    "    if delta == 0: delta = goal\n",
    "        \n",
    "    delta = int(delta/2)\n",
    "\n",
    "    # threshold to stop at\n",
    "    if want  < threshold  : return 0\n",
    "    if delta < threshold/2: return 0\n",
    "\n",
    "    torch.cuda.empty_cache()\n",
    "    x = mem_allocate_mbs(want)\n",
    "    if x is not None: # success, try more\n",
    "        del x\n",
    "        print(f\"yes: {depth} {want}, {delta} ({goal})\")\n",
    "        return max(want, get(goal, want+delta, delta, depth+1))\n",
    "    else:             # failure, try less\n",
    "        print(f\" no: {depth} {want}, {delta} ({goal})\")\n",
    "        return           get(goal, want-delta, delta, depth+1)   \n",
    "\n",
    "\n",
    "def memmap():\n",
    "    \"\"\"\n",
    "    this function finds all the blocks of memory that can be allocated, ignoring small blocks < 10MB\n",
    "    it prints out these blocks from large to small\n",
    "    \"\"\"\n",
    "    last_tried = 0\n",
    "    blocks = []\n",
    "    store  = []\n",
    "    while True:\n",
    "        avail = mem_free()\n",
    "        print(f\"have {avail}\")\n",
    "        if avail < threshold: break\n",
    "        #if avail > last_tried: break\n",
    "        #last_tried = avail\n",
    "        \n",
    "        size = get(avail)\n",
    "        print(f\"wanted to get {avail} got {size}\")\n",
    "        if not size: break\n",
    "        blocks.append(str(size))\n",
    "        store.append(mem_allocate_mbs(size))\n",
    "        print(f\"got block of size {size}\")\n",
    "        \n",
    "    # free the tmp memory\n",
    "    store = []\n",
    "    print(f\"Free blocks in MBs: {', '.join(blocks)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "free mem=1785\n",
      "have 1785\n",
      " no: 0 1785, 892 (1785)\n",
      "yes: 1 893, 446 (1785)\n",
      "yes: 2 1339, 223 (1785)\n",
      "yes: 3 1562, 111 (1785)\n",
      "yes: 4 1673, 55 (1785)\n",
      "yes: 5 1728, 27 (1785)\n",
      "yes: 6 1755, 13 (1785)\n",
      "yes: 7 1768, 6 (1785)\n",
      "wanted to get 1785 got 1768\n",
      "got block of size 1768\n",
      "have 17\n",
      " no: 0 17, 8 (17)\n",
      "wanted to get 17 got 0\n",
      "Free blocks in MBs: 1768\n",
      "free mem=1785\n",
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:04:27.356\n",
      "･ CPU:         0   14280      168 MB |\n",
      "･ GPU:         0    1768     6334 MB |\n"
     ]
    }
   ],
   "source": [
    "# XXX! try to check torch.C functions and alloc directly if possible\n",
    "\n",
    "mem_report()\n",
    "memmap()\n",
    "mem_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "free mem=7616\n",
      "free mem=7102\n",
      "free mem=6588\n",
      "free mem=7102\n"
     ]
    }
   ],
   "source": [
    "### Test the memmapper ###\n",
    "\n",
    "globals_unset(['x1', 'x2', 'x3'])\n",
    "\n",
    "mem_report()\n",
    "          \n",
    "# 1. first create a hole of 512MB\n",
    "x1 = mem_allocate_mbs(514, fatal=True)\n",
    "mem_report()\n",
    "          \n",
    "x2 = mem_allocate_mbs(514, fatal=True)    \n",
    "mem_report()\n",
    "          \n",
    "del x1\n",
    "mem_report()\n",
    "          \n",
    "x3 = mem_allocate_mbs(1500, fatal=True)    \n",
    "mem_report()\n",
    "          \n",
    "# 2. detect the hole\n",
    "memmap()\n",
    "x = get(10000)\n",
    "print(x)\n",
    "mem_report()\n",
    "          \n",
    "# cleanup\n",
    "globals_unset(['x1', 'x2', 'x3'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Attempts to create fragmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "consuming 4667MB to bring free mem to 1500MBs\n",
      "free mem=1499\n",
      "allocated 870MBs\n",
      "generated 810MBs of holes\n",
      "free mem=635\n",
      "free mem=123\n",
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 2.343s\n",
      "･ CPU:         0       0     2306 MB |\n",
      "･ GPU:         0    6114     1952 MB |\n"
     ]
    }
   ],
   "source": [
    "# \n",
    "globals_unset(['buf', 'stack', 'z'])\n",
    "\n",
    "stack = []\n",
    "\n",
    "# this ensures we always test the same thing\n",
    "buf = leave_free_mbs(1500)\n",
    "\n",
    "# this one tries to create lots of small holes\n",
    "mem_report()\n",
    "holes_total_size = 0\n",
    "for s in range(30):\n",
    "    size = s*2\n",
    "    holes_total_size += size\n",
    "    #print(f\"have {mem_free()}, want {size}\")\n",
    "    x1 = mem_allocate_mbs(size, fatal=True)\n",
    "    #print(f\"have {mem_free()}, want {size}\")\n",
    "    stack.append(mem_allocate_mbs(size, fatal=True))\n",
    "    del x1\n",
    "print(f\"allocated {holes_total_size}MBs\")\n",
    "holes_total_size -= 30*2\n",
    "print(f\"generated {holes_total_size}MBs of holes\")\n",
    "mem_report()\n",
    "\n",
    "z = mem_allocate_mbs(512, fatal=True)\n",
    "mem_report()\n",
    "\n",
    "# cleanup\n",
    "globals_unset(['buf', 'stack', 'z'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "consuming 6016MB to bring free mem to 1600MBs\n",
      "have 1600, allocating 512\n",
      "have 1088, allocating 512\n",
      "have  576, reclaiming first 512\n",
      "have 1088, allocating 1024\n",
      "have   64\n",
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:26.974\n",
      "･ CPU:         0    8192     2149 MB |\n",
      "･ GPU:         0    7552      503 MB |\n"
     ]
    }
   ],
   "source": [
    "globals_unset(['x1', 'x2', 'x3', 'buf'])\n",
    "\n",
    "# this ensures we always test the same thing\n",
    "buf = leave_free_mbs(1600)\n",
    "\n",
    "# this one tries to create one single hole\n",
    "                   # legend: [free block]  {used block}\n",
    "                   # [1600]\n",
    "x1 = mem_get(512)  # {512}[1092]\n",
    "x2 = mem_get(512)  # {512}{512}[576]\n",
    "print(f\"have {mem_free():4d}, reclaiming first 512\")\n",
    "del x1             # [512]{512}[576]\n",
    "x3 = mem_get(1024) # shouldn't be able to allocate 1024 contiguous mem\n",
    "print(f\"have {mem_free():4d}\")\n",
    "\n",
    "# cleanup\n",
    "globals_unset(['x1', 'x2', 'x3', 'buf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# All in one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "consuming 4565MB to bring free mem to 1600MBs\n",
      "have 1599, allocating 512\n",
      "have 1087, allocating 512\n",
      "have  575, reclaiming first 512\n",
      "have 1087, allocating 1024\n",
      "have   63\n",
      "･ RAM: △Consumed △Peaked  Used Total | Exec time 2.363s\n",
      "･ CPU:         0       0     2306 MB |\n",
      "･ GPU:         2    6102     1954 MB |\n"
     ]
    }
   ],
   "source": [
    "# Same as the parts of the notebook above that tries to emulate fragmentation just in one chunk\n",
    "\n",
    "import pynvml, torch, gc\n",
    "\n",
    "pynvml.nvmlInit()\n",
    "id = torch.cuda.current_device()\n",
    "def mem_free():\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    handle = pynvml.nvmlDeviceGetHandleByIndex(id)\n",
    "    info = pynvml.nvmlDeviceGetMemoryInfo(handle)\n",
    "    return int( info.free / 2**20 )\n",
    "\n",
    "def mem_report(): print(f\"free mem={mem_free()}\")\n",
    "\n",
    "def mem_allocate_mbs(n, fatal=False): \n",
    "    \" allocate n MBs, return the var holding it on success, None on failure \"\n",
    "    if n < 6: return None # don't try to allocate less than 6MB\n",
    "    try:\n",
    "        d = int(2**9*n**0.5)\n",
    "        return torch.ones((d, d)).cuda().contiguous()\n",
    "    except Exception as e:\n",
    "        if not fatal: return None\n",
    "        raise e\n",
    "        \n",
    "def leave_free_mbs(n):\n",
    "    \" consume whatever memory is needed so that n MBs are left free \"\n",
    "    avail = mem_free()\n",
    "    assert avail > n, f\"already have less available mem than desired {n}MBs\"\n",
    "    consume = avail - n\n",
    "    print(f\"consuming {consume}MB to bring free mem to {n}MBs\")\n",
    "    return mem_allocate_mbs(consume, fatal=True)\n",
    "\n",
    "def globals_unset(var_names):\n",
    "    \" this is useful for re-running the cell, so that it resets the initial state or cleanup at the end of the cell\"\n",
    "    for x in var_names: \n",
    "        if x in globals(): \n",
    "            del globals()[x]\n",
    "            \n",
    "def mem_get(n):\n",
    "    print(f\"have {mem_free():4d}, allocating {n}\")\n",
    "    return mem_allocate_mbs(n, fatal=True)\n",
    "\n",
    "globals_unset(['x1', 'x2', 'x3', 'buf'])\n",
    "_=torch.ones(1).cuda()# preload\n",
    "\n",
    "# this ensures we always test the same thing\n",
    "buf = leave_free_mbs(1600)\n",
    "    \n",
    "                   # legend: [free block]  {used block}\n",
    "                   # [1600]\n",
    "x1 = mem_get(512)  # {512}[1092]\n",
    "x2 = mem_get(512)  # {512}{512}[576]\n",
    "print(f\"have {mem_free():4d}, reclaiming first 512\")\n",
    "del x1             # [512]{512}[576]\n",
    "x3 = mem_get(1024) # shouldn't be able to allocate 1024 contiguous mem\n",
    "print(f\"have {mem_free():4d}\")\n",
    "\n",
    "# cleanup\n",
    "globals_unset(['x1', 'x2', 'x3', 'buf'])"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}