notebooks/05_cluster_size_and_distributed_IO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Distributed IO and cluster sizes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Core Lesson\n",
    "\n",
    "In a distributed file system, spreading IO to as many nodes as possible might speed up your computations."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up a Slurm cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/p/project/cecam/rath1/miniconda3_20190521/envs/dask_jobqueue_workshop/lib/python3.7/site-packages/docrep/__init__.py:341: MatplotlibDeprecationWarning: \n",
      "The dedent function was deprecated in Matplotlib 3.1 and will be removed in 3.3. Use inspect.cleandoc instead.\n",
      "  s = dedents('\\n' + '\\n'.join(lines[first:]))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://10.80.32.38:46646\n",
       "  <li><b>Dashboard: </b><a href='http://10.80.32.38:8787/status' target='_blank'>http://10.80.32.38:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>0</li>\n",
       "  <li><b>Cores: </b>0</li>\n",
       "  <li><b>Memory: </b>0 B</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://10.80.32.38:46646' processes=0 cores=0>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dask.distributed import Client\n",
    "from dask_jobqueue import SLURMCluster\n",
    "import os\n",
    "\n",
    "cluster = SLURMCluster(\n",
    "    cores=24,\n",
    "    processes=2,\n",
    "    memory=\"100GB\",\n",
    "    shebang='#!/usr/bin/env bash',\n",
    "    queue=\"batch\",\n",
    "    walltime=\"00:30:00\",\n",
    "    local_directory='/tmp',\n",
    "    death_timeout=\"15s\",\n",
    "    interface=\"ib0\",\n",
    "    log_directory=f'{os.environ[\"SCRATCH_cecam\"]}/{os.environ[\"USER\"]}/dask_jobqueue_logs/',\n",
    "    project=\"ecam\")\n",
    "\n",
    "client = Client(cluster)\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The job scripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#!/usr/bin/env bash\n",
      "\n",
      "#SBATCH -J dask-worker\n",
      "#SBATCH -e /p/scratch/cecam/rath1/dask_jobqueue_logs//dask-worker-%J.err\n",
      "#SBATCH -o /p/scratch/cecam/rath1/dask_jobqueue_logs//dask-worker-%J.out\n",
      "#SBATCH -p batch\n",
      "#SBATCH -A ecam\n",
      "#SBATCH -n 1\n",
      "#SBATCH --cpus-per-task=24\n",
      "#SBATCH --mem=94G\n",
      "#SBATCH -t 00:30:00\n",
      "JOB_ID=${SLURM_JOB_ID%;*}\n",
      "\n",
      "\n",
      "\n",
      "/p/project/cecam/rath1/miniconda3_20190521/envs/dask_jobqueue_workshop/bin/python -m distributed.cli.dask_worker tcp://10.80.32.38:46646 --nthreads 12 --nprocs 2 --memory-limit 50.00GB --name dask-worker--${JOB_ID}-- --death-timeout 15s --local-directory /tmp --interface ib0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cluster.job_script())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scale the cluster to a single node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.scale(2)  # will lead to one node"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create some data and dump it to disk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's save the target directory for the data in a variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/p/scratch/cecam/rath1/temporary_random_numbers.zarr\n"
     ]
    }
   ],
   "source": [
    "temporary_zarr_store = (\n",
    "    f'{os.environ[\"SCRATCH_cecam\"]}/{os.environ[\"USER\"]}/'\n",
    "    'temporary_random_numbers.zarr'\n",
    ")\n",
    "print(temporary_zarr_store)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean up\n",
    "!rm -rf {temporary_zarr_store}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, create a 200GB random number array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dask.array<uniform, shape=(25000000000, 1), dtype=float64, chunksize=(25000000, 1)>\n",
      "200.0 GB\n"
     ]
    }
   ],
   "source": [
    "from dask import array as da\n",
    "\n",
    "random_numbers = da.random.uniform(0, 1, size=(1/8 * 200e9, 1), chunks=(1/8 * 200e6, 1))\n",
    "\n",
    "print(random_numbers)\n",
    "print(random_numbers.nbytes / 1e9, \"GB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And store it, drawing random numbers and writing to disk in parallel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.9 s, sys: 740 ms, total: 11.6 s\n",
      "Wall time: 55.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "random_numbers.to_zarr(temporary_zarr_store);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The size on disk is:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "161G\t/p/scratch/cecam/rath1/temporary_random_numbers.zarr\n"
     ]
    }
   ],
   "source": [
    "!du -sh {temporary_zarr_store}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-read the data and time computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dask.array<zarr, shape=(25000000000, 1), dtype=float64, chunksize=(25000000, 1)>\n",
      "200.0 GB\n"
     ]
    }
   ],
   "source": [
    "reread_random_numbers = da.from_zarr(temporary_zarr_store)\n",
    "\n",
    "print(reread_random_numbers)\n",
    "print(reread_random_numbers.nbytes / 1e9, \"GB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49999982667409226\n",
      "CPU times: user 12 s, sys: 922 ms, total: 13 s\n",
      "Wall time: 31.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# should be 0.5\n",
    "print(reread_random_numbers.mean().compute())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scale the cluster to four nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.scale(8)  # will lead to four nodes\n",
    "                   # Not sure we get all up and running now."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49999982667409226\n",
      "CPU times: user 9.16 s, sys: 737 ms, total: 9.9 s\n",
      "Wall time: 16.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "print(reread_random_numbers.mean().compute())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Note that Dask did not cache anything, because we called `.compute()` which will calculate, pass on to the client and then forget the result and all intermediate info._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://10.80.32.38:46646\n",
       "  <li><b>Dashboard: </b><a href='http://10.80.32.38:8787/status' target='_blank'>http://10.80.32.38:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>48</li>\n",
       "  <li><b>Memory: </b>200.00 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://10.80.32.38:46646' processes=4 cores=48>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How far can we take this?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster.scale(40)  # will lead to 20 nodes.\n",
    "                   # Not sure we get all up and running now."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-read and average again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49999982667409226\n",
      "CPU times: user 5.4 s, sys: 420 ms, total: 5.82 s\n",
      "Wall time: 6.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "print(reread_random_numbers.mean().compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://10.80.32.38:46646\n",
       "  <li><b>Dashboard: </b><a href='http://10.80.32.38:8787/status' target='_blank'>http://10.80.32.38:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>12</li>\n",
       "  <li><b>Cores: </b>144</li>\n",
       "  <li><b>Memory: </b>600.00 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://10.80.32.38:46646' processes=12 cores=144>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary\n",
    "\n",
    "We have seen that IO bandwidth increases with the number of nodes we use for the computation.  This is obvious on one hand, but it tells us that from an analytics perspective where a lot of data lying around on disk is analysed, spreading out resources to as many nodes as possible may be the way to go."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf {temporary_zarr_store}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Complete listing of software used here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package            Version          \n",
      "------------------ -----------------\n",
      "asciitree          0.3.3            \n",
      "aspy.yaml          1.2.0            \n",
      "backcall           0.1.0            \n",
      "bokeh              1.1.0            \n",
      "certifi            2019.3.9         \n",
      "cfgv               1.6.0            \n",
      "cftime             1.0.3.4          \n",
      "Click              7.0              \n",
      "cloudpickle        1.0.0            \n",
      "cycler             0.10.0           \n",
      "cytoolz            0.9.0.1          \n",
      "dask               1.2.0            \n",
      "dask-jobqueue      0.4.1+32.g9c3371d\n",
      "decorator          4.4.0            \n",
      "distributed        1.27.1           \n",
      "docrep             0.2.5            \n",
      "fasteners          0.14.1           \n",
      "heapdict           1.0.0            \n",
      "identify           1.4.3            \n",
      "importlib-metadata 0.13             \n",
      "ipykernel          5.1.1            \n",
      "ipython            7.5.0            \n",
      "ipython-genutils   0.2.0            \n",
      "jedi               0.13.3           \n",
      "Jinja2             2.10.1           \n",
      "jupyter-client     5.2.4            \n",
      "jupyter-core       4.4.0            \n",
      "kiwisolver         1.1.0            \n",
      "llvmlite           0.28.0           \n",
      "locket             0.2.0            \n",
      "MarkupSafe         1.1.1            \n",
      "matplotlib         3.1.0            \n",
      "monotonic          1.5              \n",
      "msgpack            0.6.1            \n",
      "netCDF4            1.5.1.2          \n",
      "nodeenv            1.3.3            \n",
      "numba              0.43.1           \n",
      "numcodecs          0.6.3            \n",
      "numpy              1.16.3           \n",
      "olefile            0.46             \n",
      "packaging          19.0             \n",
      "pandas             0.24.2           \n",
      "parso              0.4.0            \n",
      "partd              0.3.9            \n",
      "patsy              0.5.1            \n",
      "pexpect            4.7.0            \n",
      "pickleshare        0.7.5            \n",
      "Pillow             6.0.0            \n",
      "pip                19.1             \n",
      "pre-commit         1.16.1           \n",
      "prompt-toolkit     2.0.9            \n",
      "psutil             5.6.2            \n",
      "ptyprocess         0.6.0            \n",
      "Pygments           2.4.0            \n",
      "pyparsing          2.4.0            \n",
      "python-dateutil    2.8.0            \n",
      "pytz               2019.1           \n",
      "PyYAML             5.1              \n",
      "pyzmq              18.0.1           \n",
      "scipy              1.3.0            \n",
      "seaborn            0.9.0            \n",
      "setuptools         41.0.1           \n",
      "six                1.12.0           \n",
      "sortedcontainers   2.1.0            \n",
      "statsmodels        0.9.0            \n",
      "tblib              1.4.0            \n",
      "toml               0.10.0           \n",
      "toolz              0.9.0            \n",
      "tornado            6.0.2            \n",
      "traitlets          4.3.2            \n",
      "virtualenv         16.6.0           \n",
      "wcwidth            0.1.7            \n",
      "wheel              0.33.4           \n",
      "xarray             0.12.1           \n",
      "zarr               2.3.1            \n",
      "zict               0.1.4            \n",
      "zipp               0.5.1            \n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# This file may be used to create an environment using:\n",
      "# $ conda create --name <env> --file <this file>\n",
      "# platform: linux-64\n",
      "@EXPLICIT\n",
      "https://conda.anaconda.org/conda-forge/linux-64/git-lfs-2.7.2-0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2019.3.9-hecc5488_0.tar.bz2\n",
      "https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-8.2.0-hdf63c60_1.tar.bz2\n",
      "https://repo.anaconda.com/pkgs/main/linux-64/libgfortran-ng-7.3.0-hdf63c60_0.tar.bz2\n",
      "https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-8.2.0-hdf63c60_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.6-h14c3975_1002.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/expat-2.2.5-hf484d3e_1002.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/icu-58.2-hf484d3e_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/jpeg-9c-h14c3975_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libffi-3.2.1-he1b5a44_1006.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.15-h516909a_1005.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.16-h14c3975_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.32.1-h14c3975_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.1-hf484d3e_1002.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.6-h6e990d7_2.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/openssl-1.1.1b-h14c3975_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pcre-8.41-hf484d3e_1003.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/perl-5.26.2-h516909a_1006.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h14c3975_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.9-h14c3975_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.3-h516909a_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.4-h14c3975_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/yaml-0.1.7-h14c3975_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h14c3975_1004.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/gettext-0.19.8.1-hc5be6a0_1002.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.13-h9a582f1_1002.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.10.4-nompi_h3c11f04_1106.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libblas-3.8.0-10_openblas.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20170329-hf8c457e_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.37-hed695b0_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.8.2-h22169c7_2.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.0.10-h648cc4a_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.13-h14c3975_1002.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.9.9-h13577e0_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/readline-7.0-hf8c457e_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.9-h84994c4_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.1-hf484d3e_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/freetype-2.10.0-he983fc9_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/glib-2.58.3-hf63aee3_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/krb5-1.16.3-h05b26f9_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.8.0-10_openblas.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.8.0-10_openblas.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.26.0-h67949de_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/dbus-1.13.6-he372182_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.13.1-he4413a7_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.14.4-h66beb1c_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.64.1-hda55be3_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/python-3.7.3-h5b0a415_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/asciitree-0.3.3-py_2.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/backcall-0.1.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/certifi-2019.3.9-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/click-7.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/cloudpickle-1.0.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/curl-7.64.1-hf8cf82a_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/dask-core-1.2.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/decorator-4.4.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/gst-plugins-base-1.14.4-hdf3bae2_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/heapdict-1.0.0-py37_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/ipython_genutils-0.2.0-py_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.1.0-py37hc9558a2_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/llvmlite-0.28.0-py37hdbcaa40_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/locket-0.2.0-py_2.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/markupsafe-1.1.1-py37h14c3975_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/monotonic-1.5-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-0.6.1-py37h6bb024c_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/numpy-1.16.3-py37he5ce36f_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/olefile-0.46-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/parso-0.4.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pickleshare-0.7.5-py37_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/psutil-5.6.2-py37h516909a_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.6.0-py_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/pyparsing-2.4.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/pytz-2019.1-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pyyaml-5.1-py37h14c3975_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pyzmq-18.0.1-py37hc4ba49a_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/sip-4.19.8-py37hf484d3e_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/six-1.12.0-py37_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.1.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/tblib-1.4.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/toolz-0.9.0-py_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/tornado-6.0.2-py37h516909a_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.1.7-py_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/cftime-1.0.3.4-py37hd352d35_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/cycler-0.10.0-py_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.9.0.1-py37h14c3975_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/fasteners-0.14.1-py_3.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/git-2.21.0-pl526h2882143_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/jedi-0.13.3-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.6.2-hbdf4f91_1001.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/numba-0.43.1-py37hf2d7682_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/numcodecs-0.6.3-py37hf484d3e_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/packaging-19.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/partd-0.3.9-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pexpect-4.7.0-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pillow-6.0.0-py37he7afcd5_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/qt-5.9.7-h52cfd70_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/scipy-1.3.0-py37hab63836_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/setuptools-41.0.1-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/traitlets-4.3.2-py37_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/zict-0.1.4-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/distributed-1.27.1-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/jinja2-2.10.1-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/jupyter_core-4.4.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.1.0-py37h5f35d83_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.5.1.2-py37had58050_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pandas-0.24.2-py37hf484d3e_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/patsy-0.5.1-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/pygments-2.4.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pyqt-5.9.2-py37hcca6a23_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/wheel-0.33.4-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/zarr-2.3.1-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/bokeh-1.1.0-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/jupyter_client-5.2.4-py_3.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/matplotlib-3.1.0-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/pip-19.1-py37_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-2.0.9-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/statsmodels-0.9.0-py37h3010b51_1000.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/xarray-0.12.1-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/dask-1.2.0-py_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/ipython-7.5.0-py37h24bf2e0_0.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/noarch/seaborn-0.9.0-py_1.tar.bz2\n",
      "https://conda.anaconda.org/conda-forge/linux-64/ipykernel-5.1.1-py37h24bf2e0_0.tar.bz2\n",
      "\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%conda list --explicit"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:dask_jobqueue_workshop]",
   "language": "python",
   "name": "conda-env-dask_jobqueue_workshop-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}