kube-ml-dask.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask.distributed import Client\n",
    "scheduler_address = '104.198.73.140:8786'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = Client(scheduler_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask_ml.datasets import make_classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_classification(n_samples=5000000, n_features=20,chunks=100000, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dask.array<normal, shape=(5000000, 20), dtype=float64, chunksize=(100000, 20)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_c = client.persist(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_c = client.persist(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install --user dask-kubernetes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.dataframe as dd\n",
    "from dask.distributed import Client\n",
    "from dask_kubernetes import KubeCluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Worker pod specification not provided. See KubeCluster docstring for ways to specify workers",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-c0b963f8b3e0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcluster\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mKubeCluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#(host='104.198.73.140', port='8786', n_workers=2)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/dask_kubernetes/core.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, pod_template, name, namespace, n_workers, host, port, env, auth, scheduler_timeout, deploy_mode, interface, protocol, dashboard_address, security, **kwargs)\u001b[0m\n\u001b[1;32m    378\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    379\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    381\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m     \u001b[0;34masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, workers, scheduler, worker, asynchronous, loop, security, silence_logs, name)\u001b[0m\n\u001b[1;32m    240\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_loop_runner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    243\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_correct_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/deploy/cluster.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m    160\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m     \u001b[0;34masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m    331\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    334\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    335\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m    315\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    316\u001b[0m                 \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_timeout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseconds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 317\u001b[0;31m             \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    318\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    319\u001b[0m             \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    733\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    734\u001b[0m                     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 735\u001b[0;31m                         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    736\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    737\u001b[0m                         \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/dask_kubernetes/core.py\u001b[0m in \u001b[0;36m_start\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    432\u001b[0m                 \u001b[0;34m\"docstring for ways to specify workers\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m             )\n\u001b[0;32m--> 434\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    435\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    436\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpod_template\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclean_pod_template\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpod_template\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Worker pod specification not provided. See KubeCluster docstring for ways to specify workers"
     ]
    }
   ],
   "source": [
    "cluster = KubeCluster(n_workers=2) #(host='104.198.73.140', port='8786', n_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "??KubeCluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask_kubernetes import KubeCluster, make_pod_spec\n",
    "pod_spec = make_pod_spec(\n",
    "                         image='daskdev/dask:latest',\n",
    "                         memory_limit='4G', memory_request='4G',\n",
    "                         cpu_limit=1, cpu_request=1,\n",
    "                         env={'EXTRA_PIP_PACKAGES': 'fastparquet git+https://github.com/dask/distributed'})\n",
    "#cluster = KubeCluster(pod_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c1e4a208>>, <Task finished coro=<SpecCluster._correct_state_internal() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:284> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n",
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c26da8d0>>, <Task finished coro=<SpecCluster._correct_state_internal() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:284> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n",
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c1deacc0>>, <Task finished coro=<SpecCluster._correct_state_internal() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:284> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n",
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c272cf60>>, <Task finished coro=<SpecCluster._correct_state_internal() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:284> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "argument of type 'NoneType' is not iterable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-2788bcb31985>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcluster\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mKubeCluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpod_spec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhost\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'104.198.73.140'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mport\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'8786'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/dask_kubernetes/core.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, pod_template, name, namespace, n_workers, host, port, env, auth, scheduler_timeout, deploy_mode, interface, protocol, dashboard_address, security, **kwargs)\u001b[0m\n\u001b[1;32m    378\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    379\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    381\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m     \u001b[0;34masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, workers, scheduler, worker, asynchronous, loop, security, silence_logs, name)\u001b[0m\n\u001b[1;32m    240\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_loop_runner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    243\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_correct_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/deploy/cluster.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m    160\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0msync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m     \u001b[0;34masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m    331\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    334\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    335\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m    315\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    316\u001b[0m                 \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_timeout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseconds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 317\u001b[0;31m             \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    318\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    319\u001b[0m             \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    733\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    734\u001b[0m                     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 735\u001b[0;31m                         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    736\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    737\u001b[0m                         \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/dask_kubernetes/core.py\u001b[0m in \u001b[0;36m_start\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    435\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    436\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpod_template\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclean_pod_template\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpod_template\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 437\u001b[0;31m         \u001b[0mawait\u001b[0m \u001b[0mClusterAuth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_first\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    439\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore_api\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkubernetes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCoreV1Api\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/dask_kubernetes/auth.py\u001b[0m in \u001b[0;36mload_first\u001b[0;34m(auth)\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mauth_instance\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m                 \u001b[0mawait\u001b[0m \u001b[0mauth_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mkubernetes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mConfigException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m                 logger.debug(\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/dask_kubernetes/auth.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;34masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         await kubernetes.config.load_kube_config(\n\u001b[0;32m--> 121\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpersist_config\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m         )\n\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/kubernetes_asyncio/config/kube_config.py\u001b[0m in \u001b[0;36mload_kube_config\u001b[0;34m(config_file, context, client_configuration, persist_config)\u001b[0m\n\u001b[1;32m    551\u001b[0m     loader = _get_kube_config_loader_for_yaml_file(\n\u001b[1;32m    552\u001b[0m         \u001b[0mconfig_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactive_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 553\u001b[0;31m         persist_config=persist_config)\n\u001b[0m\u001b[1;32m    554\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mclient_configuration\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    555\u001b[0m         \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mConfiguration\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/kubernetes_asyncio/config/kube_config.py\u001b[0m in \u001b[0;36m_get_kube_config_loader_for_yaml_file\u001b[0;34m(filename, persist_config, **kwargs)\u001b[0m\n\u001b[1;32m    519\u001b[0m         \u001b[0mconfig_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m         \u001b[0mconfig_base_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m         **kwargs)\n\u001b[0m\u001b[1;32m    522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/kubernetes_asyncio/config/kube_config.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, config_dict, active_context, get_google_credentials, config_base_path, config_persister)\u001b[0m\n\u001b[1;32m    142\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cluster\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprovider\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 144\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_active_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactive_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    145\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config_base_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig_base_path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    146\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config_persister\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig_persister\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/kubernetes_asyncio/config/kube_config.py\u001b[0m in \u001b[0;36mset_active_context\u001b[0;34m(self, context_name)\u001b[0m\n\u001b[1;32m    152\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mset_active_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcontext_name\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 154\u001b[0;31m             \u001b[0mcontext_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'current-context'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    155\u001b[0m         self._current_context = self._config['contexts'].get_with_name(\n\u001b[1;32m    156\u001b[0m             context_name)\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/kubernetes_asyncio/config/kube_config.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    403\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 405\u001b[0;31m         \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msafe_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    406\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m             raise ConfigException(\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/kubernetes_asyncio/config/kube_config.py\u001b[0m in \u001b[0;36msafe_get\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    399\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msafe_get\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    400\u001b[0m         if (isinstance(self.value, list) and isinstance(key, int) or\n\u001b[0;32m--> 401\u001b[0;31m                 key in self.value):\n\u001b[0m\u001b[1;32m    402\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    403\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: argument of type 'NoneType' is not iterable"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c1e4a208>>, <Task finished coro=<SpecCluster._close() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:355> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n",
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c26da8d0>>, <Task finished coro=<SpecCluster._close() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:355> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n",
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c1deacc0>>, <Task finished coro=<SpecCluster._close() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:355> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n",
      "tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x7f40c272cf60>>, <Task finished coro=<SpecCluster._close() done, defined at /home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py:355> exception=AttributeError(\"'KubeCluster' object has no attribute '_lock'\",)>)\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py\", line 767, in _discard_future_result\n",
      "    future.result()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 363, in _close\n",
      "    await self._correct_state()\n",
      "  File \"/home/jovyan/.local/lib/python3.6/site-packages/distributed/deploy/spec.py\", line 285, in _correct_state_internal\n",
      "    async with self._lock:\n",
      "AttributeError: 'KubeCluster' object has no attribute '_lock'\n"
     ]
    }
   ],
   "source": [
    "cluster = KubeCluster(pod_spec, n_workers=2, host='104.198.73.140', port='8786')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}