train/ranker_models/XGB-186-CLICKS-DASK-UCX-SPILL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6f7a16e2-7fa5-4993-b2df-732add0f9d7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time\n",
    "start = time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "63499de6-f8c7-4e8e-8ae0-cff87043be30",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "GPUs = ','.join([str(i) for i in range(0,4)])\n",
    "num_gpus = len(GPUs.split(','))\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = GPUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "31a873c9-73be-4fbb-af9a-7f817982f6f7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import dask, dask_cudf\n",
    "from dask.distributed import Client, wait\n",
    "from dask_cuda import LocalCUDACluster\n",
    "import subprocess\n",
    "import xgboost as xgb\n",
    "import gc\n",
    "    \n",
    "def get_ip():\n",
    "    cmd = \"hostname --all-ip-addresses\"\n",
    "    process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)\n",
    "    output, error = process.communicate()\n",
    "    IPADDR = str(output.decode()).split()[0]\n",
    "    print(IPADDR)\n",
    "    return IPADDR\n",
    "\n",
    "def get_cluster():\n",
    "    ip = get_ip()\n",
    "    cluster = LocalCUDACluster(ip=ip,\n",
    "                               device_memory_limit='10GB', \n",
    "                               jit_unspill=True,\n",
    "                               protocol=\"ucx\", \n",
    "                               rmm_pool_size=\"29GB\")\n",
    "    client = Client(cluster)\n",
    "    return client\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "11e32f06-31a8-41bc-813f-ecee45b0451c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.120.104.12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-13 18:40:48,413 - distributed.comm.ucx - WARNING - A CUDA context for device 0 (b'GPU-67be321d-9012-d150-fdc9-402004163697') already exists on process ID 16542. This is often the result of a CUDA-enabled library calling a CUDA runtime function before Dask-CUDA can spawn worker processes. Please make sure any such function calls don't happen at import time or in the global scope of a program.\n",
      "2023-04-13 18:40:49,797 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/dask-worker-space-10093/worker-wedbxe9e', purging\n",
      "2023-04-13 18:40:49,797 - distributed.diskutils - INFO - Found stale lock file and directory '/tmp/dask-worker-space-10093/worker-fhhxi8lo', purging\n",
      "2023-04-13 18:40:49,798 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize\n",
      "2023-04-13 18:40:49,798 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n",
      "2023-04-13 18:40:49,798 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize\n",
      "2023-04-13 18:40:49,798 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n",
      "2023-04-13 18:40:49,806 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize\n",
      "2023-04-13 18:40:49,806 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n",
      "2023-04-13 18:40:49,806 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize\n",
      "2023-04-13 18:40:49,806 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-662ef676-da65-11ed-809e-5cff35c18da6</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "\n",
       "        <tr>\n",
       "        \n",
       "            <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
       "            <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_cuda.LocalCUDACluster</td>\n",
       "        \n",
       "        </tr>\n",
       "\n",
       "        \n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard: </strong> <a href=\"http://10.120.104.12:8787/status\" target=\"_blank\">http://10.120.104.12:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\"></td>\n",
       "            </tr>\n",
       "        \n",
       "\n",
       "        </table>\n",
       "\n",
       "        \n",
       "\n",
       "        \n",
       "            <details>\n",
       "            <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
       "            <div class=\"jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output\">\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\">\n",
       "    </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px; margin-top: 0px;\">LocalCUDACluster</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">f97a69c2</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard:</strong> <a href=\"http://10.120.104.12:8787/status\" target=\"_blank\">http://10.120.104.12:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Workers:</strong> 4\n",
       "                </td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total threads:</strong> 4\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total memory:</strong> 1.48 TiB\n",
       "                </td>\n",
       "            </tr>\n",
       "            \n",
       "            <tr>\n",
       "    <td style=\"text-align: left;\"><strong>Status:</strong> running</td>\n",
       "    <td style=\"text-align: left;\"><strong>Using processes:</strong> True</td>\n",
       "</tr>\n",
       "\n",
       "            \n",
       "        </table>\n",
       "\n",
       "        <details>\n",
       "            <summary style=\"margin-bottom: 20px;\">\n",
       "                <h3 style=\"display: inline;\">Scheduler Info</h3>\n",
       "            </summary>\n",
       "\n",
       "            <div style=\"\">\n",
       "    <div>\n",
       "        <div style=\"width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;\"> </div>\n",
       "        <div style=\"margin-left: 48px;\">\n",
       "            <h3 style=\"margin-bottom: 0px;\">Scheduler</h3>\n",
       "            <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Scheduler-6c6d65ac-f59c-4cf5-89ce-dbf02f75b48a</p>\n",
       "            <table style=\"width: 100%; text-align: left;\">\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Comm:</strong> ucx://10.120.104.12:50739\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Workers:</strong> 4\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Dashboard:</strong> <a href=\"http://10.120.104.12:8787/status\" target=\"_blank\">http://10.120.104.12:8787/status</a>\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total threads:</strong> 4\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Started:</strong> Just now\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total memory:</strong> 1.48 TiB\n",
       "                    </td>\n",
       "                </tr>\n",
       "            </table>\n",
       "        </div>\n",
       "    </div>\n",
       "\n",
       "    <details style=\"margin-left: 48px;\">\n",
       "        <summary style=\"margin-bottom: 20px;\">\n",
       "            <h3 style=\"display: inline;\">Workers</h3>\n",
       "        </summary>\n",
       "\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 0</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> ucx://10.120.104.12:46217\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 1\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://10.120.104.12:35915/status\" target=\"_blank\">http://10.120.104.12:35915/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 377.64 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> ucx://10.120.104.12:47309\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /tmp/dask-worker-space-10093/worker-o4g5bu8b\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU: </strong>Tesla V100-SXM3-32GB\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU memory: </strong> 31.75 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 1</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> ucx://10.120.104.12:56453\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 1\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://10.120.104.12:40585/status\" target=\"_blank\">http://10.120.104.12:40585/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 377.64 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> ucx://10.120.104.12:39403\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /tmp/dask-worker-space-10093/worker-e6kobkcx\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU: </strong>Tesla V100-SXM3-32GB\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU memory: </strong> 31.75 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 2</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> ucx://10.120.104.12:32837\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 1\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://10.120.104.12:38803/status\" target=\"_blank\">http://10.120.104.12:38803/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 377.64 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> ucx://10.120.104.12:37477\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /tmp/dask-worker-space-10093/worker-tt3h_3jx\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU: </strong>Tesla V100-SXM3-32GB\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU memory: </strong> 31.75 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 3</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> ucx://10.120.104.12:39847\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 1\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://10.120.104.12:46501/status\" target=\"_blank\">http://10.120.104.12:46501/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 377.64 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> ucx://10.120.104.12:34295\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /tmp/dask-worker-space-10093/worker-vplxdrua\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU: </strong>Tesla V100-SXM3-32GB\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>GPU memory: </strong> 31.75 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "\n",
       "    </details>\n",
       "</div>\n",
       "\n",
       "        </details>\n",
       "    </div>\n",
       "</div>\n",
       "            </details>\n",
       "        \n",
       "\n",
       "    </div>\n",
       "</div>"
      ],
      "text/plain": [
       "<Client: 'ucx://10.120.104.12:50739' processes=4 threads=4, memory=1.48 TiB>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = get_cluster()\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1ea2ec45-573f-4943-ac8e-6eb928913f78",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 951 ms, sys: 513 ms, total: 1.46 s\n",
      "Wall time: 1.38 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "users = dask_cudf.read_parquet('/raid/otto/Otto-Comp/pqs/train_v152_*.pq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a9b87ed7-d1f6-4244-82b0-c596c8b0416c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "VER = 186\n",
    "USE = 'clicks'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9e9171b7-7d45-472d-a5b2-591d57da49bf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152\n",
      "['pos', 'wgt', 'score_1', 'score_3', 'score_4', 'score_5', 'score_6', 'score_7', 'score_10', 'score_11', 'score_12', 'score_13', 'score_30', 'score_14', 'score_15', 'score_16', 'score_17', 'score_18', 'score_19', 'score_20', 'score_21', 'score_22', 'score_23', 'score_24', 'score_25', 'score_26', 'buy_ratio_5', 'count_item_5', 'count_user_5', 'repeat_5', 'buy_ratio_std_5', 'item_day_std_5', 'user_day_std_5', 'buy_ratio_0_6', 'buy_ratio_1_6', 'buy_ratio_2_6', 'buy_ratio_3_6', 'buy_ratio_4_6', 'buy_ratio_5_6', 'buy_ratio_6_6', 'count_item_0_6', 'count_item_1_6', 'count_item_2_6', 'count_item_3_6', 'count_item_4_6', 'count_item_5_6', 'count_item_6_6', 'count_user_0_6', 'count_user_1_6', 'count_user_2_6', 'count_user_3_6', 'count_user_4_6', 'count_user_5_6', 'count_user_6_6', 'item_day_std_6', 'user_day_std_6', 'buy_ratio', 'count_item', 'count_user', 'hour_mean', 'day_mean', 'repeat', 'buy_ratio_std', 'hour_std', 'day_std', 'buy_ratio3', 'count_item3', 'count_user3', 'hour_mean3', 'day_mean3', 'repeat3', 'buy_ratio_std3', 'hour_std3', 'day_std3', 'prev3', 'next3', 'orders3', 'carts3', 'order_repeat3', 'cart_repeat3', 'buy_ratio2', 'count_item2', 'unique_item2', 'hour_mean2', 'day_mean2', 'repeat2', 'buy_ratio_std2', 'hour_std2', 'day_std2', 'ts_max7', 'ts_min7', 'ts_diff7', 'z1', 'z2', 'z3', 'z4', 'z5', 'z6', 'buy_ratio7', 'count_item7', 'count_user7', 'hour_mean7', 'day_mean7', 'repeat7', 'buy_ratio_std7', 'hour_std7', 'day_std7', 'prev7', 'next7', 'orders7', 'carts7', 'order_repeat7', 'cart_repeat7', 'buy_ratio8', 'count_item8', 'count_user8', 'hour_mean8', 'day_mean8', 'repeat8', 'buy_ratio_std8', 'hour_std8', 'day_std8', 'prev8', 'next8', 'orders8', 'carts8', 'order_repeat8', 'cart_repeat8', 'buy_ratio9', 'count_item9', 'count_user9', 'hour_mean9', 'day_mean9', 'repeat9', 'buy_ratio_std9', 'hour_std9', 'day_std9', 'prev9', 'next9', 'orders9', 'carts9', 'order_repeat9', 'cart_repeat9', 'buy_ratio10', 'count_item10', 'unique_item10', 'hour_mean10', 'day_mean10', 'repeat10', 'buy_ratio_std10', 'hour_std10', 'day_std10']\n",
      "['clicks']\n"
     ]
    }
   ],
   "source": [
    "FEATURES = users.columns[2:]\n",
    "TARS = [USE]\n",
    "FEATURES = [f for f in FEATURES if f not in TARS]\n",
    "print(len(FEATURES))\n",
    "print( FEATURES)\n",
    "print(TARS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c4cfe0da-c54e-40bf-85f9-7eceb8fe5d63",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.17 s, sys: 431 ms, total: 3.6 s\n",
      "Wall time: 28.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dtrain = xgb.dask.DaskQuantileDMatrix(client, users[FEATURES], users['clicks'])\n",
    "del users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c66b9aa8-c10a-4e8d-af06-df05271fb71a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "212"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cf81a032-4f50-4b3e-afea-d229db471cae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "FOLDS = 5\n",
    "SEED = 42\n",
    "\n",
    "LR = 0.1\n",
    "\n",
    "# XGB MODEL PARAMETERS\n",
    "xgb_parms = { \n",
    "    'max_depth':4, \n",
    "    'learning_rate':LR, \n",
    "    'subsample':0.7,\n",
    "    'colsample_bytree':0.5, \n",
    "    'eval_metric':'map',\n",
    "    'objective':'binary:logistic',\n",
    "    'scale_pos_weight':8,\n",
    "    'tree_method':'gpu_hist',\n",
    "    'predictor':'gpu_predictor',\n",
    "    'random_state':SEED\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7472f97c-57a2-4040-9f7c-806bffd92b19",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/raid/data/mambaforge/envs/rapids-23.04/lib/python3.10/site-packages/xgboost/dask.py:856: RuntimeWarning: coroutine 'Client._wait_for_workers' was never awaited\n",
      "  client.wait_for_workers(n_workers)\n",
      "INFO:distributed.scheduler:Receive client connection: Client-worker-7896770a-da65-11ed-81bc-5cff35c18da6\n",
      "INFO:distributed.core:Starting established connection to ucx://10.120.104.12:50739\n",
      "INFO:distributed.scheduler:Receive client connection: Client-worker-789671aa-da65-11ed-81bf-5cff35c18da6\n",
      "INFO:distributed.core:Starting established connection to ucx://10.120.104.12:50739\n",
      "INFO:distributed.scheduler:Receive client connection: Client-worker-78967094-da65-11ed-81c1-5cff35c18da6\n",
      "INFO:distributed.core:Starting established connection to ucx://10.120.104.12:50739\n",
      "INFO:distributed.scheduler:Receive client connection: Client-worker-78abab68-da65-11ed-81b9-5cff35c18da6\n",
      "INFO:distributed.core:Starting established connection to ucx://10.120.104.12:50739\n",
      "[18:41:27] task [xgboost.dask-0]:ucx://10.120.104.12:46217 got new rank 0\n",
      "[18:41:27] task [xgboost.dask-1]:ucx://10.120.104.12:56453 got new rank 1\n",
      "[18:41:27] task [xgboost.dask-2]:ucx://10.120.104.12:32837 got new rank 2\n",
      "[18:41:27] task [xgboost.dask-3]:ucx://10.120.104.12:39847 got new rank 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-map:0.16598\n",
      "[1]\ttrain-map:0.17580\n",
      "[2]\ttrain-map:0.17586\n",
      "[3]\ttrain-map:0.17439\n",
      "[4]\ttrain-map:0.17784\n",
      "[5]\ttrain-map:0.17831\n",
      "[6]\ttrain-map:0.17851\n",
      "[7]\ttrain-map:0.18267\n",
      "[8]\ttrain-map:0.18142\n",
      "[9]\ttrain-map:0.18237\n",
      "[10]\ttrain-map:0.18633\n",
      "[11]\ttrain-map:0.18673\n",
      "[12]\ttrain-map:0.18650\n",
      "[13]\ttrain-map:0.18695\n",
      "[14]\ttrain-map:0.18747\n",
      "[15]\ttrain-map:0.18766\n",
      "[16]\ttrain-map:0.18776\n",
      "[17]\ttrain-map:0.18807\n",
      "[18]\ttrain-map:0.18823\n",
      "[19]\ttrain-map:0.18845\n",
      "[20]\ttrain-map:0.18910\n",
      "[21]\ttrain-map:0.18930\n",
      "[22]\ttrain-map:0.18973\n",
      "[23]\ttrain-map:0.18993\n",
      "[24]\ttrain-map:0.18995\n",
      "[25]\ttrain-map:0.19023\n",
      "[26]\ttrain-map:0.19023\n",
      "[27]\ttrain-map:0.19032\n",
      "[28]\ttrain-map:0.19053\n",
      "[29]\ttrain-map:0.19078\n",
      "[30]\ttrain-map:0.19110\n",
      "[31]\ttrain-map:0.19147\n",
      "[32]\ttrain-map:0.19190\n",
      "[33]\ttrain-map:0.19248\n",
      "[34]\ttrain-map:0.19359\n",
      "[35]\ttrain-map:0.19463\n",
      "[36]\ttrain-map:0.19486\n",
      "[37]\ttrain-map:0.19551\n",
      "[38]\ttrain-map:0.19535\n",
      "[39]\ttrain-map:0.19565\n",
      "[40]\ttrain-map:0.19601\n",
      "[41]\ttrain-map:0.19631\n",
      "[42]\ttrain-map:0.19676\n",
      "[43]\ttrain-map:0.19690\n",
      "[44]\ttrain-map:0.19692\n",
      "[45]\ttrain-map:0.19704\n",
      "[46]\ttrain-map:0.19721\n",
      "[47]\ttrain-map:0.19728\n",
      "[48]\ttrain-map:0.19757\n",
      "[49]\ttrain-map:0.19775\n",
      "[50]\ttrain-map:0.19767\n",
      "[51]\ttrain-map:0.19781\n",
      "[52]\ttrain-map:0.19777\n",
      "[53]\ttrain-map:0.19774\n",
      "[54]\ttrain-map:0.19758\n",
      "[55]\ttrain-map:0.19774\n",
      "[56]\ttrain-map:0.19762\n",
      "[57]\ttrain-map:0.19786\n",
      "[58]\ttrain-map:0.19829\n",
      "[59]\ttrain-map:0.19829\n",
      "[60]\ttrain-map:0.19840\n",
      "[61]\ttrain-map:0.19848\n",
      "[62]\ttrain-map:0.19843\n",
      "[63]\ttrain-map:0.19854\n",
      "[64]\ttrain-map:0.19883\n",
      "[65]\ttrain-map:0.19881\n",
      "[66]\ttrain-map:0.19893\n",
      "[67]\ttrain-map:0.19919\n",
      "[68]\ttrain-map:0.19916\n",
      "[69]\ttrain-map:0.19923\n",
      "[70]\ttrain-map:0.19928\n",
      "[71]\ttrain-map:0.19931\n",
      "[72]\ttrain-map:0.19930\n",
      "[73]\ttrain-map:0.19939\n",
      "[74]\ttrain-map:0.19940\n",
      "[75]\ttrain-map:0.19953\n",
      "[76]\ttrain-map:0.19963\n",
      "[77]\ttrain-map:0.19962\n",
      "[78]\ttrain-map:0.19969\n",
      "[79]\ttrain-map:0.19978\n",
      "[80]\ttrain-map:0.19981\n",
      "[81]\ttrain-map:0.19986\n",
      "[82]\ttrain-map:0.20011\n",
      "[83]\ttrain-map:0.20016\n",
      "[84]\ttrain-map:0.20030\n",
      "[85]\ttrain-map:0.20036\n",
      "[86]\ttrain-map:0.20042\n",
      "[87]\ttrain-map:0.20041\n",
      "[88]\ttrain-map:0.20052\n",
      "[89]\ttrain-map:0.20063\n",
      "[90]\ttrain-map:0.20066\n",
      "[91]\ttrain-map:0.20088\n",
      "[92]\ttrain-map:0.20088\n",
      "[93]\ttrain-map:0.20094\n",
      "[94]\ttrain-map:0.20107\n",
      "[95]\ttrain-map:0.20110\n",
      "[96]\ttrain-map:0.20122\n",
      "[97]\ttrain-map:0.20130\n",
      "[98]\ttrain-map:0.20133\n",
      "[99]\ttrain-map:0.20137\n",
      "CPU times: user 15 s, sys: 2.12 s, total: 17.1 s\n",
      "Wall time: 2min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "output = xgb.dask.train(\n",
    "        client,\n",
    "        xgb_parms,\n",
    "        dtrain,\n",
    "        num_boost_round=100,\n",
    "        evals=[(dtrain, \"train\")],\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "35f1fd18-1ccc-4a37-b677-b5122a1c5e7b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "217.7893419265747"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "duration = time()-start\n",
    "duration"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}