notebooks/xgb_ray.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9628663-95a2-4694-8a78-f89a7d1941f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import time\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "# from tqdm import tqdm\n",
    "# import multiprocessing\n",
    "from itertools import repeat, product\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# import xgboost as xgb\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "# from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "random.seed(42)\n",
    "gc.enable()\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de4da534-3ef8-4be1-9686-1c647ccc4b0d",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f4322c1-6481-45f1-8ec3-ed5d840fb670",
   "metadata": {},
   "outputs": [],
   "source": [
    "twenty_train = fetch_20newsgroups(subset='train', shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1aac72d-eb1c-474a-880a-8adf7075b2b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(twenty_train.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e6f60b9-6cca-4eb7-a0a2-f19052bdd9b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "twenty_train.target_names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8b4c01c-d01d-47f9-9aef-287dcbb0877d",
   "metadata": {},
   "source": [
    "# modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d09e5b3a-cac9-4816-bf20-98d5eb158cbd",
   "metadata": {},
   "source": [
    "### sklearn with Ray backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9dbd9ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ray\n",
    "import joblib\n",
    "from ray.util.joblib import register_ray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e5cd157",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = twenty_train.data, twenty_train.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12628857",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b10c52f1-1f47-46c7-9e54-01b52303d2ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "%time\n",
    "register_ray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47020d8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_exp = 50\n",
    "# since these two were the fastest when using multi-processing\n",
    "t = \"exact\"\n",
    "n = -1\n",
    "\n",
    "all_exps = []\n",
    "\n",
    "\n",
    "exp_time = []\n",
    "exp_score = []\n",
    "experiment_dict ={}\n",
    "experiment_dict[\"method\"] = f\"ray_backend n_job={n}, tree_method={t}\"   \n",
    "\n",
    "print(f\"n_job={n}, tree_method={t} x {num_exp} times\")\n",
    "\n",
    "for i in tqdm(repeat(1, num_exp), total=num_exp):\n",
    "\n",
    "    xgb_model = xgb.XGBClassifier(n_jobs=n,\n",
    "                                  tree_method=t,\n",
    "                                  n_estimators=100, \n",
    "                                  random_state=42\n",
    "                                 )\n",
    "\n",
    "\n",
    "    text_clf = Pipeline([\n",
    "        ('vect', CountVectorizer(lowercase=False, ngram_range=(1,2))),\n",
    "        ('clf', xgb_model)\n",
    "    ])\n",
    "\n",
    "    with joblib.parallel_backend('ray'):        \n",
    "        start = time.time()\n",
    "        text_clf.fit(X_train, y_train)\n",
    "        end = time.time()\n",
    "    exp_time.append(end-start)\n",
    "\n",
    "    y_pred = text_clf.predict(X_test)\n",
    "    score = f1_score(y_test, y_pred, average='macro')\n",
    "    exp_score.append(score)\n",
    "    # print(score)\n",
    "\n",
    "experiment_dict[\"time_result\"] = exp_time\n",
    "experiment_dict[\"average_score\"] = exp_score\n",
    "\n",
    "all_exps.append(experiment_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a2b1e26",
   "metadata": {},
   "source": [
    "# Ray AI Runtimr (AIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "07b90ee3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ray\n",
    "from ray.air.config import ScalingConfig\n",
    "from ray.train.xgboost import XGBoostTrainer\n",
    "from ray.data.preprocessors import BatchMapper, Chain, CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e185dafc",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "78802b1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-07 16:06:53,328\tINFO worker.py:1529 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32m127.0.0.1:8265 \u001b[39m\u001b[22m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"margin-left: 50px;display: flex;flex-direction: row;align-items: center\">\n",
       "        <h3 style=\"color: var(--jp-ui-font-color0)\">Ray</h3>\n",
       "        <svg version=\"1.1\" id=\"ray\" width=\"3em\" viewBox=\"0 0 144.5 144.6\" style=\"margin-left: 3em;margin-right: 3em\">\n",
       "            <g id=\"layer-1\">\n",
       "                <path fill=\"#00a2e9\" class=\"st0\" d=\"M97.3,77.2c-3.8-1.1-6.2,0.9-8.3,5.1c-3.5,6.8-9.9,9.9-17.4,9.6S58,88.1,54.8,81.2c-1.4-3-3-4-6.3-4.1\n",
       "                    c-5.6-0.1-9.9,0.1-13.1,6.4c-3.8,7.6-13.6,10.2-21.8,7.6C5.2,88.4-0.4,80.5,0,71.7c0.1-8.4,5.7-15.8,13.8-18.2\n",
       "                    c8.4-2.6,17.5,0.7,22.3,8c1.3,1.9,1.3,5.2,3.6,5.6c3.9,0.6,8,0.2,12,0.2c1.8,0,1.9-1.6,2.4-2.8c3.5-7.8,9.7-11.8,18-11.9\n",
       "                    c8.2-0.1,14.4,3.9,17.8,11.4c1.3,2.8,2.9,3.6,5.7,3.3c1-0.1,2,0.1,3,0c2.8-0.5,6.4,1.7,8.1-2.7s-2.3-5.5-4.1-7.5\n",
       "                    c-5.1-5.7-10.9-10.8-16.1-16.3C84,38,81.9,37.1,78,38.3C66.7,42,56.2,35.7,53,24.1C50.3,14,57.3,2.8,67.7,0.5\n",
       "                    C78.4-2,89,4.7,91.5,15.3c0.1,0.3,0.1,0.5,0.2,0.8c0.7,3.4,0.7,6.9-0.8,9.8c-1.7,3.2-0.8,5,1.5,7.2c6.7,6.5,13.3,13,19.8,19.7\n",
       "                    c1.8,1.8,3,2.1,5.5,1.2c9.1-3.4,17.9-0.6,23.4,7c4.8,6.9,4.6,16.1-0.4,22.9c-5.4,7.2-14.2,9.9-23.1,6.5c-2.3-0.9-3.5-0.6-5.1,1.1\n",
       "                    c-6.7,6.9-13.6,13.7-20.5,20.4c-1.8,1.8-2.5,3.2-1.4,5.9c3.5,8.7,0.3,18.6-7.7,23.6c-7.9,5-18.2,3.8-24.8-2.9\n",
       "                    c-6.4-6.4-7.4-16.2-2.5-24.3c4.9-7.8,14.5-11,23.1-7.8c3,1.1,4.7,0.5,6.9-1.7C91.7,98.4,98,92.3,104.2,86c1.6-1.6,4.1-2.7,2.6-6.2\n",
       "                    c-1.4-3.3-3.8-2.5-6.2-2.6C99.8,77.2,98.9,77.2,97.3,77.2z M72.1,29.7c5.5,0.1,9.9-4.3,10-9.8c0-0.1,0-0.2,0-0.3\n",
       "                    C81.8,14,77,9.8,71.5,10.2c-5,0.3-9,4.2-9.3,9.2c-0.2,5.5,4,10.1,9.5,10.3C71.8,29.7,72,29.7,72.1,29.7z M72.3,62.3\n",
       "                    c-5.4-0.1-9.9,4.2-10.1,9.7c0,0.2,0,0.3,0,0.5c0.2,5.4,4.5,9.7,9.9,10c5.1,0.1,9.9-4.7,10.1-9.8c0.2-5.5-4-10-9.5-10.3\n",
       "                    C72.6,62.3,72.4,62.3,72.3,62.3z M115,72.5c0.1,5.4,4.5,9.7,9.8,9.9c5.6-0.2,10-4.8,10-10.4c-0.2-5.4-4.6-9.7-10-9.7\n",
       "                    c-5.3-0.1-9.8,4.2-9.9,9.5C115,72.1,115,72.3,115,72.5z M19.5,62.3c-5.4,0.1-9.8,4.4-10,9.8c-0.1,5.1,5.2,10.4,10.2,10.3\n",
       "                    c5.6-0.2,10-4.9,9.8-10.5c-0.1-5.4-4.5-9.7-9.9-9.6C19.6,62.3,19.5,62.3,19.5,62.3z M71.8,134.6c5.9,0.2,10.3-3.9,10.4-9.6\n",
       "                    c0.5-5.5-3.6-10.4-9.1-10.8c-5.5-0.5-10.4,3.6-10.8,9.1c0,0.5,0,0.9,0,1.4c-0.2,5.3,4,9.8,9.3,10\n",
       "                    C71.6,134.6,71.7,134.6,71.8,134.6z\"/>\n",
       "            </g>\n",
       "        </svg>\n",
       "        <table>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left\"><b>Python version:</b></td>\n",
       "                <td style=\"text-align: left\"><b>3.9.16</b></td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left\"><b>Ray version:</b></td>\n",
       "                <td style=\"text-align: left\"><b> 2.2.0</b></td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "    <td style=\"text-align: left\"><b>Dashboard:</b></td>\n",
       "    <td style=\"text-align: left\"><b><a href=\"http://127.0.0.1:8265\" target=\"_blank\">http://127.0.0.1:8265</a></b></td>\n",
       "</tr>\n",
       "\n",
       "        </table>\n",
       "    </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "RayContext(dashboard_url='127.0.0.1:8265', python_version='3.9.16', ray_version='2.2.0', ray_commit='b6af0887ee5f2e460202133791ad941a41f15beb', address_info={'node_ip_address': '127.0.0.1', 'raylet_ip_address': '127.0.0.1', 'redis_address': None, 'object_store_address': '/tmp/ray/session_2023-02-07_16-06-51_378547_41041/sockets/plasma_store', 'raylet_socket_name': '/tmp/ray/session_2023-02-07_16-06-51_378547_41041/sockets/raylet', 'webui_url': '127.0.0.1:8265', 'session_dir': '/tmp/ray/session_2023-02-07_16-06-51_378547_41041', 'metrics_export_port': 59403, 'gcs_address': '127.0.0.1:58910', 'address': '127.0.0.1:58910', 'dashboard_agent_listen_port': 52365, 'node_id': '5636e517bdd3301c6aa1c6a739f0f452cfbb7b028cb8d8b203caf14e'})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ray.init(num_cpus=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "40797788",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dask dataframe \n",
    "df = pd.DataFrame({\"text\": twenty_train.data, \"target\": twenty_train.target})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "722608c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = ray.data.from_pandas(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "473a90d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split data into train and validation.\n",
    "train_dataset, valid_dataset = ds.train_test_split(test_size=0.2, seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4647d58d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_col_names(batch: pd.DataFrame) -> pd.DataFrame:\n",
    "    batch.columns = [\n",
    "        colname.replace(\":\", \"COLON\")\n",
    "        .replace(\",\", \"COMMA\")\n",
    "        .replace(\"<\", \"LT\")\n",
    "        .replace(\">\", \"GT\")\n",
    "        .replace(\"[\", \"LBRACKET\")\n",
    "        .replace(\"]\", \"RBRACKET\")\n",
    "        for colname in batch.columns\n",
    "    ]\n",
    "    return batch\n",
    "\n",
    "\n",
    "preprocessor = Chain(   \n",
    "    CountVectorizer(columns=[\"text\"]), \n",
    "    BatchMapper(fix_col_names, batch_format=\"pandas\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6f07718a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGBoost specific params\n",
    "params = {\n",
    "    \"tree_method\": \"approx\",\n",
    "    \"objective\": \"multi:softmax\",\n",
    "    \"eval_metric\": [\"merror\"],\n",
    "    \"num_class\": df[\"target\"].nunique(),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "aad434d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = XGBoostTrainer(\n",
    "    scaling_config=ScalingConfig(num_workers=2, use_gpu=False),\n",
    "    label_column=\"target\",\n",
    "    params=params,\n",
    "    datasets={\"train\": train_dataset, \"valid\": valid_dataset},\n",
    "    preprocessor=preprocessor,\n",
    "    num_boost_round=100,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "79d6493a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2023-02-07 16:09:42</td></tr>\n",
       "<tr><td>Running for: </td><td>00:00:31.48        </td></tr>\n",
       "<tr><td>Memory:      </td><td>14.4/16.0 GiB      </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using FIFO scheduling algorithm.<br>Resources requested: 0/8 CPUs, 0/0 GPUs, 0.0/10.73 GiB heap, 0.0/2.0 GiB objects\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "<div class=\"messages\">\n",
       "  <h3>Messages</h3>\n",
       "  \n",
       "  \n",
       "  Number of errored trials: 1<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                </th><th style=\"text-align: right;\">  # failures</th><th>error file                                                                                                             </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>XGBoostTrainer_6e735_00000</td><td style=\"text-align: right;\">           1</td><td>/Users/Amiros/ray_results/XGBoostTrainer_2023-02-07_16-09-11/XGBoostTrainer_6e735_00000_0_2023-02-07_16-09-11/error.txt</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".messages {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  padding-left: 1em;\n",
       "  overflow-y: auto;\n",
       "}\n",
       ".messages h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n",
       "\n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name                </th><th>status  </th><th>loc            </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>XGBoostTrainer_6e735_00000</td><td>ERROR   </td><td>127.0.0.1:41186</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-07 16:09:42,598\tERROR trial_runner.py:1088 -- Trial XGBoostTrainer_6e735_00000: Error processing event.\n",
      "ray.exceptions.RayTaskError(IndexError): \u001b[36mray::_Inner.train()\u001b[39m (pid=41186, ip=127.0.0.1, repr=XGBoostTrainer)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/trainable.py\", line 367, in train\n",
      "    raise skipped from exception_cause(skipped)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/function_trainable.py\", line 335, in entrypoint\n",
      "    return self._trainable_func(\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 480, in _trainable_func\n",
      "    super()._trainable_func(self._merged_config, reporter, checkpoint_dir)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/function_trainable.py\", line 652, in _trainable_func\n",
      "    output = fn()\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 389, in train_func\n",
      "    trainer.preprocess_datasets()\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/gbdt_trainer.py\", line 187, in preprocess_datasets\n",
      "    super().preprocess_datasets()\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 299, in preprocess_datasets\n",
      "    self.preprocessor.fit(train_dataset)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 105, in fit\n",
      "    return self._fit(dataset)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessors/chain.py\", line 73, in _fit\n",
      "    ds = preprocessor.fit_transform(ds)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 120, in fit_transform\n",
      "    self.fit(dataset)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 105, in fit\n",
      "    return self._fit(dataset)\n",
      "  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessors/vectorizer.py\", line 233, in _fit\n",
      "    total_counts[i].update(col_value_counts)\n",
      "IndexError: list index out of range\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name                </th><th>date               </th><th>experiment_id                   </th><th>hostname               </th><th>node_ip  </th><th style=\"text-align: right;\">  pid</th><th style=\"text-align: right;\">  timestamp</th><th>trial_id   </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>XGBoostTrainer_6e735_00000</td><td>2023-02-07_16-09-13</td><td>6a286d1bc8c44ba0953e4d6d81037cf8</td><td>amirs-MacBook-Pro.local</td><td>127.0.0.1</td><td style=\"text-align: right;\">41186</td><td style=\"text-align: right;\"> 1675811353</td><td>6e735_00000</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-07 16:09:42,720\tERROR tune.py:758 -- Trials did not complete: [XGBoostTrainer_6e735_00000]\n",
      "2023-02-07 16:09:42,721\tINFO tune.py:762 -- Total run time: 31.61 seconds (31.48 seconds for the tuning loop).\n"
     ]
    },
    {
     "ename": "RayTaskError(IndexError)",
     "evalue": "\u001b[36mray::_Inner.train()\u001b[39m (pid=41186, ip=127.0.0.1, repr=XGBoostTrainer)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/trainable.py\", line 367, in train\n    raise skipped from exception_cause(skipped)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/function_trainable.py\", line 335, in entrypoint\n    return self._trainable_func(\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 480, in _trainable_func\n    super()._trainable_func(self._merged_config, reporter, checkpoint_dir)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/function_trainable.py\", line 652, in _trainable_func\n    output = fn()\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 389, in train_func\n    trainer.preprocess_datasets()\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/gbdt_trainer.py\", line 187, in preprocess_datasets\n    super().preprocess_datasets()\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 299, in preprocess_datasets\n    self.preprocessor.fit(train_dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 105, in fit\n    return self._fit(dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessors/chain.py\", line 73, in _fit\n    ds = preprocessor.fit_transform(ds)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 120, in fit_transform\n    self.fit(dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 105, in fit\n    return self._fit(dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessors/vectorizer.py\", line 233, in _fit\n    total_counts[i].update(col_value_counts)\nIndexError: list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRayTaskError(IndexError)\u001b[0m                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[23], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py:360\u001b[0m, in \u001b[0;36mBaseTrainer.fit\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    358\u001b[0m     result \u001b[38;5;241m=\u001b[39m result_grid[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    359\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result\u001b[38;5;241m.\u001b[39merror:\n\u001b[0;32m--> 360\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m result\u001b[38;5;241m.\u001b[39merror\n\u001b[1;32m    361\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m TuneError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    362\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m TrainingFailedError \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n",
      "\u001b[0;31mRayTaskError(IndexError)\u001b[0m: \u001b[36mray::_Inner.train()\u001b[39m (pid=41186, ip=127.0.0.1, repr=XGBoostTrainer)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/trainable.py\", line 367, in train\n    raise skipped from exception_cause(skipped)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/function_trainable.py\", line 335, in entrypoint\n    return self._trainable_func(\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 480, in _trainable_func\n    super()._trainable_func(self._merged_config, reporter, checkpoint_dir)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/tune/trainable/function_trainable.py\", line 652, in _trainable_func\n    output = fn()\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 389, in train_func\n    trainer.preprocess_datasets()\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/gbdt_trainer.py\", line 187, in preprocess_datasets\n    super().preprocess_datasets()\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/train/base_trainer.py\", line 299, in preprocess_datasets\n    self.preprocessor.fit(train_dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 105, in fit\n    return self._fit(dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessors/chain.py\", line 73, in _fit\n    ds = preprocessor.fit_transform(ds)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 120, in fit_transform\n    self.fit(dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessor.py\", line 105, in fit\n    return self._fit(dataset)\n  File \"/opt/anaconda3/envs/dask/lib/python3.9/site-packages/ray/data/preprocessors/vectorizer.py\", line 233, in _fit\n    total_counts[i].update(col_value_counts)\nIndexError: list index out of range"
     ]
    }
   ],
   "source": [
    "result = trainer.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8697c7e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74.53868389129639"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.metrics['time_total_s']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41d0d928",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_exp = 2\n",
    "\n",
    "all_exps = []\n",
    "\n",
    "\n",
    "exp_time = []\n",
    "exp_score = []\n",
    "experiment_dict ={}\n",
    "experiment_dict[\"method\"] = f\"ray_AIR n_job={num_exp}, tree_method='approx'\"   \n",
    "\n",
    "\n",
    "for i in repeat(1, num_exp):\n",
    "    result = trainer.fit()\n",
    "# print(result.metrics)\n",
    "\n",
    "   \n",
    "    exp_time.append(result.metrics['time_total_s'])\n",
    "\n",
    "experiment_dict[\"time_result\"] = exp_time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1238f5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1791ab19",
   "metadata": {},
   "source": [
    "### XGBoost_ray\n",
    "\n",
    "https://xgboost.readthedocs.io/en/stable/tutorials/ray.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef5d4155",
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost_ray import RayDMatrix, RayParams, train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9d81a02",
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer(lowercase=False, ngram_range=(1,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05f88c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = vect.fit_transform(df['text']).todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5b5eb5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = RayDMatrix(np.squeeze(np.asarray(X)), df['target'].to_numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b19995b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "evals_result = {}\n",
    "bst = train(\n",
    "    {\n",
    "        \"objective\": \"multi:softmax\",\n",
    "        \"eval_metric\": [\"merror\"],\n",
    "    },\n",
    "    train_set,\n",
    "    evals_result=evals_result,\n",
    "    evals=[(train_set, \"train\")],\n",
    "    verbose_eval=False,\n",
    "    ray_params=RayParams(num_actors=8, cpus_per_actor=1))\n",
    "\n",
    "bst.save_model(\"model.xgb\")\n",
    "print(\"Final training error: {:.4f}\".format(\n",
    "    evals_result[\"train\"][\"error\"][-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1271a147-12f2-494e-87b5-1273a4913545",
   "metadata": {},
   "source": [
    "# Visualize result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8faed2b-4abd-4793-9166-50463c642052",
   "metadata": {},
   "outputs": [],
   "source": [
    "## if you ran the notebook\n",
    "# df_result = pd.DataFrame(all_exps).set_index(\"method\")\n",
    "# df_result = df_result.reset_index()\n",
    "\n",
    "# if you are loading the result dataframe from file\n",
    "import ast\n",
    "\n",
    "df_result = pd.read_csv('../output/df_result_joblib_ray_29012023.csv')\n",
    "df_result['time_result'] = df_result['time_result'].apply(lambda x: ast.literal_eval(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20a607ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result['average_score'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "771d3415-c25a-463f-bbf8-11758cff9e15",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result['time_result_avg'] = df_result['time_result'].apply(lambda x: np.mean(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5916c77f-bf1a-498b-a54e-afb2705e4370",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result = df_result.sort_values(by=\"time_result_avg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8292654-110e-44e4-8992-c0453d85e4e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result = df_result.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62321107-cacc-4178-a55b-2c744c564fc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_exp = 50\n",
    "\n",
    "fig = plt.figure(figsize=(10, 10))\n",
    "plt.title(f'XGB performance benchmark for {num_exp} iterations (20 Newsgroup data)')\n",
    "plt.ylabel(\"elapsed time (sec)\")\n",
    "fig = df_result[\"time_result\"].head(2).apply(lambda x: pd.Series(x)).T.boxplot(rot=45)\n",
    "plt.tight_layout()\n",
    "fig.figure.savefig(f'../img/performance_{num_exp}_joblib_ray.png')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b122275-dcc2-4e14-8731-56ed68a96261",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 10))\n",
    "plt.title(f'XGB average time for {num_exp} (20 Newsgroup data)')\n",
    "plt.ylabel(\"elapsed time (sec)\")\n",
    "\n",
    "df_result_t = df_result[\"time_result\"].head(2).apply(lambda x: pd.Series(x)).T\n",
    "fig = df_result_t.reindex(df_result_t.mean().sort_values().index, axis=1).mean().plot(kind='bar', rot=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "# fig.figure.savefig(f'../img/average_time_{num_exp}.png')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11584f7b-9355-4404-9704-4066ed385ae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ray https://www.anyscale.com/blog/three-ways-to-speed-up-xgboost-model-training"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "0479e754db294e497611fd8d244dbddaec7d11e5bc6920d9ff3f73235eb6645f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}