analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "import os\n",
    "import pickle\n",
    "import squarify\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "from matplotlib.lines import Line2D\n",
    "\n",
    "SAVEPATH = \"./figure\"\n",
    "TRACEPATH = \"./data/job_trace\"\n",
    "PKLPATH = \"./data/utilization/util_pkl\"\n",
    "\n",
    "sns.set_style(\"ticks\")\n",
    "font = {\n",
    "    \"font.family\": \"Roboto\",\n",
    "    \"font.size\": 12,\n",
    "}\n",
    "sns.set_style(font)\n",
    "paper_rc = {\n",
    "    \"lines.linewidth\": 3,\n",
    "    \"lines.markersize\": 10,\n",
    "}\n",
    "sns.set_context(\"paper\", font_scale=2, rc=paper_rc)\n",
    "cmp = sns.color_palette(\"tab10\")\n",
    "\n",
    "\n",
    "def autolabel(rects, ax, prec=1):\n",
    "    \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n",
    "    for rect in rects:\n",
    "        height = rect.get_height()\n",
    "        ax.annotate(\n",
    "            f\"{height:.{prec}f}\",\n",
    "            xy=(rect.get_x() + rect.get_width() / 2, height),\n",
    "            xytext=(0, 3),  # 3 points vertical offset\n",
    "            textcoords=\"offset points\",\n",
    "            ha=\"center\",\n",
    "            va=\"bottom\",\n",
    "            size=16,\n",
    "        )\n",
    "\n",
    "\n",
    "def calculate_num_cdf_customized_xaxis(df: pd.DataFrame, x_axis: List, key: str):\n",
    "    \"\"\"\n",
    "    Calculate quantity percentile CDF with customized threshold of x-axis, y-axis: 0-100%,\n",
    "    \"\"\"\n",
    "    # print(\"Parsing\")\n",
    "    data = df[[key]].copy()\n",
    "    data.dropna(inplace=True)\n",
    "\n",
    "    y = [len(data[data[key] <= x]) / len(data) * 100 for x in x_axis]\n",
    "\n",
    "    return y\n",
    "\n",
    "\n",
    "def calculate_sum_cdf_customized_xaxis(df: pd.DataFrame, x_axis: List, key: str, key_to_time=None):\n",
    "    \"\"\"\n",
    "    Calculate sum CDF with customized threshold of x-axis, y-axis: 0-100%,\n",
    "    \"\"\"\n",
    "    if key_to_time is not None:\n",
    "        data = df[[key, key_to_time]].copy()\n",
    "        data[\"new\"] = data[key] * data[key_to_time]\n",
    "    else:\n",
    "        data = df[[key]].copy()\n",
    "        data[\"new\"] = data[key]\n",
    "    data.dropna(inplace=True)\n",
    "    sum = data[\"new\"].sum()\n",
    "\n",
    "    y = [data[data[key] <= x][\"new\"].sum() / sum * 100 for x in x_axis]\n",
    "\n",
    "    return y\n",
    "\n",
    "\n",
    "if not os.path.exists(SAVEPATH):\n",
    "    os.makedirs(SAVEPATH)\n",
    "\n",
    "\n",
    "data_seren = pd.read_csv(f\"{TRACEPATH}/trace_seren.csv\")\n",
    "data_kalos = pd.read_csv(f\"{TRACEPATH}/trace_kalos.csv\")\n",
    "data_philly = pd.read_csv(f\"{TRACEPATH}/trace_previous_work/philly_trace.csv\")\n",
    "data_helios = pd.read_csv(f\"{TRACEPATH}/trace_previous_work/helios_trace.csv\")\n",
    "data_pai = pd.read_csv(f\"{TRACEPATH}/trace_previous_work/pai_trace.csv\")\n",
    "\n",
    "# A few further process\n",
    "data_pai.rename(columns={\"plan_cpu\": \"cpu_num\", \"plan_gpu\": \"gpu_num\", \"wait_time\": \"queue\", \"status\": \"state\"}, inplace=True)\n",
    "data_pai[[\"cpu_num\", \"gpu_num\"]] /= 100\n",
    "data_pai[\"state\"] = data_pai[\"state\"].map({\"Failed\": \"FAILED\"})  # Not suitable for final state analysis\n",
    "data_philly[\"state\"] = data_philly[\"state\"].map({\"Pass\": \"COMPLETED\", \"Failed\": \"FAILED\", \"Killed\": \"CANCELLED\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CDF: GPU Job Duration & Utilization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = [2**i for i in range(0, 22)]\n",
    "y_gpu_seren = calculate_num_cdf_customized_xaxis(data_seren[data_seren[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",
    "y_gpu_kalos = calculate_num_cdf_customized_xaxis(data_kalos[data_kalos[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",
    "y_gpu_philly = calculate_num_cdf_customized_xaxis(data_philly[data_philly[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",
    "y_gpu_helios = calculate_num_cdf_customized_xaxis(data_helios[data_helios[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",
    "y_gpu_pai = calculate_num_cdf_customized_xaxis(data_pai[data_pai[\"gpu_num\"] > 0], x_axis=x, key=\"duration\")\n",
    "\n",
    "with open(f\"{PKLPATH}/util_gpu_seren.pkl\", \"rb\") as file:\n",
    "    x1, y1, _, _, _, _, _, _, _, _ = pickle.load(file)\n",
    "\n",
    "with open(f\"{PKLPATH}/util_gpu_kalos.pkl\", \"rb\") as file:\n",
    "    x4, y4, _, _ = pickle.load(file)\n",
    "\n",
    "with open(f\"{PKLPATH}/util_gpu_pai.pkl\", \"rb\") as file:  # Collect via Antman\n",
    "    x2, y2 = pickle.load(file)\n",
    "\n",
    "with open(f\"{PKLPATH}/util_gpu_philly.pkl\", \"rb\") as file:\n",
    "    x3, y3 = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linestyles = [\"-\", \"--\", \":\", \":\", \":\"]\n",
    "grid_params = dict(width_ratios=[1, 1])\n",
    "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",
    "\n",
    "ax1.plot(x, y_gpu_seren, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",
    "ax1.plot(x, y_gpu_kalos, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",
    "ax1.plot(x, y_gpu_philly, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",
    "ax1.plot(x, y_gpu_helios, linestyles[3], linewidth=3, alpha=0.9, color=cmp[3], label=\"Helios\")\n",
    "ax1.plot(x, y_gpu_pai, linestyles[3], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",
    "\n",
    "ax2.plot(x1, y1, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",
    "ax2.plot(x4, y4, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",
    "ax2.plot(x2, y2, linestyles[2], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",
    "ax2.plot(x3, y3, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",
    "\n",
    "ax1.set_xlabel(f\"(a) GPU Job Duration (s)\")\n",
    "ax1.set_ylabel(f\"CDF (%)\")\n",
    "ax1.set_xscale(\"log\")\n",
    "ax1.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6])\n",
    "ax1.set_xlim(1, x[-1])\n",
    "ax1.set_ylim(-0.5, 100.8)\n",
    "ax1.grid(linestyle=\":\")\n",
    "\n",
    "ax2.set_xlabel(f\"(b) GPU Utilization (%)\")\n",
    "ax2.set_ylabel(f\"CDF (%)\")\n",
    "ax2.set_xlim(-0.8, 100.8)\n",
    "ax2.set_xticks([0, 25, 50, 75, 100])\n",
    "ax2.set_ylim(0, 100.8)\n",
    "ax2.grid(linestyle=\":\")\n",
    "\n",
    "handles, labels = ax1.get_legend_handles_labels()\n",
    "fig.legend(handles=handles, labels=labels, ncols=5, bbox_to_anchor=(0.1, 1.145), loc=2, columnspacing=1.5, handletextpad=0.5)\n",
    "\n",
    "sns.despine()\n",
    "fig.savefig(f\"{SAVEPATH}/cdf_job_duration_util.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CDF: GPU Number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = [i for i in range(0, 1025)]\n",
    "y_gpu_seren = calculate_num_cdf_customized_xaxis(data_seren[data_seren[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",
    "y_gpu_kalos = calculate_num_cdf_customized_xaxis(data_kalos[data_kalos[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",
    "y_gpu_philly = calculate_num_cdf_customized_xaxis(data_philly[data_philly[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",
    "y_gpu_helios = calculate_num_cdf_customized_xaxis(data_helios[data_helios[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",
    "y_gpu_pai = calculate_num_cdf_customized_xaxis(data_pai[data_pai[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\")\n",
    "\n",
    "y_gtime_seren = calculate_sum_cdf_customized_xaxis(\n",
    "    data_seren[data_seren[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",
    ")\n",
    "y_gtime_kalos = calculate_sum_cdf_customized_xaxis(\n",
    "    data_kalos[data_kalos[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",
    ")\n",
    "y_gtime_philly = calculate_sum_cdf_customized_xaxis(\n",
    "    data_philly[data_philly[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",
    ")\n",
    "y_gtime_helios = calculate_sum_cdf_customized_xaxis(\n",
    "    data_helios[data_helios[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",
    ")\n",
    "y_gtime_pai = calculate_sum_cdf_customized_xaxis(\n",
    "    data_pai[data_pai[\"gpu_num\"] > 0], x_axis=x, key=\"gpu_num\", key_to_time=\"duration\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linestyles = [\"-\", \"--\", \":\", \":\", \":\"]\n",
    "grid_params = dict(width_ratios=[1, 1])\n",
    "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",
    "\n",
    "ax1.plot(x, y_gpu_seren, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",
    "ax1.plot(x, y_gpu_kalos, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",
    "ax1.plot(x, y_gpu_philly, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",
    "ax1.plot(x, y_gpu_helios, linestyles[3], linewidth=3, alpha=0.9, color=cmp[3], label=\"Helios\")\n",
    "ax1.plot(x, y_gpu_pai, linestyles[3], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",
    "\n",
    "ax2.plot(x, y_gtime_seren, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",
    "ax2.plot(x, y_gtime_kalos, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",
    "ax2.plot(x, y_gtime_philly, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"Philly\")\n",
    "ax2.plot(x, y_gtime_helios, linestyles[3], linewidth=3, alpha=0.9, color=cmp[3], label=\"Helios\")\n",
    "ax2.plot(x, y_gtime_pai, linestyles[3], linewidth=3, alpha=0.9, color=cmp[4], label=\"PAI\")\n",
    "\n",
    "\n",
    "ax1.set_xlabel(f\"(a) Number of GPU\")\n",
    "ax1.set_ylabel(f\"CDF of Jobs (%)\")\n",
    "ax1.set_xscale(\"log\", base=2)\n",
    "ax1.set_xticks([2**i for i in range(0, 11, 2)])\n",
    "ax1.set_xticklabels(\n",
    "    [2**i for i in range(0, 10, 2)]\n",
    "    + [\n",
    "        \"1024+\",\n",
    "    ]\n",
    ")\n",
    "ax1.set_xlim(1, x[-1] + 1)\n",
    "ax1.set_ylim(-0.5, 100.8)\n",
    "ax1.grid(linestyle=\":\")\n",
    "\n",
    "ax2.set_xlabel(f\"(b) Number of GPU\")\n",
    "ax2.set_ylabel(f\"CDF of GPU Time (%)\")\n",
    "ax2.set_xscale(\"log\", base=2)\n",
    "ax2.set_xticks([2**i for i in range(0, 11, 2)])\n",
    "ax2.set_xticklabels(\n",
    "    [2**i for i in range(0, 10, 2)]\n",
    "    + [\n",
    "        \"1024+\",\n",
    "    ]\n",
    ")\n",
    "ax2.set_xlim(1, x[-1] + 50)\n",
    "ax2.set_ylim(-0.5, 100.8)\n",
    "ax2.grid(linestyle=\":\")\n",
    "\n",
    "handles, labels = ax1.get_legend_handles_labels()\n",
    "fig.legend(handles=handles, labels=labels, ncols=5, bbox_to_anchor=(0.1, 1.145), loc=2, columnspacing=1.5, handletextpad=0.5)\n",
    "\n",
    "sns.despine()\n",
    "fig.savefig(f\"{SAVEPATH}/cdf_job_gpunum.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bar: Job Final State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./data/cluster_summary.csv\", index_col=\"id\")\n",
    "grid_params = dict(width_ratios=[1, 1])\n",
    "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",
    "\n",
    "x = np.arange(1, 3)\n",
    "width = 0.22\n",
    "p1 = ax1.bar(\n",
    "    x - width,\n",
    "    df.loc[[\"Seren\", \"Kalos\"], \"complete_rate_gpu\"] * 100,\n",
    "    width,\n",
    "    label=\"Completed\",\n",
    "    alpha=0.8,\n",
    "    linewidth=1,\n",
    "    edgecolor=\"k\",\n",
    ")\n",
    "p2 = ax1.bar(\n",
    "    x, df.loc[[\"Seren\", \"Kalos\"], \"cancel_rate_gpu\"] * 100, width, label=\"Canceled\", alpha=0.8, linewidth=1, edgecolor=\"k\"\n",
    ")\n",
    "p3 = ax1.bar(\n",
    "    x + width, df.loc[[\"Seren\", \"Kalos\"], \"fail_rate_gpu\"] * 100, width, label=\"Failed\", alpha=0.8, linewidth=1, edgecolor=\"k\"\n",
    ")\n",
    "\n",
    "p4 = ax2.bar(\n",
    "    x - width,\n",
    "    df.loc[[\"Seren\", \"Kalos\"], \"complete_rate_gpu_time\"] * 100,\n",
    "    width,\n",
    "    label=\"Completed\",\n",
    "    alpha=0.8,\n",
    "    linewidth=1,\n",
    "    edgecolor=\"k\",\n",
    ")\n",
    "p5 = ax2.bar(\n",
    "    x, df.loc[[\"Seren\", \"Kalos\"], \"cancel_rate_gpu_time\"] * 100, width, label=\"Canceled\", alpha=0.8, linewidth=1, edgecolor=\"k\"\n",
    ")\n",
    "p6 = ax2.bar(\n",
    "    x + width,\n",
    "    df.loc[[\"Seren\", \"Kalos\"], \"fail_rate_gpu_time\"] * 100,\n",
    "    width,\n",
    "    label=\"Failed\",\n",
    "    alpha=0.8,\n",
    "    linewidth=1,\n",
    "    edgecolor=\"k\",\n",
    ")\n",
    "\n",
    "autolabel(p1, ax1)\n",
    "autolabel(p2, ax1)\n",
    "autolabel(p3, ax1)\n",
    "autolabel(p4, ax2)\n",
    "autolabel(p5, ax2)\n",
    "autolabel(p6, ax2)\n",
    "\n",
    "ax1.set_xlabel(f\"(a) Job Count\")\n",
    "ax1.set_ylabel(f\"Fraction (%)\")\n",
    "ax1.set_xticks(x)\n",
    "ax1.set_xticklabels([\"Seren\", \"Kalos\"])\n",
    "ax1.set_xlim(0.5, 2.5)\n",
    "ax1.set_ylim(0, 100)\n",
    "ax1.grid(axis=\"y\", linestyle=\":\")\n",
    "\n",
    "ax2.set_xlabel(f\"(b) GPU Time\")\n",
    "ax2.set_ylabel(f\"Fraction (%)\")\n",
    "ax2.set_xticks(x)\n",
    "ax2.set_xticklabels([\"Seren\", \"Kalos\"])\n",
    "ax2.set_xlim(0.5, 2.5)\n",
    "ax2.set_ylim(0, 100)\n",
    "ax2.grid(axis=\"y\", linestyle=\":\")\n",
    "\n",
    "handles, labels = ax1.get_legend_handles_labels()\n",
    "fig.legend(handles=handles, labels=labels, ncols=5, bbox_to_anchor=(0.18, 1.145), loc=2)\n",
    "\n",
    "sns.despine()\n",
    "fig.savefig(f\"{SAVEPATH}/bar_job_state.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Treemap: Job Number Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Processing Seren\")\n",
    "datas = data_seren[data_seren[\"gpu_num\"] > 0]\n",
    "\n",
    "job_type = [\"Eval\", \"Pretrain\", \"SFT\", \"MLLM\", \"Debug\", \"Other\"]\n",
    "df = pd.DataFrame(index=job_type, columns=[\"job_count\"]).fillna(0)\n",
    "df[\"job_count\"] = df.index.map(datas.groupby(\"type\").size()).astype(int)\n",
    "df[\"gtime\"] = df.index.map(datas.groupby(\"type\")[\"gpu_time\"].sum()).astype(int)\n",
    "\n",
    "total = df[\"job_count\"].sum()\n",
    "total_gtime = df[\"gtime\"].sum()\n",
    "\n",
    "df[\"count_percent\"] = df[\"job_count\"] / total * 100\n",
    "df[\"gtime_percent\"] = df[\"gtime\"] / total_gtime * 100\n",
    "\n",
    "# For plotting\n",
    "df[\"label\"] = [x + f\"\\n{df.at[x, 'count_percent']:.1f}%\" for x in list(df.index)]\n",
    "df[\"label_gtime\"] = [x + f\"\\n{df.at[x, 'gtime_percent']:.1f}%\" for x in list(df.index)]\n",
    "df[\"label_percent\"] = [f\"{df.at[x, 'count_percent']:.1f}%\" for x in list(df.index)]\n",
    "df[\"label_gtime_percent\"] = [f\"{df.at[x, 'gtime_percent']:.1f}%\" for x in list(df.index)]\n",
    "df_s = df.copy()\n",
    "\n",
    "print(\"Processing Kalos\")\n",
    "datak = data_kalos[data_kalos[\"gpu_num\"] > 0]\n",
    "\n",
    "job_type = [\"Eval\", \"Pretrain\", \"Debug\", \"Other\"]\n",
    "df = pd.DataFrame(index=job_type, columns=[\"job_count\"]).fillna(0)\n",
    "df[\"job_count\"] = df.index.map(datak.groupby(\"type\").size()).astype(int)\n",
    "df[\"gtime\"] = df.index.map(datak.groupby(\"type\")[\"gpu_time\"].sum()).astype(int)\n",
    "\n",
    "\n",
    "total = df[\"job_count\"].sum()\n",
    "total_gtime = df[\"gtime\"].sum()\n",
    "\n",
    "df[\"count_percent\"] = df[\"job_count\"] / total * 100\n",
    "df[\"gtime_percent\"] = df[\"gtime\"] / total_gtime * 100\n",
    "\n",
    "# For plotting\n",
    "df[\"label\"] = [x + f\"\\n{df.at[x, 'count_percent']:.1f}%\" for x in list(df.index)]\n",
    "df[\"label_gtime\"] = [x + f\"\\n{df.at[x, 'gtime_percent']:.1f}%\" for x in list(df.index)]\n",
    "df[\"label_percent\"] = [f\"{df.at[x, 'count_percent']:.1f}\\n%\" for x in list(df.index)]\n",
    "df[\"label_gtime_percent\"] = [f\"{df.at[x, 'gtime_percent']:.1f}\\n%\" for x in list(df.index)]\n",
    "df_k = df.copy()\n",
    "\n",
    "# For plotting\n",
    "df_k.at[\"Pretrain\", \"label_gtime_percent\"] = df_k.at[\"Pretrain\", \"label_gtime\"]\n",
    "df_k.at[\"Eval\", \"label_percent\"] = df_k.at[\"Eval\", \"label\"]\n",
    "df_k.at[\"Other\", \"label_percent\"] = \"\"\n",
    "df_k.at[\"Eval\", \"label_gtime_percent\"] = \" \"\n",
    "\n",
    "df_s.at[\"Pretrain\", \"label_gtime_percent\"] = df_s.at[\"Pretrain\", \"label_gtime\"]\n",
    "df_s.at[\"Eval\", \"label_percent\"] = df_s.at[\"Eval\", \"label\"]\n",
    "df_s.at[\"SFT\", \"label_percent\"] = df_s.at[\"SFT\", \"label\"]\n",
    "df_s.at[\"Other\", \"label_percent\"] = df_s.at[\"Other\", \"label\"]\n",
    "df_s.at[\"Pretrain\", \"label_percent\"] = \" \"\n",
    "df_s.at[\"Debug\", \"label_gtime_percent\"] = df_s.at[\"Debug\", \"label_gtime_percent\"].replace(\"%\", \"\\n%\")\n",
    "\n",
    "\n",
    "cmp_treemap = sns.color_palette(\"pastel\")\n",
    "label = df_s.index.to_list()\n",
    "df_s[\"color\"] = cmp_treemap[: len(df_s)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, constrained_layout=True, figsize=(9, 4.2))\n",
    "FONT = 15\n",
    "\n",
    "###### Fig 1 ######\n",
    "df_s.sort_values(by=\"count_percent\", ascending=False, inplace=True)\n",
    "squarify.plot(\n",
    "    ax=ax1,\n",
    "    sizes=list(df_s[\"job_count\"].values),\n",
    "    label=df_s[\"label_percent\"],\n",
    "    text_kwargs={\"fontsize\": FONT},\n",
    "    color=df_s[\"color\"],\n",
    "    bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",
    ")\n",
    "\n",
    "\n",
    "handles, labels = ax1.get_legend_handles_labels()\n",
    "handles_new = [handles[0], handles[-1], handles[1], handles[3], handles[4], handles[2]]\n",
    "fig.legend(\n",
    "    handles=handles_new, labels=label, ncols=6, bbox_to_anchor=(0.0, 1.135), loc=2, columnspacing=0.82, handletextpad=0.2\n",
    ")\n",
    "\n",
    "###### Fig 2 ######\n",
    "df_s.sort_values(by=\"gtime\", ascending=False, inplace=True)\n",
    "squarify.plot(\n",
    "    ax=ax2,\n",
    "    sizes=list(df_s[\"gtime\"].values),\n",
    "    label=df_s[\"label_gtime_percent\"],\n",
    "    text_kwargs={\"fontsize\": FONT},\n",
    "    color=df_s[\"color\"],\n",
    "    bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",
    ")\n",
    "\n",
    "plt.tick_params(axis=\"both\", which=\"both\", bottom=False, top=False, left=False, right=False)\n",
    "\n",
    "ax1.set_xlabel(f\"(a) Job Count\", fontsize=16)\n",
    "ax2.set_xlabel(f\"(b) GPU Time\", fontsize=16)\n",
    "\n",
    "\n",
    "###### Fig 3 ######\n",
    "df_k.sort_values(by=\"count_percent\", ascending=False, inplace=True)\n",
    "df_k[\"color\"] = [df_s[\"color\"][job_name] for job_name in df_k.index]\n",
    "\n",
    "squarify.plot(\n",
    "    ax=ax3,\n",
    "    sizes=list(df_k[\"job_count\"].values),\n",
    "    label=df_k[\"label_percent\"],\n",
    "    text_kwargs={\"fontsize\": FONT},\n",
    "    color=df_k[\"color\"],\n",
    "    bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",
    ")\n",
    "\n",
    "###### Fig 4 ######\n",
    "df_k.sort_values(by=\"gtime\", ascending=False, inplace=True)\n",
    "squarify.plot(\n",
    "    ax=ax4,\n",
    "    sizes=list(df_k[\"gtime\"].values),\n",
    "    label=df_k[\"label_gtime_percent\"],\n",
    "    text_kwargs={\"fontsize\": FONT},\n",
    "    color=df_k[\"color\"],\n",
    "    bar_kwargs={\"alpha\": 0.8, \"linewidth\": 1, \"edgecolor\": \"k\"},\n",
    ")\n",
    "\n",
    "ax1.annotate(\n",
    "    df_s.at[\"Pretrain\", \"label\"].split(\"\\n\")[1],\n",
    "    xy=(97, 96),\n",
    "    xytext=(90, 70),\n",
    "    arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",
    "    color=\"black\",\n",
    "    fontsize=15,\n",
    ")\n",
    "\n",
    "ax3.annotate(\n",
    "    df_k.at[\"Other\", \"label\"].split(\"\\n\")[1],\n",
    "    xy=(98.5, 92),\n",
    "    xytext=(80, 80),\n",
    "    arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",
    "    color=\"black\",\n",
    "    fontsize=15,\n",
    ")\n",
    "\n",
    "ax4.annotate(\n",
    "    df_k.at[\"Eval\", \"label_gtime\"].split(\"\\n\")[1],\n",
    "    xy=(98.5, 93),\n",
    "    xytext=(80, 80),\n",
    "    arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",
    "    color=\"black\",\n",
    "    fontsize=15,\n",
    ")\n",
    "\n",
    "plt.tick_params(axis=\"both\", which=\"both\", bottom=False, top=False, left=False, right=False)\n",
    "\n",
    "ax3.set_xlabel(f\"(c) Job Count\", fontsize=16)\n",
    "ax4.set_xlabel(f\"(d) GPU Time\", fontsize=16, labelpad=8)\n",
    "\n",
    "ax1.set_xticks([])\n",
    "ax1.set_yticks([])\n",
    "ax2.set_xticks([])\n",
    "ax2.set_yticks([])\n",
    "ax3.set_xticks([])\n",
    "ax3.set_yticks([])\n",
    "ax4.set_xticks([])\n",
    "ax4.set_yticks([])\n",
    "\n",
    "ax1.text(0.015, 0.03, \"Seren\", transform=ax1.transAxes, size=18, fontweight=\"bold\")\n",
    "ax2.text(0.02, 0.03, \"Seren\", transform=ax2.transAxes, size=18, fontweight=\"bold\")\n",
    "ax3.text(0.02, 0.03, \"Kalos\", transform=ax3.transAxes, size=18, fontweight=\"bold\")\n",
    "ax4.text(0.02, 0.03, \"Kalos\", transform=ax4.transAxes, size=18, fontweight=\"bold\")\n",
    "fig.savefig(f\"{SAVEPATH}/treemap_job_dist.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CDF: Duration and Queuing Delay of Different Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "(a) Seren Duration (b) Seren Queuing  (c) Kalos Duration (d) Kalos Queuing\n",
    "\"\"\"\n",
    "\n",
    "# Duration part\n",
    "x = [2**i for i in range(0, 22)]\n",
    "y_gpu_seren_other = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Other\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_seren_debug = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Debug\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_seren_pretrain = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Pretrain\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_seren_eval = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Eval\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_seren_tuning = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"SFT\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_seren_mllm = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"MLLM\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "\n",
    "y_gpu_kalos_other = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Other\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_kalos_debug = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Debug\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_kalos_pretrain = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Pretrain\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_kalos_eval = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Eval\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "y_gpu_kalos_tuning = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"SFT\")], x_axis=x, key=\"duration\"\n",
    ")\n",
    "\n",
    "# Queuing part\n",
    "x2 = [2**i for i in range(0, 16)]\n",
    "y_que_s_other = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Other\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_s_debug = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Debug\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_s_pretrain = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Pretrain\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_s_eval = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"Eval\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_s_tuning = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"SFT\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_s_mllm = calculate_num_cdf_customized_xaxis(\n",
    "    data_seren[(data_seren[\"gpu_num\"] > 0) & (data_seren[\"type\"] == \"MLLM\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "\n",
    "y_que_ali_other = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Other\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_ali_debug = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Debug\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_ali_pretrain = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Pretrain\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_ali_eval = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"Eval\")], x_axis=x2, key=\"queue\"\n",
    ")\n",
    "y_que_ali_tuning = calculate_num_cdf_customized_xaxis(\n",
    "    data_kalos[(data_kalos[\"gpu_num\"] > 0) & (data_kalos[\"type\"] == \"SFT\")], x_axis=x2, key=\"queue\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linestyles = [\"--\", \"-.\", \":\", \"--\", \"-.\", \":\"]\n",
    "grid_params = dict(width_ratios=[1, 1])\n",
    "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, constrained_layout=True, figsize=(9, 7))\n",
    "\n",
    "# (a) Seren Duration\n",
    "ax1.plot(x, y_gpu_seren_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",
    "ax1.plot(x, y_gpu_seren_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",
    "ax1.plot(x, y_gpu_seren_tuning, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"SFT\")\n",
    "ax1.plot(x, y_gpu_seren_mllm, linestyles[0], linewidth=3, alpha=0.9, color=cmp[3], label=\"MLLM\")\n",
    "ax1.plot(x, y_gpu_seren_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",
    "ax1.plot(x, y_gpu_seren_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",
    "\n",
    "\n",
    "# (b) Seren Queuing\n",
    "ax2.plot(x2, y_que_s_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",
    "ax2.plot(x2, y_que_s_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",
    "ax2.plot(x2, y_que_s_tuning, linestyles[2], linewidth=3, alpha=0.9, color=cmp[2], label=\"SFT\")\n",
    "ax2.plot(x2, y_que_s_mllm, linestyles[0], linewidth=3, alpha=0.9, color=cmp[3], label=\"MLLM\")\n",
    "ax2.plot(x2, y_que_s_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",
    "ax2.plot(x2, y_que_s_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",
    "\n",
    "# (c) Kalos Duration\n",
    "ax3.plot(x, y_gpu_kalos_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",
    "ax3.plot(x, y_gpu_kalos_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",
    "ax3.plot(x, y_gpu_kalos_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",
    "ax3.plot(x, y_gpu_kalos_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",
    "\n",
    "\n",
    "# (d) Kalos Queuing\n",
    "ax4.plot(x2, y_que_ali_eval, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Evaluation\")\n",
    "ax4.plot(x2, y_que_ali_pretrain, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Pretrain\")\n",
    "ax4.plot(x2, y_que_ali_debug, linestyles[1], linewidth=3, alpha=0.9, color=cmp[4], label=\"Debug\")\n",
    "ax4.plot(x2, y_que_ali_other, linestyles[2], linewidth=3, alpha=0.9, color=cmp[5], label=\"Other\")\n",
    "\n",
    "ax1.set_xlabel(f\"(a) Job Duration (s)\")\n",
    "ax1.set_ylabel(f\"CDF (%)\")\n",
    "ax1.set_xscale(\"log\")\n",
    "ax1.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6])\n",
    "ax1.set_xlim(1, x[-1])\n",
    "ax1.set_ylim(-0.5, 100.8)\n",
    "handles, labels = ax1.get_legend_handles_labels()\n",
    "fig.legend(handles=handles, labels=labels, ncols=6, bbox_to_anchor=(-0.01, 1.08), loc=2, columnspacing=0.9, handletextpad=0.2)\n",
    "ax1.grid(linestyle=\":\")\n",
    "\n",
    "ax2.set_xlabel(f\"(b) Job Queuing Delay (s)\")\n",
    "ax2.set_ylabel(f\"CDF (%)\")\n",
    "ax2.set_xscale(\"log\")\n",
    "ax2.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4])\n",
    "ax2.set_xlim(1, x2[-1])\n",
    "ax2.set_ylim(-0.5, 100.8)\n",
    "ax2.grid(linestyle=\":\")\n",
    "\n",
    "ax3.set_xlabel(f\"(c) Job Duration (s)\")\n",
    "ax3.set_ylabel(f\"CDF (%)\")\n",
    "ax3.set_xscale(\"log\")\n",
    "ax3.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6])\n",
    "ax3.set_xlim(1, x[-1])\n",
    "ax3.set_ylim(-0.5, 100.8)\n",
    "ax3.grid(linestyle=\":\")\n",
    "\n",
    "ax4.set_xlabel(f\"(d) Job Queuing Delay (s)\")\n",
    "ax4.set_ylabel(f\"CDF (%)\")\n",
    "ax4.set_xscale(\"log\")\n",
    "ax4.set_xticks([1e0, 1e1, 1e2, 1e3, 1e4])\n",
    "ax4.set_xlim(1, x2[-1])\n",
    "ax4.set_ylim(-0.5, 100.8)\n",
    "ax4.grid(linestyle=\":\")\n",
    "\n",
    "# 1 hour and 1 day\n",
    "ax1.axvline(x=3600, ls=\"--\", alpha=0.6, c=\"gray\", ymax=0.94, lw=1.5)\n",
    "ax1.axvline(x=3600 * 24, ls=\"--\", alpha=0.9, c=\"gray\", ymax=0.94, lw=1.5)\n",
    "ax3.axvline(x=3600, ls=\"--\", alpha=0.6, c=\"gray\", ymax=0.94, lw=1.5)\n",
    "ax3.axvline(x=3600 * 24, ls=\"--\", alpha=0.9, c=\"gray\", ymax=0.94, lw=1.5)\n",
    "\n",
    "sns.despine()\n",
    "ax1.text(0.78, 0.03, \"Seren\", transform=ax1.transAxes, size=20, fontweight=\"bold\")\n",
    "ax2.text(0.78, 0.03, \"Seren\", transform=ax2.transAxes, size=20, fontweight=\"bold\")\n",
    "ax3.text(0.78, 0.03, \"Kalos\", transform=ax3.transAxes, size=20, fontweight=\"bold\")\n",
    "ax4.text(0.78, 0.03, \"Kalos\", transform=ax4.transAxes, size=20, fontweight=\"bold\")\n",
    "\n",
    "fig.savefig(f\"{SAVEPATH}/cdf_job_duration_queue.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Box Plot: Request GPU number Different Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmap = sns.color_palette(\"pastel\")\n",
    "fig, (ax1, ax2) = plt.subplots(\n",
    "    ncols=2,\n",
    "    nrows=1,\n",
    "    gridspec_kw={\"width_ratios\": [4.2, 3]},\n",
    "    constrained_layout=True,\n",
    "    figsize=(9, 3.75),\n",
    ")\n",
    "\n",
    "############ Fig 1 ############\n",
    "data_seren.sort_values(by=\"gpu_num\", ascending=False, inplace=True)\n",
    "data_seren[\"type\"].replace(\"SFT\", \"SFT\", inplace=True)\n",
    "\n",
    "x_ticks = [\n",
    "    \"Eval\",\n",
    "    \"Pretrain\",\n",
    "    \"SFT\",\n",
    "    \"MLLM\",\n",
    "    \"Debug\",\n",
    "    \"Other\",\n",
    "]\n",
    "\n",
    "flierprops = dict(marker=\".\", markerfacecolor=\"k\", markersize=2, linestyle=\"none\")\n",
    "sns.boxplot(\n",
    "    x=\"type\",\n",
    "    y=\"gpu_num\",\n",
    "    data=data_seren,\n",
    "    flierprops=flierprops,\n",
    "    width=0.6,\n",
    "    linewidth=2.2,\n",
    "    saturation=2,\n",
    "    palette=cmap,\n",
    "    ax=ax1,\n",
    "    order=x_ticks,\n",
    "    boxprops=dict(alpha=1),\n",
    ")\n",
    "sns.color_palette(\"tab10\")\n",
    "ax1.set_xlabel(\"(a) Seren\")\n",
    "ax1.set_xticklabels(ax1.get_xticklabels(), rotation=0)\n",
    "ax1.set_ylabel(f\"Number of GPUs\")\n",
    "ax1.set_yscale(\"log\")\n",
    "ax1.grid(axis=\"y\", linestyle=\":\")\n",
    "\n",
    "\n",
    "############ Fig 2 ############\n",
    "data_kalos.sort_values(by=\"gpu_num\", ascending=False, inplace=True)\n",
    "data_kalos = data_kalos[data_kalos[\"type\"] != \"SFT\"]\n",
    "x_ticks_k = [\n",
    "    \"Eval\",\n",
    "    \"Pretrain\",\n",
    "    \"Debug\",\n",
    "    \"Other\",\n",
    "]\n",
    "my_pal = [cmap[0], cmap[1], cmap[4], cmap[5]]\n",
    "\n",
    "flierprops = dict(marker=\".\", markerfacecolor=\"k\", markersize=3, linestyle=\"none\")\n",
    "sns.boxplot(\n",
    "    x=\"type\",\n",
    "    y=\"gpu_num\",\n",
    "    data=data_kalos,\n",
    "    flierprops=flierprops,\n",
    "    width=0.6,\n",
    "    linewidth=2.2,\n",
    "    saturation=2,\n",
    "    palette=my_pal,\n",
    "    ax=ax2,\n",
    "    order=x_ticks_k,\n",
    "    boxprops=dict(alpha=1),\n",
    ")\n",
    "sns.color_palette(\"tab10\")\n",
    "ax2.set_xlabel(\"(b) Kalos\")\n",
    "ax2.set_ylabel(None)\n",
    "ax2.set_xticklabels(ax2.get_xticklabels(), rotation=0)\n",
    "ax2.set_yscale(\"log\")\n",
    "ax2.grid(axis=\"y\", linestyle=\":\")\n",
    "\n",
    "sns.despine()\n",
    "fig.savefig(f\"{SAVEPATH}/box_gpu_num.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CDF: Resource Utilization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"{PKLPATH}/util_gpu_seren.pkl\", \"rb\") as file:\n",
    "    _, _, x2, y2, x3, y3, x4, y4, x5, y5 = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/util_gpu_kalos_full.pkl\", \"rb\") as file:\n",
    "    _, _, x2_k, y2_k, x3_k, y3_k, x4_k, y4_k, x5_k, y5_k = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/util_cpu_mem_seren.pkl\", \"rb\") as file:\n",
    "    x6, y6, x7, y7 = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/util_cpu_mem_kalos.pkl\", \"rb\") as file:\n",
    "    x6_k, y6_k, x7_k, y7_k = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/ib_seren.pkl\", \"rb\") as file:\n",
    "    x8, y8, x9, y9 = pickle.load(file)\n",
    "\n",
    "x8 = x8 / x8.max() * 100\n",
    "x9 = x9 / x9.max() * 100\n",
    "\n",
    "linestyles = [\"--\", \":\", \"--\", \"-.\", \":\"]\n",
    "grid_params = dict(width_ratios=[1, 1])\n",
    "fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, constrained_layout=True, figsize=(9, 7))\n",
    "\n",
    "############ Fig 1: SM, Occupancy ############\n",
    "ax1.plot(x3, y3, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren SM Activity\")\n",
    "ax1.plot(x5, y5, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren Occupancy\")\n",
    "ax1.plot(x3_k, y3_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos SM Activity\")\n",
    "ax1.plot(x5_k, y5_k, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos Occupancy\")\n",
    "\n",
    "############ Fig 2: CPU mem usage, GPU mem usage ############\n",
    "ax2.plot(x7, y7, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren CPU Mem\")\n",
    "ax2.plot(x2, y2, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren GPU Mem\")\n",
    "ax2.plot(x7_k, y7_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos CPU Mem\")\n",
    "ax2.plot(x2_k, y2_k, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos GPU Mem\")\n",
    "\n",
    "############ Fig 3: CPU util ############\n",
    "ax3.plot(x6, y6, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",
    "ax3.plot(x6_k, y6_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",
    "\n",
    "############ Fig 4: IB send, receive ############\n",
    "ax4.plot(x8, y8, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"IB Send\")\n",
    "ax4.plot(x9, y9, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"IB Receive\")\n",
    "\n",
    "ax1.set_xlabel(f\"(a) GPU DCGM Metric (%)\")\n",
    "ax1.set_ylabel(f\"CDF (%)\")\n",
    "ax1.set_xlim(-0.8, 100.8)\n",
    "ax1.set_ylim(0, 100.8)\n",
    "ax1.set_xticks([0, 25, 50, 75, 100])\n",
    "ax1.grid(linestyle=\":\")\n",
    "\n",
    "ax2.set_xlabel(f\"(b) Memory Footprint (%)\")\n",
    "ax2.set_ylabel(f\"CDF (%)\")\n",
    "ax2.set_xlim(-0.8, 100.8)\n",
    "ax2.set_xticks([0, 25, 50, 75, 100])\n",
    "ax2.set_ylim(0, 100.8)\n",
    "ax2.grid(linestyle=\":\")\n",
    "\n",
    "ax3.set_xlabel(f\"(c) CPU Utilization (%)\")\n",
    "ax3.set_ylabel(f\"CDF (%)\")\n",
    "ax3.set_xlim(-0.8, 100.8)\n",
    "ax3.set_xticks([0, 25, 50, 75, 100])\n",
    "ax3.set_ylim(0, 100.8)\n",
    "ax3.legend(loc=\"lower right\")\n",
    "ax3.grid(linestyle=\":\")\n",
    "\n",
    "ax4.set_xlabel(f\"(d) Network (%)\")\n",
    "ax4.set_ylabel(f\"CDF (%)\")\n",
    "ax4.set_xlim(-0.8, 100.8)\n",
    "ax4.set_xticks([0, 25, 50, 75, 100])\n",
    "ax4.set_ylim(0, 100.8)\n",
    "ax4.legend(loc=\"lower right\")\n",
    "ax4.grid(linestyle=\":\")\n",
    "sns.despine()\n",
    "\n",
    "\n",
    "S = mpatches.Patch(facecolor=cmp[0], alpha=0.9)\n",
    "K = mpatches.Patch(facecolor=cmp[1], alpha=0.9)\n",
    "A = (Line2D([0], [0], color=\"black\", lw=3, ls=\"--\"),)\n",
    "B = (Line2D([0], [0], color=\"black\", lw=3, ls=\":\"),)\n",
    "\n",
    "legend1 = ax1.legend([S, K], [\"Seren\", \"Kalos\"], bbox_to_anchor=(0.5, 0.62), loc=2, ncol=1, fontsize=17, frameon=False)\n",
    "\n",
    "ax1.add_artist(legend1)\n",
    "\n",
    "ax1.legend([A, B], [\"SM Activity\", \"TC Activity\"], bbox_to_anchor=(0.3, 0.36), loc=2, ncol=1)\n",
    "\n",
    "ax2.legend([A, B], [\"CPU Memory\", \"GPU Memory\"], bbox_to_anchor=(0.25, 0.36), loc=2, ncol=1)\n",
    "\n",
    "fig.savefig(f\"{SAVEPATH}/cdf_resource_util.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CDF: Temperature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We use August data for GPU temperature and power\n",
    "with open(f\"{PKLPATH}/gpu_temp_seren.pkl\", \"rb\") as file:\n",
    "    x, y1, x2, y2 = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/gpu_temp_kalos.pkl\", \"rb\") as file:\n",
    "    x1_k, y1_k, x2_k, y2_k = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/gpu_power_seren.pkl\", \"rb\") as file:\n",
    "    x3, y3 = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/gpu_power_kalos.pkl\", \"rb\") as file:\n",
    "    x3_k, y3_k = pickle.load(file)\n",
    "\n",
    "linestyles = [\"-\", \":\", \":\", \"-\"]\n",
    "fig, ax1 = plt.subplots(ncols=1, nrows=1, constrained_layout=True, figsize=(5, 3.75))\n",
    "\n",
    "############ Fig 1: Temperature ############\n",
    "ax1.plot(x, y1, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren GPU Temp\")\n",
    "ax1.plot(x2, y2, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren GPU Mem Temp\")\n",
    "ax1.plot(x1_k, y1_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos GPU Temp\")\n",
    "ax1.plot(x2_k, y2_k, linestyles[1], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos GPU Mem Temp\")\n",
    "\n",
    "ax1.set_xlabel(f\"Temperature (°C)\")\n",
    "ax1.set_ylabel(f\"CDF (%)\")\n",
    "ax1.set_xlim(20, 85)\n",
    "ax1.set_ylim(0, 100.8)\n",
    "ax1.grid(linestyle=\":\")\n",
    "\n",
    "S = mpatches.Patch(facecolor=cmp[0], alpha=0.9)\n",
    "K = mpatches.Patch(facecolor=cmp[1], alpha=0.9)\n",
    "A = (Line2D([0], [0], color=\"black\", lw=3, ls=\"-\"),)\n",
    "B = (Line2D([0], [0], color=\"black\", lw=3, ls=\":\"),)\n",
    "\n",
    "legend1 = ax1.legend([S, K], [\"Seren\", \"Kalos\"], bbox_to_anchor=(0.6, 0.62), loc=2, ncol=1, fontsize=17, frameon=False)\n",
    "\n",
    "ax1.add_artist(legend1)\n",
    "\n",
    "ax1.legend(\n",
    "    [A, B],\n",
    "    [\"GPU Temp.\", \"GMem Temp.\"],\n",
    "    bbox_to_anchor=(0.4, 0.32),\n",
    "    loc=2,\n",
    "    ncol=1,\n",
    "    fontsize=17,\n",
    ")\n",
    "\n",
    "sns.despine()\n",
    "fig.savefig(f\"{SAVEPATH}/cdf_temperature.pdf\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CDF: Power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"{PKLPATH}/server_power.pkl\", \"rb\") as file:\n",
    "    x1, y1, x2, y2 = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/gpu_power_seren.pkl\", \"rb\") as file:\n",
    "    x3, y3 = pickle.load(file)\n",
    "with open(f\"{PKLPATH}/gpu_power_kalos.pkl\", \"rb\") as file:\n",
    "    x3_k, y3_k = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linestyles = [\"--\", \":\", \":\", \"-\"]\n",
    "grid_params = dict(width_ratios=[1, 1])\n",
    "fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(9, 3.75))\n",
    "\n",
    "############ Fig 1: GPU power ############\n",
    "ax1.plot(x3, y3, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"Seren\")\n",
    "ax1.plot(x3_k, y3_k, linestyles[0], linewidth=3, alpha=0.9, color=cmp[1], label=\"Kalos\")\n",
    "ax1.axvline(x=400, ls=\"--\", alpha=0.6, c=\"gray\", ymax=100, lw=1.5)\n",
    "ax1.annotate(\n",
    "    \"A100 TDP\",\n",
    "    xy=(400, 33),\n",
    "    xytext=(420, 20),\n",
    "    arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",
    "    color=\"black\",\n",
    "    fontsize=16,\n",
    ")\n",
    "ax1.annotate(\n",
    "    \"Max=600\",\n",
    "    xy=(600, 100),\n",
    "    xytext=(430, 85),\n",
    "    arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",
    "    color=\"black\",\n",
    "    fontsize=16,\n",
    ")\n",
    "\n",
    "############ Fig 2: Server power ############\n",
    "ax2.plot(x1, y1, linestyles[0], linewidth=3, alpha=0.9, color=cmp[0], label=\"GPU Node\")\n",
    "ax2.plot(x2, y2, linestyles[1], linewidth=3, alpha=0.9, color=cmp[0], label=\"CPU Node\")\n",
    "ax2.annotate(\n",
    "    \"Max=960\",\n",
    "    xy=(960, 100),\n",
    "    xytext=(1200, 90),\n",
    "    arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",
    "    color=\"black\",\n",
    "    fontsize=16,\n",
    ")\n",
    "ax2.annotate(\n",
    "    \"Max=6550\",\n",
    "    xy=(6550, 100),\n",
    "    xytext=(4500, 70),\n",
    "    arrowprops=dict(facecolor=\"black\", width=2.5, headwidth=8),\n",
    "    color=\"black\",\n",
    "    fontsize=16,\n",
    ")\n",
    "\n",
    "ax1.set_xlabel(f\"(a) GPU Power (W)\")\n",
    "ax1.set_ylabel(f\"CDF (%)\")\n",
    "ax1.set_xlim(-0.8, 610)\n",
    "ax1.set_ylim(0, 100.8)\n",
    "ax1.legend()\n",
    "ax1.grid(linestyle=\":\")\n",
    "ax1.xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([60]))\n",
    "ax1.xaxis.set_minor_formatter(matplotlib.ticker.FixedFormatter([60]))\n",
    "ax1.tick_params(axis=\"x\", which=\"minor\", labelsize=15)\n",
    "\n",
    "ax2.set_xlabel(f\"(b) Server Power in Seren (W)\")\n",
    "ax2.set_ylabel(f\"CDF (%)\")\n",
    "ax2.set_xlim(-0.8, x1.max())\n",
    "ax2.set_ylim(0, 100.8)\n",
    "ax2.legend(loc=\"lower right\")\n",
    "ax2.grid(linestyle=\":\")\n",
    "ax2.xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([520]))\n",
    "ax2.xaxis.set_minor_formatter(matplotlib.ticker.FixedFormatter([520]))\n",
    "ax2.tick_params(axis=\"x\", which=\"minor\", labelsize=15)\n",
    "sns.despine()\n",
    "\n",
    "fig.savefig(f\"{SAVEPATH}/cdf_power.pdf\", bbox_inches=\"tight\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}