Experimental_trials.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import multiprocessing as mp\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from CBAI import MAB, instance_generator, Algo, sigmoid, safe_monotonic_bai"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Safe-Linear Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Total number of arms: 10\n",
    "- Initial points: 0.1\n",
    "- Boundary values (`M`): 1.5\n",
    "- Noise parameters (`s_mu` and `s_theta`): $\\sim {\\cal N}(0,0.5^2)$\n",
    "- reward parameters (`theta`): [1,0.9,1,...,1]\n",
    "- safety parameters (`mu`): [1,1.5,5,5...,5]\n",
    "- safety threshold: 1\n",
    "- True optimal arm is at index **0**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed 0 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 1 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 2 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 3 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 4 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 5 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 6 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 7 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 8 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 9 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 10 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 11 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 12 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 13 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 14 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 15 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 16 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 17 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 7931.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 18 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 6634.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n",
      "seed 19 dim 10 completed\n",
      "{'dim': 10, 'gap': [0.0, 0.4, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8], 'min_gap': 0.4, 'total arm pulls': 4040.0, 'unsafe pulls': 0, 'true optimal': 0, 'current optimal': 0}\n"
     ]
    }
   ],
   "source": [
    "regret_sum = {}\n",
    "\n",
    "## We run trials for 20 different seeds\n",
    "for seed in range(20):\n",
    "    dim=10\n",
    "    seed=seed\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    init_points = np.array([0.1] + [0.1] *(dim-1)) \n",
    "    theta_list = np.array([1, 0.9] + [1] * (dim-2))\n",
    "    mu_list = np.array([1, 1.5] + [5] * (dim-2))\n",
    "    M_i_list = np.ones(dim) * 1.5\n",
    "\n",
    "    bandit = MAB(dim, seed=seed, init_points=init_points, theta_list=theta_list, mu_list=mu_list, M_i_list=M_i_list, s_mu=0.5, s_theta=0.5)\n",
    "    opt_reward_list = [min(bandit.gamma / bandit.mu_list[i], bandit.M_i_list[i]) * bandit.theta_list[i] for i in range(bandit.d)]\n",
    "\n",
    "    algo = Algo(bandit, opt_reward_list)\n",
    "    _ = algo.run_algo()\n",
    "    res = algo.return_key_stats()\n",
    "    print(res)\n",
    "    \n",
    "    ## You can compute the regret by uncommenting the code block below\n",
    "#     max_regret = [1-i[0] for i in algo.simple_regret]\n",
    "#     T = [i[1] for i in algo.simple_regret]\n",
    "#     regret_sum[seed] = [max_regret,T]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Safe-Monotonic Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Total number of arms: 5\n",
    "- Seed: 5 (we run 10 trials on same seed to get average, the same cam be done in Safe-Linear but we leave it to the interested researchers to try out)\n",
    "- Initial points (`a0`): -1.1\n",
    "- Drug response model: sigmoid / logit function with offset = 0 (`params_f_b0` and `params_g_b0`)\n",
    "- reward parameters (`params_f_b1`): [1,10,...,10]\n",
    "- safety parameters (`params_g_b1`): [1,1,...,1]\n",
    "- safety threshold (`gamma`): 0.3\n",
    "- safety tolerance (`epssafe`): 0.2\n",
    "- confidence: 1 - 0.01\n",
    "- True optimal arm is at index **0**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min_gap: 0.2997910023653128\n",
      "total arm pulls: 1.227770e+04\n",
      "number of unsafe pulls:0\n",
      "correctness: 10/10\n"
     ]
    }
   ],
   "source": [
    "d = 5\n",
    "seed = 5\n",
    "np.random.seed(seed)\n",
    "delta = 0.01\n",
    "epssafe = 0.2\n",
    "gamma = 0.3\n",
    "\n",
    "a = -1/1*np.log(1/gamma-1)\n",
    "a0 = [-1.1] * d\n",
    "\n",
    "best_true = 0\n",
    "\n",
    "params_f_b1 = np.array([1, 10] + [10] * (d-2))  # reward params\n",
    "params_f_b0 = np.array([0] * d)\n",
    "params_g_b1 = np.array([1, 1] + [1] * (d-2)) # cost params\n",
    "params_g_b0 = np.array([0] * d)\n",
    "f_func = []\n",
    "g_func = []\n",
    "\n",
    "max_val = -1000\n",
    "best_true = -1\n",
    "asafe_true = np.zeros(d)\n",
    "value = np.zeros(d)\n",
    "for i in range(d):\n",
    "    a = -1/params_g_b1[i]*(params_g_b0[i] + np.log(1/gamma-1))\n",
    "    asafe_true[i] = a\n",
    "    f_func.append(sigmoid)\n",
    "    g_func.append(sigmoid)\n",
    "    value[i] = sigmoid(a,params_f_b1[i],params_f_b0[i])\n",
    "    if sigmoid(a,params_f_b1[i],params_f_b0[i]) > max_val:\n",
    "        max_val = sigmoid(a,params_f_b1[i],params_f_b0[i])\n",
    "        best_true = i\n",
    "        \n",
    "\n",
    "\n",
    "value[best_true] = -1000\n",
    "gap = max_val - np.max(value)\n",
    "print('min_gap: ' + str(gap))\n",
    "\n",
    "N = 10\n",
    "correct = 0\n",
    "total = 0 \n",
    "total_unsafe = 0\n",
    "total_epoch = []\n",
    "total_pulls = []\n",
    "reward_list = []\n",
    "regret_list = []\n",
    "for t in range(N):\n",
    "    T, best, unsafe, epoch_idx, reward = safe_monotonic_bai(a0,gamma,delta,epssafe,f_func,g_func,params_f_b1,params_f_b0,params_g_b1,params_g_b0,noise_var=0.01)\n",
    "    if best == best_true:\n",
    "        correct += 1\n",
    "    total += T / N\n",
    "    total_unsafe += unsafe / N\n",
    "    total_epoch += [epoch_idx]\n",
    "    total_pulls += [T]\n",
    "    reward_list += [reward]\n",
    "\n",
    "print(\"total arm pulls: {:e}\".format(total))\n",
    "print(f'number of unsafe pulls:{unsafe}')\n",
    "print(f'correctness: {correct}/{N}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}