mab_testing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bb674a39",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Solution planning\n",
    "\n",
    "**Business problem**\n",
    "\n",
    "- Check if new sign up button in the website has a conversion rate better, equal or worse than current one\n",
    "\n",
    "Observations\n",
    "\n",
    "* Conversion rate would be related to number of clicks in sign up button. That would be number of sign up clicks (success) divided by total number of visits\n",
    "* The current page has a blue color button and the alternative has a red color.\n",
    "\n",
    "## Input\n",
    "\n",
    "- Data acquired through website\n",
    "\n",
    "\n",
    "## Tasks\n",
    "\n",
    "- Design what type of test will be used (Multi-Armed Bandit with Thompson agent)\n",
    "- Choose of metric of interest\n",
    "- Definition of posteriori distribution (beta-binomial distribution)\n",
    "- Definition of priori distribution (beta distribution)\n",
    "- Define maximum accepted loss ( 5% )\n",
    "- Design test infrastructure (define endpoint, API, two websites, agent)\n",
    "- Design two 'fake' websites with different buttons colors\n",
    "- Create an API in flask to run websites\n",
    "- Create a webscrapping\n",
    "- Getting data\n",
    "- Calculate probability of page B be better than page A"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4f9dfa5",
   "metadata": {},
   "source": [
    "# Imports and helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "77e173d0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-19T14:05:04.982838Z",
     "start_time": "2023-04-19T14:05:03.431332Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_68205/357792895.py:119: UserWarning: frames=None which we can infer the length of, did not pass an explicit *save_count* and passed cache_frame_data=True.  To avoid a possibly unbounded cache, frame data caching has been disabled. To suppress this warning either pass `cache_frame_data=False` or `save_count=MAX_FRAMES`.\n",
      "  ani = FuncAnimation(plt.gcf(), animate, interval=1000)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import beta\n",
    "from scipy.stats import norm\n",
    "from flask import Flask, render_template, redirect, url_for, request\n",
    "from matplotlib.animation import FuncAnimation\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np \n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "#Function that calculates probability of B being better than A\n",
    "def bayesian_inference(data):\n",
    "    N_mc = 10000\n",
    "    proba_b_better_a = []\n",
    "    expected_loss_a = []\n",
    "    expected_loss_b = []\n",
    "    for day in range (len(data)):\n",
    "        u_a, var_a = beta.stats(a = 1 + data.loc[day, 'acc_click_A'],\n",
    "                                      b = 1 + (data.loc[day,'acc_visit_A'] -  data.loc[day, 'acc_click_A']),\n",
    "                                      moments = 'mv'\n",
    "        )\n",
    "        \n",
    "        u_b, var_b = beta.stats(a = 1 + data.loc[day, 'acc_click_B'],\n",
    "                                      b = 1 + (data.loc[day,'acc_visit_B'] -  data.loc[day, 'acc_click_B']),\n",
    "                                      moments = 'mv'\n",
    "        )\n",
    "        \n",
    "        # Sampling of a normal distribution using u_a and var_a\n",
    "        x_a = np.random.normal(loc = u_a,\n",
    "                               scale = 1.25*np.sqrt(var_a),\n",
    "                               size = N_mc\n",
    "        )\n",
    "\n",
    "         # Sampling of a normal distribution using u_b and var_b\n",
    "        x_b = np.random.normal(loc = u_b,\n",
    "                               scale = 1.25*np.sqrt(var_b),\n",
    "                               size = N_mc\n",
    "        )\n",
    "\n",
    "        # Beta distribution function of page A\n",
    "        pdf_beta_a = beta.pdf(x_a,\n",
    "                            a = 1 + data.loc[day, 'acc_click_A'],\n",
    "                            b = 1 + (data.loc[day,'acc_visit_A'] -  data.loc[day, 'acc_click_A'])\n",
    "        )\n",
    "\n",
    "        pdf_beta_b = beta.pdf(x_b,\n",
    "                            a = 1 + data.loc[day, 'acc_click_B'],\n",
    "                            b = 1 + (data.loc[day,'acc_visit_B'] -  data.loc[day, 'acc_click_B'])\n",
    "        )\n",
    "\n",
    "        # Normal distribution function of page A\n",
    "        pdf_normal_a = norm.pdf(x_a,\n",
    "                                      loc = u_a,\n",
    "                                      scale = 1.25*np.sqrt(var_a)\n",
    "        )\n",
    "\n",
    "         # Normal distribution function of page B\n",
    "        pdf_normal_b = norm.pdf(x_b,\n",
    "                                      loc = u_b,\n",
    "                                      scale = 1.25*np.sqrt(var_b)\n",
    "        )\n",
    "\n",
    "        # Beta / Normal\n",
    "        y = (pdf_beta_a*pdf_beta_b) / (pdf_normal_a*pdf_normal_b)\n",
    "\n",
    "        # Values where B is better than A\n",
    "        y_b = y[x_b >= x_a]\n",
    "\n",
    "        # Probability of B being A\n",
    "        p = (1/N_mc) * np.sum(y_b)\n",
    "\n",
    "        # Expected error\n",
    "        expected_loss_A = (1/N_mc) * np.sum(((x_b - x_a) * y)[x_b >= x_a])\n",
    "        expected_loss_B = (1/N_mc) * np.sum(((x_a - x_b) * y)[x_a >= x_b])\n",
    "\n",
    "        proba_b_better_a.append(p)\n",
    "        expected_loss_a.append(expected_loss_A)\n",
    "        expected_loss_b.append(expected_loss_B)\n",
    "\n",
    "    return proba_b_better_a, expected_loss_a, expected_loss_b\n",
    "\n",
    "\n",
    "def animate( i ):\n",
    "    data = pd.read_csv('data_experiment.csv')\n",
    "\n",
    "    # dtypes\n",
    "    data['click'] = data['click'].astype(int)\n",
    "    data['visit'] = data['visit'].astype(int)\n",
    "\n",
    "    data = data.reset_index().rename(columns={'index':'day'})\n",
    "    data = data.pivot(index='day', columns='group', values=['click', 'visit']).fillna(0)\n",
    "    data.columns = ['click_control', 'click_treatment','visit_control', 'visit_treatment']\n",
    "    data = data.reset_index(drop=True)\n",
    "\n",
    "    data['acc_visit_A'] = data['visit_control'].cumsum()\n",
    "    data['acc_click_A'] = data['click_control'].cumsum()\n",
    "    data['acc_visit_B'] = data['visit_treatment'].cumsum()\n",
    "    data['acc_click_B'] = data['click_treatment'].cumsum()\n",
    "\n",
    "    # inference bayesian\n",
    "    p, expected_loss_a, expected_loss_b = bayesian_inference(data)\n",
    "\n",
    "    x1 = np.arange(len(p))\n",
    "    \n",
    "\n",
    "    plt.cla()\n",
    "    plt.plot(x1,p, label='Probability B better A')\n",
    "    plt.plot(x1,expected_loss_a, label='Risk Choosing A')\n",
    "    plt.plot(x1,expected_loss_b, label='Risk Choosing B')\n",
    "\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "056b79cb",
   "metadata": {},
   "source": [
    "# Design API "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f17e988b",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-17T15:36:59.226Z"
    }
   },
   "outputs": [],
   "source": [
    "app = Flask( __name__)\n",
    "\n",
    "@app.route( '/')\n",
    "def index():\n",
    "    # get data\n",
    "    df = pd.read_csv('data_experiment.csv')\n",
    "\n",
    "    df['no_click'] = df['visit'] - df['click']\n",
    "    click_array = df.groupby('group').sum().reset_index()[['click', 'no_click']].T.to_numpy()\n",
    "\n",
    "    # Thompson Agent\n",
    "    prob_reward = np.random.beta( click_array[0], click_array[1])\n",
    "\n",
    "    if np.argmax(prob_reward) == 0:\n",
    "        return render_template( 'page01.html')\n",
    "    else:\n",
    "        return render_template( 'page02.html')\n",
    "\n",
    "\n",
    "@app.route( '/yes', methods=['POST'] )\n",
    "def yes_event():\n",
    "    df = pd.read_csv('data_experiment.csv')\n",
    "    \n",
    "    if request.form['yescheckbox'] == 'red':\n",
    "        click = 1\n",
    "        visit = 1\n",
    "        group = 'treatment'\n",
    "    else:\n",
    "        click = 1\n",
    "        visit = 1\n",
    "        group = 'control'\n",
    "\n",
    "    df_raw = pd.DataFrame({'click': click, 'visit': visit, 'group':group}, index= [0])\n",
    "    df = pd.concat([df,df_raw])\n",
    "    df.to_csv('data_experiment.csv', index=False)\n",
    "\n",
    "    return redirect( url_for ( 'index') )\n",
    "\n",
    "\n",
    "@app.route( '/no', methods=['POST'] )\n",
    "def no_event():\n",
    "    df = pd.read_csv('data_experiment.csv')\n",
    "\n",
    "    if request.form['nocheckbox'] == 'red':\n",
    "        click = 0\n",
    "        visit = 1\n",
    "        group = 'treatment'\n",
    "    else:\n",
    "        click = 0\n",
    "        visit = 1\n",
    "        group = 'control'\n",
    "\n",
    "    df_raw = pd.DataFrame({'click': click, 'visit': visit, 'group':group}, index = [0])\n",
    "    df = pd.concat([df,df_raw])\n",
    "    df.to_csv('data_experiment.csv', index=False)\n",
    "\n",
    "    return redirect( url_for ( 'index') )\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    app.run(port=5000)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f62b5d66",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-17T14:27:58.040941Z",
     "start_time": "2023-04-17T14:27:58.038408Z"
    }
   },
   "source": [
    "# Design webscrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fac7a18",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-17T15:37:14.406Z"
    }
   },
   "outputs": [],
   "source": [
    "path_webdriver ='/home/dbcordeiro/repos/isketch_website_mab_testing/chromedriver_linux64/chromedriver'\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_experimental_option(\"detach\", True)\n",
    "driver = webdriver.Chrome(service=Service(path_webdriver), options=options )\n",
    "\n",
    "driver.get('http://127.0.0.1:5000/')\n",
    "\n",
    "clicks=10000\n",
    "for click in range(clicks):\n",
    "    button_color = driver.find_element('name', 'yescheckbox').get_attribute('value')\n",
    "\n",
    "    if button_color == 'blue':\n",
    "        if np.random.random() < 0.3:\n",
    "            driver.find_element( 'name', 'yescheckbox').click()\n",
    "            driver.find_element('id', 'yesbtn').click()\n",
    "            time.sleep(1)\n",
    "        else:\n",
    "            driver.find_element( 'name', 'nocheckbox').click()\n",
    "            driver.find_element('id', 'nobtn').click()\n",
    "            time.sleep(1)\n",
    "    else:\n",
    "        if np.random.random() < 0.35:\n",
    "            driver.find_element( 'name', 'yescheckbox').click()\n",
    "            driver.find_element('id', 'yesbtn').click()\n",
    "            time.sleep(1)\n",
    "        else:\n",
    "            driver.find_element( 'name', 'nocheckbox').click()\n",
    "            driver.find_element('id', 'nobtn').click()\n",
    "            time.sleep(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2f71325",
   "metadata": {},
   "source": [
    "# Decision plot of probability of B better than A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c6636d64",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-19T14:05:17.331941Z",
     "start_time": "2023-04-19T14:05:17.326577Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_68205/4265140949.py:1: UserWarning: frames=None which we can infer the length of, did not pass an explicit *save_count* and passed cache_frame_data=True.  To avoid a possibly unbounded cache, frame data caching has been disabled. To suppress this warning either pass `cache_frame_data=False` or `save_count=MAX_FRAMES`.\n",
      "  ani = FuncAnimation(plt.gcf(), animate, interval=1000)\n",
      "/home/dbcordeiro/anaconda3/lib/python3.9/site-packages/matplotlib/animation.py:880: UserWarning: Animation was deleted without rendering anything. This is most likely not intended. To prevent deletion, assign the Animation to a variable, e.g. `anim`, that exists until you output the Animation using `plt.show()` or `anim.save()`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ani = FuncAnimation(plt.gcf(), animate, interval=1000) \n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "1610045a2a0d141a8db9a4528e143c190fd5d487e641245758a73bb33ce00219"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}