p-values and t-test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d389a66-14d2-4d21-81bf-ca71ef4cafce",
   "metadata": {},
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "df1a6ac8-d2ea-4c60-92c4-e4b9f70d19f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "689c2dca-9be1-4115-bf9d-8967fc4275c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from zipfile import ZipFile\n",
    "import random "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76ff167a-c6ac-40f0-aa09-4b058dd27dae",
   "metadata": {},
   "source": [
    "# DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "434dc0e5-ab34-4958-9386-90af25427a72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'G:\\\\My Drive\\\\JOBI\\\\Data_science\\\\PUBLICADO\\\\Statistical_Experiments_and_Significance_Testing'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "21fe5afb-b371-4c0c-a3c3-f5ad88ae04fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DATA_LAB', 'ab_data.zip', 'diet_anova.zip']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(\"../../DATA_SET\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "968b2d1d-89b5-4313-84ab-292001c1314c",
   "metadata": {},
   "outputs": [],
   "source": [
    "path =  \"../../DATA_SET/ab_data.zip\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c0b52023-bf83-497e-a5e4-87a6a1a0c23e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File Name                                             Modified             Size\n",
      "ab_data.csv                                    2020-08-03 19:50:04     15901933\n"
     ]
    }
   ],
   "source": [
    "with ZipFile(path , \"r\") as zip:\n",
    "    zip.printdir()\n",
    "    with zip.open(\"ab_data.csv\") as f:\n",
    "        data = pd.read_csv(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "630adb4f-2326-439f-b13a-224edfab7167",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>851104</td>\n",
       "      <td>2017-01-21 22:11:48.556739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>804228</td>\n",
       "      <td>2017-01-12 08:01:45.159739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>661590</td>\n",
       "      <td>2017-01-11 16:55:06.154213</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>853541</td>\n",
       "      <td>2017-01-08 18:28:03.143765</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>864975</td>\n",
       "      <td>2017-01-21 01:52:26.210827</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id                   timestamp      group landing_page  converted\n",
       "0   851104  2017-01-21 22:11:48.556739    control     old_page          0\n",
       "1   804228  2017-01-12 08:01:45.159739    control     old_page          0\n",
       "2   661590  2017-01-11 16:55:06.154213  treatment     new_page          0\n",
       "3   853541  2017-01-08 18:28:03.143765  treatment     new_page          0\n",
       "4   864975  2017-01-21 01:52:26.210827    control     old_page          1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dca34f44-e243-4f09-9e01-9bbc0498b07f",
   "metadata": {},
   "source": [
    "# Convertion Rate "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db4cd563-0c33-405b-aad8-d15a8a40623c",
   "metadata": {},
   "source": [
    "**Definition:** The conversion rate is the percentage of users who take a desired action. The archetypical example of conversion rate is the percentage of website visitors who buy something on the site.\n",
    "\n",
    "To determine a conversion rate, divide the number of goals achieved in a given time frame by the total number of visitors to your website, then multiply that number by 100. So if your landing page had $16,982$ visitors and of those, $3,604$ took a desired action, then your conversion rate is $21.22\\%$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8caa132-be1f-46ee-8280-f3a447bf6e91",
   "metadata": {},
   "source": [
    "For our Dependent Variable (i.e. what we are trying to measure), we are interested in capturing the conversion rate. A way we can code this is by each user session with a binary variable:\n",
    "\n",
    "* 0 - The user did not buy the product during this user session\n",
    "\n",
    "* 1 - The user bought the product during this user session"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b6315a5-c185-40f4-9a2b-6afccf2e08e2",
   "metadata": {},
   "source": [
    "The following permutation procedure asks, “If the two prices share the same conversion rate, could chance variation produce a difference as big as 5%?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c083d073-4409-484b-bd8c-2050a7c67ef5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>converted</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>group</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>control</th>\n",
       "      <td>0.120399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>treatment</th>\n",
       "      <td>0.118920</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          converted\n",
       "               mean\n",
       "group              \n",
       "control    0.120399\n",
       "treatment  0.118920"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[[\"group\",\"converted\"]].groupby(\"group\").agg([\"mean\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "146379b8-50de-4b2d-a5eb-590e895f90f9",
   "metadata": {},
   "source": [
    "There are $12.0399\\%$ and $11.8920\\%$ convertion rates for group A and B respectively. The difference is "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "363b67bb-b59d-416a-b217-059de1d255eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observed difference: 0.1479%\n"
     ]
    }
   ],
   "source": [
    "obs_pct_diff = (0.120399-0.118920)*100\n",
    "print(f'Observed difference: {obs_pct_diff:.4f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82e3f351-18a3-4c19-9a87-a4d69060e73d",
   "metadata": {},
   "source": [
    "less than$1\\%$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29c85cf7-c93d-4b1c-a662-768174b33671",
   "metadata": {},
   "source": [
    "# Resampling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee0095d0-e827-4681-bd5a-0907e2881e1a",
   "metadata": {},
   "source": [
    "## Funcion resampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ad26940b-28d1-40a6-9528-a795497d409e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def perm_fun(x, nA, nB): \n",
    "    n = nA + nB \n",
    "    idx_B = set(random.sample(range(n), nB)) \n",
    "    idx_A = set(range(n)) - idx_B \n",
    "    return x.loc[list(idx_B)].mean() - x.loc[list(idx_A)].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e180ce4-ef86-4b90-a7bc-212c5d3b75b3",
   "metadata": {},
   "source": [
    "## Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "60d4823e-b4b0-4cee-b4cb-488dafec6716",
   "metadata": {},
   "outputs": [],
   "source": [
    "table = pd.DataFrame(\n",
    "    {\n",
    "        \"outcome\":[\"converted\",\"no converted\"],\n",
    "        \"control\":[len(data[(data[\"group\"]==\"control\")&(data[\"converted\"]==1)]),\n",
    "                  len(data[(data[\"group\"]==\"control\")&(data[\"converted\"]==0)])],\n",
    "        \"treatment\":[len(data[(data[\"group\"]==\"treatment\")&(data[\"converted\"]==1)]),\n",
    "                     len(data[(data[\"group\"]==\"treatment\")&(data[\"converted\"]==0)])]\n",
    "    }\n",
    ").set_index(\"outcome\")\n",
    "#df2 = pd.DataFrame({\"control\": [len(data[data[\"group\"]==\"control\"])],\n",
    "#                    \"treatment\": [len(data[data[\"group\"]==\"treatment\"])]},\n",
    "#                   index=[\"total\"])\n",
    "\n",
    "#table = pd.concat([df1,df2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "18c1c87d-7bbd-4054-b995-21530544c34c",
   "metadata": {},
   "outputs": [],
   "source": [
    "table.loc[\"Total group\"] = table.sum(axis=0)\n",
    "table[\"total converted\"] = table.sum(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e987b77-b15c-480f-bafb-709a915fa22d",
   "metadata": {},
   "source": [
    "## Procedure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92a2e19a-0697-4f9c-bfef-fbd31657f067",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>control</th>\n",
       "      <th>treatment</th>\n",
       "      <th>total converted</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>outcome</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>converted</th>\n",
       "      <td>17723</td>\n",
       "      <td>17514</td>\n",
       "      <td>35237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no converted</th>\n",
       "      <td>129479</td>\n",
       "      <td>129762</td>\n",
       "      <td>259241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total group</th>\n",
       "      <td>147202</td>\n",
       "      <td>147276</td>\n",
       "      <td>294478</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              control  treatment  total converted\n",
       "outcome                                          \n",
       "converted       17723      17514            35237\n",
       "no converted   129479     129762           259241\n",
       "Total group    147202     147276           294478"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06113d48-eb58-4ed0-bdf0-ff36a3d0b26e",
   "metadata": {},
   "source": [
    "there are 35237 converted 1 and 259241 no converted 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c6bb0d5f-0370-4736-850c-0670fb4749fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_conversion = [0]*259241\n",
    "table_conversion.extend([1]*35237)\n",
    "table_conversion = pd.Series(table_conversion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a98a188b-3c5a-4403-b7f5-e9f57d537602",
   "metadata": {},
   "outputs": [],
   "source": [
    "perm_diffs = [100*perm_fun(table_conversion, 147202, 147276) for _ in range(500)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "494a5495-1ac0-4e0e-9d78-04a9cd675008",
   "metadata": {},
   "source": [
    "# Histrogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "301f79d2-9038-48c1-8e3a-10bc9be9404f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observed difference in resampling is: 0.0005%\n"
     ]
    }
   ],
   "source": [
    "print(f'Observed difference in resampling is: {np.mean(perm_diffs):.4f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "bae791d6-ff8f-4f5a-b2be-151a043cf926",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observed difference: 0.1479%\n"
     ]
    }
   ],
   "source": [
    "print(f'Observed difference: {obs_pct_diff:.4f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3380f414-6255-42a4-833d-2850a67bfa92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAE9CAYAAACP0jAFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXG0lEQVR4nO3de7BlZX3m8e9DgwIqA4QGOwg2RqKihREOFirjDU0EIpcoUctLa4iME+MlE0dbY9SpKavIxHhJOUbROLRKQEAFFHVsW5AYI9CtiNwMXlokdOiOUVHHAsHf/LHWgW27u89+D2efvZvz/VTt2uu+fuf04eFdt3elqpAkjWanSRcgSTsSQ1OSGhiaktTA0JSkBoamJDUwNCWpwc6TLuCe2GeffWrlypWTLkMTtGHDBgAOP/zwCVeyMO5tP8+OasOGDf9eVcuHzcuOfJ/mzMxMrV+/ftJlaIKSALAj/x0Purf9PDuqJBuqambYPA/PJamBoSlJDQxNSWpgaEpSA0NTkhoYmpLUwNCUpAaGpiQ1MDQlqYGhKUkNDE1JarBDd9ghzVq5+qIF29bG045bsG3p3seWpiQ1MDQlqYGhKUkNDE1JamBoSlIDQ1OSGhiaktRgbKGZ5INJNie5emDa3knWJrmh/95rYN7rk3wryTeT/N646pKke2KcLc0zgGdsNW01sK6qDgbW9eMkOQR4LvDIfp33JFk2xtokaV7GFppVdSnwH1tNPgFY0w+vAU4cmH52Vd1WVd8FvgU8dly1SdJ8LfY5zf2qahNA/71vP31/4PsDy93UT5OkqTItF4IyZNrQFz8nOTXJ+iTrt2zZMuayJOlXLXZo3pJkBUD/vbmffhNwwMByDwJuHraBqjq9qmaqamb58uVjLVaStrbYoXkhsKofXgVcMDD9uUnum+Qg4GDg8kWuTZLmNLau4ZKcBTwZ2CfJTcCbgdOAc5KcAtwInAxQVdckOQe4FrgDeHlV3Tmu2iRpvsYWmlX1vG3MOnoby78VeOu46pGkhWAnxBq7hewgeJYdBWtSpuXquSTtEAxNSWpgaEpSA0NTkhoYmpLUwNCUpAaGpiQ1MDQlqYGhKUkNDE1JamBoSlIDQ1OSGhiaktTA0JSkBoamJDUwNCWpgaEpSQ0MTUlqYGhKUgNDU5IaGJqS1MDQlKQGhqYkNTA0JamBoSlJDQxNSWpgaEpSA0NTkhoYmpLUwNCUpAaGpiQ1MDQlqYGhKUkNDE1JamBoSlIDQ1OSGhiaktTA0JSkBoamJDUwNCWpwURCM8mfJbkmydVJzkqya5K9k6xNckP/vdckapOk7Vn00EyyP/BKYKaqHgUsA54LrAbWVdXBwLp+XJKmyqQOz3cGdkuyM7A7cDNwArCmn78GOHEypUnSti16aFbVvwJvA24ENgE/rqrPAftV1aZ+mU3AvotdmyTNZRKH53vRtSoPAn4TuF+SFzSsf2qS9UnWb9myZVxlStJQkzg8fxrw3araUlW/AD4OPB64JckKgP5787CVq+r0qpqpqpnly5cvWtGSBJMJzRuBI5PsniTA0cB1wIXAqn6ZVcAFE6hNkrZr58XeYVVdluQ84KvAHcDXgNOB+wPnJDmFLlhPXuzaJGkuix6aAFX1ZuDNW02+ja7VKUlTyyeCJKmBoSlJDQxNSWpgaEpSA0NTkhoYmpLUwNCUpAYTuU9T2lGsXH3Rgm9z42nHLfg2tXhsaUpSA0NTkhoYmpLUwNCUpAaGpiQ1MDQlqYG3HAnw1hppVLY0JamBoSlJDQxNSWpgaEpSA0NTkhoYmpLUwNCUpAaGpiQ1MDQlqYGhKUkNDE1JamBoSlIDQ1OSGhiaktTA0JSkBoamJDUwNCWpwUihmeRR4y5EknYEo7Y035vk8iR/kmTPcRYkSdNspNCsqqOA5wMHAOuT/EOSp4+1MkmaQiOf06yqG4A3Aq8DngT8bZLrk/zBuIqTpGkz6jnNQ5O8A7gOeCrwzKp6RD/8jjHWJ0lTZdRX+L4beD/whqr6+ezEqro5yRvHUpkkTaFRQ/NY4OdVdSdAkp2AXavq/1XVh8dWnSRNmVHPaX4e2G1gfPd+miQtKaOG5q5V9dPZkX549/nuNMmeSc7rLyRdl+RxSfZOsjbJDf33XvPdviSNy6ih+bMkh82OJDkc+Pl2lp/Lu4DPVtXDgUfTXWBaDayrqoOBdf24JE2VUc9pvho4N8nN/fgK4Dnz2WGSPYAnAi8GqKrbgduTnAA8uV9sDXAJ3e1NkjQ1RgrNqroiycOBhwEBrq+qX8xznw8BtgD/J8mjgQ3Aq4D9qmpTv79NSfad5/YlaWxGbWkCHAGs7Nd5TBKq6kPz3OdhwCuq6rIk76LhUDzJqcCpAAceeOA8di9Nh5WrL5rXvO3ZeNpx8y1HIxr15vYPA28DjqILzyOAmXnu8ybgpqq6rB8/jy5Eb0myot/fCmDzsJWr6vSqmqmqmeXLl8+zBEman1FbmjPAIVVV93SHVfVvSb6f5GFV9U3gaODa/rMKOK3/vuCe7kuSFtqooXk18EBg0wLt9xXAmUnuA3wHeAldq/ecJKcANwInL9C+JGnBjBqa+wDXJrkcuG12YlUdP5+dVtWVDD+8P3o+25OkxTJqaL5lnEVI0o5i1FuOvpjkwcDBVfX5JLsDy8ZbmiRNn1Gvnr+U7ir3+/pJ+wPnj6kmSZpaoz5G+XLgCcCtcFeHxN58LmnJGTU0b+sfdwQgyc7APb79SJJ2NKOG5heTvAHYrX830LnAJ8dXliRNp1FDczXd8+LfAP4L8Gm69wVJ0pIy6tXzX9K97uL94y1HkqbbSKGZ5LsMOYdZVQ9Z8IokaYq1PHs+a1e6Rxz3XvhyJGm6jXROs6p+MPD516p6J93reyVpSRn18PywgdGd6FqeDxhLRZI0xUY9PP+bgeE7gI3AHy54NZI05Ua9ev6UcRciSTuCUQ/P/9v25lfV2xemHEmabi1Xz48ALuzHnwlcCnx/HEVJ0rRq6YT4sKr6CUCStwDnVtUfj6swSZpGoz5GeSBw+8D47XRvppSkJWXUluaHgcuTfILuyaCTgPm8vleSdmijXj1/a5LPAP+5n/SSqvra+MqSpOk06uE5wO7ArVX1LuCmJAeNqSZJmlqjvu7izcDrgNf3k3YBPjKuoiRpWo3a0jwJOB74GUBV3YyPUUpagkYNzdurqui7h0tyv/GVJEnTa9TQPCfJ+4A9+zdTfh47JJa0BM159TxJgI8CD6d7G+XDgDdV1dox1yZJU2fO0KyqSnJ+VR0OGJSSlrRRD8+/kuSIsVYiSTuAUZ8IegrwsiQb6a6gh64Reui4CpOkabTd0ExyYFXdCByzSPVI0lSbq6V5Pl3vRt9L8rGqetYi1CRJU2uuc5oZGPZ1vZKWvLlCs7YxLElL0lyH549Ocitdi3O3fhjuvhC0x1irk6Qps93QrKpli1WIJO0IWrqGk6Qlz9CUpAaGpiQ1MDQlqcGoj1FqCqxcfdGCbm/jacct6PakpWBiLc0ky5J8Lcmn+vG9k6xNckP/vdekapOkbZnk4fmrgOsGxlcD66rqYGBdPy5JU2UioZnkQcBxwAcGJp8ArOmH1wAnLnJZkjSnSbU03wm8FvjlwLT9qmoTQP+97wTqkqTtWvTQTPL7wOaq2jDP9U9Nsj7J+i1btixwdZK0fZNoaT4BOL7v0Phs4KlJPgLckmQFQP+9edjKVXV6Vc1U1czy5csXq2ZJAiYQmlX1+qp6UFWtBJ4LfKGqXgBcCKzqF1sFXLDYtUnSXKbp5vbTgKcnuQF4ej8uSVNloje3V9UlwCX98A+AoydZjyTNZZpampI09QxNSWpgaEpSA0NTkhoYmpLUwNCUpAaGpiQ1MDQlqYGhKUkNDE1JamBoSlIDQ1OSGhiaktTA0JSkBoamJDUwNCWpgaEpSQ0MTUlqYGhKUgNDU5IaGJqS1MDQlKQGhqYkNTA0JamBoSlJDQxNSWpgaEpSA0NTkhoYmpLUwNCUpAaGpiQ1MDQlqYGhKUkNDE1JamBoSlIDQ1OSGhiaktTA0JSkBoamJDUwNCWpwc6TLuDeZOXqixZ8mxtPO27Btylp/ha9pZnkgCQXJ7kuyTVJXtVP3zvJ2iQ39N97LXZtkjSXSRye3wH8eVU9AjgSeHmSQ4DVwLqqOhhY149L0lRZ9NCsqk1V9dV++CfAdcD+wAnAmn6xNcCJi12bJM1loheCkqwEHgNcBuxXVZugC1Zg322sc2qS9UnWb9myZdFqlSSYYGgmuT/wMeDVVXXrqOtV1elVNVNVM8uXLx9fgZI0xERCM8kudIF5ZlV9vJ98S5IV/fwVwOZJ1CZJ2zOJq+cB/h64rqrePjDrQmBVP7wKuGCxa5OkuUziPs0nAC8EvpHkyn7aG4DTgHOSnALcCJw8gdokabsWPTSr6ktAtjH76MWsRZJa+RilJDUwNCWpgaEpSQ0MTUlqYGhKUgNDU5IaGJqS1MBOiKV7ITvEHh9bmpLUwNCUpAaGpiQ1MDQlqYGhKUkNDE1JamBoSlIDQ1OSGhiaktTA0JSkBoamJDUwNCWpgaEpSQ0MTUlqYGhKUgNDU5IaGJqS1MDQlKQGhqYkNTA0JamBoSlJDQxNSWpgaEpSA997LqnJUn+nui1NSWpgaEpSgyV3eL7UDy0k3TO2NCWpgaEpSQ0MTUlqYGhKUoMldyFI0nSb9ou1U9fSTPKMJN9M8q0kqyddjyQNmqrQTLIM+N/AMcAhwPOSHDLZqiTpblMVmsBjgW9V1Xeq6nbgbOCECdckSXeZttDcH/j+wPhN/TRJmgqpqknXcJckJwO/V1V/3I+/EHhsVb1iYJlTgVP70YcB3xxzWfsA/z7mfYzKWoazluGsZbhRanlwVS0fNmParp7fBBwwMP4g4ObBBarqdOD0xSooyfqqmlms/W2PtQxnLcNZy3D3tJZpOzy/Ajg4yUFJ7gM8F7hwwjVJ0l2mqqVZVXck+VPg/wLLgA9W1TUTLkuS7jJVoQlQVZ8GPj3pOgYs2qmAEVjLcNYynLUMd49qmaoLQZI07abtnKYkTTVDcytJ9k6yNskN/fdeQ5bZNcnlSb6e5Jok/2OCtRyQ5OIk1/W1vGpStfTLfTDJ5iRXL/D+t/t4bTp/28+/KslhC7n/xloenuSfk9yW5DXjqmPEWp7f/z6uSvLlJI+ecD0n9LVcmWR9kqMmVcvAckckuTPJs0facFX5GfgA/wtY3Q+vBv5qyDIB7t8P7wJcBhw5oVpWAIf1ww8A/gU4ZBK19POeCBwGXL2A+14GfBt4CHAf4Otb/4zAscBn+n+bI4HLxvT3MUot+wJHAG8FXjPGv9VRank8sFc/fMy4fi8N9dyfu08LHgpcP6laBpb7At11lGePsm1bmr/uBGBNP7wGOHHrBarz0350l/4zjpPDo9Syqaq+2g//BLiO8TxFNWctfQ2XAv+xwPse5fHaE4AP9f82XwH2TLJigesYqZaq2lxVVwC/GMP+W2v5clX9sB/9Ct29z5Os56fVpxVwP8bz381ItfReAXwM2Dzqhg3NX7dfVW2CLpDoWg2/JsmyJFfS/bLXVtVlk6ploKaVwGPoWr4TrWWBjfJ47WI9gjtNj/q21nIKXWt8ovUkOSnJ9cBFwB9NqpYk+wMnAe9t2fDU3XK0GJJ8HnjgkFl/Meo2qupO4HeS7Al8Ismjqqr5PN5C1NJv5/50/8d8dVXd2lrHQtYyBhkybesWyijLLFYti2XkWpI8hS40x3YOcdR6quoTdP/NPBH4n8DTJlTLO4HXVdWdybDFh1uSoVlV2/xHSnJLkhVVtak/vNtus72qfpTkEuAZQHNoLkQtSXahC8wzq+rjrTUsZC1jMufjtSMus1i1LJaRaklyKPAB4Jiq+sGk65lVVZcm+a0k+1TVQj+XPkotM8DZfWDuAxyb5I6qOn97G/bw/NddCKzqh1cBF2y9QJLlfQuTJLvR/Z/y+gnVEuDvgeuq6u1jqGHkWsZolMdrLwRe1F9FPxL48ezphAnUsljmrCXJgcDHgRdW1b9MQT0P7f9m6e9wuA8wjiCfs5aqOqiqVlbVSuA84E/mCszZFf386tW03wDWATf033v3038T+HTdfdXva8BVdK3LN02wlqPoDjuuAq7sP8dOopZ+/CxgE91FkJuAUxZo/8fS3RnwbeAv+mkvA17WD4euA+tvA98AZsb4NzJXLQ/sf/ZbgR/1w3tMqJYPAD8c+NtYP67fy4j1vA64pq/ln4GjJlXLVsuewYhXz30iSJIaeHguSQ0MTUlqYGhKUgNDU5IaGJqS1MDQXGKSPDDJ2Um+neTaJJ9O8tuTrmtWkuO31yPNmPb56iS7z2O9d/ZPtUyNJCcmOWRg/G1JnjrJmu5tvOVoCelvKv4ysKaq3ttP+x3gAVX1j4tcy7LqHkVdjH2F7m/9l9uYv5Huvs6Rn0pJsjfd/alHLkB9O1fVHfd0O/22zgA+VVXn9eMPBt5fVb+7ENuXLc2l5inAL2YDE6Cqrqyqf+yfpPnrJFcn+UaS5wAkeXKSS5Kcl+T6JGf2yx6T5JzZ7fTLfbIf/t10/Ul+Ncm5/XPxJNmY5E1JvgScnOSVfWv3qiRn98u8OMm7++EHJ1nXz1/XP91CkjPS9Z355STfyZB+EJOsTNfH6HuArwIHJPm7dH043tUHapJX0t2gf3GSi7dX/1aeDXx2YH8bk/xVun5WL0/y0H768iQfS3JF/3lCP/0tSU5P8jngQ0n2S/KJdH20fj3J4/vlXtBv78ok70uyrJ/+0yRv7Zf9Sr/+44Hjgb/ul/+tqvoe8BtJhvUpoPkY59MBfqbrA7wSeMc25j0LWEvXv+B+wI10fXU+Gfgx3bO7O9E/xUHXb8GNwP369f8OeAHdM7yXDkx/Hf0TU8BG4LUD+7wZuG8/vGf//WLg3f3wJ4FV/fAfAef3w2cA5/b1HELXBdjWP89K4JcM9HPK3U8xLQMuAQ4dqGuffnib9W+1/TXAMwfGN3L3UycvomvtAfwD/VMvwIF0j7sCvAXYAOzWj3+UrrOV2fr+E/CI/newSz/9PcCL+uGa3T9dX6dvHPjdPHurWt8PPGvSf3/3ls+S7LBDQx0FnFXdIfMtSb5I15HurcDlVXUTQLru8FZW1ZeSfBZ4ZpLzgOOA1wJPoguyf+qOirkPXdDO+ujA8FXAmUnOB84fUtPjgD/ohz9MFw6zzq/ucPvaJPtt42f6XnV9a876wySn0gX+ir7Oq7Za58g56p+1Atiy1bSzBr7f0Q8/DTgkd/eis0eSB/TDF1bVz/vhp9KFLf2/wY+TvBA4HLiiX3837u4o5XbgU/3wBuDpQ2qctZmuNa0FYGguLdfQHVYOs72+sW4bGL6Tu/9uPgq8nK7T4Suq6if9+cO1VfW8bWzrZwPDx9H19H488JdJHjlH/YMn4Adr2lbtd+0ryUHAa4AjquqH/bm/XYesM1f9s34+ZP0aMrwT8LiBcJyt51fq24bQnX9+/ZB5v6i+Gcmv/psMs2tfrxaA5zSXli8A903y0tkJ6d6P8iS6Q9LnpOtceTldmF0+x/YuoXu1xUu5uwX5FeAJA+f0ds+Qq/NJdgIOqKqL6Vqoe9K9CmHQl+l6pwF4PvClEX/OYfagC6kf9y3TYwbm/YTuVSEj10/XQ/5Dt5r2nIHv2dbp54A/nV0g3YW3YdYB/7VfZlmSPfppz06ybz9973QXdrZn8GeZ9dvMo9tCDWdoLiF9y+Qk4Onpbjm6hu7c2s3AJ+gOVb9OF66vrap/m2N7d9IdIh7Tf1NVW+jOS56V5Cq6EHr4kNWXAR9J8g26HqPeUVU/2mqZVwIv6bfzQmDeL42rqq/3+7kG+CDwTwOzTwc+k+TihvovojvfO+i+SS7r6/yzgZ9hpr+YdS1dLzvDvAp4Sv/72AA8sqquBd4IfK6vZS3daYHtORv470m+lq6vyl3own39HOtpRN5yJM1TfxfA71fXEfVGGm9bWgxJTqJ78d5fTrqWewtbmtL8/TndFfFptjPwN5Mu4t7ElqYkNbClKUkNDE1JamBoSlIDQ1OSGhiaktTA0JSkBv8fu3w+XYlpBDMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "fig, ax = plt.subplots(figsize=(5, 5))\n",
    "ax.hist(perm_diffs, bins=11, rwidth=0.9) \n",
    "ax.axvline(x=obs_pct_diff, color='black', lw=2) \n",
    "ax.axvline(x=np.mean(perm_diffs), color='black', lw=2)\n",
    "#ax.text(0.06, 200, 'Observed\\ndifference', bbox={'facecolor':'white'}) \n",
    "ax.set_xlabel('Conversion rate (percent)') \n",
    "ax.set_ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2d58395-0902-40aa-a53c-6220ca23fa36",
   "metadata": {},
   "source": [
    "# p-values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c89261b1-05b2-4d8c-9b85-484020aa7cba",
   "metadata": {},
   "source": [
    "How often was the difference >= 0.1479%?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "4d0bb44b-a7ff-494d-806d-cda84e81cdf3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.122"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(np.array(perm_diffs)>0.1479)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb134dd1-f9aa-4cff-b7e3-0be237734ad4",
   "metadata": {},
   "source": [
    "shows that mean difference of random permutations exceeds the observed difference (the vertical line) in   $12.2\\%$  of the cases"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70d02d5b-3a4b-4eb0-882c-5197ed7306ac",
   "metadata": {},
   "source": [
    "## Using scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f7dbce3b-2258-44ab-851e-556a3c91d576",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "04bc4419-fd20-4bd0-bf64-dd0f3367d212",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value for single sided test: 0.1091\n"
     ]
    }
   ],
   "source": [
    "survivors = np.array([[17723, 129479], [17514, 129762]]) \n",
    "chi2, p_value, df, _ = stats.chi2_contingency(survivors) \n",
    "print(f'p-value for single sided test: {p_value / 2:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63df978d-632c-4e33-80d3-c39875bca3b3",
   "metadata": {},
   "source": [
    "The normal approximation yields a p-value of $0.1091$, which is close to the p-value obtained from the permutation test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91792242-228f-4f4e-abcc-ac0a2bb4c8d2",
   "metadata": {},
   "source": [
    "# t-test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bf6c068-6e07-43fe-859d-8a1c650de49b",
   "metadata": {},
   "source": [
    "## session time in minutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "0ecb615f-686b-4fce-b507-339691ec0a8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "54414a33-0cca-44ea-b331-109256ad5cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"timestamp\"] = data[\"timestamp\"].apply(lambda x: x.split(\".\")[0])\n",
    "data[\"timestamp\"] = data[\"timestamp\"].apply(lambda x:\n",
    "                                            datetime.strptime(x,'%Y-%m-%d %H:%M:%S').hour*60+\n",
    "                                            datetime.strptime(x,'%Y-%m-%d %H:%M:%S').minute\n",
    "                                            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "402ac102-c4b3-4dd9-9e09-eb76ff2e3b9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp_min</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>851104</td>\n",
       "      <td>1331</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>804228</td>\n",
       "      <td>481</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>661590</td>\n",
       "      <td>1015</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>853541</td>\n",
       "      <td>1108</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>864975</td>\n",
       "      <td>112</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  timestamp_min      group landing_page  converted\n",
       "0   851104           1331    control     old_page          0\n",
       "1   804228            481    control     old_page          0\n",
       "2   661590           1015  treatment     new_page          0\n",
       "3   853541           1108  treatment     new_page          0\n",
       "4   864975            112    control     old_page          1"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.rename(columns={\"timestamp\":\"timestamp_min\"})\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c79d915b-82ed-438b-b8ba-dcbbba9d2610",
   "metadata": {},
   "source": [
    "## Using scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e0eda1af-235f-4337-a26d-3d5d0e57253c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value for single sided test: 0.4184\n"
     ]
    }
   ],
   "source": [
    "res = stats.ttest_ind(data[data.group == 'control'].timestamp_min,\n",
    "                      data[data.group == 'treatment'].timestamp_min,\n",
    "                      equal_var=False)\n",
    "print(f'p-value for single sided test: {res.pvalue / 2:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0326a3f-11f4-4576-9b0b-c2e158636c2d",
   "metadata": {},
   "source": [
    "Remenber that  mean difference of random permutations (resampling procedure) often exceeds the observed difference in session times  $42.2\\%$ of the cases (see notebook Resampling), using t-test we get $41.84\\%$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48b34682-0a5a-4824-bdc6-83612c913304",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}