Kaggle_Python3_Bronze_Medal_Submission_and_the_Inference_of_LightGBM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.016477,
     "end_time": "2020-12-23T13:52:38.233051",
     "exception": false,
     "start_time": "2020-12-23T13:52:38.216574",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Source Kernel\n",
    "This kernel generates and submits a prediction using the model: [\"LightGBM on GPU with Feature Engineering, Optuna, and , Visualization\"](https://github.com/satorushibata0627/Publishment/blob/main/Kaggle_Python3_LightGBM_on_GPU_with_Feature_Engineering_Optuna_and_Visualization.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T13:52:38.270827Z",
     "iopub.status.busy": "2020-12-23T13:52:38.270146Z",
     "iopub.status.idle": "2020-12-23T13:52:39.249789Z",
     "shell.execute_reply": "2020-12-23T13:52:39.249141Z"
    },
    "lines_to_next_cell": 2,
    "papermill": {
     "duration": 1.001369,
     "end_time": "2020-12-23T13:52:39.249937",
     "exception": false,
     "start_time": "2020-12-23T13:52:38.248568",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from collections import defaultdict\n",
    "import gc\n",
    "from tqdm.notebook import tqdm\n",
    "import riiideducation\n",
    "from scipy import stats\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.015231,
     "end_time": "2020-12-23T13:52:39.281518",
     "exception": false,
     "start_time": "2020-12-23T13:52:39.266287",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "This idea of random seeed generation was based on this notebook: [Riiid model LGBM](https://www.kaggle.com/ragnar123/riiid-model-lgbm)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T13:52:39.319096Z",
     "iopub.status.busy": "2020-12-23T13:52:39.318315Z",
     "iopub.status.idle": "2020-12-23T13:52:39.320686Z",
     "shell.execute_reply": "2020-12-23T13:52:39.321211Z"
    },
    "papermill": {
     "duration": 0.024528,
     "end_time": "2020-12-23T13:52:39.321345",
     "exception": false,
     "start_time": "2020-12-23T13:52:39.296817",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Random seed\n",
    "SEED = 123\n",
    "# Function to seed everything\n",
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "seed_everything(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.015007,
     "end_time": "2020-12-23T13:52:39.352067",
     "exception": false,
     "start_time": "2020-12-23T13:52:39.337060",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Load the model and parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T13:52:39.388225Z",
     "iopub.status.busy": "2020-12-23T13:52:39.387524Z",
     "iopub.status.idle": "2020-12-23T13:52:40.001679Z",
     "shell.execute_reply": "2020-12-23T13:52:40.002568Z"
    },
    "papermill": {
     "duration": 0.635463,
     "end_time": "2020-12-23T13:52:40.002793",
     "exception": false,
     "start_time": "2020-12-23T13:52:39.367330",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load model: <lightgbm.basic.Booster object at 0x7fe42cfdf450>\n"
     ]
    }
   ],
   "source": [
    "model = pd.read_pickle('../input/the-best-of-model-on-lightgbm/The_best_of_model_on_LightGBM.pickle')\n",
    "print('Load model:', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T13:52:40.051027Z",
     "iopub.status.busy": "2020-12-23T13:52:40.050227Z",
     "iopub.status.idle": "2020-12-23T13:52:40.062069Z",
     "shell.execute_reply": "2020-12-23T13:52:40.061310Z"
    },
    "papermill": {
     "duration": 0.03766,
     "end_time": "2020-12-23T13:52:40.062197",
     "exception": false,
     "start_time": "2020-12-23T13:52:40.024537",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters: {'objective': 'binary', 'seed': 123, 'metric': 'AUC', 'boosting_type': 'gbdt', 'lambda_l1': 0.23094442978103413, 'lambda_l2': 0.10970746111051044, 'num_leaves': 206, 'feature_fraction': 0.7548698723252225, 'bagging_fraction': 0.7186994268081899, 'bagging_freq': 7, 'min_child_samples': 45}\n"
     ]
    }
   ],
   "source": [
    "params = pd.read_pickle('../input/the-best-of-all-parameters-on-lightgbm/The_best_of_all_parameters_on_LightGBM.pickle')\n",
    "print('Parameters:', params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.020657,
     "end_time": "2020-12-23T13:52:40.103829",
     "exception": false,
     "start_time": "2020-12-23T13:52:40.083172",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Setting\n",
    "CV files are generated by [this notebook](https://www.kaggle.com/its7171/cv-strategy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T13:52:40.154482Z",
     "iopub.status.busy": "2020-12-23T13:52:40.153534Z",
     "iopub.status.idle": "2020-12-23T13:52:40.155843Z",
     "shell.execute_reply": "2020-12-23T13:52:40.155187Z"
    },
    "papermill": {
     "duration": 0.030866,
     "end_time": "2020-12-23T13:52:40.155983",
     "exception": false,
     "start_time": "2020-12-23T13:52:40.125117",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_pickle  = '../input/riiid-cross-validation-files/cv1_train.pickle'\n",
    "valid_pickle  = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n",
    "question_file = '../input/riiid-test-answer-prediction/questions.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.018706,
     "end_time": "2020-12-23T13:52:40.195880",
     "exception": false,
     "start_time": "2020-12-23T13:52:40.177174",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T13:52:40.247428Z",
     "iopub.status.busy": "2020-12-23T13:52:40.246749Z",
     "iopub.status.idle": "2020-12-23T13:52:40.248606Z",
     "shell.execute_reply": "2020-12-23T13:52:40.249088Z"
    },
    "papermill": {
     "duration": 0.036378,
     "end_time": "2020-12-23T13:52:40.249224",
     "exception": false,
     "start_time": "2020-12-23T13:52:40.212846",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# funcs for user stats with loop\n",
    "def add_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):\n",
    "    acsu = np.zeros(len(df), dtype=np.int32)\n",
    "    cu = np.zeros(len(df), dtype=np.int32)\n",
    "    for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)):\n",
    "        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n",
    "        cu[cnt] = count_u_dict[row[0]]\n",
    "        answered_correctly_sum_u_dict[row[0]] += row[1]\n",
    "        count_u_dict[row[0]] += 1\n",
    "    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n",
    "    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n",
    "    df = pd.concat([df, user_feats_df], axis=1)\n",
    "    return df\n",
    "\n",
    "def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):\n",
    "    acsu = np.zeros(len(df), dtype=np.int32)\n",
    "    cu = np.zeros(len(df), dtype=np.int32)\n",
    "    for cnt,row in enumerate(df[['user_id']].values):\n",
    "        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n",
    "        cu[cnt] = count_u_dict[row[0]]\n",
    "    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n",
    "    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n",
    "    df = pd.concat([df, user_feats_df], axis=1)\n",
    "    return df\n",
    "\n",
    "def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):\n",
    "    for row in df[['user_id','answered_correctly','content_type_id']].values:\n",
    "        if row[2] == 0:\n",
    "            answered_correctly_sum_u_dict[row[0]] += row[1]\n",
    "            count_u_dict[row[0]] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T13:52:40.286264Z",
     "iopub.status.busy": "2020-12-23T13:52:40.285554Z",
     "iopub.status.idle": "2020-12-23T14:02:01.311066Z",
     "shell.execute_reply": "2020-12-23T14:02:01.310463Z"
    },
    "papermill": {
     "duration": 561.04479,
     "end_time": "2020-12-23T14:02:01.311217",
     "exception": false,
     "start_time": "2020-12-23T13:52:40.266427",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "050e2a1c913a4de084f107f63df61876",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=96817414.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4eda9ed6f7c24ad5a2774ee152633b5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2453886.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# read data\n",
    "feld_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']\n",
    "train = pd.read_pickle(train_pickle)[feld_needed]\n",
    "valid = pd.read_pickle(valid_pickle)[feld_needed]\n",
    "train = train.loc[train.content_type_id == False].reset_index(drop=True)\n",
    "valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)\n",
    "\n",
    "# answered correctly average for each content\n",
    "content_df = train[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean']).reset_index()\n",
    "content_df.columns = ['content_id', 'answered_correctly_avg_c']\n",
    "train = pd.merge(train, content_df, on=['content_id'], how=\"left\")\n",
    "valid = pd.merge(valid, content_df, on=['content_id'], how=\"left\")\n",
    "\n",
    "# user stats features with loops\n",
    "answered_correctly_sum_u_dict = defaultdict(int)\n",
    "count_u_dict = defaultdict(int)\n",
    "train = add_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)\n",
    "valid = add_user_feats(valid, answered_correctly_sum_u_dict, count_u_dict)\n",
    "\n",
    "# fill with mean value for prior_question_elapsed_time\n",
    "# note that `train.prior_question_elapsed_time.mean()` dose not work!\n",
    "# please refer https://www.kaggle.com/its7171/can-we-trust-pandas-mean for detail.\n",
    "prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()\n",
    "train['prior_question_elapsed_time_mean'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n",
    "valid['prior_question_elapsed_time_mean'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n",
    "\n",
    "# part\n",
    "questions_df = pd.read_csv(question_file)\n",
    "train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n",
    "valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n",
    "\n",
    "# changing dtype to avoid lightgbm error\n",
    "train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n",
    "valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:01.361045Z",
     "iopub.status.busy": "2020-12-23T14:02:01.359500Z",
     "iopub.status.idle": "2020-12-23T14:02:09.720560Z",
     "shell.execute_reply": "2020-12-23T14:02:09.721716Z"
    },
    "papermill": {
     "duration": 8.390438,
     "end_time": "2020-12-23T14:02:09.721951",
     "exception": false,
     "start_time": "2020-12-23T14:02:01.331513",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "TARGET = 'answered_correctly'\n",
    "FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']\n",
    "dro_cols = list(set(train.columns) - set(FEATS))\n",
    "y_tr = train[TARGET]\n",
    "y_va = valid[TARGET]\n",
    "train.drop(dro_cols, axis=1, inplace=True)\n",
    "valid.drop(dro_cols, axis=1, inplace=True)\n",
    "_=gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.018996,
     "end_time": "2020-12-23T14:02:09.766625",
     "exception": false,
     "start_time": "2020-12-23T14:02:09.747629",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:09.830376Z",
     "iopub.status.busy": "2020-12-23T14:02:09.829281Z",
     "iopub.status.idle": "2020-12-23T14:02:09.832017Z",
     "shell.execute_reply": "2020-12-23T14:02:09.832539Z"
    },
    "papermill": {
     "duration": 0.047138,
     "end_time": "2020-12-23T14:02:09.832680",
     "exception": false,
     "start_time": "2020-12-23T14:02:09.785542",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Iter_Valid(object):\n",
    "    def __init__(self, df, max_user=1000):\n",
    "        df = df.reset_index(drop=True)\n",
    "        self.df = df\n",
    "        self.user_answer = df['user_answer'].astype(str).values\n",
    "        self.answered_correctly = df['answered_correctly'].astype(str).values\n",
    "        df['prior_group_responses'] = \"[]\"\n",
    "        df['prior_group_answers_correct'] = \"[]\"\n",
    "        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n",
    "        self.sample_df['answered_correctly'] = 0\n",
    "        self.len = len(df)\n",
    "        self.user_id = df.user_id.values\n",
    "        self.task_container_id = df.task_container_id.values\n",
    "        self.content_type_id = df.content_type_id.values\n",
    "        self.max_user = max_user\n",
    "        self.current = 0\n",
    "        self.pre_user_answer_list = []\n",
    "        self.pre_answered_correctly_list = []\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "    \n",
    "    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n",
    "        df= self.df[pre_start:self.current].copy()\n",
    "        sample_df = self.sample_df[pre_start:self.current].copy()\n",
    "        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n",
    "        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n",
    "        self.pre_user_answer_list = user_answer_list\n",
    "        self.pre_answered_correctly_list = answered_correctly_list\n",
    "        return df, sample_df\n",
    "\n",
    "    def __next__(self):\n",
    "        added_user = set()\n",
    "        pre_start = self.current\n",
    "        pre_added_user = -1\n",
    "        pre_task_container_id = -1\n",
    "        pre_content_type_id = -1\n",
    "        user_answer_list = []\n",
    "        answered_correctly_list = []\n",
    "        while self.current < self.len:\n",
    "            crr_user_id = self.user_id[self.current]\n",
    "            crr_task_container_id = self.task_container_id[self.current]\n",
    "            crr_content_type_id = self.content_type_id[self.current]\n",
    "            if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n",
    "                # known user(not prev user or (differnt task container and both question))\n",
    "                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "            if len(added_user) == self.max_user:\n",
    "                if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n",
    "                    user_answer_list.append(self.user_answer[self.current])\n",
    "                    answered_correctly_list.append(self.answered_correctly[self.current])\n",
    "                    self.current += 1\n",
    "                    continue\n",
    "                else:\n",
    "                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "            added_user.add(crr_user_id)\n",
    "            pre_added_user = crr_user_id\n",
    "            pre_task_container_id = crr_task_container_id\n",
    "            pre_content_type_id = crr_content_type_id\n",
    "            user_answer_list.append(self.user_answer[self.current])\n",
    "            answered_correctly_list.append(self.answered_correctly[self.current])\n",
    "            self.current += 1\n",
    "        if pre_start < self.current:\n",
    "            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "        else:\n",
    "            raise StopIteration()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.018387,
     "end_time": "2020-12-23T14:02:09.870781",
     "exception": false,
     "start_time": "2020-12-23T14:02:09.852394",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Prediction and submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:09.912056Z",
     "iopub.status.busy": "2020-12-23T14:02:09.911080Z",
     "iopub.status.idle": "2020-12-23T14:02:09.915800Z",
     "shell.execute_reply": "2020-12-23T14:02:09.916391Z"
    },
    "papermill": {
     "duration": 0.027025,
     "end_time": "2020-12-23T14:02:09.916524",
     "exception": false,
     "start_time": "2020-12-23T14:02:09.889499",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "env = riiideducation.make_env()\n",
    "iter_test = env.iter_test()\n",
    "set_predict = env.predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:09.957575Z",
     "iopub.status.busy": "2020-12-23T14:02:09.956601Z",
     "iopub.status.idle": "2020-12-23T14:02:10.134403Z",
     "shell.execute_reply": "2020-12-23T14:02:10.135106Z"
    },
    "papermill": {
     "duration": 0.200071,
     "end_time": "2020-12-23T14:02:10.135290",
     "exception": false,
     "start_time": "2020-12-23T14:02:09.935219",
     "status": "completed"
    },
    "scrolled": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "previous_test_df = None\n",
    "for (test_df, sample_prediction_df) in iter_test:\n",
    "    if previous_test_df is not None:\n",
    "        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n",
    "        update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)\n",
    "    previous_test_df = test_df.copy()\n",
    "    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n",
    "    test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)\n",
    "    test_df = pd.merge(test_df, content_df, on='content_id',  how=\"left\")\n",
    "    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n",
    "    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n",
    "    test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)\n",
    "    # Prediction with LightGBM\n",
    "    test_df[TARGET] = model.predict(test_df[FEATS], num_iteration=model.best_iteration)\n",
    "    # Submit\n",
    "    set_predict(test_df[['row_id', TARGET]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.023817,
     "end_time": "2020-12-23T14:02:10.183778",
     "exception": false,
     "start_time": "2020-12-23T14:02:10.159961",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Empirical analysis\n",
    "- [Bootstrap](https://qiita.com/katsu1110/items/23b8a37ac400dd0eaad2#%E3%83%96%E3%83%BC%E3%83%88%E3%82%B9%E3%83%88%E3%83%A9%E3%83%83%E3%83%97%E3%81%A8%E3%81%AF)\n",
    "- [Histogram](https://seaborn.pydata.org/generated/seaborn.distplot.html)\n",
    "- [統計的有意差検定](https://qiita.com/suaaa7/items/745ac1ca0a8d6753cf60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:10.244830Z",
     "iopub.status.busy": "2020-12-23T14:02:10.244030Z",
     "iopub.status.idle": "2020-12-23T14:02:10.251056Z",
     "shell.execute_reply": "2020-12-23T14:02:10.251640Z"
    },
    "papermill": {
     "duration": 0.043465,
     "end_time": "2020-12-23T14:02:10.251835",
     "exception": false,
     "start_time": "2020-12-23T14:02:10.208370",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.403990\n",
       "1    0.888715\n",
       "2    0.747603\n",
       "3    0.857809\n",
       "4    0.356513\n",
       "Name: answered_correctly, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simulation = pd.read_csv('./submission.csv')\n",
    "simulation = simulation['answered_correctly']\n",
    "simulation.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:10.315807Z",
     "iopub.status.busy": "2020-12-23T14:02:10.315010Z",
     "iopub.status.idle": "2020-12-23T14:02:10.319744Z",
     "shell.execute_reply": "2020-12-23T14:02:10.320452Z"
    },
    "papermill": {
     "duration": 0.04373,
     "end_time": "2020-12-23T14:02:10.320607",
     "exception": false,
     "start_time": "2020-12-23T14:02:10.276877",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Bootstrap to compute SEM of the median\n",
    "def bootstrap(x, repeats):\n",
    "    # Compute mean, SEM(Standard Error of the Mean) and, median\n",
    "    mean_x = np.average(x)\n",
    "    sem_x = np.std(x)/np.sqrt(len(x))\n",
    "    median_x = np.median(x)\n",
    "    # Placeholder (column1: mean, column2: median)\n",
    "    vec = np.zeros((2, repeats))\n",
    "    for i in np.arange(repeats):\n",
    "        # Resample data with replacement\n",
    "        re = np.random.choice(len(x),len(x),replace=True)\n",
    "        re_x = x[re]\n",
    "        # Compute mean and median of the \"new\" dataset\n",
    "        vec[0,i] = np.mean(re_x)\n",
    "        vec[1,i] = np.median(re_x)\n",
    "    # Histogram of median from resampled datasets\n",
    "    sns.distplot(vec[1,:], kde=True, rug=True, hist=True, norm_hist=True, axlabel='Frequency distribution')\n",
    "    plt.show()\n",
    "    # Compute bootstrapped standard error of the mean and standard error of the median\n",
    "    b_mean_sem = np.std(vec[0,:])\n",
    "    b_median_sem = np.std(vec[1,:])\n",
    "    return b_mean_sem, b_median_sem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:10.373355Z",
     "iopub.status.busy": "2020-12-23T14:02:10.372356Z",
     "iopub.status.idle": "2020-12-23T14:02:10.375962Z",
     "shell.execute_reply": "2020-12-23T14:02:10.375432Z"
    },
    "papermill": {
     "duration": 0.035954,
     "end_time": "2020-12-23T14:02:10.376104",
     "exception": false,
     "start_time": "2020-12-23T14:02:10.340150",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def Statistical_significance_test(A, B):\n",
    "    A = np.array(A)\n",
    "    B = np.array(B)\n",
    "    # Paired t-test\n",
    "    print('Paired t-test:', stats.ttest_rel(A, B))\n",
    "    # Student's t-test\n",
    "    A_var = np.var(A, ddof=1)  # Aの不偏分散\n",
    "    B_var = np.var(B, ddof=1)  # Bの不偏分散\n",
    "    A_df = len(A) - 1  # Aの自由度\n",
    "    B_df = len(B) - 1  # Bの自由度\n",
    "    f = A_var / B_var  # F比の値\n",
    "    one_sided_pval1 = stats.f.cdf(f, A_df, B_df)  # 片側検定のp値 1\n",
    "    one_sided_pval2 = stats.f.sf(f, A_df, B_df)   # 片側検定のp値 2\n",
    "    two_sided_pval = min(one_sided_pval1, one_sided_pval2) * 2  # 両側検定のp値\n",
    "    print(\"Student's t-test:\")\n",
    "    print('F:       ', round(f, 3))\n",
    "    print('p-value: ', round(two_sided_pval, 3))\n",
    "    # Welch's t-test\n",
    "    two_sided_pval = min(one_sided_pval1, one_sided_pval2) * 2  # 両側検定のp値\n",
    "    print(\"Welch's t-test:\")\n",
    "    print('F:       ', round(f, 3))\n",
    "    print('p-value: ', round(two_sided_pval, 4))\n",
    "    # Mann-Whitney's U-test\n",
    "    print(\"Mann-Whitney's U-test:\", stats.mannwhitneyu(A, B, alternative='two-sided'))\n",
    "    # Wilcoxon signed rank sum test\n",
    "    print(\"Wilcoxon signed rank sum test:\", stats.wilcoxon(A, B))\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T14:02:10.422435Z",
     "iopub.status.busy": "2020-12-23T14:02:10.421296Z",
     "iopub.status.idle": "2020-12-23T14:04:00.231039Z",
     "shell.execute_reply": "2020-12-23T14:04:00.230125Z"
    },
    "papermill": {
     "duration": 109.834654,
     "end_time": "2020-12-23T14:04:00.231239",
     "exception": false,
     "start_time": "2020-12-23T14:02:10.396585",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEGCAYAAAB8Ys7jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xcdXn48c8z970lm002IVfCHQEhSAAp2npBpbbWy8+K2lpqbaH91V/196r9vai1Vm1taeulWquVWtRWsWgRr0hFBBGEQIKRAAFyIQkJSXazm2Svc39+f5xzZmdm53Jmb7Mz53m/XvvamTPnzHxncubZJ8/5XkRVMcYY03pCzW6AMcaYmbEAbowxLcoCuDHGtCgL4MYY06IsgBtjTIuKLOSLrVixQjdu3LiQL2mMMS1v27Ztx1S1v3z7ggbwjRs3snXr1oV8SWOMaXkisr/SdiuhGGNMi7IAbowxLcoCuDHGtCgL4MYY06IsgBtjTIuyAG6MMS3KArgxxrQoC+DGGNOiLIAb08I+9j9P87+/uq3ZzTBNsqAjMY0xc+vRA8d57vhEs5thmsQycGNa2PB4msl0rtnNME1iAdyYFjY0nmbCAnhgWQnFmBalqhwfT5PNK/m8EgpJs5tkFphl4Ma0qJHJLNm8syh5MmtZeBBZADemRR0bTxVuWxklmCyAG9OihsfThdt2ITOYLIAb06KGxqYCuGXgwWQB3JgWVZyBj6ezTWyJaRYL4Ma0qOGiGriVUILJArgxLWpo3EooQVc3gItIQkQeFpFfiMgTIvJhd/uHROSQiGx3f147/801xniGx9OI2/V7wkoogeRnIE8KeIWqjolIFLhfRH7gPvZJVf3Y/DXPGFPN0FiaVT0JjowkrYQSUHUzcHWMuXej7o/Oa6uMMXUNjadZ39cBWAklqHzVwEUkLCLbgQHgLlXd4j70bhF5TERuFpFlVY69TkS2isjWwcHBOWq2MWZ4PMW6ZZ2AlVCCylcAV9Wcqm4C1gGXicgFwOeAM4BNwGHg41WOvUlVN6vq5v7+/jlqtjHBpqoMj6dZtSRBOCSWgQdUQ71QVPUEcC9wtaoedQN7Hvg34LJ5aJ8xpoLRVJZMTlneFaMzFrYAHlB+eqH0i0ive7sDuAp4SkRWF+32RuDx+WmiMabcsDsKs88N4HYRM5j89EJZDXxZRMI4Af/rqvo9EflPEdmEc0FzH3D9/DXTGFPM6wPe1x2jMxZhImMBPIjqBnBVfQy4uML2d8xLi4wxdQ2NOaMwV3TF6YiGmbSLmIFkIzGNaUHDJRl4mPGUZeBBZAHcmBbklVCWd8XoiIWthBJQFsCNaUHD42k6Y2ES0TBdsYiVUALKArgxLWh4PE1fVwzAuhEGmAVwY1rQ0Hia5W4A77BuhIFlAdyYFjQ0lirJwG1Bh2CyAG5MCxoeT7O8Ow5ARyxCMpMnn7c55oLGArgxLUZVS0ooXbEwAJPWEyVwLIAb02Im0jnS2TzLikoo3nYTLBbAjWkxI8kMAEs7ooBTQgFbFzOILIAb02JGk84Fy56EE7gLGXjGLmQGjQVwY1qMF8C7404A73ADuA2nDx4L4Ma0mLFUWQYedS9iWgklcCyAG9NixgolFKcG3uVm4rasWvBYADemxYy6FzHLSyjWjTB4LIAb02K8Ekp3+UVMK6EEjgVwY1qMdxGzK+bVwJ3f4ykroQSNBXBjWsxoMkt3PEI4JEBRCcUy8MDxs6hxQkQeFpFfiMgTIvJhd3ufiNwlIrvc38vmv7nGmLFUplD/BohFQkRCYos6BJCfDDwFvEJVLwI2AVeLyIuBG4C7VfUs4G73vjFmno2lsoX6t8dWpg+mugFcHWPu3aj7o8DrgS+7278MvGFeWmiMKTGazBb6gHs6YxHrRhhAvmrgIhIWke3AAHCXqm4BVqnqYQD398oqx14nIltFZOvg4OBctduYwPJq4MVsVZ5g8hXAVTWnqpuAdcBlInKB3xdQ1ZtUdbOqbu7v759pO40xrrHU9Ay8wwJ4IDXUC0VVTwD3AlcDR0VkNYD7e2DOW2eMmWasagZuJZSg8dMLpV9Eet3bHcBVwFPAd4Br3d2uBb49X400xkwZTWYKw+g9HbGIXcQMoEj9XVgNfFlEwjgB/+uq+j0ReRD4uoi8CzgA/OY8ttMYA+Tyyng6Ny0D74qFOXzCAnjQ1A3gqvoYcHGF7UPAK+ejUcaY6W7ZcoCk29d719FRbtlyAIC3X77BauABZSMxjWkhXgBPuFPIejpjYZvMKoAsgBvTQpLZPADxaQE8YnOhBJAFcGNaSMrNsuOR0q9uRzRMKpsnl9dmNMs0iQVwY1pIys3AE2UBvCtuc4IHkQVwY1qIVwMvL6F4K9NbX/BgsQBuTAtJZdwMvLwGbutiBpIFcGNaSDJbuQbeaSvTB5IFcGNaSDKTR3DmAC82tS6mlVCCxAK4MS0klc0Ri4QIiZRs90oqXonFBIMFcGNaSCqTn1b/hqIAnrUAHiQWwI1pIclsblr9G6Zq4knrRhgoFsCNaSHVMnAvgFsGHiwWwI1pIdUy8KkSimXgQWIB3JgWksrkpw3igeISimXgQWIB3JgWkszmpg2jh6mRmZaBB4sFcGNaSNVeKF4N3DLwQLEAbkyLyKuSzuUr1sAj4RDhkBRGappgsABuTIvwsutKNXBw6uCWgQeLBXBjWoSXXVeqgYPTE8W6EQaLn1Xp14vIPSKyU0SeEJH3uNs/JCKHRGS7+/Pa+W+uMcHlJwO3gTzB4mdV+izwp6r6qIj0ANtE5C73sU+q6sfmr3nGGE9hPcwqGXg8ErIMPGD8rEp/GDjs3h4VkZ3A2vlumDGmlNdFsFIvFIB4JGzdCAOmoRq4iGwELga2uJveLSKPicjNIrKsyjHXichWEdk6ODg4q8YaE2SFBY2r1sAtAw8a3wFcRLqB24D3quoI8DngDGATTob+8UrHqepNqrpZVTf39/fPQZONCaZqNfBbthzgli0HODmZYf/QROG+aX++AriIRHGC91dV9ZsAqnpUVXOqmgf+Dbhs/pppjElVWY3HEwmHyOYsAw8SP71QBPh3YKeqfqJo++qi3d4IPD73zTPGeDJucI6GK39toyEhm9eFbJJpMj+9UK4E3gHsEJHt7rb3A28TkU2AAvuA6+elhcYYALI5JSQQDknFxyPhEJmcBfAg8dML5X6g0hlzx9w3xxhTTTavVYM3QCQkZPNWQgkSG4lpTIvI5vNEQtW/stFwiKxl4IFiAdyYFpHNKZFwjQw8LIU6uQkGC+DGtIhsXonULKGE7CJmwFgAN6ZFOAG8VglFyOWVvFoQDwoL4Ma0iGwuX6eEEnL3swAeFBbAjWkRubolFOcx64kSHBbATVtKZnJ84Fs7GBpLNbspcyaT00KWXUnUMvDAsQBu2tKTh0f4ykMHuH/3sWY3Zc7k8vnaGXjYy8AtgAeFBXDTlkYmMwCcmMg0uSVzp34vFOcx60oYHBbATVsaSWaBNgvgVkIxZSyAm7ZUyMAn001uydzJ+i6hWAYeFBbATVsaSbZpCaVWN0K3j7hNaBUcFsBNWxqZ9EoobZSB55RwnYE8zn6WgQeFBXDTlgoZ+GQ7ZeB5ojVLKG4Gbr1QAsMCuGlLbdkLJaeEa5RQvOBuGXhwWAA3bWmqF0p7lFCyuTwKNedCsaH0wWMB3LSlk24GfnIyQ77BksITz59k7+DYfDRrxrzV5mv1QvEy8Iz1QgkMC+CmLY26ATyvMOpm437dcNsOPvzdJ+ejWTOW9gK4TWZlivhZ1Hi9iNwjIjtF5AkReY+7vU9E7hKRXe7vZfPfXGP8GUlmSESd07vRvuCjyQz7h8bno1kzNpWB1yqhWAYeNH4y8Czwp6r6AuDFwB+LyHnADcDdqnoWcLd735imU1VGJrNs6OsEGr+QOZnJcejEJLlF1JvDTwYeEiEsYhl4gNQN4Kp6WFUfdW+PAjuBtcDrgS+7u30ZeMN8NdKYRqSyedK5fCGAH2/wQuZkOkcmpxwZSc5H82Yklc0BtWvg4AR464USHA3VwEVkI3AxsAVYpaqHwQnywMoqx1wnIltFZOvg4ODsWmuMD14Xwg19XcDUBU2/khknAB4Ympjbhs2CnxKK87hYP/AA8R3ARaQbuA14r6qO+D1OVW9S1c2qurm/v38mbTSmId4gng19HUBjJZRszsneAZ47vggDeI0SCtjK9EHjK4CLSBQneH9VVb/pbj4qIqvdx1cDA/PTRGMac9IdRr9+BiWUyUyucPu54cUTwNM+uhGCW0Kxi5iB4acXigD/DuxU1U8UPfQd4Fr39rXAt+e+ecY0zsvAl3XF6ElEGsrAF2sA910DD4VsMqsAifjY50rgHcAOEdnubns/cCPwdRF5F3AA+M35aaIxjfFq4EsSUXo7ow3VwJPpqez1wCIK4FO9UGrnXFG7iBkodQO4qt4PVPuz/8q5bY4xs+cNo1/SEaG3IzajEkpPPMJzxyfnpX0z4dXAw3VLKCFbUi1AbCSmaTvlGfhMSihnn9LD4GiKyXSuzhELw8vAo3Uy8EhIbEm1ALEAbtrOyGSGeCREIhqmtzPWUAnFC9hnr+oB4OAi6YniNwO3XijBYgHctJ2RZIYlHVEAlnVGGyqhJN0M/JxV3cDiqYN7FzFrzQcO1gslaCyAm7YzMpllScK5vNPbEW1oRsKJ9FQJBRZPTxSvhFJrPnCwXihBYwHctJ3iDHxpZwxtYEZCrwa+rreTjmiYA8ONX8jM5vK86bMPcMeOww0fW43fkZjWCyVYLICbtjMymWFJYqqEAv4H83gBvCMWZkNf54xGYz53fJJHD5zgk3c9g+rcZMPpbB7BRy+UkFgvlADx0w/cmJYyksyyYbkzD0qvG8DrrY15y5YDAPxs9zEAvveL5xGZWQllz4CzGMSugTF+tmeIK89c0fBzlEtlc3WH0YPTjdB6oQSHZeCm7TgZuJObLO2IAf4zcC/4RcIh+rpiPDc80XAWvffYmPvaUb74wL6Gjq0mnc3XLZ+AU0LJK4tqKlwzfyyAm7aiqoUa+C1bDvDALiej/sGOw4Usu5ZMTgmHhHBI6OuKMZ7OMTCaaqgNewbGWd4V49orTuXup47OyeIQqWzeXwbuBnkL4MFgAdy0lWQmTyanhRp4RywMTPUuqSedyxN1A+X6Zc5kWNv2H2+oDXuPjXFGfze/9eJTCYvwHw/ub+j4iu3K5uvOgwJTsxXahcxgsABu2oo3kdWSDqeEkog2FsAz2Twxd7Tjmt4OOqJhHn52uKE27B0c5/T+LlYtSfDyc1fywyePNHR8JSnfJRRnH5sTPBgsgJu2UjyMHpxeG4loyPeQeCcDDxWOveTUZQ0F8BMTaYbG05ze71xEXb00wchkY4sqV+K/hGIZeJBYADdtZSoDjxa2dcYiJdPE1pLJKbHI1NfistP62HlkxPdw/D2DTr37jH5nJGd3PMJ4Kjvr7oSpbM5nCcUy8CCxboSmrZwsZOARDrnbOqJhJtL+suBMUQYOTgBXhW37h3nFuasK26tdEPXq56e7AbwrHiGbV1LZfKGcMxPpbL7uVLLFr28ZeDBYBm7aileuKM7A49EQqYy/gJbJTl3EBNi0vpdoWNjis4yyZ3CcaFhYv8xZzq3H7c44lppdGSXl9yKmWye34fTBYBm4aSuFEkpiKoAnImGGU/77gXfEio6NhrloXa/vOvh9zwzS2xnj61sPArDj4EkAxpJZVnTHfT1HJX57oRQycJvQKhAsAzdtpXARs2MqN4lHQoXZ/OpJ53TanNuXndbHjoMnfV0IHRxN0V8UqONuPb1WBp7N5bn25od5cM9Q1X1S2RxhHyUUr8xiU8oGgwVw01ZGklkS0RDxyFS9OR4NkfRbQsnlSy5ighPAs3nl5wdq9wfP5ZXh8TT9PUUB3K171wrgwxNpfvLMYM0sP53L151KFqZ6odhw+mCwAG7aymgyQ3c8WrItHgmTzuZ99QRJZ/PTMvBLTl2GCGytM6Dn+ESanGpJqaSQgdeYDdGbKXEsVb2nSyqTrzuRFUz1A7cJrYLBz6r0N4vIgIg8XrTtQyJySES2uz+vnd9mGuPPRDpHZ6y0t0c8EiKn6iuoZXJ5YmX9rXsSUdYt62CXO0kVOH8oPv+TPSUr9gy6Q+77u2NFr+20ZbxGLxiv7FMrS0/n/PVC8fqKWwYeDH4y8C8BV1fY/klV3eT+3DG3zTJmZioGcLeM4c2pXU3eDfKV1p08s7+b3UUB/Nlj4+wfnuCWLQeYSGXJ5PL8dNcgkZCwckli6rXdDLzWfOTeIswjNfZJZXxexAxZDTxI/KxKf5+IbJz/phgze5PpXGH+E48XRFN1BvN4Qa9iAF/Zzc/2DJHLO5NdHRlJIsBoKsvXtz1HWIT9QxNcc+n6kv7e8ajzXOM1sutCBl4jgDsZeANzoVgJJRBmUwN/t4g85pZYllXbSUSuE5GtIrJ1cHBwFi9nTH0T6ey0DDzhBfA6GXjaLTtEI9O/Fmf0d5PK5nn+hLNCz9GTSVb0xPm1F67mmaNj7DwyyusuWsOF63pLjouFQwi1yyNTNfDK+2RzeXJ59ZWBh+0iZqDMNIB/DjgD2AQcBj5ebUdVvUlVN6vq5v7+/hm+nDH+TKRzdERL/2Ppt4TiBb3yGjg4GThQKKMcGUlyypIEl5/Wx8vPWcnrLlrDi09fPu04ESEWCdUM4F7f9dFk5YuY3h8WP5NZhUScVXmshBIIMwrgqnpUVXOqmgf+DbhsbptlzMxMZipfxIT6JZSMG+CrlVDACeBjqSzHJzKsWpJARHjVeau4okLw9iSi4ZrlkXolFG8UqZ8SirefDeQJhhkFcBFZXXT3jcDj1fY1ZiFV7oXiLwNPFzLw6V+L3s4YK7pj7B4Y4+kjowCcUnSxshb/GXjlfRrJwL39LAMPhroXMUXka8DLgBUichD4K+BlIrIJUGAfcP08ttEY32pdxEzWGY3pzR9SqQYOTh1892BRAF/qL4An6gTwQg08nSWfV0Jlte5CBu6jBg7OcPq01cADwU8vlLdV2Pzv89AWY2ZFVStexPR6gtSb0MqrgVcqoYBTRvneY4d5+sgIsXCosGByPfFouHYG7pZQVGEik6M7Xvq1TOecPzx+SyidsYjv2RdNa7ORmKZtpLJ58uoEsGJeT5C6JZRCDbxyoDxzZTcnJzM8sGeIVUvihMRfQI1HQrW7ERaVTipdyEw2mIF3xcO+VyAyrc0CuGkb3mRTHWXzbns9QepNaJWpUQOHqUUadg+Mscpn/RucGny9i5je34JK+xVq4D5GYoKXgVsADwIL4KZtTLi9TMpLKOD0BKnfjbD6QB6Y6okC/uvf4GTg1Uoot2w5wNGRZKFsctujh6YtFuGVfvzMhQLO+6+V8Zv2YQHctI1Jt+7bGZ9+aScWCdXvRuhl4FUuYq5emqDL/ePQUAYedQJ4tcm0kpk8ve4CFMkKbSwMMPIdwCOksvlCSci0Lwvgpm14ZYPOCkuXJSIh/yMxq2TgIsIZbhbutwshOCWUvFJxXc5cXknn8vR2OhNgVWqj94fHfwnFef8nJv0tYmFalwVw0zYKAbxCCSXup4SSzROS2qWK89csZd2yDroqZPnV1FrUwcu4/WTgfksoXtuOj/tbiNm0Lgvgpm0ULmJWCuCRUMXgWKx8QeNK/vy15/L1669oqF2JaPU5wb02LXW7JFYq83g18Hpt83h/wIbHLQNvd7YmpmkbUxn49NPaW9ShlnROq/ZA8SxJREvW2/SjMCd4anpw9roILvUy8AptbDQDL5RQJiyAtzvLwE3b8AavVCyhREI+RmLmq47CnI3CnOAVVtyZzEz90al2odXb5vciZpf7B2zYAnjbswBu2sZErRJKNEQqU3tZNaeE4i9INsLLwGuVUBLREIlIqHYG7nskpvN6x62E0vYsgJu2UfMiZiSMQs3FjZ3l1OYhA/cWdagwvH0qgIedC601auC+J7MKh4hFQgzbRcy2ZwHctA2vH3giUrmEApXLGJ5KCxrPhVoLG3sBvCMarpmB1+sdU64rFrYaeABYADdtw1nMITxtNj+Y6glS6UKiJ5OrvB7mbBVKKBVeezKTR3AGDyWqZeDZfOE5/OqMRawGHgAWwE3bmKiwmIOnVh3ak56ni5jRsBAOCWMVsv9kJkc8GiIk4nZ1rJCBZ/NVR4dW0xkLWw08ACyAm7ZRaS5wj58SilMDn/uLmCJCVyxcpRthrrAIsjNfS6UMPFdov19d8QjHJ6wG3u6sH7hpG5XmAvdU64v9hZ/uZe+xcS5Ys5TMPNXAAXoS0Yor7iQzucLsifEqNfDUDDPwPQOWgbc7y8BN25hI5+ioMIgHpnqCFJcxvvLQfv7m+zu5ZcsBBkaT81YDB+iORyqWUCYz+UIGHo86g43yZV0dnRp44wF8NJW1Ca3anAVw0zYm07mKE1lB8XwkTgZ+5+OH+ctvP84Vpy9HBB47eNLXUPqZ6or7K6HA9JWDnBp44xcxwSa0and+1sS8Gfh1YEBVL3C39QG3Ahtx1sR8i6oen79mGlPfRDrH6qWVh7l7JZT7dx1D88rf3LGTdb0dvOb8Uzg6kuTRA8dRqk8lO1vdiSgnJytfxEy4r+n9Lq+DzyQDL57QamWP/5kTTWvxc1Z8Cbi6bNsNwN2qehZwt3vfmKaazFS/iBkNCyFxguORkSTpbJ6XnNVPLBLionW9nHAv+M3HSEyA7njlRRaS2RyJ2FQJBaYPNkpnczOqgYNNaNXu6p4VqnofMFy2+fXAl93bXwbeMMftMqZhtS5iFpZVy+Q5dGISgLW9HQCcv3YJYXdNs/kYiQluDbzsImY+r6Qy+cJFzLnMwG1Cq2CY6dm6SlUPA7i/V85dk4yZmYl0ruJMhJ5ExOmm9/yJJB3RMMvcKVw7YxHOWuUs1DB/NfDItAx8NJVFoeQiJlTKwGdQQrEJrQJh3rsRish1wHUAGzZsmO+XMwFWqx84uBNaZfOcmEixpjeBFK0qf+G6Xp46MjqtVFG+PuVM9cQjjKWz5PNaGCk64tbEy2vg5bMmzmwkpk1oFQQzTTeOishqAPf3QLUdVfUmVd2sqpv7+/tn+HLG1JbO5snmtWovFHAuZI6nnBq4Vz7xvHDtUl534eqShYvnUncigurUwstAoV94eQZe3gtlMp0rdIP0KxIO0RUL24RWbW6mAfw7wLXu7WuBb89Nc4yZmVqr8XjikRCHTkyQyytrygJ4OCRcccaKeS2hACVllJGkE1y9NnvztZSvHDQymaG3I9bway7rilkNvM3VPVtF5GvAg8A5InJQRN4F3Ai8SkR2Aa9y7xvTNBMZJzDWWqsyHgmRyTmDZMoz8PnW7bareDRmoYTiZt6xcAih9CJmJpdnNJUtrNjTiGWdMauBt7m6NXBVfVuVh145x20xZkZu2XKAwdEUAD8/cIJqazbEo1OZbl9X4xntbHRXzMCd214vFBEhHi0dTu8F+d7OGQTwrpjVwNucjcQ0bcFbtaZWN0DvIuGapR0lFzAXQo+7jqZXNoHpFzGd26VTyp6cRQDv64zahFZtzgK4aQvenB+1Brx4w9EXunwCsKLbyfi9/ykAHBtLERIKA3nA6SlT3I3whBvAl8yghNLbaRl4u7MAbtpCppCBV8+svYuEa5YtfABfucQZzj5QFMCPjqToSUQJFf1vIB4pnVL2pJtB984ggC/vijGayk67KGrahwVw0xa8DLzWggzeBc71yzoXpE3FuuMRumJhBkamAvjAaJKeROllqMS0DNzJoHs7G6/Zn7qiC4B9Q+MzabJpATYfuGkLfmrgF65dSn93fMEvYHpWLklwdDRZuD/gZuDFEtEwQ2NTZQ8vA59JL5Qz+p0AvntgjHNPWTKTJptFzjJw0xb81MAj4RDr+xY++/b098QZLMvAl5Rl4F3xCGNFPVUKNfBE47nW6Su6EYE9A5aBtysL4KYtZHxk4M22akmCATcDT2VzHJ/ITMvAe+IRUtl8oW59YiJDTyJCZAbvqyMWZm1vB7sHx2bfeLMoWQnFtIWUjxp4M92y5QDDYykOnZjkqw/tL0xfW55Ze/3Fj42lWLesk5HJzIzKJ54zV3azZ8ACeLtanGe7MQ3KZPNEQlLSo2OxWdIRJZNTUtk8o25/8PIMfCqAO3XwE5OZGfUB95zR383eY2Pk81VGN5mWZhm4aQvpXOML/y40r8fJaDJbGIVZ3gul271/27aDPPn8CLsHxoiFQzOaFdHJ+tMkM3k+d+8elrkXb99+uc0K2i4W9xlvjE/pbH5R17+hdDSml4GXD9DxMnBv8YfJdK5koE+jVvTEgdL+56Z9LO4z3hif0rn8oq1/e8oz8JAwbQUhr6/6qNsTZSJTfaFmP1a6AXxwzAJ4O1rcZ7wxPmVyiz8DX+Jm4KPJDKPJ7LRRmOCsCJSIhhhLZVFVJtPZmlPk1tMVj9AZCzNY1P/ctI/FfcYb41M6u/hr4PFIiGhYGE1mGU1mptW/Pd3xKGOpLOlsnrxOzVY4U/098ZI5WEz7WNxnvDE+pVsgAxcRehJRRpIZRpLT+4B7vAWQJ92+4NUWavZrZU/cauBtanGf8cb4lM7qoq+Bg1MHdzLwbNXRld0JZzTmhLvKUGK2GXh3nIl0btqiyqb1Lf4z3hgfWqEGDk5PlOMTaSbSudoZeCozZxl4f48zE6KVUdrP4j/jjfHBqYEv3kE8niWJSNVRmJ7ueIRkJl9Yfm02FzFhqifKwROTs3oes/hYADctL5nJkcrmiEdmF+gWQnHWXS0D9y5uej1HZnsRs7czyoa+Tu7fNViYM2b3wBjX/+dWW/S4xc1qJKaI7ANGgRyQVdXNc9EoYyqpNhrx/t3HyCucv2bxT5la3POkei8UL4A7JY/O2OwGTIsIrz5/FV/46bM8tHeIay5dz7tveZSnjoxy9QUDvPHidbN6ftM8c5GBv1xVN1nwNs0wlspy/+5jXLBmCeuasFBDo5YUZd3VlknzAvjAaIpwSIjWWGXIr9NXdHP2qm7ufXqQv7j9cZ46MkosHOLhZ4/P+rlN81gJxbS0nzw9QCab56rzVjW7Kb54WXelUZgebz6UobE0HdHwnC3A/OrzTmEyk+O2Rw/yBy89jV86c80ohsEAABFISURBVDmP7Buek+c2zTHbAK7AD0Vkm4hcV2kHEblORLaKyNbBwcFZvpwxU05MpHno2WEuOXUZK92eFoudl4FXGoXp8TLwnOqsL2AWW9PbweWn9XHlmcv5s9ecy6Ub+9g9MMaQDbNvWbMN4Feq6ouAXwX+WER+uXwHVb1JVTer6ub+/v5ZvpwxU37+3AlyeeUV565sdlN8S0RDREJStf4NznD6uNunfTbzoFTy+k1r+cq7LicWCXH5aX0APLLPyiitalYBXFWfd38PALcDl81Fo4zx45kjo6zt7ZjRgr/NIiIs6YiW1MIr8QL8XGbgxW0AeOG6pcQiISujtLAZB3AR6RKRHu828Grg8blqmDG1TKZzHBie4OxVPc1uSsN+85J1vLpOzd4ro8y2C2Et8UiYTet7LYC3sNn0T1oF3O7+NY8At6jqnXPSKmPq2DUwigLnrOpudlMaduryrrr7eAF8tqMw67lsYx+f+8kexlNZvr39+ZLH8qo8sPsYmzf28a6XnDav7TAzM+MArqp7gYvmsC3G1HTkZJL+njjhkPDM0TE6omHWNXGV+fnk9USZzWIOflx6Wh+fuWc3W/cf58jJJCPJDOuXdRIOCd/Y9hxPPD/CnU8c4cWn93H+mqXz2hbTOFtSzbSELXuH+PSPd/Hi05fz6xeuZtfRUc5a1b2o18CcjUIGPg8llOIBUclMDgHe+cWH8ZbNFJwJtJKZHFe9YBVb9w/zt3fs5CvvunzOujSauWEB3DTd0FiKkEhhzcZKbt36HAAP7R2iIxpiNJVtyfq3X91x5yJnxyxHYdaTiIb5lXP6GZnMcPqKbpZ0RDkwPM6Rk0k2b+zj7FU9XHnmcj783Se59+lBXt5CPX6CwAK4aapcXnnL5x9kIp3jB+95acUeJaPJDD/YcYRLNizj6GiSe552xhOctbL16t9+FXqhzONFTM+rzzul5P6ZZZ/rb11+Kv/x4H4+esdOXnrWCiItMOtjUNi/hGmq7+84zJ7BcQ6fTHLDbTtQ1en7PHaYyUyOS0/r462XbiARDbG2t6PqZFDt4NTlnVy4bikbFkGNPxYJ8b5Xn8PugTF+uutYs5tjilgGbprmKw/t559/vIv+njgv2rCMO584wntv3c6n3npxyX7f2HaQM/q7WL+sAxHhupeeQTjU3rXYzliEt166odnNAJyaeTaXJx4J8S/37ObwSWeWxLdfvjjaF2QWwE3TPHV4hKMjKd6yeR0Xrutlz+AY3/3F8zxzdIyzVnYXMtBt+49zw6+eW7iAdsrS1hg2304i4RBnr+ph5+ER8qpte/G41VgAN02hqtzz9CB9XTFeuLaXkAhv3byen+4+Rkhg2/7jfOcXTr/kcEh408Vr+dHOgSa3OtjOX7OEHYdOcmBogo0r6vdlN/PPArhpiv/edpBDJyZ508VrC+WQzniE15x/SuG/5odPTrJl7zCJaJiVSyzrbrazV/UQDglPHh6xAL5IWAA3C+7g8Qk+/N0n2bi8ixeduqzqfquXdvCGi9cuYMtMLYlomDP6u3ji+ZP86gWn1D/AzDvrhWIWVD6vvO8bv0BVefMl66yW2mLOX72U4xMZjowkm90Ug2XgZgGNp7J89I6dPLR3mH/4XxeSzU/vMgjVl04zzXfu6h5kO9x8/7Pctu0gpy7v4l/fcQlLq6wuZOaXVOp3O182b96sW7dundPn/ORdz/B/X3X2rPbxHr/yxrt54IZXVj3u7L+4g2c++tqSY6/5/IPcev0VfPKuZ3ho7xC3Xn9FyeMbb/g++278tZJjN97wfdb2Jjh0Ism+G3+tsE/x7/e88qzCa2+84fsA7Lvx10qes/w1Km331Nq/0vMDrO1N8MANr5z2PJXuX35aH1ueHa75fJX87RtfyPtv31F322x5z1nvuf08DvhqX1ggV+PrtbQjwsnJbM3nKG+3389mNp9hPCKkslMNf8W5K/nxUwOF3543vWgt33z0UOF++b998Xl06ERy2nagcJ5XOq5c8b498TA7Pnz1tHPxms8/yJZnh6veh9Lvzwv/6k5GU7mSc9177uJji4+Hqe89TMWJaz7/IAePT/DmS9bXjUmNEpFtlZatbPkSyqfu3jXrfbzHvZOs2nHpCt9G7x/4U3fvKvnHLld+bPlrVWtTM9VrY7Fa773Y9b98+kyb03JqBW+gbvBuluLgDRSCdnHwBvjEWzaV3P/Uj3ZVvF3rPGrkPC/edzSVq7hP+XlY77z0nqe4jd62WscWP+a1a8uzwxw6kVzQ727LB3CzuB06MVly389UqqY1ffJHz1S8beaPBXAzb5KZHH/4n9ua3QwzT8qvVfx+0Zzhv/tLGysec2IiPZ9NChwL4GbevP/2Hew4dLLZzTAL5PT+qUmwqs0UefFf37VQzQkEC+Bm3nzz0UO896qzmt0Ms4hUWoD6wNBEE1rSHiyAm3lz1QtW8ievsABuprzy3Olrgb784/cufEPahAVwM6de88n7Crd/6YwV/NcjzzWxNaYVXHZaX+H2i6zE0pBZBXARuVpEnhaR3SJyw1w1yixejx44XnL/6n+6r+T+c8en/jucWIDFCEzre92Fawq3T6kz583vfemRwu2PfPfJksceO3hibhvWAmY8ElNEwsC/AK8CDgKPiMh3VPXJ2kcGjzdYKpXNkc+XPjY4mgLgebe73f6h8cJjP901yGhyqq/w00dGCyu17Ds2zlgqSzrnPOFDe4cA+Mkzg+TyeXJlr3Pn40dY2hHl5GQGgM/euxuAz927p7DPn39zB08+X3rR8VWf+EnJ/Td99mcl93Nloyn/32vO5UPffWLaZ2CMH7/94lNrDkAq7ot+8wPPljz2G595oOT+P5V1Zdyyd6iQVOwdHCMWKc1f09mpL02+yijhxWY2Q+kvA3a7q9MjIv8FvB6Y8wD+1997kq89XH149Qv+8s6qjynOP8S5f/mDmq/hPV6+X/lzF9/3pvE474NT28774J2oQk61cBKc9ud3AHDOB6a389KP/giAX7rxxwD8yj/eW3jsHf/+cMm+rynKdl/2sXtLHnvrTQ8BcO3Npcd4/vArpd35/uHOpwH4+zufKmz71s8Psbpsru3yk/x3rnCW1/K888rTSr5w5fsbM5f+5g0X8IFvPQ7AO6/cyBcf2Fd47A2b1vKt7VMjQ//pR6UDaq5xvyMAr/h4aWICcPYHpr77p7//jmmPn/n+OxABQabtf84HpscNLz7kVVGFL1y7mZee1V/vLTZkxkPpReTNwNWq+vvu/XcAl6vqu8v2uw64zr17DvD0zJvbNCuAIK8lFfT3D/YZBP39Q3M/g1NVdVr0n00GXmkauWl/DVT1JuCmWbxO04nI1krzEARF0N8/2GcQ9PcPi/MzmM3/dw8C64vurwOen11zjDHG+DWbAP4IcJaInCYiMeCtwHfmplnGGGPqmXEJRVWzIvJu4H+AMHCzqrZr94OWLgHNgaC/f7DPIOjvHxbhZ7Cg84EbY4yZO9bnyxhjWpQFcGOMaVGBDuD1pgIQkZeJyEkR2e7+fLDosX0issPdPrfrxC0gP9MhuJ/DdhF5QkR+0sixi90s338gzgER+bOi78DjIpITkT4/x7aCWb7/5p4DqhrIH5wLr3uA04EY8AvgvLJ9XgZ8r8rx+4AVzX4fC/AZ9OKMrt3g3l/p99jF/jOb9x+kc6Bs/9cBPw7SOVDt/S+GcyDIGXhhKgBVTQPeVABB4uczeDvwTVU9AKCqAw0cu9jN5v23i0b/Hd8GfG2Gxy5Gs3n/TRfkAL4WKJ7r9KC7rdwVIvILEfmBiJxftF2BH4rINne6gFbk5zM4G1gmIve67/V3Gjh2sZvN+4fgnAMAiEgncDVwW6PHLmKzef/Q5HNgNkPpW52fqQAexZmDYExEXgt8C/BWKLhSVZ8XkZXAXSLylKreR2vx8xlEgEuAVwIdwIMi8pDPYxe7Gb9/VX2G4JwDntcBD6iqtyR7UM4BT/n7hyafA0HOwOtOBaCqI6o65t6+A4iKyAr3/vPu7wHgdpz/irUaP9MhHATuVNVxVT0G3Adc5PPYxW427z9I54DnrZSWD4JyDnjK33/zz4FmX0Ro1g9OZrUXOI2pixfnl+1zClODnS4DDuD8xe4CetztXcDPcGZmbPr7mofP4AXA3e6+ncDjwAV+jl3sP7N8/4E5B9z9lgLDQFejxy7mn1m+/6afA4EtoWiVqQBE5A/dx/8VeDPwRyKSBSaBt6qqisgq4HZxJvyNALeoavVJyRcpP5+Bqu4UkTuBx4A88AVVfRyg1adSmM37F5HTCcg54O76RuCHqjpe79iFfQezM5v3DzQ9DthQemOMaVFBroEbY0xLswBujDEtygK4Mca0KAvgxhjToiyAG2NMi7IAbnxzZ2HbXvSzsdltmg/usPnN7u07RKS3xr7vdYdYV3v8CyJynnt7rMF2bHJHAHv3f6NVZ/wz88O6ERrfRGRMVburPCY451N+gZs150TkXuB9qlp3elAR2QdsVmeUZvljYVXNFd2v+vlVee7fdZ/73X6PMcFiGbiZMRHZKCI7ReSzOPPGrHfnTn5ERB4TkQ8X7fsX7pzLPxKRr4nI+9ztxdnuCjcgIiJhEfnHoue63t3+MveY/xaRp0Tkq+4fD0TkUhH5mTv52MMi0iMiPxWRTUXteEBELix7Hx0i8l/u69yKM+eJ99g+t11dIvJ997kfF5FrRORPgDXAPSJyj7v/mIh8RES24EyEVnh/7uMfF5FHReRuEemv9hmIs1D4R4Br3P/tXCMivysin3H3O9V9jsfc3xvc7V8SkU+7n8NeEXnzHPxTm0XKArhpREdR+eR2d9s5wH+o6sXu7bNwph3YBFwiIr8sIpfgzCNxMfAm4FIfr/Uu4KSqXuru/wcicpr72MXAe4HzcOZxvtINeLcC71HVi4CrcEbPfgH4XQARORuIq+pjZa/1R8CEql4IfBRn8qpyVwPPq+pFqnoBzvwon8aZN+Plqvpyd78u4HFVvVxV7y97ji7gUVV9EfAT4K+qvXl1pjb9IHCrqm5S1VvLdvkMzud+IfBV4NNFj60GXgL8OnBjtdcwrc8CuGnEpBtMNqnqG91t+1X1Iff2q92fn+Nk5OfiBPSXArer6oSqjgDf8fFarwZ+R0S2A1uA5UzNBPmwqh50yzXbgY04fzwOq+ojUJiILAt8A/h1EYkCvwd8qcJr/TLwFfe4x3CGzZfbAVwlIn8vIi9V1ZNV2p2jdLrRYnmcPzK4r/eSKvv5cQVwi3v7P8ue61uqmlfVJ3GGe5s2Fdi5UMycKZ4bQoC/U9XPF+8gIu+l+hSdWaYSiUTZc/0fVf2fsud6GZAq2pTDOY+l0muo6oSI3IUzSf9bgM3l+3i7VtnuPc8z7v8kXgv8nYj8UFU/UmHXZHHduw7vNat9Bo0obn/x51NpulTTJiwDN3Ppf4DfE5FuABFZK848yfcBb3RrzT048yp79jFVsnhz2XP9kZs5IyJni0hXjdd+ClgjIpe6+/eIiJegfAGnxPCIls7l7LkP+C33uAuAC8t3EJE1OGWWrwAfA17kPjQK9NRoV7EQU+/x7YBXYtlH5c+g1nP/DKcshdv28nKNCQDLwM2cUdUfisgLcBY9ABgDfltVH3UvDm4H9gM/LTrsY8DXReQdwI+Ltn8BpzTyqHuRchB4Q43XTovINcA/i0gHTv37KmBMVbeJyAjwxSqHfw74oog85rbx4Qr7vBD4RxHJAxmcujnATcAPRORwUR28mnHgfBHZBpwErnG3V/sM7gFucMtIf1f2XH8C3Cwif4bz2byzzmubNmTdCM2CE5EP4QTWjy3Q660B7gXObYdujsZ4rIRi2po4a1huAf7CgrdpN5aBG2NMi7IM3BhjWpQFcGOMaVEWwI0xpkVZADfGmBZlAdwYY1rU/wepGbKr16UpywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Paired t-test: Ttest_relResult(statistic=33.879514529499076, pvalue=1.1703467677300927e-57)\n",
      "Student's t-test:\n",
      "F:        7.36077410064683e+32\n",
      "p-value:  0.0\n",
      "Welch's t-test:\n",
      "F:        7.36077410064683e+32\n",
      "p-value:  0.0\n",
      "Mann-Whitney's U-test: MannwhitneyuResult(statistic=10816.0, pvalue=1.7928395567775997e-40)\n",
      "Wilcoxon signed rank sum test: WilcoxonResult(statistic=0.0, pvalue=8.530137789278572e-19)\n"
     ]
    }
   ],
   "source": [
    "b_mean_sem, b_median_sem = bootstrap(np.array(simulation), 1000000) \n",
    "Statistical_significance_test(A = simulation, B = np.repeat(b_mean_sem, len(simulation)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-23T14:04:00.279796Z",
     "iopub.status.busy": "2020-12-23T14:04:00.279140Z",
     "iopub.status.idle": "2020-12-23T14:04:00.282600Z",
     "shell.execute_reply": "2020-12-23T14:04:00.283185Z"
    },
    "papermill": {
     "duration": 0.030218,
     "end_time": "2020-12-23T14:04:00.283321",
     "exception": false,
     "start_time": "2020-12-23T14:04:00.253103",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Operation completed.\n"
     ]
    }
   ],
   "source": [
    "print('Operation completed.')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 686.798168,
   "end_time": "2020-12-23T14:04:00.414529",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-12-23T13:52:33.616361",
   "version": "2.1.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "050e2a1c913a4de084f107f63df61876": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5f3ceeb2474e49d389f7dd9ef3f71232",
        "IPY_MODEL_63ac44ad2eab4b1a89213447732dd333"
       ],
       "layout": "IPY_MODEL_375f1748d79d4f0b8d1e5b87bdc5c98b"
      }
     },
     "10b7a972fc254992ac9c7680240b8ba7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "13721207b517460caeb868f2181ef35f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "31b1f521bbed45d28c27800af60c68ad": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "375f1748d79d4f0b8d1e5b87bdc5c98b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4eda9ed6f7c24ad5a2774ee152633b5c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_7e388f3365c8499bb40184cc5a44c3db",
        "IPY_MODEL_f99e9a38b1724d9eac5d444e8a597bae"
       ],
       "layout": "IPY_MODEL_579dc10806f6484eaf6affdcb73874c8"
      }
     },
     "56d1b2ebcb18483195b765f9feb3eef7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "579dc10806f6484eaf6affdcb73874c8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5f3ceeb2474e49d389f7dd9ef3f71232": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bbf4a21b6d2e404da822cef229d21296",
       "max": 96817414.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_e5c0df6b44cc49afb8008d35cb3668e1",
       "value": 96817414.0
      }
     },
     "63ac44ad2eab4b1a89213447732dd333": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_31b1f521bbed45d28c27800af60c68ad",
       "placeholder": "​",
       "style": "IPY_MODEL_f8bd1732fe864405a2aba31a6a5e9ad4",
       "value": " 96817414/96817414 [07:57&lt;00:00, 202711.86it/s]"
      }
     },
     "7e388f3365c8499bb40184cc5a44c3db": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_10b7a972fc254992ac9c7680240b8ba7",
       "max": 2453886.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_13721207b517460caeb868f2181ef35f",
       "value": 2453886.0
      }
     },
     "bbf4a21b6d2e404da822cef229d21296": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c911a7803fe646a789b5a98903605cb0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e5c0df6b44cc49afb8008d35cb3668e1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "f8bd1732fe864405a2aba31a6a5e9ad4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f99e9a38b1724d9eac5d444e8a597bae": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c911a7803fe646a789b5a98903605cb0",
       "placeholder": "​",
       "style": "IPY_MODEL_56d1b2ebcb18483195b765f9feb3eef7",
       "value": " 2453886/2453886 [00:12&lt;00:00, 203483.88it/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}