.ipynb_checkpoints/Untitled-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "import re\n",
    "import datetime\n",
    "import sqlite3\n",
    "import time\n",
    "\n",
    "all_links = []\n",
    "location = []\n",
    "events = []\n",
    "f1 = []\n",
    "f2 = []\n",
    "winner = []\n",
    "f1_odds = []\n",
    "f2_odds = []\n",
    "label = []\n",
    "favourite = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up page to extract table\n",
    "data = requests.get(\"https://www.betmma.tips/mma_betting_favorites_vs_underdogs.php?Org=1\")\n",
    "soup = BeautifulSoup(data.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "# table with 98% width \n",
    "table = soup.find('table', {'width': \"98%\"})\n",
    "# find all links in that table\n",
    "links = table.find_all('a', href=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "# append all links to a list \n",
    "for link in links:\n",
    "    all_links.append(\"https://www.betmma.tips/\"+link.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=173\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=151\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=150\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=144\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=143\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=142\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=114\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=113\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=112\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=110\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=109\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=107\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=106\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=105\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=97\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=95\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=94\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=93\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=92\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=88\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=87\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=85\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=81\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=79\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=77\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=75\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=73\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=72\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=71\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=1\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=2\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=3\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=5\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=6\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=7\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=9\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=10\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=11\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=12\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=13\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=14\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=15\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=16\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=17\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=18\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=19\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=20\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=21\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=22\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=23\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=24\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=25\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=26\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=27\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=28\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=29\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=30\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=31\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=32\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=33\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=34\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=35\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=36\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=37\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=38\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=39\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=42\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=40\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=44\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=45\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=46\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=47\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=48\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=49\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=55\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=54\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=50\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=51\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=52\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=61\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=66\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=100\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=101\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=102\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=119\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=120\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=124\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=123\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=126\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=165\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=121\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=170\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=166\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=167\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=176\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=125\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=171\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=180\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=177\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=181\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=190\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=184\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=191\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=186\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=185\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=194\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=192\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=196\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=197\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=199\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=203\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=204\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=212\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=205\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=206\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=209\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=207\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=210\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=208\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=217\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=220\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=221\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=219\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=226\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=223\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=224\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=225\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=231\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=235\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=234\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=233\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=237\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=236\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=241\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=243\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=240\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=251\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=246\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=250\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=252\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=263\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=264\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=273\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=274\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=275\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=280\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=279\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=285\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=286\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=295\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=303\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=302\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=299\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=294\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=304\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=305\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=306\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=312\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=311\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=320\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=319\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=321\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=322\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=323\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=326\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=328\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=332\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=343\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=337\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=329\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=360\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=361\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=327\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=359\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=352\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=358\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=344\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=381\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=380\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=382\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=383\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=391\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=393\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=392\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=394\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=408\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=409\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=403\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=407\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=406\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=423\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=399\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=433\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=434\n",
      "Now currently in link: https://www.betmma.tips/mma_event_betting_history.php?Event=435\n"
     ]
    }
   ],
   "source": [
    "# test for one use case\n",
    "for link in all_links:\n",
    "    print(f\"Now currently scraping link: {link}\")\n",
    "    \n",
    "    data = requests.get(link)\n",
    "    soup = BeautifulSoup(data.text, 'html.parser')\n",
    "    \n",
    "    # specific table with the information\n",
    "    rows = soup.find_all('table', {'cellspacing': \"5\"})\n",
    "    \n",
    "    for row in rows:\n",
    "        \n",
    "        # check for draw, if draw, then skip\n",
    "        # dictionary of won and lost\n",
    "        odds = row.find_all('td', {'align': \"center\", 'valign': \"middle\"})\n",
    "        if odds[0].text not in ['WON', 'LOST']:\n",
    "            continue\n",
    "        \n",
    "        # event name\n",
    "        h1 = soup.find(\"h1\")\n",
    "        # location and date\n",
    "        h2 = soup.find(\"h2\")\n",
    "\n",
    "        events.append(h1.text)\n",
    "        location.append(h2.text)\n",
    "        \n",
    "        odds_f1 = float(odds[2].text.strip(\" @\"))\n",
    "        odds_f2 = float(odds[3].text.strip(\" @\"))\n",
    "        \n",
    "        f1_odds.append(odds_f1)\n",
    "        f2_odds.append(odds_f2)\n",
    "        \n",
    "        # how to generate label\n",
    "        odds_dict = {}\n",
    "        odds_dict[odds[0].text] = odds_f1\n",
    "        odds_dict[odds[1].text] = odds_f2 \n",
    "        \n",
    "        if odds_dict[\"WON\"] > odds_dict[\"LOST\"]:\n",
    "            label.append(\"Underdog\")\n",
    "        else:\n",
    "            label.append(\"Favourite\")\n",
    "        \n",
    "        if odds_f1 > odds_f2:\n",
    "            favourite.append(\"f2\")\n",
    "        else:\n",
    "            favourite.append(\"f1\")\n",
    "        \n",
    "        \n",
    "        fighters = row.find_all('a', attrs={'href': re.compile(\"^fighter_profile.php\")})\n",
    "        f1.append(fighters[0].text)\n",
    "        f2.append(fighters[1].text)\n",
    "        winner.append(fighters[2].text)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Events list: ['UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC 159 - Jones vs. Sonnen', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC on FX 8 - Belfort vs. Rockhold', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2', 'UFC 160 - Velasquez vs. Bigfoot 2']\n",
      "Location list: ['Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Newark, New Jersey; 27th Apr 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Santa Catarina, Brazil; 18th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013', 'Las Vegas, Nevada; 25th May 2013']\n",
      "f1 list: ['Jon Jones', 'Michael Bisping', 'Roy Nelson', 'Phil Davis', 'Pat Healy', 'Rustam Khabilov', 'Ovince St Preux', 'Sara McMann', 'Bryan Caraway', 'Cody McKenzie', 'Steven Siler', 'Vitor Belfort', 'Ronaldo Souza', 'Rafael dos Anjos', 'Rafael Natal', 'Nik Lentz', 'Francisco Trinaldo', 'Gleison Tibau', 'Paulo Thiago', 'Yuri Alcantara', 'Fabio Maldonado', 'John Lineker', 'Jussier Formiga', 'Lucas Martins', 'Cain Velasquez', 'Junior dos Santos', 'Glover Teixeira', 'T.J. Grant', 'Donald Cerrone', 'Mike Pyle', 'Dennis Bermudez', 'Robert Whittaker', 'Khabib Nurmagomedov', 'Stephen Thompson', 'George Roop', 'Jeremy Stephens']\n",
      "f1 odds list: [1.13, 1.57, 1.43, 1.36, 3.4, 1.54, 1.69, 1.18, 2.15, 2.93, 1.95, 1.95, 1.17, 1.5, 1.36, 1.99, 1.33, 1.42, 1.57, 1.2, 1.38, 2.1, 1.65, 1.33, 1.16, 1.25, 1.33, 2.65, 1.37, 2.5, 1.4, 2.75, 1.4, 1.63, 3.66, 1.6]\n",
      "f2 list: ['Chael Sonnen', 'Alan Belcher', 'Cheick Kongo', 'Vinny Magalhaes', 'Jim Miller', 'Yancy Medeiros', 'Gian Villante', 'Sheila Gaff', 'Johnny Bedford', 'Leonard Garcia', 'Kurt Holobaugh', 'Luke Rockhold', 'Chris Camozzi', 'Evan Dunham', 'Joao Zeferino', 'Hacran Dias', 'Mike Rio', 'John Cholish', 'Michel Prazeres', 'Iliarde Santos', 'Roger Hollett', 'Azamat Gashimov', 'Chris Cariaso', 'Jeremy Larsen', 'Antonio Silva', 'Mark Hunt', 'James Te Huna', 'Gray Maynard', 'K.J. Noons', 'Rick Story', 'Max Holloway', 'Colton Smith', 'Abel Trujillo', 'Nah-Shon Burrell', 'Brian Bowles', 'Estevan Payan']\n",
      "f2 odds list: [9.0, 4.5, 3.2, 3.55, 1.4, 3.4, 2.4, 5.8, 1.91, 1.53, 2.06, 2.04, 6.0, 2.91, 3.7, 2.1, 3.9, 3.35, 2.76, 5.5, 3.45, 1.87, 2.55, 3.85, 6.75, 5.25, 4.75, 1.74, 3.48, 1.67, 3.65, 1.71, 3.75, 2.65, 1.36, 2.9]\n",
      "winners list: ['Jon Jones', 'Michael Bisping', 'Roy Nelson', 'Phil Davis', 'Pat Healy', 'Rustam Khabilov', 'Ovince St Preux', 'Sara McMann', 'Bryan Caraway', 'Cody McKenzie', 'Steven Siler', 'Vitor Belfort', 'Ronaldo Souza', 'Rafael dos Anjos', 'Rafael Natal', 'Nik Lentz', 'Francisco Trinaldo', 'Gleison Tibau', 'Paulo Thiago', 'Yuri Alcantara', 'Fabio Maldonado', 'John Lineker', 'Jussier Formiga', 'Lucas Martins', 'Cain Velasquez', 'Junior dos Santos', 'Glover Teixeira', 'T.J. Grant', 'Donald Cerrone', 'Mike Pyle', 'Dennis Bermudez', 'Robert Whittaker', 'Khabib Nurmagomedov', 'Stephen Thompson', 'George Roop', 'Jeremy Stephens']\n",
      "labels list: ['Favourite', 'Favourite', 'Favourite', 'Favourite', 'Underdog', 'Favourite', 'Favourite', 'Favourite', 'Underdog', 'Underdog', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Underdog', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Favourite', 'Underdog', 'Favourite', 'Underdog', 'Favourite', 'Underdog', 'Favourite', 'Favourite', 'Underdog', 'Favourite']\n",
      "favourite is: ['f1', 'f1', 'f1', 'f1', 'f2', 'f1', 'f1', 'f1', 'f2', 'f2', 'f1', 'f1', 'f1', 'f1', 'f1', 'f1', 'f1', 'f1', 'f1', 'f1', 'f1', 'f2', 'f1', 'f1', 'f1', 'f1', 'f1', 'f2', 'f1', 'f2', 'f1', 'f2', 'f1', 'f1', 'f2', 'f1']\n"
     ]
    }
   ],
   "source": [
    "print(f\"Events list: {events}\")\n",
    "print(f\"Location list: {location}\")\n",
    "print(f\"f1 list: {f1}\")\n",
    "print(f\"f1 odds list: {f1_odds}\")\n",
    "print(f\"f2 list: {f2}\")\n",
    "print(f\"f2 odds list: {f2_odds}\")\n",
    "print(f\"winners list: {winner}\")\n",
    "print(f\"labels list: {label}\")\n",
    "print(f\"favourite is: {favourite}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataframe\n",
    "df = pd.DataFrame()\n",
    "df[\"Events\"] = events\n",
    "df[\"Location\"] = location\n",
    "df[\"Fighter1\"] = f1\n",
    "df[\"Fighter2\"] = f2\n",
    "df[\"Winner\"] = winner\n",
    "df[\"fighter1_odds\"] = f1_odds\n",
    "df[\"fighter2_odds\"] = f2_odds\n",
    "df[\"Favourite\"] = favourite\n",
    "df[\"Label\"] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully scraped 2265 fights and last fight card was UFC 234 Melbourne; 9th Feb 2019\n"
     ]
    }
   ],
   "source": [
    "print(f\"Successfully scraped {df.shape[0]} fights and last fight card was {df.iloc[-1, :]['Events']} {df.iloc[-1, :]['Location']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Favourite    0.654746\n",
       "Underdog     0.345254\n",
       "Name: Label, dtype: float64"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"Label\"].value_counts()/2265"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Events</th>\n",
       "      <th>Location</th>\n",
       "      <th>Fighter1</th>\n",
       "      <th>Fighter2</th>\n",
       "      <th>Winner</th>\n",
       "      <th>fighter1_odds</th>\n",
       "      <th>fighter2_odds</th>\n",
       "      <th>Favourite</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UFC 159 - Jones vs. Sonnen</td>\n",
       "      <td>Newark, New Jersey; 27th Apr 2013</td>\n",
       "      <td>Jon Jones</td>\n",
       "      <td>Chael Sonnen</td>\n",
       "      <td>Jon Jones</td>\n",
       "      <td>1.13</td>\n",
       "      <td>9.00</td>\n",
       "      <td>f1</td>\n",
       "      <td>Favourite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UFC 159 - Jones vs. Sonnen</td>\n",
       "      <td>Newark, New Jersey; 27th Apr 2013</td>\n",
       "      <td>Michael Bisping</td>\n",
       "      <td>Alan Belcher</td>\n",
       "      <td>Michael Bisping</td>\n",
       "      <td>1.57</td>\n",
       "      <td>4.50</td>\n",
       "      <td>f1</td>\n",
       "      <td>Favourite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>UFC 159 - Jones vs. Sonnen</td>\n",
       "      <td>Newark, New Jersey; 27th Apr 2013</td>\n",
       "      <td>Roy Nelson</td>\n",
       "      <td>Cheick Kongo</td>\n",
       "      <td>Roy Nelson</td>\n",
       "      <td>1.43</td>\n",
       "      <td>3.20</td>\n",
       "      <td>f1</td>\n",
       "      <td>Favourite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UFC 159 - Jones vs. Sonnen</td>\n",
       "      <td>Newark, New Jersey; 27th Apr 2013</td>\n",
       "      <td>Phil Davis</td>\n",
       "      <td>Vinny Magalhaes</td>\n",
       "      <td>Phil Davis</td>\n",
       "      <td>1.36</td>\n",
       "      <td>3.55</td>\n",
       "      <td>f1</td>\n",
       "      <td>Favourite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>UFC 159 - Jones vs. Sonnen</td>\n",
       "      <td>Newark, New Jersey; 27th Apr 2013</td>\n",
       "      <td>Pat Healy</td>\n",
       "      <td>Jim Miller</td>\n",
       "      <td>Pat Healy</td>\n",
       "      <td>3.40</td>\n",
       "      <td>1.40</td>\n",
       "      <td>f2</td>\n",
       "      <td>Underdog</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Events                           Location  \\\n",
       "0  UFC 159 - Jones vs. Sonnen  Newark, New Jersey; 27th Apr 2013   \n",
       "1  UFC 159 - Jones vs. Sonnen  Newark, New Jersey; 27th Apr 2013   \n",
       "2  UFC 159 - Jones vs. Sonnen  Newark, New Jersey; 27th Apr 2013   \n",
       "3  UFC 159 - Jones vs. Sonnen  Newark, New Jersey; 27th Apr 2013   \n",
       "4  UFC 159 - Jones vs. Sonnen  Newark, New Jersey; 27th Apr 2013   \n",
       "\n",
       "          Fighter1         Fighter2           Winner  fighter1_odds  \\\n",
       "0        Jon Jones     Chael Sonnen        Jon Jones           1.13   \n",
       "1  Michael Bisping     Alan Belcher  Michael Bisping           1.57   \n",
       "2       Roy Nelson     Cheick Kongo       Roy Nelson           1.43   \n",
       "3       Phil Davis  Vinny Magalhaes       Phil Davis           1.36   \n",
       "4        Pat Healy       Jim Miller        Pat Healy           3.40   \n",
       "\n",
       "   fighter2_odds Favourite      Label  \n",
       "0           9.00        f1  Favourite  \n",
       "1           4.50        f1  Favourite  \n",
       "2           3.20        f1  Favourite  \n",
       "3           3.55        f1  Favourite  \n",
       "4           1.40        f2   Underdog  "
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nTODO\\n\\n- merge df above with df with fighters information\\n- fuzzy match instead of exact match due to different sources\\n- compute the deltas\\n- save it in sqlite and push it to app\\n- separate ML from main app\\n\\n'"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "TODO\n",
    "\n",
    "- merge df above with df with fighters information\n",
    "- fuzzy match instead of exact match due to different sources\n",
    "- compute the deltas\n",
    "- save it in sqlite and push it to app\n",
    "- separate ML from main app\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}