4_modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autotime extension is already loaded. To reload it, use:\n",
      "  %reload_ext autotime\n",
      "time: 1.06 ms (started: 2021-12-26 15:13:03 +00:00)\n"
     ]
    }
   ],
   "source": [
    "import sys, os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Scikit-learn\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "# Explicitly require this experimental feature\n",
    "from sklearn.experimental import enable_halving_search_cv # noqa\n",
    "# Now you can import normally from model_selection\n",
    "from sklearn.model_selection import HalvingGridSearchCV\n",
    "\n",
    "# Random Forest\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "    \n",
    "# Timings\n",
    "%load_ext autotime\n",
    "# %unload_ext autotime\n",
    "\n",
    "# Progress bar\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read in cleaned data from local directory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(Temporary dataset *Will be updated soon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8048, 47)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>finished</th>\n",
       "      <th>value</th>\n",
       "      <th>chance_of_playing_next_round</th>\n",
       "      <th>home_flag</th>\n",
       "      <th>goalkeeper_flag</th>\n",
       "      <th>defender_flag</th>\n",
       "      <th>midfielder_flag</th>\n",
       "      <th>forward_flag</th>\n",
       "      <th>goals_WMA</th>\n",
       "      <th>shots_WMA</th>\n",
       "      <th>...</th>\n",
       "      <th>xGBuildup_pgw</th>\n",
       "      <th>team_xG_pgw</th>\n",
       "      <th>team_goals_pgw</th>\n",
       "      <th>team_xGA_pgw</th>\n",
       "      <th>team_goals_against_pgw</th>\n",
       "      <th>opponent_xG_pgw</th>\n",
       "      <th>opponent_goals_pgw</th>\n",
       "      <th>opponent_xGA_pgw</th>\n",
       "      <th>opponent_goals_against_pgw</th>\n",
       "      <th>total_points</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>player_name</th>\n",
       "      <th>position</th>\n",
       "      <th>team_title</th>\n",
       "      <th>event</th>\n",
       "      <th>opponent_team_title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">Bernd Leno</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">goalkeeper</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">Arsenal</th>\n",
       "      <th>2.0</th>\n",
       "      <th>Chelsea</th>\n",
       "      <td>True</td>\n",
       "      <td>50.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.097028</td>\n",
       "      <td>1.023850</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.888180</td>\n",
       "      <td>2.00</td>\n",
       "      <td>1.187090</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.321701</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <th>Manchester City</th>\n",
       "      <td>True</td>\n",
       "      <td>49.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.048514</td>\n",
       "      <td>0.826975</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.764750</td>\n",
       "      <td>2.00</td>\n",
       "      <td>2.272565</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>0.542574</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <th>Norwich</th>\n",
       "      <td>True</td>\n",
       "      <td>49.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.032343</td>\n",
       "      <td>0.590251</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3.334077</td>\n",
       "      <td>3.00</td>\n",
       "      <td>0.933050</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.833423</td>\n",
       "      <td>3.333333</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <th>Burnley</th>\n",
       "      <td>True</td>\n",
       "      <td>49.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.024257</td>\n",
       "      <td>1.113976</td>\n",
       "      <td>0.25</td>\n",
       "      <td>2.644881</td>\n",
       "      <td>2.25</td>\n",
       "      <td>1.504393</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>1.912348</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                             finished  value  \\\n",
       "player_name position   team_title event opponent_team_title                    \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                  True   50.0   \n",
       "                                  3.0   Manchester City          True   49.0   \n",
       "                                  4.0   Norwich                  True   49.0   \n",
       "                                  5.0   Burnley                  True   49.0   \n",
       "\n",
       "                                                             chance_of_playing_next_round  \\\n",
       "player_name position   team_title event opponent_team_title                                 \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                                     100.0   \n",
       "                                  3.0   Manchester City                             100.0   \n",
       "                                  4.0   Norwich                                     100.0   \n",
       "                                  5.0   Burnley                                     100.0   \n",
       "\n",
       "                                                             home_flag  \\\n",
       "player_name position   team_title event opponent_team_title              \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                      1   \n",
       "                                  3.0   Manchester City              0   \n",
       "                                  4.0   Norwich                      1   \n",
       "                                  5.0   Burnley                      0   \n",
       "\n",
       "                                                             goalkeeper_flag  \\\n",
       "player_name position   team_title event opponent_team_title                    \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                            1   \n",
       "                                  3.0   Manchester City                    1   \n",
       "                                  4.0   Norwich                            1   \n",
       "                                  5.0   Burnley                            1   \n",
       "\n",
       "                                                             defender_flag  \\\n",
       "player_name position   team_title event opponent_team_title                  \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                          0   \n",
       "                                  3.0   Manchester City                  0   \n",
       "                                  4.0   Norwich                          0   \n",
       "                                  5.0   Burnley                          0   \n",
       "\n",
       "                                                             midfielder_flag  \\\n",
       "player_name position   team_title event opponent_team_title                    \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                            0   \n",
       "                                  3.0   Manchester City                    0   \n",
       "                                  4.0   Norwich                            0   \n",
       "                                  5.0   Burnley                            0   \n",
       "\n",
       "                                                             forward_flag  \\\n",
       "player_name position   team_title event opponent_team_title                 \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                         0   \n",
       "                                  3.0   Manchester City                 0   \n",
       "                                  4.0   Norwich                         0   \n",
       "                                  5.0   Burnley                         0   \n",
       "\n",
       "                                                             goals_WMA  \\\n",
       "player_name position   team_title event opponent_team_title              \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                    0.0   \n",
       "                                  3.0   Manchester City            0.0   \n",
       "                                  4.0   Norwich                    0.0   \n",
       "                                  5.0   Burnley                    0.0   \n",
       "\n",
       "                                                             shots_WMA  ...  \\\n",
       "player_name position   team_title event opponent_team_title             ...   \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                    0.0  ...   \n",
       "                                  3.0   Manchester City            0.0  ...   \n",
       "                                  4.0   Norwich                    0.0  ...   \n",
       "                                  5.0   Burnley                    0.0  ...   \n",
       "\n",
       "                                                             xGBuildup_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                  \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                   0.097028   \n",
       "                                  3.0   Manchester City           0.048514   \n",
       "                                  4.0   Norwich                   0.032343   \n",
       "                                  5.0   Burnley                   0.024257   \n",
       "\n",
       "                                                             team_xG_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                 1.023850   \n",
       "                                  3.0   Manchester City         0.826975   \n",
       "                                  4.0   Norwich                 0.590251   \n",
       "                                  5.0   Burnley                 1.113976   \n",
       "\n",
       "                                                             team_goals_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                   \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                        0.00   \n",
       "                                  3.0   Manchester City                0.00   \n",
       "                                  4.0   Norwich                        0.00   \n",
       "                                  5.0   Burnley                        0.25   \n",
       "\n",
       "                                                             team_xGA_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                 \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                  1.888180   \n",
       "                                  3.0   Manchester City          2.764750   \n",
       "                                  4.0   Norwich                  3.334077   \n",
       "                                  5.0   Burnley                  2.644881   \n",
       "\n",
       "                                                             team_goals_against_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                           \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                                2.00   \n",
       "                                  3.0   Manchester City                        2.00   \n",
       "                                  4.0   Norwich                                3.00   \n",
       "                                  5.0   Burnley                                2.25   \n",
       "\n",
       "                                                             opponent_xG_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                    \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                     1.187090   \n",
       "                                  3.0   Manchester City             2.272565   \n",
       "                                  4.0   Norwich                     0.933050   \n",
       "                                  5.0   Burnley                     1.504393   \n",
       "\n",
       "                                                             opponent_goals_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                       \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                        3.000000   \n",
       "                                  3.0   Manchester City                2.500000   \n",
       "                                  4.0   Norwich                        0.333333   \n",
       "                                  5.0   Burnley                        0.750000   \n",
       "\n",
       "                                                             opponent_xGA_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                     \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                      0.321701   \n",
       "                                  3.0   Manchester City              0.542574   \n",
       "                                  4.0   Norwich                      1.833423   \n",
       "                                  5.0   Burnley                      1.912348   \n",
       "\n",
       "                                                             opponent_goals_against_pgw  \\\n",
       "player_name position   team_title event opponent_team_title                               \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                                0.000000   \n",
       "                                  3.0   Manchester City                        0.500000   \n",
       "                                  4.0   Norwich                                3.333333   \n",
       "                                  5.0   Burnley                                2.000000   \n",
       "\n",
       "                                                             total_points  \n",
       "player_name position   team_title event opponent_team_title                \n",
       "Bernd Leno  goalkeeper Arsenal    2.0   Chelsea                       2.0  \n",
       "                                  3.0   Manchester City               1.0  \n",
       "                                  4.0   Norwich                       0.0  \n",
       "                                  5.0   Burnley                       0.0  \n",
       "\n",
       "[4 rows x 47 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 97.8 ms (started: 2021-12-26 15:13:05 +00:00)\n"
     ]
    }
   ],
   "source": [
    "# Redefine Index\n",
    "df = pd.read_csv(\"/Users/samharrison/Documents/data_sci/fpl_points_predictor/data/cleaned_data.csv\")\n",
    "df = df.set_index(['player_name','position','team_title','event','opponent_team_title'])\n",
    "\n",
    "print(df.shape)\n",
    "df.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define:\n",
    "* The upcoming **gameweek number**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19.0\n",
      "time: 4.17 ms (started: 2021-12-26 15:13:08 +00:00)\n"
     ]
    }
   ],
   "source": [
    "# Find the upcoming gameweek\n",
    "gameweek_num = df[df['finished']==False].index.get_level_values('event').min()\n",
    "print(gameweek_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.94 ms (started: 2021-12-26 15:13:17 +00:00)\n"
     ]
    }
   ],
   "source": [
    "def get_preds(gameweek_num):\n",
    "\n",
    "    \"\"\"Returns the predictions for the upcoming gameweeks.\n",
    "    \n",
    "    :param: int64 gameweek_num: The upcoming gameweek number, and \n",
    "            the first gameweek in the test range. E.g. if we're at \n",
    "            the gameweek 5 is next, we predict gameweeks 5,6,7,8 and 9. \n",
    "\n",
    "    :rtype: DataFrame train: Training dataset containing predictions.\n",
    "            DataFrame test: Test dataset containing predictions.\n",
    "    \"\"\"    \n",
    "\n",
    "    # Initialise gameweek ranges\n",
    "    prev_gw = int(gameweek_num)-1\n",
    "    all_gameweeks = list(range(0,prev_gw+6))\n",
    "    train_gameweeks = list(range(0,prev_gw+1))\n",
    "    test_gameweeks = list(range(prev_gw+1,prev_gw+6))\n",
    "\n",
    "    # Get all gameweeks in both sets of ranges\n",
    "    df_all_gameweeks = df[(df.index.get_level_values('event').isin(all_gameweeks))]\n",
    "\n",
    "    # Rename target variable\n",
    "    df_all_gameweeks = df_all_gameweeks.rename(columns={'total_points':'total_points_actual'})\n",
    "\n",
    "    # Drop \"chance variables\". \n",
    "    # Note: These will be reintroduced later\n",
    "    df_all_gameweeks = df_all_gameweeks.drop(columns={'finished','value','chance_of_playing_next_round'})\n",
    "\n",
    "    # Standardise the independent/predictor variables\n",
    "    feat_to_not_scale = ['home_flag','goalkeeper_flag','defender_flag','midfielder_flag', 'forward_flag','total_points_actual']\n",
    "    df_all_gameweeks.loc[:, ~df_all_gameweeks.columns.isin(feat_to_not_scale)] = StandardScaler().fit_transform(\n",
    "                                            df_all_gameweeks.loc[:, ~df_all_gameweeks.columns.isin(feat_to_not_scale)])\n",
    "    # Train-Test split the data\n",
    "    df_train = df_all_gameweeks[(df_all_gameweeks.index.get_level_values('event').isin(train_gameweeks))]\n",
    "    df_test = df_all_gameweeks[(df_all_gameweeks.index.get_level_values('event').isin(test_gameweeks))]\n",
    "\n",
    "    # Standardise the dependent/target variable (i.e. we will inverse this later)\n",
    "    target_scaler = MinMaxScaler()\n",
    "    target_scaler.fit(df_train[['total_points_actual']])\n",
    "    y_train = target_scaler.transform(df_train[['total_points_actual']])\n",
    "    y_test = target_scaler.transform(df_test[['total_points_actual']])\n",
    "\n",
    "    # Define independent/predictor variables\n",
    "    X_train = df_train.loc[:, df_train.columns != 'total_points_actual']\n",
    "    X_test = df_test.loc[:, df_test.columns != 'total_points_actual']\n",
    "\n",
    "    # Create Random Forest with best settings     \n",
    "    rf = RandomForestRegressor(bootstrap=True, max_depth=8, max_features=7, min_samples_leaf=2,\n",
    "                               min_samples_split=5, n_estimators=200)\n",
    "\n",
    "    # Fit Random Forest to training data\n",
    "    model = rf.fit(X_train, y_train)   \n",
    "    \n",
    "#     # Initialise XGBoost\n",
    "#     xbgr = xgb.XGBRegressor()\n",
    "\n",
    "#     # Setup search heuristic using parameter grid from earlier\n",
    "#     sh = HalvingGridSearchCV(xbgr, param_grid, cv = 5, factor = 5, \n",
    "#                 min_resources ='exhaust', n_jobs = -1, verbose = 2, random_state = 42).fit(X_train, y_train) \n",
    "\n",
    "#     # Fit XGBoost best estimator to training data\n",
    "#     model = sh.best_estimator_.fit(X_train, y_train) \n",
    "\n",
    "    # Obtain predictions on training data via. Cross Validation \n",
    "    y_train_pred = cross_val_predict(model, X_train, y_train, cv=10)\n",
    "\n",
    "    # Invert transform on predictions for both datasets\n",
    "    y_test_pred = model.predict(X_test)\n",
    "    y_test_pred = target_scaler.inverse_transform([y_test_pred])\n",
    "    y_train_pred = target_scaler.inverse_transform([y_train_pred])\n",
    "\n",
    "    # Create prediction column for train/test DataFrames\n",
    "    df_train['total_points_predicted'] = y_train_pred[0]\n",
    "    df_test['total_points_predicted'] = y_test_pred[0]\n",
    "\n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Call function. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 18.6 s (started: 2021-12-26 15:13:19 +00:00)\n"
     ]
    }
   ],
   "source": [
    "# Get predictions\n",
    "df_train, df_test = get_preds(gameweek_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reintroduce \"chance_of_playing_next_round\" and multiply the predictions made by this 'chance'. Also reintroduce player values\n",
    "**Recall:** In the 3_research section, we dropped the variable **chance_of_playing_next_round** since it did not reflect the gameweek defined. However, now, we can use these variables to our advantage. In particular, we can **multiply the predictions made by these features(/probabilities)**. \n",
    "\n",
    "**Example:** Let's say: \n",
    "\n",
    "<center> 'Jamie Vardy is 75% likely to play next week (due to injury) but if does play we expect him to score 4 points. <br> Instead, we predict Jamie Vardy should get: 0.75 x 4 = 3 points.'</center>\n",
    "\n",
    "I believe using the features this way should only improve the final predictions we make. However, I also believe it is important to give the users **visibility** of this process. If a FPL Manager (i.e. a User) wants to take the risk to play Jamie Vardy in the above example, I would like them to be able to explicitly see the above calculation. This way, the FPL Manager is more informed about whether to play Jamie Vardy!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 750 µs (started: 2021-12-26 15:13:43 +00:00)\n"
     ]
    }
   ],
   "source": [
    "def adjust_test_preds(df_test):\n",
    "    \n",
    "    \"\"\"Adjusts the test predictions for the upcoming gameweek; namely,\n",
    "    \"chance_of_playing_next_round\" is used to modify predictions so that \n",
    "    they better reflect things like injuries etc.\n",
    "    \n",
    "    :param: DataFrame df_test: The test data pre-adjustment.\n",
    "\n",
    "    :rtype: DataFrame df_test: The test data, now adjusted.\n",
    "    \"\"\"    \n",
    "\n",
    "    # Reset indexes and join chance & value features\n",
    "    df_test = pd.merge(df_test.reset_index(), \n",
    "                       df.reset_index()[['player_name','position','team_title','event',\n",
    "                       'opponent_team_title','value','chance_of_playing_next_round']], \n",
    "                       how='left', \n",
    "                       on=['player_name','position','team_title','event','opponent_team_title'])\n",
    "\n",
    "    # Before preceding further, we round our predictions to 2 d.p. for clarity with the user\n",
    "    df_test['total_points_predicted'] = round(df_test['total_points_predicted'], 2)\n",
    "\n",
    "    # Initialise new cols\n",
    "    adj_total_points_predicted_col = []\n",
    "    points_calculation_col = []\n",
    "\n",
    "    # Create new adjusted total points predicted column (created row by row)\n",
    "    for i in range(0, len(df_test)):\n",
    "        game = df_test.iloc[i]\n",
    "\n",
    "        # If the game is 'next gameweek' (i.e. upcoming gameweek+1) - multiply the prev. points predicted by chance\n",
    "        if (game['event']==gameweek_num)&(game['chance_of_playing_next_round'] != 100):\n",
    "            adj_total_points_predicted = round(game['total_points_predicted']*game['chance_of_playing_next_round']/100, 2)\n",
    "            adj_total_points_predicted_col.append(game['total_points_predicted']*game['chance_of_playing_next_round']/100)\n",
    "\n",
    "            # Explain calcuation to user\n",
    "            points_calculation_col.append(''.join((\n",
    "                                          \"Player has \"+str(game['chance_of_playing_next_round'])+\"% chance of playing. \",\n",
    "                                          \"The model's prediction of \"+str(game['total_points_predicted'])+\" points has been adjusted to \",\n",
    "                                          str(game['chance_of_playing_next_round']/100)+\"*\"+str(game['total_points_predicted'])+\"=\",\n",
    "                                          str(adj_total_points_predicted))))\n",
    "\n",
    "        # Otherwise, we just append the prev. points predicted\n",
    "        else:\n",
    "            adj_total_points_predicted_col.append(game['total_points_predicted']) \n",
    "            points_calculation_col.append('NaN')\n",
    "    df_test['adj_total_points_predicted'] = adj_total_points_predicted_col\n",
    "    df_test['points_calculation'] = points_calculation_col\n",
    "    \n",
    "    return df_test\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Call function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>player_name</th>\n",
       "      <th>position</th>\n",
       "      <th>team_title</th>\n",
       "      <th>event</th>\n",
       "      <th>opponent_team_title</th>\n",
       "      <th>home_flag</th>\n",
       "      <th>goalkeeper_flag</th>\n",
       "      <th>defender_flag</th>\n",
       "      <th>midfielder_flag</th>\n",
       "      <th>forward_flag</th>\n",
       "      <th>...</th>\n",
       "      <th>opponent_xG_pgw</th>\n",
       "      <th>opponent_goals_pgw</th>\n",
       "      <th>opponent_xGA_pgw</th>\n",
       "      <th>opponent_goals_against_pgw</th>\n",
       "      <th>total_points_actual</th>\n",
       "      <th>total_points_predicted</th>\n",
       "      <th>value</th>\n",
       "      <th>chance_of_playing_next_round</th>\n",
       "      <th>adj_total_points_predicted</th>\n",
       "      <th>points_calculation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pierre-Emerick Aubameyang</td>\n",
       "      <td>forward</td>\n",
       "      <td>Arsenal</td>\n",
       "      <td>19.0</td>\n",
       "      <td>Norwich</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.099764</td>\n",
       "      <td>-1.295752</td>\n",
       "      <td>0.762269</td>\n",
       "      <td>0.866442</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.0</td>\n",
       "      <td>2.49</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pierre-Emerick Aubameyang</td>\n",
       "      <td>forward</td>\n",
       "      <td>Arsenal</td>\n",
       "      <td>20.0</td>\n",
       "      <td>Wolverhampton Wanderers</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.714349</td>\n",
       "      <td>-0.948818</td>\n",
       "      <td>-0.169636</td>\n",
       "      <td>-0.926127</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.0</td>\n",
       "      <td>2.30</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pierre-Emerick Aubameyang</td>\n",
       "      <td>forward</td>\n",
       "      <td>Arsenal</td>\n",
       "      <td>21.0</td>\n",
       "      <td>Manchester City</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.743960</td>\n",
       "      <td>1.244483</td>\n",
       "      <td>-1.964045</td>\n",
       "      <td>-1.372450</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.77</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1.77</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pierre-Emerick Aubameyang</td>\n",
       "      <td>forward</td>\n",
       "      <td>Arsenal</td>\n",
       "      <td>22.0</td>\n",
       "      <td>Tottenham</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.774183</td>\n",
       "      <td>-0.698590</td>\n",
       "      <td>-0.831544</td>\n",
       "      <td>-0.664846</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100.0</td>\n",
       "      <td>2.09</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 player_name position team_title  event  \\\n",
       "0  Pierre-Emerick Aubameyang  forward    Arsenal   19.0   \n",
       "1  Pierre-Emerick Aubameyang  forward    Arsenal   20.0   \n",
       "2  Pierre-Emerick Aubameyang  forward    Arsenal   21.0   \n",
       "3  Pierre-Emerick Aubameyang  forward    Arsenal   22.0   \n",
       "\n",
       "       opponent_team_title  home_flag  goalkeeper_flag  defender_flag  \\\n",
       "0                  Norwich          0                0              0   \n",
       "1  Wolverhampton Wanderers          1                0              0   \n",
       "2          Manchester City          1                0              0   \n",
       "3                Tottenham          0                0              0   \n",
       "\n",
       "   midfielder_flag  forward_flag  ...  opponent_xG_pgw  opponent_goals_pgw  \\\n",
       "0                0             1  ...        -1.099764           -1.295752   \n",
       "1                0             1  ...        -0.714349           -0.948818   \n",
       "2                0             1  ...         1.743960            1.244483   \n",
       "3                0             1  ...        -0.774183           -0.698590   \n",
       "\n",
       "   opponent_xGA_pgw  opponent_goals_against_pgw  total_points_actual  \\\n",
       "0          0.762269                    0.866442                  NaN   \n",
       "1         -0.169636                   -0.926127                  NaN   \n",
       "2         -1.964045                   -1.372450                  NaN   \n",
       "3         -0.831544                   -0.664846                  NaN   \n",
       "\n",
       "   total_points_predicted  value  chance_of_playing_next_round  \\\n",
       "0                    2.49    NaN                         100.0   \n",
       "1                    2.30    NaN                         100.0   \n",
       "2                    1.77    NaN                         100.0   \n",
       "3                    2.09    NaN                         100.0   \n",
       "\n",
       "   adj_total_points_predicted  points_calculation  \n",
       "0                        2.49                 NaN  \n",
       "1                        2.30                 NaN  \n",
       "2                        1.77                 NaN  \n",
       "3                        2.09                 NaN  \n",
       "\n",
       "[4 rows x 54 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 295 ms (started: 2021-12-26 15:13:49 +00:00)\n"
     ]
    }
   ],
   "source": [
    "# Get (final) predictions\n",
    "df_test = adjust_test_preds(df_test)\n",
    "df_test.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine the predictions made on both datasets and prepare for export\n",
    "\n",
    "All we're doing here is combining the two datasets now containing predictions. We join them together for ease when we export next. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2.63 ms (started: 2021-12-26 15:13:59 +00:00)\n"
     ]
    }
   ],
   "source": [
    "def prepare_data_for_export(df_train, df_test):\n",
    "\n",
    "    # Reset indexes and join value feature\n",
    "    df_train = pd.merge(df_train.reset_index(), \n",
    "                       df.reset_index()[['player_name','position','team_title','event',\n",
    "                       'opponent_team_title','value']], \n",
    "                       how='left', \n",
    "                       on=['player_name','position','team_title','event','opponent_team_title'])\n",
    "\n",
    "    # Redefine training DataFrame (and test DataFrame) with only variables we want to output\n",
    "    df_train = df_train.reset_index()[['player_name','position','team_title','event','opponent_team_title','value',\n",
    "                        'total_points_actual','total_points_predicted']]\n",
    "    df_test = df_test[['player_name','position','team_title','event','opponent_team_title','value',\n",
    "            'total_points_actual','adj_total_points_predicted','points_calculation']]\n",
    "\n",
    "    # Rename 'adj_total_points_predicted' column, and create new (empty) 'points_calculation' col - since we're about to concat\n",
    "    df_test = df_test.rename(columns={'adj_total_points_predicted':'total_points_predicted'})\n",
    "    df_train['points_calculation'] = np.nan\n",
    "\n",
    "    # Create 'train_test' column for filtering later on\n",
    "    df_test['train_test'] = 'test'\n",
    "    df_train['train_test'] = 'train'\n",
    "\n",
    "    # Concatenate DataFrames and sort by player and gameweek\n",
    "    df_predictions = pd.concat([df_test, df_train]).sort_values(by=['player_name','event']).reset_index(drop=True)\n",
    "    df_predictions.insert(0, 'train_test', df_predictions.pop('train_test'))\n",
    "    df_predictions.head(6)\n",
    "\n",
    "    # Update market value 'value' col to historic valuations\n",
    "    value_col = []\n",
    "    for i in range(0,len(df_predictions)):\n",
    "        row = df_predictions.iloc[i]\n",
    "\n",
    "        # If training data append historic market value\n",
    "        if row['train_test']=='train':\n",
    "            value_col.append(row['value'])\n",
    "            # Temporarily set value for future assignment\n",
    "            value_temp = row['value']\n",
    "\n",
    "        # If test data - assign value to most recent \n",
    "        elif row['train_test']=='test':\n",
    "            value_col.append(value_temp)\n",
    "        else:\n",
    "            pass\n",
    "    df_predictions['value'] = value_col\n",
    "\n",
    "    # Create 'more_info_flag' col (helps to format injury info in Tableau)\n",
    "    more_info_flag_col = []\n",
    "    for i in range(0,len(df_predictions)):\n",
    "        row = df_predictions.iloc[i]\n",
    "\n",
    "        # If that player has any sort of injury:\n",
    "        if len(df_predictions[df_predictions['player_name']==row['player_name']]['points_calculation'].unique())>2:\n",
    "    #         print(df_predictions[df_predictions['player_name']==row['player_name']]['points_calculation'].unique())\n",
    "            more_info_flag_col.append(1)\n",
    "        else:\n",
    "            more_info_flag_col.append(0)\n",
    "            pass\n",
    "    df_predictions['more_info_flag'] = more_info_flag_col\n",
    "    \n",
    "    # Divide 'value' by 10 for correct scale\n",
    "    df_predictions['value'] = df_predictions['value']/10\n",
    "    \n",
    "    # Replace long-team names with abbreviations (for better display)\n",
    "    for col in ['team_title','opponent_team_title']:\n",
    "        df_predictions[col] = df_predictions[col].replace('Arsenal','ARS')\n",
    "        df_predictions[col] = df_predictions[col].replace('Brighton','BRI')\n",
    "        df_predictions[col] = df_predictions[col].replace('Burnley','BUR')\n",
    "        df_predictions[col] = df_predictions[col].replace('Chelsea','CHE')\n",
    "        df_predictions[col] = df_predictions[col].replace('Crystal Palace','CRY')\n",
    "        df_predictions[col] = df_predictions[col].replace('Everton','EVE')\n",
    "        df_predictions[col] = df_predictions[col].replace('Norwich','NOR')\n",
    "        df_predictions[col] = df_predictions[col].replace('Brentford','BRE')\n",
    "        df_predictions[col] = df_predictions[col].replace('Leicester','LEI')\n",
    "        df_predictions[col] = df_predictions[col].replace('Liverpool','LIV')\n",
    "        df_predictions[col] = df_predictions[col].replace('Wolverhampton Wanderers','WOV')\n",
    "        df_predictions[col] = df_predictions[col].replace('Leeds','LEE')\n",
    "        df_predictions[col] = df_predictions[col].replace('Watford','WAT')\n",
    "        df_predictions[col] = df_predictions[col].replace('Tottenham','TOT')\n",
    "        df_predictions[col] = df_predictions[col].replace('West Ham','WHU')\n",
    "        df_predictions[col] = df_predictions[col].replace('Manchester United','MUN')\n",
    "        df_predictions[col] = df_predictions[col].replace('Manchester City','MCI')\n",
    "        df_predictions[col] = df_predictions[col].replace('Newcastle United','NEW')\n",
    "        df_predictions[col] = df_predictions[col].replace('Aston Villa','AVL')\n",
    "        df_predictions[col] = df_predictions[col].replace('Southampton','SOU')\n",
    "    \n",
    "    return df_predictions\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Call function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_test</th>\n",
       "      <th>player_name</th>\n",
       "      <th>position</th>\n",
       "      <th>team_title</th>\n",
       "      <th>event</th>\n",
       "      <th>opponent_team_title</th>\n",
       "      <th>value</th>\n",
       "      <th>total_points_actual</th>\n",
       "      <th>total_points_predicted</th>\n",
       "      <th>points_calculation</th>\n",
       "      <th>more_info_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>3.0</td>\n",
       "      <td>EVE</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.608768</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>4.0</td>\n",
       "      <td>BRE</td>\n",
       "      <td>5.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.354054</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LEI</td>\n",
       "      <td>5.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.568184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>6.0</td>\n",
       "      <td>CRY</td>\n",
       "      <td>5.3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.268184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  train_test     player_name position team_title  event opponent_team_title  \\\n",
       "0      train  Aaron Connolly  forward        BRI    3.0                 EVE   \n",
       "1      train  Aaron Connolly  forward        BRI    4.0                 BRE   \n",
       "2      train  Aaron Connolly  forward        BRI    5.0                 LEI   \n",
       "3      train  Aaron Connolly  forward        BRI    6.0                 CRY   \n",
       "\n",
       "   value  total_points_actual  total_points_predicted points_calculation  \\\n",
       "0    5.5                  0.0                1.608768                NaN   \n",
       "1    5.4                  0.0                1.354054                NaN   \n",
       "2    5.4                  0.0                1.568184                NaN   \n",
       "3    5.3                  1.0                1.268184                NaN   \n",
       "\n",
       "   more_info_flag  \n",
       "0               0  \n",
       "1               0  \n",
       "2               0  \n",
       "3               0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 7.87 s (started: 2021-12-26 15:14:02 +00:00)\n"
     ]
    }
   ],
   "source": [
    "# Prepare data for export to Tableau\n",
    "df_predictions = prepare_data_for_export(df_train, df_test)\n",
    "df_predictions.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_test</th>\n",
       "      <th>player_name</th>\n",
       "      <th>position</th>\n",
       "      <th>team_title</th>\n",
       "      <th>event</th>\n",
       "      <th>opponent_team_title</th>\n",
       "      <th>value</th>\n",
       "      <th>total_points_actual</th>\n",
       "      <th>total_points_predicted</th>\n",
       "      <th>points_calculation</th>\n",
       "      <th>more_info_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>3.0</td>\n",
       "      <td>EVE</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.608768</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>4.0</td>\n",
       "      <td>BRE</td>\n",
       "      <td>5.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.354054</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LEI</td>\n",
       "      <td>5.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.568184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>6.0</td>\n",
       "      <td>CRY</td>\n",
       "      <td>5.3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.268184</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>7.0</td>\n",
       "      <td>ARS</td>\n",
       "      <td>5.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.233944</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NOR</td>\n",
       "      <td>5.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.299814</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>9.0</td>\n",
       "      <td>MCI</td>\n",
       "      <td>5.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.903236</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>10.0</td>\n",
       "      <td>LIV</td>\n",
       "      <td>5.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.832384</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>11.0</td>\n",
       "      <td>NEW</td>\n",
       "      <td>5.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.831310</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>12.0</td>\n",
       "      <td>AVL</td>\n",
       "      <td>5.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.849608</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>17.0</td>\n",
       "      <td>WOV</td>\n",
       "      <td>5.3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.561482</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>test</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>19.0</td>\n",
       "      <td>BRE</td>\n",
       "      <td>5.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.430000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>test</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>20.0</td>\n",
       "      <td>CHE</td>\n",
       "      <td>5.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.350000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>test</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>21.0</td>\n",
       "      <td>EVE</td>\n",
       "      <td>5.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.560000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>test</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>22.0</td>\n",
       "      <td>CRY</td>\n",
       "      <td>5.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.440000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>test</td>\n",
       "      <td>Aaron Connolly</td>\n",
       "      <td>forward</td>\n",
       "      <td>BRI</td>\n",
       "      <td>23.0</td>\n",
       "      <td>LEI</td>\n",
       "      <td>5.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Cresswell</td>\n",
       "      <td>defender</td>\n",
       "      <td>WHU</td>\n",
       "      <td>2.0</td>\n",
       "      <td>LEI</td>\n",
       "      <td>5.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.721634</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Cresswell</td>\n",
       "      <td>defender</td>\n",
       "      <td>WHU</td>\n",
       "      <td>3.0</td>\n",
       "      <td>CRY</td>\n",
       "      <td>5.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.431549</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Cresswell</td>\n",
       "      <td>defender</td>\n",
       "      <td>WHU</td>\n",
       "      <td>4.0</td>\n",
       "      <td>SOU</td>\n",
       "      <td>5.5</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.904851</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>train</td>\n",
       "      <td>Aaron Cresswell</td>\n",
       "      <td>defender</td>\n",
       "      <td>WHU</td>\n",
       "      <td>5.0</td>\n",
       "      <td>MUN</td>\n",
       "      <td>5.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.306083</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_test      player_name  position team_title  event  \\\n",
       "0       train   Aaron Connolly   forward        BRI    3.0   \n",
       "1       train   Aaron Connolly   forward        BRI    4.0   \n",
       "2       train   Aaron Connolly   forward        BRI    5.0   \n",
       "3       train   Aaron Connolly   forward        BRI    6.0   \n",
       "4       train   Aaron Connolly   forward        BRI    7.0   \n",
       "5       train   Aaron Connolly   forward        BRI    8.0   \n",
       "6       train   Aaron Connolly   forward        BRI    9.0   \n",
       "7       train   Aaron Connolly   forward        BRI   10.0   \n",
       "8       train   Aaron Connolly   forward        BRI   11.0   \n",
       "9       train   Aaron Connolly   forward        BRI   12.0   \n",
       "10      train   Aaron Connolly   forward        BRI   17.0   \n",
       "11       test   Aaron Connolly   forward        BRI   19.0   \n",
       "12       test   Aaron Connolly   forward        BRI   20.0   \n",
       "13       test   Aaron Connolly   forward        BRI   21.0   \n",
       "14       test   Aaron Connolly   forward        BRI   22.0   \n",
       "15       test   Aaron Connolly   forward        BRI   23.0   \n",
       "16      train  Aaron Cresswell  defender        WHU    2.0   \n",
       "17      train  Aaron Cresswell  defender        WHU    3.0   \n",
       "18      train  Aaron Cresswell  defender        WHU    4.0   \n",
       "19      train  Aaron Cresswell  defender        WHU    5.0   \n",
       "\n",
       "   opponent_team_title  value  total_points_actual  total_points_predicted  \\\n",
       "0                  EVE    5.5                  0.0                1.608768   \n",
       "1                  BRE    5.4                  0.0                1.354054   \n",
       "2                  LEI    5.4                  0.0                1.568184   \n",
       "3                  CRY    5.3                  1.0                1.268184   \n",
       "4                  ARS    5.3                  0.0                1.233944   \n",
       "5                  NOR    5.3                  0.0                1.299814   \n",
       "6                  MCI    5.3                  0.0                0.903236   \n",
       "7                  LIV    5.3                  0.0                0.832384   \n",
       "8                  NEW    5.3                  0.0                0.831310   \n",
       "9                  AVL    5.3                  0.0                0.849608   \n",
       "10                 WOV    5.3                  2.0                1.561482   \n",
       "11                 BRE    5.3                  NaN                1.430000   \n",
       "12                 CHE    5.3                  NaN                1.350000   \n",
       "13                 EVE    5.3                  NaN                1.560000   \n",
       "14                 CRY    5.3                  NaN                1.440000   \n",
       "15                 LEI    5.3                  NaN                1.500000   \n",
       "16                 LEI    5.5                  2.0                4.721634   \n",
       "17                 CRY    5.5                  5.0                4.431549   \n",
       "18                 SOU    5.5                  8.0                2.904851   \n",
       "19                 MUN    5.5                  1.0                2.306083   \n",
       "\n",
       "   points_calculation  more_info_flag  \n",
       "0                 NaN               0  \n",
       "1                 NaN               0  \n",
       "2                 NaN               0  \n",
       "3                 NaN               0  \n",
       "4                 NaN               0  \n",
       "5                 NaN               0  \n",
       "6                 NaN               0  \n",
       "7                 NaN               0  \n",
       "8                 NaN               0  \n",
       "9                 NaN               0  \n",
       "10                NaN               0  \n",
       "11                NaN               0  \n",
       "12                NaN               0  \n",
       "13                NaN               0  \n",
       "14                NaN               0  \n",
       "15                NaN               0  \n",
       "16                NaN               1  \n",
       "17                NaN               1  \n",
       "18                NaN               1  \n",
       "19                NaN               1  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 25.5 ms (started: 2021-12-26 15:21:00 +00:00)\n"
     ]
    }
   ],
   "source": [
    "df_predictions.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overwrite prediction data in local directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 60.4 ms (started: 2021-12-26 15:14:24 +00:00)\n"
     ]
    }
   ],
   "source": [
    "df_predictions.to_csv(index=True, path_or_buf=\"/Users/samharrison/Documents/data_sci/fpl_points_predictor/data/predictions.csv\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also store a historic version of the prediction data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 59.6 ms (started: 2021-12-26 15:14:28 +00:00)\n"
     ]
    }
   ],
   "source": [
    "df_predictions.to_csv(index=True, path_or_buf=\"/Users/samharrison/Documents/data_sci/fpl_points_predictor/data/data_archive/predictions_gw\"+str(gameweek_num)+\".csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we also overwrite the .csv in Google Drive (this will update the Google Sheet)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 73.2 ms (started: 2021-12-26 15:14:31 +00:00)\n"
     ]
    }
   ],
   "source": [
    "df_predictions.to_csv(index=True, path_or_buf=\"/Users/samharrison/My Drive/fpl_points_predictor/predictions.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}