get_riders_urls.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding: utf8\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "import random\n",
    "import os\n",
    "import re\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ### Variables\n",
    "\n",
    "user_agent = {'User-agent': 'Mozilla/5.0'}\n",
    "\n",
    "# ### Useful functions\n",
    "\n",
    "# Base function to return HTML from an URL using BeautifulSoup\n",
    "def download_soup(url):\n",
    "    print('Starting request for: ' + url)\n",
    "    response = requests.get(url, headers=user_agent)\n",
    "    if response.status_code == 200:\n",
    "        page = response.text\n",
    "        soup = BeautifulSoup(page, 'lxml')\n",
    "        print('Successful request!')\n",
    "        return soup\n",
    "    else:\n",
    "        print('Unsuccessful request, status code: '+ response.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ### Get riders pages URLs from team pages\n",
    "# Uses the file created with get_team_urls,\n",
    "# download each team page and scrape through the page to find riders URLs,\n",
    "# creates its own file of URLs\n",
    "\n",
    "def get_riders_urls(start_year, end_year):\n",
    "    print(\"Starting get_riders_urls\")\n",
    "    df_rider_urls = pd.DataFrame(columns=[\"rider_url\"])\n",
    "    for year in range(start_year, end_year+1):\n",
    "        df_team_urls = pd.read_csv(\"data/team_urls/team_urls_\" + str(year) + \".csv\")\n",
    "        list_rider_urls = []\n",
    "        for idx, row in df_team_urls.iterrows():\n",
    "            url = row[\"team_url\"]\n",
    "            timer = 0.5 + 0.5 * random.random()\n",
    "            time.sleep(timer)\n",
    "            riders = download_soup(url).find_all('a', class_ = 'rider')\n",
    "            urls = [x['href'] for x in riders]\n",
    "            for u in urls:\n",
    "#                 list_rider_urls.append('https://www.procyclingstats.com/' + u)\n",
    "                list_rider_urls.append(u[6:])\n",
    "        tmp_rider_urls = pd.DataFrame()\n",
    "        tmp_rider_urls[\"rider_url\"] = list_rider_urls\n",
    "        df_rider_urls = pd.concat([df_rider_urls, tmp_rider_urls])\n",
    "        df_rider_urls = df_rider_urls.drop_duplicates(\"rider_url\")\n",
    "    df_rider_urls.to_csv(\"data/rider_urls.csv\")\n",
    "    print('Finished get_riders_url')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting get_riders_urls\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bmc-racing-team-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/euskaltel-euskadi-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fdj-bigmat-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lampre-isd-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/liquigas-cannondale-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-belisol-team-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/movistar-team-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/omega-pharma-quick-step-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/orica-greenedge-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/rabobank-cycling-team-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/radioshack-nissan-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/sky-procycling-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-garmin-barracuda-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-saxo-bank-tinkoff-bank-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/vacansoleil-dcm-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/accent.jobs-willems-veranda-s-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/acqua-sapone-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/andalucia-caja-granada-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-giocattoli-venezuela-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/argos-shimano-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bretagne-schuller-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/champion-system-pro-cycling-team-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-le-credit-en-ligne-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/colnago-csf-bardiani-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/colombia-coldeportes-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/farnese-vini-neri-sottoli-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/landbouwkrediet-euphony-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/rusvelo-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/saur-sojasun-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/spidertech-powered-by-c10-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-europcar-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-netapp-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-type-1-sanofi-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/topsport-vlaanderen-mercator-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/unitedhealthcare-presented-by-maxxis-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/utensilnord-named-2012\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/blanco-pro-cycling-team-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bmc-racing-team-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cannondale-pro-cycling-team-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/euskaltel-euskadi-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fdj.fr-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/garmin-sharp-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lampre-merida-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-belisol-team-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/movistar-team-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/omega-pharma-quick-step-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/orica-greenedge-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/radioshack-leopard-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/sky-procycling-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-argos-shimano-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-saxo-tinkoff-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/vacansoleil-dcm-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/accent.jobs-wanty-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-giocattoli-venezuela-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bardiani-valvole-csf-inox-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bretagne-seche-environnement-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ccc-polsat-polkowice-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/champion-system-pro-cycling-team-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-solutions-credits-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/crelan-euphony-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/iam-cycling-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/mtn-qhubeka-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/rusvelo-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/sojasun-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-colombia-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-europcar-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-netapp-endura-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-novo-nordisk-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/topsport-vlaanderen-baloise-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/unitedhealthcare-presented-by-maxxis-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/vini-fantini-2013\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2014\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/belkin-pro-cycling-team-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bmc-racing-team-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cannondale-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fdj.fr-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/garmin-sharp-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lampre-merida-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-belisol-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/movistar-team-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/omega-pharma-quick-step-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/orica-greenedge-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-europcar-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-giant-shimano-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sky-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/tinkoff-saxo-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/trek-factory-racing-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-giocattoli-venezuela-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bardiani-csf-pro-team-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bretagne-seche-environnement-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-seguros-rga-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ccc-polsat-polkowice-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-solutions-credits-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/drapac-professional-cycling-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/iam-cycling-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/mtn-qhubeka-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/neri-sottoli-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/rusvelo-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-colombia-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-netapp-endura-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-novo-nordisk-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/topsport-vlaanderen-baloise-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/unitedhealthcare-pro-cycling-team-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wanty-groupe-gobert-2014\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bmc-racing-team-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/etixx-quick-step-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fdj-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/iam-cycling-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lampre-merida-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-soudal-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/orica-greenedge-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-cannondale-garmin-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-giant-alpecin-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-lottonl-jumbo-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sky-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/tinkoff-saxo-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/trek-factory-racing-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-giocattoli-sidermec-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bardiani-csf-pro-team-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bora-argon-18-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bretagne-seche-environnement-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-seguros-rga-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ccc-sprandi-polkowice-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-solutions-credits-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cult-energy-pro-cycling-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/drapac-professional-cycling-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/mtn-qhubeka-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/nippo-vini-fantini-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/roompot-oranje-peloton-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/rusvelo-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/southeast-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-colombia-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-europcar-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-novo-nordisk-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/topsport-vlaanderen-baloise-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/unitedhealthcare-pro-cycling-team-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wanty-groupe-gobert-2015\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bmc-racing-team-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cannondale-pro-cycling-team-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/etixx-quick-step-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fdj-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/iam-cycling-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lampre-merida-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-soudal-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/orica-greenedge-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/dimension-data-2016\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-giant-alpecin-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-lottonl-jumbo-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sky-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/tinkoff-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/trek-segafredo-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-giocattoli-sidermec-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bardiani-csf-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bora-argon-18-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-seguros-rga-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ccc-sprandi-polkowice-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-solutions-credits-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/delko-marseille-provence-ktm-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/direct-energie-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/drapac-professional-cycling-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fortuneo-vital-concept-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/funvic-soul-cycles-carrefour-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/gazprom-rusvelo-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/nippo-vini-fantini-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/one-pro-cycling-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/roompot-oranje-peloton-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/stolting-service-group-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-novo-nordisk-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-roth-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/topsport-vlaanderen-baloise-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/unitedhealthcare-pro-cycling-team-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/verva-activejet-pro-cycling-team-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wanty-groupe-gobert-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wilier-southeast-2016\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bahrain-merida-pro-cycling-team-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bmc-racing-team-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bora-hansgrohe-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cannondale-drapac-pro-cycling-team-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fdj-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-soudal-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/orica-scott-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/quick-step-floors-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/dimension-data-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-alpecin-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-lottonl-jumbo-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sky-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sunweb-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/trek-segafredo-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/uae-team-emirates-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-sidermec-bottecchia-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/aqua-blue-sport-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bardiani-csf-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-seguros-rga-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ccc-sprandi-polkowice-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-solutions-credits-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/delko-marseille-provence-ktm-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/direct-energie-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fortuneo-oscaro-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/gazprom-rusvelo-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/israel-cycling-academy-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/manzana-postobon-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/nippo-vini-fantini-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/roompot-nederlandse-loterij-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/soul-brasil-pro-cycling-team-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/sport-vlaanderen-baloise-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-novo-nordisk-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/unitedhealthcare-pro-cycling-team-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/verandas-willems-crelan-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wanty-groupe-gobert-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wb-veranclassic-aqua-protect-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wilier-triestina-selle-italia-2017\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bahrain-merida-pro-cycling-team-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bmc-racing-team-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bora-hansgrohe-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/groupama-fdj-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-soudal-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/mitchelton-scott-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/movistar-team-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/quick-step-floors-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-dimension-data-2018\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-ef-education-first-drapac-p-b-cannondale-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-alpecin-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-lottonl-jumbo-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sky-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sunweb-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/trek-segafredo-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/uae-team-emirates-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-sidermec-bottecchia-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/aqua-blue-sport-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bardiani-csf-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/burgos-bh-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-seguros-rga-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ccc-sprandi-polkowice-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-solutions-credits-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/delko-marseille-provence-ktm-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/direct-energie-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/euskadi-murias-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/gazprom-rusvelo-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/hagens-berman-axeon-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/holowesko-citadel-pb-arapahoe-resources-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/israel-cycling-academy-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/manzana-postobon-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/nippo-vini-fantini-europa-ovini-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/rally-cycling-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/roompot-nederlandse-loterij-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/sport-vlaanderen-baloise-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/fortuneo-samsic-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-novo-nordisk-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/unitedhealthcare-pro-cycling-team-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/verandas-willems-crelan-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/vital-concept-cycling-club-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wanty-groupe-gobert-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wb-aqua-protect-veranclassic-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wilier-triestina-selle-italia-2018\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ag2r-la-mondiale-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/astana-pro-team-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bahrain-merida-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bora-hansgrohe-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ccc-team-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/deceuninck-quick-step-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/ef-education-first-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/groupama-fdj-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/lotto-soudal-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/mitchelton-scott-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/movistar-team-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-dimension-data-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-ineos-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-jumbo-visma-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-katusha-alpecin-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-sunweb-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/trek-segafredo-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/uae-team-emirates-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/androni-sidermec-bottecchia-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/bardiani-csf-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/burgos-bh-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/caja-rural-seguros-rga-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/cofidis-solutions-credits-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/corendon-circus-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/delko-marseille-provence-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/euskadi-basque-country-murias-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/gazprom-rusvelo-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/hagens-berman-axeon-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/israel-cycling-academy-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/neri-sottoli-selle-italia-ktm-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/nippo-vini-fantini-faizane-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/rally-uhc-cycling-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/riwal-ceramicspeed-cycling-team-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/roompot-charles-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/sport-vlaanderen-baloise-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-arkea-samsic-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-novo-nordisk-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/team-total-direct-energie-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/vital-concept-bb-hotels-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/w52-fc-porto-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wallonie-bruxelles-2019\n",
      "Successful request!\n",
      "Starting request for: https://www.procyclingstats.com/team/wanty-gobert-cycling-team-2019\n",
      "Successful request!\n",
      "Finished get_riders_url\n"
     ]
    }
   ],
   "source": [
    "get_riders_urls(2012, 2019)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}