From ea41c6d4a47786ab6ec56b704ede98771c553cd8 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 18:00:24 +0000 Subject: [PATCH 01/22] Update .gitignore .swp --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2bb111d..b87ff25 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,5 @@ venv.bak/ .vscode/ config.json + +*.swp From 474a8f2e52b73dd859b84e1a01817c30127b1d56 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 18:00:47 +0000 Subject: [PATCH 02/22] Add understat_general_player_data() function Returns a dict containing general player data retrieved from https://understat.com/. --- FPLbot/bot.py | 1 - FPLbot/utils.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/FPLbot/bot.py b/FPLbot/bot.py index 5ec89eb..5372fcf 100644 --- a/FPLbot/bot.py +++ b/FPLbot/bot.py @@ -93,4 +93,3 @@ async def main(config): loop = asyncio.get_event_loop() loop.run_until_complete(main(config)) loop.close() - diff --git a/FPLbot/utils.py b/FPLbot/utils.py index cb7ea77..4e0d973 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -1,8 +1,12 @@ import asyncio +import codecs +import json import logging import os +import re import aiohttp +from bs4 import BeautifulSoup from fpl import FPL from fpl.utils import position_converter, team_converter from pymongo import MongoClient, ReplaceOne @@ -39,6 +43,33 @@ def create_logger(): return logger +async def fetch(session, url): + async with session.get(url) as response: + return await response.text() + + +async def understat_general_player_data(): + """Returns a dict containing general player data retrieved from + https://understat.com/. + """ + async with aiohttp.ClientSession() as session: + html = await fetch(session, "https://understat.com/league/EPL/") + + soup = BeautifulSoup(html, "html.parser") + scripts = soup.find_all("script") + pattern = re.compile(r"var\s+playersData\s+=\s+JSON.parse\(\'(.*?)\'\);") + + for script in scripts: + match = re.search(pattern, script.string) + if match: + break + + byte_data = codecs.escape_decode(match.group(1)) + player_data = json.loads(byte_data[0].decode("utf-8")) + + return player_data + + async def update_players(): """Updates all players in the database.""" async with aiohttp.ClientSession() as session: @@ -77,4 +108,3 @@ def get_player_table(players, risers=True): loop = asyncio.get_event_loop() loop.run_until_complete(update_players()) loop.close() - From 9d886edf5c0f60ab4222858ba6680760166ecec4 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 18:39:56 +0000 Subject: [PATCH 03/22] Add mapping Understat name -> FPL name --- FPLbot/constants.py | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 FPLbot/constants.py diff --git a/FPLbot/constants.py b/FPLbot/constants.py new file mode 100644 index 0000000..9609d96 --- /dev/null +++ b/FPLbot/constants.py @@ -0,0 +1,58 @@ +understat_to_fpl = { + "Fred": "Frederico Rodrigues de Paula Santos", + "Ki Sung-yueng": "Sung-yueng Ki", + "Solly March": "Solomon March", + "Jonny": "Jonathan Castro Otto", + "Felipe Anderson": "Felipe Anderson Pereira Gomes", + "Mat Ryan": "Mathew Ryan", + "Kenedy": "Robert Kenedy Nunes do Nascimento", + "Jorginho": "Jorge Luiz Frello Filho", + "Bernard": "Bernard Anício Caldeira Duarte", + "Romain Saiss": "Romain Saïss", + "Bernardo Silva": "Bernardo Mota Veiga de Carvalho e Silva", + "N'Golo Kanté": "N'Golo Kanté", + "João Moutinho": "João Filipe Iria Santos Moutinho", + "Franck Zambo": "André-Frank Zambo Anguissa", + "Fousseni Diabate": "Fousseni Diabaté", + "Jazz Richards": "Ashley Darel Jazz Richards", + "Danilo": "Danilo Luiz da Silva", + "Richarlison": "Richarlison de Andrade", + "Bernardo": "Bernardo Fernandes da Silva Junior", + "Fernandinho": "Fernando Luiz Rosa", + "Joselu": "Jose Luis Mato Sanmartín", + "Son Heung-Min": "Heung-Min Son", + "Diogo Dalot": "José Diogo Dalot Teixeira", + "José Izquierdo": "José Heriberto Izquierdo Mena", + "Fabri": "Fabricio Agosto Ramírez", + "Eddie Nketiah": "Edward Nketiah", + "Rui Patrício": "Rui Pedro dos Santos Patrício", + "Greg Cunningham": "Greg Cunninghamm", + "Junior Hoilett": "David Junior Hoilett", + "Isaac Success": "Isaac Success Ajayi", + "Xande Silva": "Alexandre Nascimento Costa Silva", + "Bruno": "Bruno Saltor Grau", + "Léo Bonatini": "Bonatini Lohner Maia Bonatini", + "André Gomes": "André Filipe Tavares Gomes", + "Kiko Femenía": "Francisco Femenía Far", + "Dele Alli": "Bamidele Alli", + "Ricardo Pereira": "Ricardo Domingos Barbosa Pereira", + "Sokratis": "Sokratis Papastathopoulos", + "Alisson": "Alisson Ramses Becker", + "Fabinho": "Fabio Henrique Tavares", + "Adrien Silva": "Adrien Sebastian Perruchet Silva", + "David de Gea": "David De Gea", + "Gabriel Jesus": "Gabriel Fernando de Jesus", + "Pedro": "Pedro Rodríguez Ledesma", + "Zanka": "Mathias Jorgensen", + "David Luiz": "David Luiz Moreira Marinho", + "Rúben Neves": "Rúben Diogo da Silva Neves", + "Ben Chilwell": "Benjamin Chilwell", + "Kepa": "Kepa Arrizabalaga", + "Emerson": "Emerson Palmieri dos Santos", + "Ederson": "Ederson Santana de Moraes", + "Chicharito": "Javier Hernández Balcázar", + "Rúben Vinagre": "Rúben Gonçalo Silva Nascimento Vinagre", + "Oriol Romeu": "Oriol Romeu Vidal", + "Lucas Moura": "Lucas Rodrigues Moura da Silva", + "Willian": "Willian Borges Da Silva", +} From 6160d08679ec3a005f70c880eec147b0de602821 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 18:40:28 +0000 Subject: [PATCH 04/22] Update understat_general_player_data() to include mapping --- FPLbot/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/FPLbot/utils.py b/FPLbot/utils.py index 4e0d973..c05d8b8 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -11,6 +11,8 @@ from fpl.utils import position_converter, team_converter from pymongo import MongoClient, ReplaceOne +from constants import understat_to_fpl + client = MongoClient() database = client.fpl logger = logging.getLogger("FPLbot") @@ -67,6 +69,11 @@ async def understat_general_player_data(): byte_data = codecs.escape_decode(match.group(1)) player_data = json.loads(byte_data[0].decode("utf-8")) + # Convert Understat player name to FPL player name + for player in player_data: + if player["player_name"] in understat_to_fpl.keys(): + player["player_name"] = understat_to_fpl[player["player_name"]] + return player_data From e545638e3aef2a34ef8a7622fe2ae8353c311eca Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 19:53:24 +0000 Subject: [PATCH 05/22] Add understat_matches_data() & understat_players() * understat_players() retrieves matches data for all players. * understat_players() returns a list of dicts of all players. --- FPLbot/utils.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/FPLbot/utils.py b/FPLbot/utils.py index c05d8b8..aa2a279 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -50,12 +50,11 @@ async def fetch(session, url): return await response.text() -async def understat_general_player_data(): +async def understat_players_data(session): """Returns a dict containing general player data retrieved from https://understat.com/. """ - async with aiohttp.ClientSession() as session: - html = await fetch(session, "https://understat.com/league/EPL/") + html = await fetch(session, "https://understat.com/league/EPL/") soup = BeautifulSoup(html, "html.parser") scripts = soup.find_all("script") @@ -77,6 +76,44 @@ async def understat_general_player_data(): return player_data +async def understat_matches_data(session, player): + """Sets the 'matches' attribute of the given player to the data found on + https://understat.com/player/. + """ + html = await fetch(session, f"https://understat.com/player/{player['id']}") + + soup = BeautifulSoup(html, "html.parser") + scripts = soup.find_all("script") + pattern = re.compile(r"var\s+matchesData\s+=\s+JSON.parse\(\'(.*?)\'\);") + + for script in scripts: + match = re.search(pattern, script.string) + if match: + break + + # If no match could be found, simply return an empty dict + try: + byte_data = codecs.escape_decode(match.group(1)) + matches_data = json.loads(byte_data[0].decode("utf-8")) + + player["matches"] = matches_data + except UnboundLocalError: + player["matches"] = {} + + +async def understat_players(): + """Returns a list of dicts containing all information available on + https://understat.com/ for Premier League players. + """ + async with aiohttp.ClientSession() as session: + players_data = await understat_players_data(session) + tasks = [asyncio.ensure_future(understat_matches_data(session, player)) + for player in players_data] + await asyncio.gather(*tasks) + + return players_data + + async def update_players(): """Updates all players in the database.""" async with aiohttp.ClientSession() as session: From c4dc59a5ede813b8d6cd10f8f762f5f345dbdb3a Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 21:44:51 +0000 Subject: [PATCH 06/22] Add desired_attributes constant --- FPLbot/constants.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/FPLbot/constants.py b/FPLbot/constants.py index 9609d96..e2217a6 100644 --- a/FPLbot/constants.py +++ b/FPLbot/constants.py @@ -56,3 +56,15 @@ "Lucas Moura": "Lucas Rodrigues Moura da Silva", "Willian": "Willian Borges Da Silva", } + +desired_attributes = [ + "xG", + "xA", + "key_passes", + "npg", + "npxG", + "xGChain", + "xGBuildup", + "shots", + "understat_history" +] From 461ebdc1f7dd25511514a65843bf5178d648ac02 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 21:44:59 +0000 Subject: [PATCH 07/22] Update update_players() for Understat Now also retrieves information from https://understat.com/ and saves this to the database. --- FPLbot/utils.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/FPLbot/utils.py b/FPLbot/utils.py index aa2a279..e71b2da 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -11,7 +11,7 @@ from fpl.utils import position_converter, team_converter from pymongo import MongoClient, ReplaceOne -from constants import understat_to_fpl +from constants import understat_to_fpl, desired_attributes client = MongoClient() database = client.fpl @@ -54,6 +54,7 @@ async def understat_players_data(session): """Returns a dict containing general player data retrieved from https://understat.com/. """ + logger.info("Getting Understat players data.") html = await fetch(session, "https://understat.com/league/EPL/") soup = BeautifulSoup(html, "html.parser") @@ -80,6 +81,7 @@ async def understat_matches_data(session, player): """Sets the 'matches' attribute of the given player to the data found on https://understat.com/player/. """ + logger.info(f"Getting {player['player_name']} Understat matches data.") html = await fetch(session, f"https://understat.com/player/{player['id']}") soup = BeautifulSoup(html, "html.parser") @@ -96,15 +98,17 @@ async def understat_matches_data(session, player): byte_data = codecs.escape_decode(match.group(1)) matches_data = json.loads(byte_data[0].decode("utf-8")) - player["matches"] = matches_data + player["understat_history"] = matches_data except UnboundLocalError: - player["matches"] = {} + player["understat_history"] = {} -async def understat_players(): +async def get_understat_players(): """Returns a list of dicts containing all information available on https://understat.com/ for Premier League players. """ + logger.info("Retrieving player information from https://understat.com/.") + async with aiohttp.ClientSession() as session: players_data = await understat_players_data(session) tasks = [asyncio.ensure_future(understat_matches_data(session, player)) @@ -120,6 +124,25 @@ async def update_players(): fpl = FPL(session) players = await fpl.get_players(include_summary=True, return_json=True) + understat_players = await get_understat_players() + + for player in players: + fpl_name = f"{player['first_name']} {player['second_name']}" + try: + understat_player = next( + understat_player for understat_player in understat_players + if fpl_name == understat_player["player_name"]) + except StopIteration: + # FPL player not available on Understat + continue + + # Only update FPL player with desired attributes + understat_attributes = { + attribute: value for attribute, value in understat_player.items() + if attribute in desired_attributes + } + player.update(understat_attributes) + requests = [ReplaceOne({"id": player["id"]}, player, upsert=True) for player in players] From 282a5741a05c43ee388ca5b51cf476483687ba94 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 23:40:42 +0000 Subject: [PATCH 08/22] Add team and player name converters --- FPLbot/constants.py | 31 +++++++++++++++++++++---------- FPLbot/utils.py | 20 +++++++++++++++++--- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/FPLbot/constants.py b/FPLbot/constants.py index e2217a6..3aa1280 100644 --- a/FPLbot/constants.py +++ b/FPLbot/constants.py @@ -1,4 +1,6 @@ -understat_to_fpl = { +import re + +player_dict = { "Fred": "Frederico Rodrigues de Paula Santos", "Ki Sung-yueng": "Sung-yueng Ki", "Solly March": "Solomon March", @@ -57,14 +59,23 @@ "Willian": "Willian Borges Da Silva", } +team_dict = { + "Manchester City": "Man City", + "Tottenham": "Spurs", + "Manchester United": "Man Utd", + "Wolverhampton Wanderers": "Wolves" +} + desired_attributes = [ - "xG", - "xA", - "key_passes", - "npg", - "npxG", - "xGChain", - "xGBuildup", - "shots", - "understat_history" + "xG", + "xA", + "key_passes", + "npg", + "npxG", + "xGChain", + "xGBuildup", + "shots", + "understat_history" ] + +versus_pattern = re.compile(r"!fplbot\s+([^\W\d_]+)\s+(?:vs.|vs)\s+([a-zA-Z ]+)") diff --git a/FPLbot/utils.py b/FPLbot/utils.py index e71b2da..528fc2f 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -11,7 +11,7 @@ from fpl.utils import position_converter, team_converter from pymongo import MongoClient, ReplaceOne -from constants import understat_to_fpl, desired_attributes +from constants import player_dict, team_dict, desired_attributes client = MongoClient() database = client.fpl @@ -71,8 +71,8 @@ async def understat_players_data(session): # Convert Understat player name to FPL player name for player in player_data: - if player["player_name"] in understat_to_fpl.keys(): - player["player_name"] = understat_to_fpl[player["player_name"]] + player["team_title"] = understat_team_converter(player["team_title"]) + player["player_name"] = understat_player_converter(player["player_name"]) return player_data @@ -168,6 +168,20 @@ def get_player_table(players, risers=True): return table_header + table_body +def understat_player_converter(player_name): + try: + return player_dict[player_name] + except KeyError: + return player_name + + +def understat_team_converter(team_name): + try: + return team_dict[team_name] + except KeyError: + return team_name + + if __name__ == "__main__": try: asyncio.run(update_players()) From 4cc601e130ed395712b1b47ca69401d8a568840b Mon Sep 17 00:00:00 2001 From: amosbastian Date: Thu, 7 Feb 2019 23:45:16 +0000 Subject: [PATCH 09/22] Add team name conversion Understat matches --- FPLbot/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/FPLbot/utils.py b/FPLbot/utils.py index 528fc2f..f48a151 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -98,6 +98,10 @@ async def understat_matches_data(session, player): byte_data = codecs.escape_decode(match.group(1)) matches_data = json.loads(byte_data[0].decode("utf-8")) + for fixture in matches_data: + fixture["h_team"] = understat_team_converter(fixture["h_team"]) + fixture["a_team"] = understat_team_converter(fixture["a_team"]) + player["understat_history"] = matches_data except UnboundLocalError: player["understat_history"] = {} From 7850ccd2307303a6b1b3eb811f19b9496da82299 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 20:52:48 +0000 Subject: [PATCH 10/22] Add versus pattern --- FPLbot/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FPLbot/constants.py b/FPLbot/constants.py index 3aa1280..b96e047 100644 --- a/FPLbot/constants.py +++ b/FPLbot/constants.py @@ -78,4 +78,4 @@ "understat_history" ] -versus_pattern = re.compile(r"!fplbot\s+([^\W\d_]+)\s+(?:vs.|vs)\s+([a-zA-Z ]+)") +versus_pattern = re.compile(r"!fplbot\s+([^\W\d_]+)\s+(?:vs.|vs)\s+([a-zA-Z ]+)(\d+)?") From c1569c116c7a264782c24b0c27657eb6ae83e2bc Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 20:53:18 +0000 Subject: [PATCH 11/22] Add fpl_team_names list & to_fpl_team_dict dict --- FPLbot/constants.py | 104 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/FPLbot/constants.py b/FPLbot/constants.py index b96e047..0f29045 100644 --- a/FPLbot/constants.py +++ b/FPLbot/constants.py @@ -79,3 +79,107 @@ ] versus_pattern = re.compile(r"!fplbot\s+([^\W\d_]+)\s+(?:vs.|vs)\s+([a-zA-Z ]+)(\d+)?") + +to_fpl_team_dict = { + "arsenal fc": "arsenal", + "the gunners": "arsenal", + "afc bournemouth": "bournemouth", + "the cherries": "bournemouth", + "boscombe": "bournemouth", + "the seagulls": "brighton", + "albion": "brighton", + "brighton and hove albion": "brighton", + "brighton & hove albion": "brighton", + "brighton fc": "brighton", + "bha": "brighton", + "burnley fc": "burnley", + "the clarets": "burnley", + "cardiff city": "cardiff", + "cardiff city fc": "cardiff", + "ccfc": "cardiff", + "car": "cardiff", + "the bluebirds": "cardiff", + "chelsea fc": "chelsea", + "cfc": "chelsea", + "che": "chelsea", + "the pensioners": "chelsea", + "crystal palace fc": "crystal palace", + "cpfc": "crystal palace", + "cp": "crystal palace", + "the eagles": "crystal palace", + "the glaziers": "crystal palace", + "everton fc": "everton", + "the toffees": "everton", + "fulham fc": "fulham", + "the cottagers": "fulham", + "huddersfield town": "huddersfield", + "huddersfield town afc": "huddersfield", + "huddersfield afc": "huddersfield", + "the terriers": "huddersfield", + "leicester city": "leicester", + "leicester city fc": "leicester", + "the foxes": "leicester", + "lfc": "liverpool", + "liverpool fc": "liverpool", + "mcfc": "man city", + "manchester city": "man city", + "manchester city fc": "man city", + "man city fc": "man city", + "citizens": "man city", + "mufc": "man utd", + "manchester united": "man utd", + "manchester utd": "man utd", + "man u": "man utd", + "man united": "man utd", + "the red devils": "man utd", + "red devils": "man utd", + "newcastle united": "newcastle", + "newcastle united fc": "newcastle", + "nufc": "newcastle", + "newcastle utd": "newcastle", + "the magpies": "newcastle", + "southampton fc": "southampton", + "the saints": "southampton", + "tottenham": "spurs", + "thfc": "spurs", + "tottenham hotspur": "spurs", + "tottenham hotspurs": "spurs", + "tottenham fc": "spurs", + "watford fc": "watford", + "wfc": "watford", + "the hornets": "watford", + "west ham united": "west ham", + "west ham utd": "west ham", + "the hammers": "west ham", + "west ham fc": "west ham", + "west ham united fc": "west ham", + "wolverhampton": "wolves", + "wolverhampton wanderers": "wolves", + "wolves fc": "wolves", + "wolverhampton fc": "wolves", + "wolverhampton wanderers fc": "wolves", + "the wanderers": "wolves" +} + +fpl_team_names = [ + "arsenal", + "bournemouth", + "brighton", + "burnley", + "cardiff", + "chelsea", + "crystal palace", + "everton", + "fulham", + "huddersfield", + "leicester", + "liverpool", + "man city", + "man utd", + "newcastle", + "southampton", + "spurs", + "watford", + "west ham", + "wolves" +] From 5cb19c7b89684a5d354bf76fa56bf72d8e3c8a6f Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 20:53:58 +0000 Subject: [PATCH 12/22] Add to_fpl_team() --- FPLbot/utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/FPLbot/utils.py b/FPLbot/utils.py index f48a151..99a234a 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -11,7 +11,8 @@ from fpl.utils import position_converter, team_converter from pymongo import MongoClient, ReplaceOne -from constants import player_dict, team_dict, desired_attributes +from constants import (desired_attributes, player_dict, team_dict, + to_fpl_team_dict) client = MongoClient() database = client.fpl @@ -172,6 +173,13 @@ def get_player_table(players, risers=True): return table_header + table_body +def to_fpl_team(team_name): + try: + return to_fpl_team_dict[team_name] + except KeyError: + return team_name + + def understat_player_converter(player_name): try: return player_dict[player_name] From e452a144754f51b24eaf3b0effd5be5d1229ed32 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 20:54:41 +0000 Subject: [PATCH 13/22] Add functions for handling player vs. team comments * comment_handler() for generic comment handling * player_vs_team_table() for creating the Markdown table * versus_team_handler() for handling player vs. team comments --- FPLbot/bot.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/FPLbot/bot.py b/FPLbot/bot.py index 5372fcf..d9cba73 100644 --- a/FPLbot/bot.py +++ b/FPLbot/bot.py @@ -2,15 +2,17 @@ import json import logging import os +import re from datetime import datetime import aiohttp import praw from fpl import FPL -from fpl.utils import position_converter +from fpl.utils import position_converter, team_converter from pymongo import MongoClient -from utils import create_logger, get_player_table, update_players +from constants import fpl_team_names, versus_pattern +from utils import create_logger, get_player_table, to_fpl_team, update_players dirname = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() @@ -78,6 +80,83 @@ async def post_price_changes(self): await update_players() +def player_vs_team_table(fixtures): + """Returns a Markdown table showing the player's performance in the given + fixtures. + """ + table = ("|Fixture|Date|MP|G|xG|A|xA|NPG|NPxG|KP|\n" + "|:-|:-:|-:|-:|-:|-:|-:|-:|-:|-:|\n") + + for fixture in fixtures: + home_team = f"{fixture['h_team']} {fixture['h_goals']}" + away_team = f"{fixture['a_goals']} {fixture['a_team']}" + + # Highlight the winning team + if int(fixture["h_goals"]) > int(fixture["a_goals"]): + home_team = f"**{home_team}**" + elif int(fixture["h_goals"]) < int(fixture["a_goals"]): + away_team = f"**{away_team}**" + + # Highlight whether the player was a starter or not + if fixture["position"].lower() != "sub": + fixture["time"] = f"**{fixture['time']}**" + + table += ( + f"|{home_team}-{away_team}" + f"|{fixture['date']}" + f"|{fixture['time']}" + f"|{fixture['goals']}" + f"|{float(fixture['xG']):.2f}" + f"|{fixture['assists']}" + f"|{float(fixture['xA']):.2f}" + f"|{float(fixture['npg']):.2f}" + f"|{float(fixture['npxG']):.2f}" + f"|{fixture['key_passes']}|\n" + ) + + return table + + +def versus_team_handler(player_name, team_name, number_of_fixtures): + """Function for handling player vs. team comment.""" + client = MongoClient() + player = client.fpl.players.find_one({"$text": {"$search": player_name}}) + + if not number_of_fixtures: + number_of_fixtures = len(player["understat_history"]) + + fixture_count = 0 + relevant_fixtures = [] + for fixture in player["understat_history"]: + if fixture_count >= int(number_of_fixtures): + break + + if (team_name != fixture["h_team"].lower() and + team_name != fixture["a_team"].lower()): + continue + + fixture_count += 1 + relevant_fixtures.append(fixture) + + table = player_vs_team_table(relevant_fixtures) + print(table) + + +def comment_handler(comment): + """Generic comment handler.""" + match = re.search(versus_pattern, comment) + + if not match: + return + + player_name = match.group(1).lower().strip() + opponent_name = match.group(2).lower().replace(".", "").strip() + number = match.group(3) + + if to_fpl_team(opponent_name) in fpl_team_names: + versus_team_handler(player_name, opponent_name, number) + + async def main(config): async with aiohttp.ClientSession() as session: fpl_bot = FPLBot(config, session) From 7ad56623c5f1d071021bd6fcc7c8b5ecde7cb799 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 21:03:48 +0000 Subject: [PATCH 14/22] Refactor functions into FPLBot class --- FPLbot/bot.py | 141 ++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 72 deletions(-) diff --git a/FPLbot/bot.py b/FPLbot/bot.py index d9cba73..1b80bce 100644 --- a/FPLbot/bot.py +++ b/FPLbot/bot.py @@ -16,11 +16,12 @@ dirname = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() +client = MongoClient() class FPLBot: def __init__(self, config, session): - self.client = MongoClient() + self.database = client.fpl self.fpl = FPL(session) self.reddit = praw.Reddit( client_id=config.get("CLIENT_ID"), @@ -36,7 +37,7 @@ async def get_price_changers(self): """ logger.info("Retrieving risers and fallers.") new_players = await self.fpl.get_players(include_summary=True) - old_players = [player for player in self.client.fpl.players.find()] + old_players = [player for player in self.database.players.find()] risers = [] fallers = [] @@ -79,82 +80,78 @@ async def post_price_changes(self): self.subreddit.submit(post_title, selftext=post_body) await update_players() + def player_vs_team_table(self, fixtures): + """Returns a Markdown table showing the player's performance in the + given fixtures. + """ + table = ("|Fixture|Date|MP|G|xG|A|xA|NPG|NPxG|KP|\n" + "|:-|:-:|-:|-:|-:|-:|-:|-:|-:|-:|\n") + + for fixture in fixtures: + home_team = f"{fixture['h_team']} {fixture['h_goals']}" + away_team = f"{fixture['a_goals']} {fixture['a_team']}" + + # Highlight the winning team + if int(fixture["h_goals"]) > int(fixture["a_goals"]): + home_team = f"**{home_team}**" + elif int(fixture["h_goals"]) < int(fixture["a_goals"]): + away_team = f"**{away_team}**" + + # Highlight whether the player was a starter or not + if fixture["position"].lower() != "sub": + fixture["time"] = f"**{fixture['time']}**" + + table += ( + f"|{home_team}-{away_team}" + f"|{fixture['date']}" + f"|{fixture['time']}" + f"|{fixture['goals']}" + f"|{float(fixture['xG']):.2f}" + f"|{fixture['assists']}" + f"|{float(fixture['xA']):.2f}" + f"|{float(fixture['npg']):.2f}" + f"|{float(fixture['npxG']):.2f}" + f"|{fixture['key_passes']}|\n" + ) + + return table + + def versus_team_handler(self, player_name, team_name, number_of_fixtures): + """Function for handling player vs. team comment.""" + player = self.database.players.find_one({"$text": {"$search": player_name}}) + + if not number_of_fixtures: + number_of_fixtures = len(player["understat_history"]) + + fixture_count = 0 + relevant_fixtures = [] + for fixture in player["understat_history"]: + if fixture_count >= int(number_of_fixtures): + break + + if (team_name != fixture["h_team"].lower() and + team_name != fixture["a_team"].lower()): + continue -def player_vs_team_table(fixtures): - """Returns a Markdown table showing the player's performance in the given - fixtures. - """ - table = ("|Fixture|Date|MP|G|xG|A|xA|NPG|NPxG|KP|\n" - "|:-|:-:|-:|-:|-:|-:|-:|-:|-:|-:|\n") - - for fixture in fixtures: - home_team = f"{fixture['h_team']} {fixture['h_goals']}" - away_team = f"{fixture['a_goals']} {fixture['a_team']}" - - # Highlight the winning team - if int(fixture["h_goals"]) > int(fixture["a_goals"]): - home_team = f"**{home_team}**" - elif int(fixture["h_goals"]) < int(fixture["a_goals"]): - away_team = f"**{away_team}**" - - # Highlight whether the player was a starter or not - if fixture["position"].lower() != "sub": - fixture["time"] = f"**{fixture['time']}**" - - table += ( - f"|{home_team}-{away_team}" - f"|{fixture['date']}" - f"|{fixture['time']}" - f"|{fixture['goals']}" - f"|{float(fixture['xG']):.2f}" - f"|{fixture['assists']}" - f"|{float(fixture['xA']):.2f}" - f"|{float(fixture['npg']):.2f}" - f"|{float(fixture['npxG']):.2f}" - f"|{fixture['key_passes']}|\n" - ) - - return table - - -def versus_team_handler(player_name, team_name, number_of_fixtures): - """Function for handling player vs. team comment.""" - client = MongoClient() - player = client.fpl.players.find_one({"$text": {"$search": player_name}}) - - if not number_of_fixtures: - number_of_fixtures = len(player["understat_history"]) - - fixture_count = 0 - relevant_fixtures = [] - for fixture in player["understat_history"]: - if fixture_count >= int(number_of_fixtures): - break - - if (team_name != fixture["h_team"].lower() and - team_name != fixture["a_team"].lower()): - continue - - fixture_count += 1 - relevant_fixtures.append(fixture) - - table = player_vs_team_table(relevant_fixtures) - print(table) + fixture_count += 1 + relevant_fixtures.append(fixture) + table = self.player_vs_team_table(relevant_fixtures) + print(table) -def comment_handler(comment): - """Generic comment handler.""" - match = re.search(versus_pattern, comment) + def comment_handler(self, comment): + """Generic comment handler.""" + match = re.search(versus_pattern, comment) - if not match: - return + if not match: + return - player_name = match.group(1).lower().strip() - opponent_name = match.group(2).lower().replace(".", "").strip() - number = match.group(3) + player_name = match.group(1).lower().strip() + opponent_name = match.group(2).lower().replace(".", "").strip() + number = match.group(3) - if to_fpl_team(opponent_name) in fpl_team_names: - versus_team_handler(player_name, opponent_name, number) + if to_fpl_team(opponent_name) in fpl_team_names: + self.versus_team_handler(player_name, opponent_name, number) async def main(config): From 3b4da89fce3466ad033171bf30348badc8cba947 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 22:48:46 +0000 Subject: [PATCH 15/22] Fix formatting NPG --- FPLbot/bot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/FPLbot/bot.py b/FPLbot/bot.py index 1b80bce..7fe2395 100644 --- a/FPLbot/bot.py +++ b/FPLbot/bot.py @@ -109,7 +109,7 @@ def player_vs_team_table(self, fixtures): f"|{float(fixture['xG']):.2f}" f"|{fixture['assists']}" f"|{float(fixture['xA']):.2f}" - f"|{float(fixture['npg']):.2f}" + f"|{fixture['npg']}" f"|{float(fixture['npxG']):.2f}" f"|{fixture['key_passes']}|\n" ) @@ -119,7 +119,6 @@ def player_vs_team_table(self, fixtures): def versus_team_handler(self, player_name, team_name, number_of_fixtures): """Function for handling player vs. team comment.""" player = self.database.players.find_one({"$text": {"$search": player_name}}) - if not number_of_fixtures: number_of_fixtures = len(player["understat_history"]) From 60bd73004e4822eb3d666cb528068877c63f8ae4 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 22:49:25 +0000 Subject: [PATCH 16/22] Update versus_pattern for first + second name usage --- FPLbot/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FPLbot/constants.py b/FPLbot/constants.py index 0f29045..c2a1717 100644 --- a/FPLbot/constants.py +++ b/FPLbot/constants.py @@ -78,7 +78,7 @@ "understat_history" ] -versus_pattern = re.compile(r"!fplbot\s+([^\W\d_]+)\s+(?:vs.|vs)\s+([a-zA-Z ]+)(\d+)?") +versus_pattern = re.compile(r"!fplbot\s+([^\W\d_]+(?:\s[^\W\d_]+)?)\s+(?:vs.|vs)\s+([a-zA-Z ]+)(\d+)?") to_fpl_team_dict = { "arsenal fc": "arsenal", From 5a49e201e0c2f22cbd0532296f233cfb584b8288 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Fri, 8 Feb 2019 22:51:23 +0000 Subject: [PATCH 17/22] Refactor update_players() robustness --- FPLbot/utils.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/FPLbot/utils.py b/FPLbot/utils.py index 99a234a..8f3c433 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -94,7 +94,7 @@ async def understat_matches_data(session, player): if match: break - # If no match could be found, simply return an empty dict + # If no match could be found, retry (probably rate limited?) try: byte_data = codecs.escape_decode(match.group(1)) matches_data = json.loads(byte_data[0].decode("utf-8")) @@ -105,7 +105,9 @@ async def understat_matches_data(session, player): player["understat_history"] = matches_data except UnboundLocalError: - player["understat_history"] = {} + await understat_matches_data(session, player) + + return player async def get_understat_players(): @@ -118,9 +120,9 @@ async def get_understat_players(): players_data = await understat_players_data(session) tasks = [asyncio.ensure_future(understat_matches_data(session, player)) for player in players_data] - await asyncio.gather(*tasks) + players = await asyncio.gather(*tasks) - return players_data + return players async def update_players(): @@ -129,30 +131,26 @@ async def update_players(): fpl = FPL(session) players = await fpl.get_players(include_summary=True, return_json=True) - understat_players = await get_understat_players() + requests = [ReplaceOne({"id": player["id"]}, player, upsert=True) + for player in players] - for player in players: - fpl_name = f"{player['first_name']} {player['second_name']}" - try: - understat_player = next( - understat_player for understat_player in understat_players - if fpl_name == understat_player["player_name"]) - except StopIteration: - # FPL player not available on Understat - continue + logger.info("Updating FPL players in database.") + database.players.bulk_write(requests) + understat_players = await get_understat_players() + + logger.info("Adding Understat data to players in database.") + for understat_player in understat_players: # Only update FPL player with desired attributes understat_attributes = { attribute: value for attribute, value in understat_player.items() if attribute in desired_attributes } - player.update(understat_attributes) - requests = [ReplaceOne({"id": player["id"]}, player, upsert=True) - for player in players] - - logger.info("Updating players in database.") - database.players.bulk_write(requests) + player = database.players.find_one_and_update( + {"$text": {"$search": understat_player["player_name"]}}, + {"$set": understat_attributes} + ) def get_player_table(players, risers=True): From c5ec7a0ea02800cb50e4e422367ca53edb7fe2b5 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Sat, 9 Feb 2019 15:55:10 +0000 Subject: [PATCH 18/22] Update update_players() to use text search Now uses the player's full name and team to try and find the correct player, and then updates the player in the database with their respective Understat attributes. --- FPLbot/utils.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/FPLbot/utils.py b/FPLbot/utils.py index 8f3c433..cf04072 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -127,28 +127,37 @@ async def get_understat_players(): async def update_players(): """Updates all players in the database.""" + logger.info("Updating FPL players in database.") async with aiohttp.ClientSession() as session: fpl = FPL(session) players = await fpl.get_players(include_summary=True, return_json=True) + for player in players: + player["team"] = team_converter(player["team"]) requests = [ReplaceOne({"id": player["id"]}, player, upsert=True) for player in players] - - logger.info("Updating FPL players in database.") database.players.bulk_write(requests) + logger.info("Adding Understat data to players in database.") understat_players = await get_understat_players() - logger.info("Adding Understat data to players in database.") - for understat_player in understat_players: + for player in understat_players: # Only update FPL player with desired attributes understat_attributes = { - attribute: value for attribute, value in understat_player.items() + attribute: value for attribute, value in player.items() if attribute in desired_attributes } - player = database.players.find_one_and_update( - {"$text": {"$search": understat_player["player_name"]}}, + # Use player's full name and team to try and find the correct player + search_string = f"{player['player_name']} {player['team_title']}" + players = database.players.find( + {"$text": {"$search": search_string}}, + {"score": {"$meta": "textScore"}} + ).sort([("score", {"$meta": "textScore"})]) + relevant_player = list(players)[0] + + database.players.update_one( + {"id": relevant_player["id"]}, {"$set": understat_attributes} ) From 189843c4815a5cbd9db5b81cb13931af578f75eb Mon Sep 17 00:00:00 2001 From: amosbastian Date: Sat, 9 Feb 2019 15:56:23 +0000 Subject: [PATCH 19/22] Update versus_team_handler() to use text search Now uses text search with textScore to try and find the correct player in the database. --- FPLbot/bot.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/FPLbot/bot.py b/FPLbot/bot.py index 7fe2395..60e8b6b 100644 --- a/FPLbot/bot.py +++ b/FPLbot/bot.py @@ -118,7 +118,13 @@ def player_vs_team_table(self, fixtures): def versus_team_handler(self, player_name, team_name, number_of_fixtures): """Function for handling player vs. team comment.""" - player = self.database.players.find_one({"$text": {"$search": player_name}}) + # Find most relevant player using text search + players = self.database.players.find( + {"$text": {"$search": player_name}}, + {"score": {"$meta": "textScore"}} + ).sort([("score", {"$meta": "textScore"})]) + player = list(players.limit(1))[0] + if not number_of_fixtures: number_of_fixtures = len(player["understat_history"]) @@ -157,7 +163,7 @@ async def main(config): async with aiohttp.ClientSession() as session: fpl_bot = FPLBot(config, session) - await fpl_bot.post_price_changes() + fpl_bot.comment_handler("!fplbot mohamed salah vs. bournemouth") if __name__ == "__main__": From 8bbc2e9b52e61cd6e179e180582f7049be8a58d8 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Sat, 9 Feb 2019 20:00:15 +0000 Subject: [PATCH 20/22] Add Reddit comment handling / streaming --- FPLbot/bot.py | 59 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/FPLbot/bot.py b/FPLbot/bot.py index 60e8b6b..7143f0a 100644 --- a/FPLbot/bot.py +++ b/FPLbot/bot.py @@ -8,7 +8,7 @@ import aiohttp import praw from fpl import FPL -from fpl.utils import position_converter, team_converter +from fpl.utils import position_converter from pymongo import MongoClient from constants import fpl_team_names, versus_pattern @@ -21,6 +21,7 @@ class FPLBot: def __init__(self, config, session): + self.config = config self.database = client.fpl self.fpl = FPL(session) self.reddit = praw.Reddit( @@ -29,7 +30,7 @@ def __init__(self, config, session): password=config.get("PASSWORD"), user_agent=config.get("USER_AGENT"), username=config.get("USERNAME")) - self.subreddit = self.reddit.subreddit(config.get("SUBREDDIT")) + self.subreddit = self.reddit.subreddit(self.config.get("SUBREDDIT")) async def get_price_changers(self): """Returns a list of players whose price has changed since the last @@ -123,13 +124,19 @@ def versus_team_handler(self, player_name, team_name, number_of_fixtures): {"$text": {"$search": player_name}}, {"score": {"$meta": "textScore"}} ).sort([("score", {"$meta": "textScore"})]) - player = list(players.limit(1))[0] + + try: + player = list(players.limit(1))[0] + except IndexError: + logger.error(f"Player {player_name} could not be found!") + return if not number_of_fixtures: number_of_fixtures = len(player["understat_history"]) fixture_count = 0 relevant_fixtures = [] + team_name = to_fpl_team(team_name) for fixture in player["understat_history"]: if fixture_count >= int(number_of_fixtures): break @@ -141,14 +148,24 @@ def versus_team_handler(self, player_name, team_name, number_of_fixtures): fixture_count += 1 relevant_fixtures.append(fixture) - table = self.player_vs_team_table(relevant_fixtures) - print(table) + player_vs_team_table = self.player_vs_team_table(relevant_fixtures) + return player_vs_team_table + + def add_comment_to_database(self, comment): + logger.info(f"Adding comment with ID {comment.id} to the database.") + self.database.comments.update_one( + {"comment_id": comment.id}, + {"$set": {"comment_id": comment.id}}, + upsert=True + ) def comment_handler(self, comment): """Generic comment handler.""" - match = re.search(versus_pattern, comment) + logger.info(f"Handling COMMENT with ID {comment.id}.") + match = re.search(versus_pattern, comment.body.lower()) if not match: + logger.info(f"Comment with ID {comment.id} does not match pattern.") return player_name = match.group(1).lower().strip() @@ -156,14 +173,40 @@ def comment_handler(self, comment): number = match.group(3) if to_fpl_team(opponent_name) in fpl_team_names: - self.versus_team_handler(player_name, opponent_name, number) + reply_text = self.versus_team_handler( + player_name, opponent_name, number) + else: + return + + if reply_text: + logger.info(f"Replying ({player_name} vs. {opponent_name}) to " + f"comment with ID {comment.id}.") + comment.reply(reply_text) + self.add_comment_to_database(comment) + + def is_new_comment(self, comment_id): + if self.database.comments.count_documents({"comment_id": comment_id}) < 1: + return True + return False + + def run(self): + for comment in self.subreddit.stream.comments(): + body = comment.body.lower() + if self.config.get("BOT_PREFIX") in body: + if not self.is_new_comment(comment.id): + continue + + try: + self.comment_handler(comment) + except Exception as error: + logger.error(f"Something went wrong: {error}") async def main(config): async with aiohttp.ClientSession() as session: fpl_bot = FPLBot(config, session) - fpl_bot.comment_handler("!fplbot mohamed salah vs. bournemouth") + fpl_bot.run() if __name__ == "__main__": From 2c861a6287bccc9f1410845d42bef2ad50b900fe Mon Sep 17 00:00:00 2001 From: amosbastian Date: Sat, 9 Feb 2019 20:00:28 +0000 Subject: [PATCH 21/22] Update config.json.example --- config.json.example | 1 + 1 file changed, 1 insertion(+) diff --git a/config.json.example b/config.json.example index 5645674..18cf3b3 100644 --- a/config.json.example +++ b/config.json.example @@ -5,4 +5,5 @@ "CLIENT_SECRET": "gko_LXELoV07ZBNUXrvWZfzE3aI", "USER_AGENT": "The original FPLbot.", "SUBREDDIT": "FantasyPL", + "BOT_PREFIX": "!fplbot", } From e1a5ea039b165de68ef041dd4a7f8980853993b9 Mon Sep 17 00:00:00 2001 From: amosbastian Date: Sat, 9 Feb 2019 20:00:53 +0000 Subject: [PATCH 22/22] Update pattern for '-' compatibility --- FPLbot/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FPLbot/constants.py b/FPLbot/constants.py index c2a1717..cfa0e5b 100644 --- a/FPLbot/constants.py +++ b/FPLbot/constants.py @@ -78,7 +78,7 @@ "understat_history" ] -versus_pattern = re.compile(r"!fplbot\s+([^\W\d_]+(?:\s[^\W\d_]+)?)\s+(?:vs.|vs)\s+([a-zA-Z ]+)(\d+)?") +versus_pattern = re.compile(r"!fplbot\s+([^\W\d]+(?:[\s-][^\W\d]+)*)\s+(?:vs.|vs)\s+([a-zA-Z ]+)(\d+)?") to_fpl_team_dict = { "arsenal fc": "arsenal",