diff --git a/.gitignore b/.gitignore index 2bb111d..b87ff25 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,5 @@ venv.bak/ .vscode/ config.json + +*.swp diff --git a/FPLbot/bot.py b/FPLbot/bot.py index 5ec89eb..7143f0a 100644 --- a/FPLbot/bot.py +++ b/FPLbot/bot.py @@ -2,6 +2,7 @@ import json import logging import os +import re from datetime import datetime import aiohttp @@ -10,15 +11,18 @@ from fpl.utils import position_converter from pymongo import MongoClient -from utils import create_logger, get_player_table, update_players +from constants import fpl_team_names, versus_pattern +from utils import create_logger, get_player_table, to_fpl_team, update_players dirname = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() +client = MongoClient() class FPLBot: def __init__(self, config, session): - self.client = MongoClient() + self.config = config + self.database = client.fpl self.fpl = FPL(session) self.reddit = praw.Reddit( client_id=config.get("CLIENT_ID"), @@ -26,7 +30,7 @@ def __init__(self, config, session): password=config.get("PASSWORD"), user_agent=config.get("USER_AGENT"), username=config.get("USERNAME")) - self.subreddit = self.reddit.subreddit(config.get("SUBREDDIT")) + self.subreddit = self.reddit.subreddit(self.config.get("SUBREDDIT")) async def get_price_changers(self): """Returns a list of players whose price has changed since the last @@ -34,7 +38,7 @@ async def get_price_changers(self): """ logger.info("Retrieving risers and fallers.") new_players = await self.fpl.get_players(include_summary=True) - old_players = [player for player in self.client.fpl.players.find()] + old_players = [player for player in self.database.players.find()] risers = [] fallers = [] @@ -77,12 +81,132 @@ async def post_price_changes(self): self.subreddit.submit(post_title, selftext=post_body) await update_players() + def player_vs_team_table(self, fixtures): + """Returns a Markdown table showing the player's performance in the + given fixtures. + """ + table = ("|Fixture|Date|MP|G|xG|A|xA|NPG|NPxG|KP|\n" + "|:-|:-:|-:|-:|-:|-:|-:|-:|-:|-:|\n") + + for fixture in fixtures: + home_team = f"{fixture['h_team']} {fixture['h_goals']}" + away_team = f"{fixture['a_goals']} {fixture['a_team']}" + + # Highlight the winning team + if int(fixture["h_goals"]) > int(fixture["a_goals"]): + home_team = f"**{home_team}**" + elif int(fixture["h_goals"]) < int(fixture["a_goals"]): + away_team = f"**{away_team}**" + + # Highlight whether the player was a starter or not + if fixture["position"].lower() != "sub": + fixture["time"] = f"**{fixture['time']}**" + + table += ( + f"|{home_team}-{away_team}" + f"|{fixture['date']}" + f"|{fixture['time']}" + f"|{fixture['goals']}" + f"|{float(fixture['xG']):.2f}" + f"|{fixture['assists']}" + f"|{float(fixture['xA']):.2f}" + f"|{fixture['npg']}" + f"|{float(fixture['npxG']):.2f}" + f"|{fixture['key_passes']}|\n" + ) + + return table + + def versus_team_handler(self, player_name, team_name, number_of_fixtures): + """Function for handling player vs. team comment.""" + # Find most relevant player using text search + players = self.database.players.find( + {"$text": {"$search": player_name}}, + {"score": {"$meta": "textScore"}} + ).sort([("score", {"$meta": "textScore"})]) + + try: + player = list(players.limit(1))[0] + except IndexError: + logger.error(f"Player {player_name} could not be found!") + return + + if not number_of_fixtures: + number_of_fixtures = len(player["understat_history"]) + + fixture_count = 0 + relevant_fixtures = [] + team_name = to_fpl_team(team_name) + for fixture in player["understat_history"]: + if fixture_count >= int(number_of_fixtures): + break + + if (team_name != fixture["h_team"].lower() and + team_name != fixture["a_team"].lower()): + continue + + fixture_count += 1 + relevant_fixtures.append(fixture) + + player_vs_team_table = self.player_vs_team_table(relevant_fixtures) + return player_vs_team_table + + def add_comment_to_database(self, comment): + logger.info(f"Adding comment with ID {comment.id} to the database.") + self.database.comments.update_one( + {"comment_id": comment.id}, + {"$set": {"comment_id": comment.id}}, + upsert=True + ) + + def comment_handler(self, comment): + """Generic comment handler.""" + logger.info(f"Handling COMMENT with ID {comment.id}.") + match = re.search(versus_pattern, comment.body.lower()) + + if not match: + logger.info(f"Comment with ID {comment.id} does not match pattern.") + return + + player_name = match.group(1).lower().strip() + opponent_name = match.group(2).lower().replace(".", "").strip() + number = match.group(3) + + if to_fpl_team(opponent_name) in fpl_team_names: + reply_text = self.versus_team_handler( + player_name, opponent_name, number) + else: + return + + if reply_text: + logger.info(f"Replying ({player_name} vs. {opponent_name}) to " + f"comment with ID {comment.id}.") + comment.reply(reply_text) + self.add_comment_to_database(comment) + + def is_new_comment(self, comment_id): + if self.database.comments.count_documents({"comment_id": comment_id}) < 1: + return True + return False + + def run(self): + for comment in self.subreddit.stream.comments(): + body = comment.body.lower() + if self.config.get("BOT_PREFIX") in body: + if not self.is_new_comment(comment.id): + continue + + try: + self.comment_handler(comment) + except Exception as error: + logger.error(f"Something went wrong: {error}") + async def main(config): async with aiohttp.ClientSession() as session: fpl_bot = FPLBot(config, session) - await fpl_bot.post_price_changes() + fpl_bot.run() if __name__ == "__main__": @@ -93,4 +217,3 @@ async def main(config): loop = asyncio.get_event_loop() loop.run_until_complete(main(config)) loop.close() - diff --git a/FPLbot/constants.py b/FPLbot/constants.py new file mode 100644 index 0000000..cfa0e5b --- /dev/null +++ b/FPLbot/constants.py @@ -0,0 +1,185 @@ +import re + +player_dict = { + "Fred": "Frederico Rodrigues de Paula Santos", + "Ki Sung-yueng": "Sung-yueng Ki", + "Solly March": "Solomon March", + "Jonny": "Jonathan Castro Otto", + "Felipe Anderson": "Felipe Anderson Pereira Gomes", + "Mat Ryan": "Mathew Ryan", + "Kenedy": "Robert Kenedy Nunes do Nascimento", + "Jorginho": "Jorge Luiz Frello Filho", + "Bernard": "Bernard Anício Caldeira Duarte", + "Romain Saiss": "Romain Saïss", + "Bernardo Silva": "Bernardo Mota Veiga de Carvalho e Silva", + "N'Golo Kanté": "N'Golo Kanté", + "João Moutinho": "João Filipe Iria Santos Moutinho", + "Franck Zambo": "André-Frank Zambo Anguissa", + "Fousseni Diabate": "Fousseni Diabaté", + "Jazz Richards": "Ashley Darel Jazz Richards", + "Danilo": "Danilo Luiz da Silva", + "Richarlison": "Richarlison de Andrade", + "Bernardo": "Bernardo Fernandes da Silva Junior", + "Fernandinho": "Fernando Luiz Rosa", + "Joselu": "Jose Luis Mato Sanmartín", + "Son Heung-Min": "Heung-Min Son", + "Diogo Dalot": "José Diogo Dalot Teixeira", + "José Izquierdo": "José Heriberto Izquierdo Mena", + "Fabri": "Fabricio Agosto Ramírez", + "Eddie Nketiah": "Edward Nketiah", + "Rui Patrício": "Rui Pedro dos Santos Patrício", + "Greg Cunningham": "Greg Cunninghamm", + "Junior Hoilett": "David Junior Hoilett", + "Isaac Success": "Isaac Success Ajayi", + "Xande Silva": "Alexandre Nascimento Costa Silva", + "Bruno": "Bruno Saltor Grau", + "Léo Bonatini": "Bonatini Lohner Maia Bonatini", + "André Gomes": "André Filipe Tavares Gomes", + "Kiko Femenía": "Francisco Femenía Far", + "Dele Alli": "Bamidele Alli", + "Ricardo Pereira": "Ricardo Domingos Barbosa Pereira", + "Sokratis": "Sokratis Papastathopoulos", + "Alisson": "Alisson Ramses Becker", + "Fabinho": "Fabio Henrique Tavares", + "Adrien Silva": "Adrien Sebastian Perruchet Silva", + "David de Gea": "David De Gea", + "Gabriel Jesus": "Gabriel Fernando de Jesus", + "Pedro": "Pedro Rodríguez Ledesma", + "Zanka": "Mathias Jorgensen", + "David Luiz": "David Luiz Moreira Marinho", + "Rúben Neves": "Rúben Diogo da Silva Neves", + "Ben Chilwell": "Benjamin Chilwell", + "Kepa": "Kepa Arrizabalaga", + "Emerson": "Emerson Palmieri dos Santos", + "Ederson": "Ederson Santana de Moraes", + "Chicharito": "Javier Hernández Balcázar", + "Rúben Vinagre": "Rúben Gonçalo Silva Nascimento Vinagre", + "Oriol Romeu": "Oriol Romeu Vidal", + "Lucas Moura": "Lucas Rodrigues Moura da Silva", + "Willian": "Willian Borges Da Silva", +} + +team_dict = { + "Manchester City": "Man City", + "Tottenham": "Spurs", + "Manchester United": "Man Utd", + "Wolverhampton Wanderers": "Wolves" +} + +desired_attributes = [ + "xG", + "xA", + "key_passes", + "npg", + "npxG", + "xGChain", + "xGBuildup", + "shots", + "understat_history" +] + +versus_pattern = re.compile(r"!fplbot\s+([^\W\d]+(?:[\s-][^\W\d]+)*)\s+(?:vs.|vs)\s+([a-zA-Z ]+)(\d+)?") + +to_fpl_team_dict = { + "arsenal fc": "arsenal", + "the gunners": "arsenal", + "afc bournemouth": "bournemouth", + "the cherries": "bournemouth", + "boscombe": "bournemouth", + "the seagulls": "brighton", + "albion": "brighton", + "brighton and hove albion": "brighton", + "brighton & hove albion": "brighton", + "brighton fc": "brighton", + "bha": "brighton", + "burnley fc": "burnley", + "the clarets": "burnley", + "cardiff city": "cardiff", + "cardiff city fc": "cardiff", + "ccfc": "cardiff", + "car": "cardiff", + "the bluebirds": "cardiff", + "chelsea fc": "chelsea", + "cfc": "chelsea", + "che": "chelsea", + "the pensioners": "chelsea", + "crystal palace fc": "crystal palace", + "cpfc": "crystal palace", + "cp": "crystal palace", + "the eagles": "crystal palace", + "the glaziers": "crystal palace", + "everton fc": "everton", + "the toffees": "everton", + "fulham fc": "fulham", + "the cottagers": "fulham", + "huddersfield town": "huddersfield", + "huddersfield town afc": "huddersfield", + "huddersfield afc": "huddersfield", + "the terriers": "huddersfield", + "leicester city": "leicester", + "leicester city fc": "leicester", + "the foxes": "leicester", + "lfc": "liverpool", + "liverpool fc": "liverpool", + "mcfc": "man city", + "manchester city": "man city", + "manchester city fc": "man city", + "man city fc": "man city", + "citizens": "man city", + "mufc": "man utd", + "manchester united": "man utd", + "manchester utd": "man utd", + "man u": "man utd", + "man united": "man utd", + "the red devils": "man utd", + "red devils": "man utd", + "newcastle united": "newcastle", + "newcastle united fc": "newcastle", + "nufc": "newcastle", + "newcastle utd": "newcastle", + "the magpies": "newcastle", + "southampton fc": "southampton", + "the saints": "southampton", + "tottenham": "spurs", + "thfc": "spurs", + "tottenham hotspur": "spurs", + "tottenham hotspurs": "spurs", + "tottenham fc": "spurs", + "watford fc": "watford", + "wfc": "watford", + "the hornets": "watford", + "west ham united": "west ham", + "west ham utd": "west ham", + "the hammers": "west ham", + "west ham fc": "west ham", + "west ham united fc": "west ham", + "wolverhampton": "wolves", + "wolverhampton wanderers": "wolves", + "wolves fc": "wolves", + "wolverhampton fc": "wolves", + "wolverhampton wanderers fc": "wolves", + "the wanderers": "wolves" +} + +fpl_team_names = [ + "arsenal", + "bournemouth", + "brighton", + "burnley", + "cardiff", + "chelsea", + "crystal palace", + "everton", + "fulham", + "huddersfield", + "leicester", + "liverpool", + "man city", + "man utd", + "newcastle", + "southampton", + "spurs", + "watford", + "west ham", + "wolves" +] diff --git a/FPLbot/utils.py b/FPLbot/utils.py index cb7ea77..cf04072 100644 --- a/FPLbot/utils.py +++ b/FPLbot/utils.py @@ -1,12 +1,19 @@ import asyncio +import codecs +import json import logging import os +import re import aiohttp +from bs4 import BeautifulSoup from fpl import FPL from fpl.utils import position_converter, team_converter from pymongo import MongoClient, ReplaceOne +from constants import (desired_attributes, player_dict, team_dict, + to_fpl_team_dict) + client = MongoClient() database = client.fpl logger = logging.getLogger("FPLbot") @@ -39,18 +46,121 @@ def create_logger(): return logger +async def fetch(session, url): + async with session.get(url) as response: + return await response.text() + + +async def understat_players_data(session): + """Returns a dict containing general player data retrieved from + https://understat.com/. + """ + logger.info("Getting Understat players data.") + html = await fetch(session, "https://understat.com/league/EPL/") + + soup = BeautifulSoup(html, "html.parser") + scripts = soup.find_all("script") + pattern = re.compile(r"var\s+playersData\s+=\s+JSON.parse\(\'(.*?)\'\);") + + for script in scripts: + match = re.search(pattern, script.string) + if match: + break + + byte_data = codecs.escape_decode(match.group(1)) + player_data = json.loads(byte_data[0].decode("utf-8")) + + # Convert Understat player name to FPL player name + for player in player_data: + player["team_title"] = understat_team_converter(player["team_title"]) + player["player_name"] = understat_player_converter(player["player_name"]) + + return player_data + + +async def understat_matches_data(session, player): + """Sets the 'matches' attribute of the given player to the data found on + https://understat.com/player/. + """ + logger.info(f"Getting {player['player_name']} Understat matches data.") + html = await fetch(session, f"https://understat.com/player/{player['id']}") + + soup = BeautifulSoup(html, "html.parser") + scripts = soup.find_all("script") + pattern = re.compile(r"var\s+matchesData\s+=\s+JSON.parse\(\'(.*?)\'\);") + + for script in scripts: + match = re.search(pattern, script.string) + if match: + break + + # If no match could be found, retry (probably rate limited?) + try: + byte_data = codecs.escape_decode(match.group(1)) + matches_data = json.loads(byte_data[0].decode("utf-8")) + + for fixture in matches_data: + fixture["h_team"] = understat_team_converter(fixture["h_team"]) + fixture["a_team"] = understat_team_converter(fixture["a_team"]) + + player["understat_history"] = matches_data + except UnboundLocalError: + await understat_matches_data(session, player) + + return player + + +async def get_understat_players(): + """Returns a list of dicts containing all information available on + https://understat.com/ for Premier League players. + """ + logger.info("Retrieving player information from https://understat.com/.") + + async with aiohttp.ClientSession() as session: + players_data = await understat_players_data(session) + tasks = [asyncio.ensure_future(understat_matches_data(session, player)) + for player in players_data] + players = await asyncio.gather(*tasks) + + return players + + async def update_players(): """Updates all players in the database.""" + logger.info("Updating FPL players in database.") async with aiohttp.ClientSession() as session: fpl = FPL(session) players = await fpl.get_players(include_summary=True, return_json=True) + for player in players: + player["team"] = team_converter(player["team"]) requests = [ReplaceOne({"id": player["id"]}, player, upsert=True) for player in players] - - logger.info("Updating players in database.") database.players.bulk_write(requests) + logger.info("Adding Understat data to players in database.") + understat_players = await get_understat_players() + + for player in understat_players: + # Only update FPL player with desired attributes + understat_attributes = { + attribute: value for attribute, value in player.items() + if attribute in desired_attributes + } + + # Use player's full name and team to try and find the correct player + search_string = f"{player['player_name']} {player['team_title']}" + players = database.players.find( + {"$text": {"$search": search_string}}, + {"score": {"$meta": "textScore"}} + ).sort([("score", {"$meta": "textScore"})]) + relevant_player = list(players)[0] + + database.players.update_one( + {"id": relevant_player["id"]}, + {"$set": understat_attributes} + ) + def get_player_table(players, risers=True): """Returns the table used in the player price change posts on Reddit.""" @@ -70,6 +180,27 @@ def get_player_table(players, risers=True): return table_header + table_body +def to_fpl_team(team_name): + try: + return to_fpl_team_dict[team_name] + except KeyError: + return team_name + + +def understat_player_converter(player_name): + try: + return player_dict[player_name] + except KeyError: + return player_name + + +def understat_team_converter(team_name): + try: + return team_dict[team_name] + except KeyError: + return team_name + + if __name__ == "__main__": try: asyncio.run(update_players()) @@ -77,4 +208,3 @@ def get_player_table(players, risers=True): loop = asyncio.get_event_loop() loop.run_until_complete(update_players()) loop.close() - diff --git a/config.json.example b/config.json.example index 5645674..18cf3b3 100644 --- a/config.json.example +++ b/config.json.example @@ -5,4 +5,5 @@ "CLIENT_SECRET": "gko_LXELoV07ZBNUXrvWZfzE3aI", "USER_AGENT": "The original FPLbot.", "SUBREDDIT": "FantasyPL", + "BOT_PREFIX": "!fplbot", }