diff --git a/README.md b/README.md index c7e8434..90f277c 100644 --- a/README.md +++ b/README.md @@ -11,17 +11,17 @@ Retrieve player statistics in Fortnite Battle Royale * [Setup](#setup) ## Features -- Asks if you want to see current squad stats when joining Fortnite discord channel +- Asks if you want to see current squad stats when joining Fortnite discord channel -Fortnite Tracker search (Default): +Fortnite Tracker search (Default): - Calculates player current season statistics in different Game Modes (Solo, Duos, Trios, Squads) - Calculates player's overall statistics > Statistics include: - > - KD + > - KD > - Wins > - Win Percentage > - Matches Played - > - TRN + > - TRN ---------------------------------------------------------------------------------- Fortnite API search (Fall back): If player profile is not found in fortnitetracker.com, fall back to using Fortnite API @@ -29,12 +29,12 @@ If player profile is not found in fortnitetracker.com, fall back to using Fortni - Calculates player overall season statistics in different Game Modes (Solo, Duos, Squads) - Calculates player's overall statistics > Statistics include: - > - KD + > - KD > - Wins > - Win Percentage - > - Kills + > - Kills > - Matches Played -- Link to player's Twitch stream if currently streaming +- Link to player's Twitch stream if currently streaming ---------------------------------------------------------------------------------- - Link to Fortnite Tracker player profile (click on username to navigate) @@ -72,7 +72,7 @@ If player profile is not found in fortnitetracker.com, fall back to using Fortni !stats enemy !upgrade -!gold +!gold !chests !loot @@ -132,7 +132,7 @@ To install the Fortnite Replay Reader pip package, you need to install the packa To do so, run: ``` -pip install bitstring +pip install wheel bitstring pycryptodome ``` Then: diff --git a/clients/fortnite_api.py b/clients/fortnite_api.py index 69f5ff6..38b3926 100644 --- a/clients/fortnite_api.py +++ b/clients/fortnite_api.py @@ -3,6 +3,8 @@ import aiohttp import discord +from exceptions import UserDoesNotExist + FORTNITE_API_TOKEN = os.getenv("FORTNITE_API_TOKEN") TWITCH_CLIENT_ID = os.getenv("TWITCH_CLIENT_ID") @@ -62,20 +64,22 @@ async def _get_player_account_id(session, player_name, platform): """ Get account id given player name and platform """ - raw_response = await session.get( + resp = await session.get( FORTNITE_ACCOUNT_ID_URL.format(username=player_name, platform=platform), headers={"Authorization": FORTNITE_API_TOKEN} ) - return await raw_response.json() + if resp.status == 404: + raise UserDoesNotExist(f"Username not found in FN API: {player_name}") + return await resp.json() async def _get_player_stats(session, account_id): """ Get player stats given account id """ - raw_response = await session.get( + resp = await session.get( FORTNITE_PLAYER_STATS_URL.format(accountid=account_id), headers={"Authorization": FORTNITE_API_TOKEN} ) - return await raw_response.json() + return await resp.json() def _calculate_stats(game_mode): diff --git a/clients/fortnite_tracker.py b/clients/fortnite_tracker.py index fcf047e..6dc11fb 100644 --- a/clients/fortnite_tracker.py +++ b/clients/fortnite_tracker.py @@ -5,18 +5,20 @@ from collections import defaultdict from urllib.parse import unquote -import aiohttp from bs4 import BeautifulSoup import utils.discord as discord_utils from database.mysql import MySQL from exceptions import UserDoesNotExist, NoSeasonDataError +from utils.cloudscraper import cloudscrape, Method from utils.dates import get_playing_session_date ACCOUNT_SEARCH_URL = "https://fortnitetracker.com/profile/search?q={username}" ACCOUNT_PROFILE_URL = "https://fortnitetracker.com/profile/all/{username}?season={season}" -STATS_REGEX = "var imp_data = (.[\s\S]*);" +STATS_PATTERN = "const profile = " +STATS_REGEX = f"\s*{STATS_PATTERN}(.*);" + MODES = [ "solo", @@ -61,16 +63,11 @@ async def _search_username(player_name): """ Returns the player's username """ url = ACCOUNT_SEARCH_URL.format(username=player_name) - async with aiohttp.ClientSession() as client: - async with client.get(url, headers=HEADERS, allow_redirects=False) as r: - if r.status == 302: - r = r.headers - else: - raise UserDoesNotExist("Username not found in FN Tracker") - - name = unquote(r["Location"].split("/")[-1]) + resp = await cloudscrape(Method.GET, url, headers=HEADERS, allow_redirects=False) + if resp.status_code == 302: + return unquote(resp.headers["Location"].split("/")[-1]) - return name + raise UserDoesNotExist(f"Username not found in FN Tracker: {player_name}") async def _get_player_season_dataset(username): @@ -98,32 +95,38 @@ async def _get_player_profile_html(username): """ Get the player stats page in HTML """ url = ACCOUNT_PROFILE_URL.format(username=username, season=_get_season_id()) - async with aiohttp.ClientSession() as client: - async with client.get(url, headers=HEADERS) as r: - assert r.status == 200 - return await r.text() + resp = await cloudscrape(Method.GET, url, headers=HEADERS) + resp.raise_for_status() + return resp.text def _find_stats_segment(soup): """ Find the stats dataset from within the script's scripts JS """ - pattern = re.compile(STATS_REGEX) - scripts = soup.find_all("script", type="text/javascript") - stats = None + re_pattern = re.compile(STATS_REGEX) + script_tags = soup.find_all("script") + retrieved_stats = None + + for script in script_tags: + script_text = script.text + + if STATS_PATTERN not in script_text: + continue + + re_match = re_pattern.match(script_text) - for script in scripts: - if pattern.match(str(script.string)): - data = pattern.match(script.string) - stats = json.loads(data.groups()[0]) + if re_match is not None: + retrieved_stats = json.loads(re_match.groups()[0]) + break - if stats is None: - raise ValueError("Site changed, bot broke :/") + if retrieved_stats is None: + raise ValueError("Site changed and stats JSON is no longer there, bot broke :/") - return stats + return retrieved_stats def _find_latest_season_id(segments): """ Returns the latest season ID with available data """ - return max([seg['season'] for seg in segments if seg['season'] is not None]) + return max([seg["season"] for seg in segments if seg["season"] is not None]) def _newer_season_available(latest_season_id): diff --git a/fortnite_bot.py b/fortnite_bot.py index 25a6cf9..3a02bfa 100644 --- a/fortnite_bot.py +++ b/fortnite_bot.py @@ -8,6 +8,7 @@ from functools import partial from threading import Thread +import discord from discord.ext.commands import Bot from flask import Flask, jsonify, request from werkzeug.exceptions import BadRequest @@ -23,12 +24,15 @@ DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN") -SQUAD_PLAYERS_LIST = [] +SQUAD_PLAYERS_LIST = [] FORTNITE_DISCORD_ROLE_USERS_DICT = ast.literal_eval(str(os.getenv("FORTNITE_DISCORD_ROLE_USERS_DICT"))) logger = configure_logger() -bot = Bot(command_prefix="!") +# TODO: Explicitly enable the required privileged intents +# then change: intents = discord.Intents.all() +intents = discord.Intents.default() +bot = Bot(command_prefix="!", intents=intents) app = Flask(__name__) initialize_error_handlers(app) @@ -99,11 +103,11 @@ async def on_voice_state_update(member, before, after): if member.display_name in FORTNITE_DISCORD_ROLE_USERS_DICT: if FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name] not in SQUAD_PLAYERS_LIST: SQUAD_PLAYERS_LIST.append(FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name]) - + if interactions.should_remove_player_from_squad_player_session_list(member, before, after): if member.display_name in FORTNITE_DISCORD_ROLE_USERS_DICT: if FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name] in SQUAD_PLAYERS_LIST: - SQUAD_PLAYERS_LIST.pop(FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name]) + SQUAD_PLAYERS_LIST.pop(FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name]) if not interactions.send_track_question(member, before, after): return @@ -133,7 +137,7 @@ async def player_search(ctx, *player_name, guid=False, silent=False): player_name = " ".join(player_name) logger = get_logger_with_context(ctx) - logger.info("Looking up stats for '%s' ", player_name) + logger.info("Searching for player stats: %s", player_name) if not player_name: await ctx.send("Please specify an Epic username after the command, " @@ -142,15 +146,19 @@ async def player_search(ctx, *player_name, guid=False, silent=False): try: await fortnite_tracker.get_player_stats(ctx, player_name, silent) - except Exception as e: - logger.warning(e, exc_info=_should_log_traceback(e)) + except Exception as ft_exc: + logger.warning(ft_exc, exc_info=_should_log_traceback(ft_exc)) # Fortnite API stats are unnecessary in silent mode if silent: return - logger.warning(f"Falling back to Fortnite API for '{player_name}'..") - await fortnite_api.get_player_stats(ctx, player_name, guid) + logger.warning("Falling back to Fortnite API: %s", player_name) + try: + await fortnite_api.get_player_stats(ctx, player_name, guid) + except Exception as fa_exc: + logger.warning(fa_exc, exc_info=_should_log_traceback(fa_exc)) + await ctx.send(f"Player not found: {player_name}") @bot.command(name=commands.TRACK_COMMAND, @@ -273,7 +281,7 @@ async def replays_operations(ctx, *params): username = None if len(params) == 2: username = params.pop(1) - command = params.pop(0) + command = params.pop(0) if username not in SQUAD_PLAYERS_LIST: await ctx.send(f"{username} provided is not a valid squad player") return diff --git a/requirements.txt b/requirements.txt index 3ed43b0..0040283 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ -aiomysql~=0.0.21 -beautifulsoup4~=4.9.3 -bitstring~=3.1.7 +aiomysql~=0.1.1 +beautifulsoup4~=4.11.2 +bitstring~=4.0.1 +cloudscraper~=1.2.69 discord.py~=1.7.1 -flask~=1.1.2 +flask~=2.2.3 fortnite-replay-reader~=0.3.0 -pycryptodome~=3.10.1 -python-dotenv~=0.17.0 -requests~=2.25.1 +pycryptodome~=3.17 +python-dotenv~=1.0.0 +requests~=2.28.2 diff --git a/utils/cloudscraper.py b/utils/cloudscraper.py new file mode 100644 index 0000000..17272b0 --- /dev/null +++ b/utils/cloudscraper.py @@ -0,0 +1,26 @@ +import asyncio +from enum import Enum + +import cloudscraper + + +class Method(Enum): + """ Supported Cloudscrape methods. All request methods are + technically supported. + """ + GET = "get" + + +async def cloudscrape(method, url, headers, allow_redirects=True): + """ Cloudscraper is used to bypass Cloudflare's bot detection. + However, Cloudscraper does not support async so event loops are used. + """ + def _method(): + scraper = cloudscraper.create_scraper() + func = getattr(scraper, method.value) + return func(url, headers=headers, allow_redirects=allow_redirects) + + return await asyncio.get_event_loop().run_in_executor( + executor=None, + func=_method + )