Added Cloudscraper, fixed both stats sources, upgraded dependencies (#64

)

* Fixed broken things

* Typo

README.md

@@ -11,30 +11,30 @@ Retrieve player statistics in Fortnite Battle Royale
 * [Setup](#setup)
 
 ## Features
-- Asks if you want to see current squad stats when joining Fortnite discord channel 
+- Asks if you want to see current squad stats when joining Fortnite discord channel
 
-Fortnite Tracker search (Default): 
+Fortnite Tracker search (Default):
 - Calculates player current season statistics in different Game Modes (Solo, Duos, Trios, Squads)
 - Calculates player's overall statistics
     > Statistics include:
-    >    - KD 
+    >    - KD
     >    - Wins
     >    - Win Percentage
     >    - Matches Played
-    >    - TRN 
+    >    - TRN
 ----------------------------------------------------------------------------------
 Fortnite API search (Fall back):
 If player profile is not found in fortnitetracker.com, fall back to using Fortnite API
 - Show player's level
 - Calculates player overall season statistics in different Game Modes (Solo, Duos, Squads)
 - Calculates player's overall statistics
     > Statistics include:
-    >    - KD 
+    >    - KD
     >    - Wins
     >    - Win Percentage
-    >    - Kills 
+    >    - Kills
     >    - Matches Played
-- Link to player's Twitch stream if currently streaming    
+- Link to player's Twitch stream if currently streaming
 ----------------------------------------------------------------------------------
 
 - Link to Fortnite Tracker player profile (click on username to navigate)
@@ -72,7 +72,7 @@ If player profile is not found in fortnitetracker.com, fall back to using Fortni
 !stats enemy
 
 !upgrade
-!gold 
+!gold
 
 !chests
 !loot
@@ -132,7 +132,7 @@ To install the Fortnite Replay Reader pip package, you need to install the packa
 
 To do so, run:
 ```
-pip install bitstring
+pip install wheel bitstring pycryptodome
 ```
 
 Then:

clients/fortnite_api.py

@@ -3,6 +3,8 @@
 import aiohttp
 import discord
 
+from exceptions import UserDoesNotExist
+
 
 FORTNITE_API_TOKEN = os.getenv("FORTNITE_API_TOKEN")
 TWITCH_CLIENT_ID = os.getenv("TWITCH_CLIENT_ID")
@@ -62,20 +64,22 @@ async def _get_player_account_id(session, player_name, platform):
     """
     Get account id given player name and platform
     """
-    raw_response = await session.get(
+    resp = await session.get(
         FORTNITE_ACCOUNT_ID_URL.format(username=player_name, platform=platform), headers={"Authorization": FORTNITE_API_TOKEN}
     )
-    return await raw_response.json()
+    if resp.status == 404:
+        raise UserDoesNotExist(f"Username not found in FN API: {player_name}")
+    return await resp.json()
 
 
 async def _get_player_stats(session, account_id):
     """
     Get player stats given account id
     """
-    raw_response = await session.get(
+    resp = await session.get(
         FORTNITE_PLAYER_STATS_URL.format(accountid=account_id), headers={"Authorization": FORTNITE_API_TOKEN}
     )
-    return await raw_response.json()
+    return await resp.json()
 
 
 def _calculate_stats(game_mode):

clients/fortnite_tracker.py

@@ -5,18 +5,20 @@
 from collections import defaultdict
 from urllib.parse import unquote
 
-import aiohttp
 from bs4 import BeautifulSoup
 
 import utils.discord as discord_utils
 from database.mysql import MySQL
 from exceptions import UserDoesNotExist, NoSeasonDataError
+from utils.cloudscraper import cloudscrape, Method
 from utils.dates import get_playing_session_date
 
 
 ACCOUNT_SEARCH_URL = "https://fortnitetracker.com/profile/search?q={username}"
 ACCOUNT_PROFILE_URL = "https://fortnitetracker.com/profile/all/{username}?season={season}"
-STATS_REGEX = "var imp_data = (.[\s\S]*);"
+STATS_PATTERN = "const profile = "
+STATS_REGEX = f"\s*{STATS_PATTERN}(.*);"
+
 
 MODES = [
     "solo",
@@ -61,16 +63,11 @@ async def _search_username(player_name):
     """ Returns the player's username """
     url = ACCOUNT_SEARCH_URL.format(username=player_name)
 
-    async with aiohttp.ClientSession() as client:
-        async with client.get(url, headers=HEADERS, allow_redirects=False) as r:
-            if r.status == 302:
-                r = r.headers
-            else:
-                raise UserDoesNotExist("Username not found in FN Tracker")
-
-    name = unquote(r["Location"].split("/")[-1])
+    resp = await cloudscrape(Method.GET, url, headers=HEADERS, allow_redirects=False)
+    if resp.status_code == 302:
+        return unquote(resp.headers["Location"].split("/")[-1])
 
-    return name
+    raise UserDoesNotExist(f"Username not found in FN Tracker: {player_name}")
 
 
 async def _get_player_season_dataset(username):
@@ -98,32 +95,38 @@ async def _get_player_profile_html(username):
     """ Get the player stats page in HTML """
     url = ACCOUNT_PROFILE_URL.format(username=username, season=_get_season_id())
 
-    async with aiohttp.ClientSession() as client:
-        async with client.get(url, headers=HEADERS) as r:
-            assert r.status == 200
-            return await r.text()
+    resp = await cloudscrape(Method.GET, url, headers=HEADERS)
+    resp.raise_for_status()
+    return resp.text
 
 
 def _find_stats_segment(soup):
     """ Find the stats dataset from within the script's scripts JS """
-    pattern = re.compile(STATS_REGEX)
-    scripts = soup.find_all("script", type="text/javascript")
-    stats = None
+    re_pattern = re.compile(STATS_REGEX)
+    script_tags = soup.find_all("script")
+    retrieved_stats = None
+
+    for script in script_tags:
+        script_text = script.text
+
+        if STATS_PATTERN not in script_text:
+            continue
+
+        re_match = re_pattern.match(script_text)
 
-    for script in scripts:
-        if pattern.match(str(script.string)):
-            data = pattern.match(script.string)
-            stats = json.loads(data.groups()[0])
+        if re_match is not None:
+            retrieved_stats = json.loads(re_match.groups()[0])
+            break
 
-    if stats is None:
-        raise ValueError("Site changed, bot broke :/")
+    if retrieved_stats is None:
+        raise ValueError("Site changed and stats JSON is no longer there, bot broke :/")
 
-    return stats
+    return retrieved_stats
 
 
 def _find_latest_season_id(segments):
     """ Returns the latest season ID with available data """
-    return max([seg['season'] for seg in segments if seg['season'] is not None])
+    return max([seg["season"] for seg in segments if seg["season"] is not None])
 
 
 def _newer_season_available(latest_season_id):

fortnite_bot.py

@@ -8,6 +8,7 @@
 from functools import partial
 from threading import Thread
 
+import discord
 from discord.ext.commands import Bot
 from flask import Flask, jsonify, request
 from werkzeug.exceptions import BadRequest
@@ -23,12 +24,15 @@
 
 
 DISCORD_BOT_TOKEN = os.getenv("DISCORD_BOT_TOKEN")
-SQUAD_PLAYERS_LIST = [] 
+SQUAD_PLAYERS_LIST = []
 FORTNITE_DISCORD_ROLE_USERS_DICT = ast.literal_eval(str(os.getenv("FORTNITE_DISCORD_ROLE_USERS_DICT")))
 
 logger = configure_logger()
 
-bot = Bot(command_prefix="!")
+# TODO: Explicitly enable the required privileged intents
+#       then change: intents = discord.Intents.all()
+intents = discord.Intents.default()
+bot = Bot(command_prefix="!", intents=intents)
 
 app = Flask(__name__)
 initialize_error_handlers(app)
@@ -99,11 +103,11 @@ async def on_voice_state_update(member, before, after):
         if member.display_name in FORTNITE_DISCORD_ROLE_USERS_DICT:
             if FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name] not in SQUAD_PLAYERS_LIST:
                 SQUAD_PLAYERS_LIST.append(FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name])
-    
+
     if interactions.should_remove_player_from_squad_player_session_list(member, before, after):
         if member.display_name in FORTNITE_DISCORD_ROLE_USERS_DICT:
             if FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name] in SQUAD_PLAYERS_LIST:
-                SQUAD_PLAYERS_LIST.pop(FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name]) 
+                SQUAD_PLAYERS_LIST.pop(FORTNITE_DISCORD_ROLE_USERS_DICT[member.display_name])
 
     if not interactions.send_track_question(member, before, after):
         return
@@ -133,7 +137,7 @@ async def player_search(ctx, *player_name, guid=False, silent=False):
     player_name = " ".join(player_name)
 
     logger = get_logger_with_context(ctx)
-    logger.info("Looking up stats for '%s' ", player_name)
+    logger.info("Searching for player stats: %s", player_name)
 
     if not player_name:
         await ctx.send("Please specify an Epic username after the command, "
@@ -142,15 +146,19 @@ async def player_search(ctx, *player_name, guid=False, silent=False):
 
     try:
         await fortnite_tracker.get_player_stats(ctx, player_name, silent)
-    except Exception as e:
-        logger.warning(e, exc_info=_should_log_traceback(e))
+    except Exception as ft_exc:
+        logger.warning(ft_exc, exc_info=_should_log_traceback(ft_exc))
 
         # Fortnite API stats are unnecessary in silent mode
         if silent:
             return
 
-        logger.warning(f"Falling back to Fortnite API for '{player_name}'..")
-        await fortnite_api.get_player_stats(ctx, player_name, guid)
+        logger.warning("Falling back to Fortnite API: %s", player_name)
+        try:
+            await fortnite_api.get_player_stats(ctx, player_name, guid)
+        except Exception as fa_exc:
+            logger.warning(fa_exc, exc_info=_should_log_traceback(fa_exc))
+            await ctx.send(f"Player not found: {player_name}")
 
 
 @bot.command(name=commands.TRACK_COMMAND,
@@ -273,7 +281,7 @@ async def replays_operations(ctx, *params):
     username = None
     if len(params) == 2:
         username = params.pop(1)
-        command = params.pop(0) 
+        command = params.pop(0)
         if username not in SQUAD_PLAYERS_LIST:
             await ctx.send(f"{username} provided is not a valid squad player")
             return

requirements.txt

@@ -1,9 +1,10 @@
-aiomysql~=0.0.21
-beautifulsoup4~=4.9.3
-bitstring~=3.1.7
+aiomysql~=0.1.1
+beautifulsoup4~=4.11.2
+bitstring~=4.0.1
+cloudscraper~=1.2.69
 discord.py~=1.7.1
-flask~=1.1.2
+flask~=2.2.3
 fortnite-replay-reader~=0.3.0
-pycryptodome~=3.10.1
-python-dotenv~=0.17.0
-requests~=2.25.1
+pycryptodome~=3.17
+python-dotenv~=1.0.0
+requests~=2.28.2

utils/cloudscraper.py

@@ -0,0 +1,26 @@
+import asyncio
+from enum import Enum
+
+import cloudscraper
+
+
+class Method(Enum):
+    """ Supported Cloudscrape methods. All request methods are
+    technically supported.
+    """
+    GET = "get"
+
+
+async def cloudscrape(method, url, headers, allow_redirects=True):
+    """ Cloudscraper is used to bypass Cloudflare's bot detection.
+    However, Cloudscraper does not support async so event loops are used.
+    """
+    def _method():
+        scraper = cloudscraper.create_scraper()
+        func = getattr(scraper, method.value)
+        return func(url, headers=headers, allow_redirects=allow_redirects)
+
+    return await asyncio.get_event_loop().run_in_executor(
+        executor=None,
+        func=_method
+    )