Make gatherling scraper slightly more resilient to timeouts?

decksite/scrapers/gatherling.py

@@ -31,7 +31,7 @@ def scrape(limit: int = 50) -> None:
             return
 
 def tournament(url: str, name: str) -> int:
-    s = fetcher.internal.fetch(url, character_encoding='utf-8')
+    s = fetcher.internal.fetch(url, character_encoding='utf-8', retry=True)
 
     # Tournament details
     soup = BeautifulSoup(s, 'html.parser')
@@ -188,7 +188,7 @@ def tournament_deck(cells: ResultSet, competition_id: int, date: datetime.dateti
 
 def tournament_matches(d: deck.Deck) -> List[bs4.element.Tag]:
     url = 'https://gatherling.com/deck.php?mode=view&id={identifier}'.format(identifier=d.identifier)
-    s = fetcher.internal.fetch(url, character_encoding='utf-8')
+    s = fetcher.internal.fetch(url, character_encoding='utf-8', retry=True)
     soup = BeautifulSoup(s, 'html.parser')
     anchor = soup.find(string='MATCHUPS')
     if anchor is None:

shared/fetcher_internal.py

@@ -18,7 +18,7 @@
     'http://whatsinstandard.com',
     CacheControlAdapter(heuristic=ExpiresAfter(days=14)))
 
-def fetch(url: str, character_encoding: Optional[str] = None, force: bool = False) -> str:
+def fetch(url: str, character_encoding: Optional[str] = None, force: bool = False, retry: bool = False) -> str:
     headers = {}
     if force:
         headers['Cache-Control'] = 'no-cache'
@@ -38,6 +38,8 @@ def fetch(url: str, character_encoding: Optional[str] = None, force: bool = Fals
             print('Getting text from response was very slow. Setting an explicit character_encoding may help.')
         return t
     except (urllib.error.HTTPError, requests.exceptions.ConnectionError) as e: # type: ignore # urllib isn't fully stubbed
+        if retry:
+            return fetch(url, character_encoding, force, retry=False)
         raise FetchException(e)
 
 async def fetch_async(url: str) -> str: