Retry executemany statement if deadlock is detected

server/db/__init__.py

@@ -1,4 +1,10 @@
+import asyncio
+import logging
+
 from aiomysql.sa import create_engine
+from pymysql.err import OperationalError
+
+logger = logging.getLogger(__name__)
 
 
 class FAFDatabase:
@@ -40,3 +46,25 @@ async def close(self):
         self.engine.close()
         await self.engine.wait_closed()
         self.engine = None
+
+
+async def deadlock_retry_execute(conn, *args, max_attempts=3):
+    for attempt in range(max_attempts - 1):
+        try:
+            return await conn.execute(*args)
+        except OperationalError as e:
+            if any(msg in e.message for msg in (
+                "Deadlock found",
+                "Lock wait timeout exceeded"
+            )):
+                logger.warning(
+                    "Encountered deadlock during SQL execution. Attempts: %d",
+                    attempt + 1
+                )
+                # Exponential backoff
+                await asyncio.sleep(0.3 * 2 ** attempt)
+            else:
+                raise
+
+    # On the final attempt we don't do any error handling
+    return await conn.execute(*args)

server/games/game.py

@@ -10,6 +10,7 @@
 from sqlalchemy.sql.functions import now as sql_now
 
 from server.config import FFA_TEAM
+from server.db import deadlock_retry_execute
 from server.db.models import (
     game_player_stats,
     game_stats,
@@ -524,16 +525,16 @@ async def persist_results(self):
                 )
 
             update_statement = game_player_stats.update().where(
-                    and_(
-                        game_player_stats.c.gameId == bindparam("game_id"),
-                        game_player_stats.c.playerId == bindparam("player_id"),
-                    )
-                ).values(
-                    score=bindparam("score"),
-                    scoreTime=sql_now(),
-                    result=bindparam("result"),
+                and_(
+                    game_player_stats.c.gameId == bindparam("game_id"),
+                    game_player_stats.c.playerId == bindparam("player_id"),
                 )
-            await conn.execute(update_statement, rows)
+            ).values(
+                score=bindparam("score"),
+                scoreTime=sql_now(),
+                result=bindparam("result"),
+            )
+            await deadlock_retry_execute(conn, update_statement, rows)
 
     def get_basic_info(self) -> BasicGameInfo:
         return BasicGameInfo(