fix: reject heartbeats on lock timeout instead of proceeding unsafely

aw_server/rest.py

@@ -271,8 +271,12 @@ def delete(self, bucket_id: str, event_id: int):
 
 @api.route("/0/buckets/<string:bucket_id>/heartbeat")
 class HeartbeatResource(Resource):
+    # Class-level lock shared across all instances.
+    # Flask-RESTX creates a new Resource instance per request, so an
+    # instance-level lock would provide no mutual exclusion.
+    lock = Lock()
+
     def __init__(self, *args, **kwargs):
-        self.lock = Lock()
         super().__init__(*args, **kwargs)
 
     @api.expect(event, validate=True)
@@ -291,11 +295,12 @@ def post(self, bucket_id):
         # This lock is meant to ensure that only one heartbeat is processed at a time,
         # as the heartbeat function is not thread-safe.
         # This should maybe be moved into the api.py file instead (but would be very messy).
-        aquired = self.lock.acquire(timeout=1)
-        if not aquired:
+        acquired = self.lock.acquire(timeout=10)
+        if not acquired:
             logger.warning(
-                "Heartbeat lock could not be aquired within a reasonable time, this likely indicates a bug."
+                "Heartbeat lock could not be acquired within timeout, rejecting request."
             )
+            return {"message": "Server busy, try again later"}, 503
         try:
             event = current_app.api.heartbeat(bucket_id, heartbeat, pulsetime)
         finally: