[Pagination] Fix Spammy Pagination Logs on Token Expiry (#5612)

* Less spammy logs

* CR

---------

Co-authored-by: quaark <a.melnick@icloud.com>

mlrun/errors.py

@@ -155,6 +155,10 @@ class MLRunNotFoundError(MLRunHTTPStatusError):
     error_status_code = HTTPStatus.NOT_FOUND.value
 
 
+class MLRunPaginationEndOfResultsError(MLRunNotFoundError):
+    pass
+
+
 class MLRunBadRequestError(MLRunHTTPStatusError):
     error_status_code = HTTPStatus.BAD_REQUEST.value
 
@@ -240,3 +244,5 @@ def __init__(
     HTTPStatus.SERVICE_UNAVAILABLE.value: MLRunServiceUnavailableError,
     HTTPStatus.NOT_IMPLEMENTED.value: MLRunNotImplementedServerError,
 }
+
+EXPECTED_ERRORS = (MLRunPaginationEndOfResultsError,)

server/api/main.py

@@ -146,13 +146,24 @@ async def http_status_error_handler(
         request_id = request.state.request_id
     status_code = exc.response.status_code
     error_message = repr(exc)
-    logger.warning(
-        "Request handling returned error status",
-        error_message=error_message,
-        status_code=status_code,
-        traceback=traceback.format_exc(),
-        request_id=request_id,
-    )
+    log_message = "Request handling returned error status"
+
+    if isinstance(exc, mlrun.errors.EXPECTED_ERRORS):
+        logger.debug(
+            log_message,
+            error_message=error_message,
+            status_code=status_code,
+            request_id=request_id,
+        )
+    else:
+        logger.warning(
+            log_message,
+            error_message=error_message,
+            status_code=status_code,
+            traceback=traceback.format_exc(),
+            request_id=request_id,
+        )
+
     return await http_exception_handler(
         request,
         fastapi.HTTPException(status_code=status_code, detail=error_message),

server/api/utils/pagination.py

@@ -218,7 +218,7 @@ def _create_or_update_pagination_cache_record(
                 self._pagination_cache.get_pagination_cache_record(session, key=token)
             )
             if pagination_cache_record is None:
-                raise mlrun.errors.MLRunNotFoundError(
+                raise mlrun.errors.MLRunPaginationEndOfResultsError(
                     f"Token {token} not found in pagination cache"
                 )
             method = PaginatedMethods.get_method(pagination_cache_record.function)