Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Iguazio] Retry on session verification [1.6.x] #5630

Merged
merged 3 commits into from
May 27, 2024
Merged

[Iguazio] Retry on session verification [1.6.x] #5630

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e99cbef
[Iguazio] Retry on session verification [1.6.x]
alonmr May 26, 2024
3457d8e
Merge branch '1.6.x' of github.com:mlrun/mlrun into ML-6560
alonmr May 26, 2024
8d93196
minor readability
alonmr May 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 29 additions & 6 deletions server/api/utils/clients/iguazio.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,22 @@ class Client(
):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
retry_on_exception = (
mlrun.mlconf.httpdb.projects.retry_leader_request_on_exception
== mlrun.common.schemas.HTTPSessionRetryMode.enabled.value
)
self._session = mlrun.utils.HTTPSessionWithRetry(
retry_on_exception=mlrun.mlconf.httpdb.projects.retry_leader_request_on_exception
== mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
retry_on_exception=retry_on_exception,
verbose=True,
)
self._retry_on_post_session = None
if retry_on_exception:
self._retry_on_post_session = mlrun.utils.HTTPSessionWithRetry(
retry_on_exception=mlrun.mlconf.httpdb.projects.retry_leader_request_on_exception
== mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
retry_on_post=True,
verbose=True,
)
self._api_url = mlrun.mlconf.iguazio_api_url
# The job is expected to be completed in less than 5 seconds. If 10 seconds have passed and the job
# has not been completed, increase the interval to retry every 5 seconds
Expand Down Expand Up @@ -171,6 +182,7 @@ def verify_request_session(
"authorization": request.headers.get("authorization"),
"cookie": request.headers.get("cookie"),
},
retry_on_post=True,
)
return self._generate_auth_info_from_session_verification_response(
response.headers, response.json()
Expand All @@ -181,7 +193,8 @@ def verify_session(self, session: str) -> mlrun.common.schemas.AuthInfo:
"POST",
mlrun.mlconf.httpdb.authentication.iguazio.session_verification_endpoint,
"Failed verifying iguazio session",
session,
session=session,
retry_on_post=True,
)
return self._generate_auth_info_from_session_verification_response(
response.headers, response.json()
Expand Down Expand Up @@ -215,7 +228,8 @@ def get_or_create_access_key(
"POST",
"self/get_or_create_access_key",
"Failed getting or creating iguazio access key",
session,
session=session,
retry_on_post=True,
json=body,
)
if response.status_code == http.HTTPStatus.CREATED.value:
Expand Down Expand Up @@ -638,11 +652,20 @@ def _get_job_from_iguazio(self, session: str, job_id: str) -> dict:
return response.json()

def _send_request_to_api(
self, method, path, error_message: str, session=None, **kwargs
self,
method,
path,
error_message: str,
session=None,
retry_on_post=False,
**kwargs,
):
url = f"{self._api_url}/api/{path}"
self._prepare_request_kwargs(session, path, kwargs=kwargs)
response = self._session.request(
http_session = self._session
if retry_on_post and self._retry_on_post_session:
http_session = self._retry_on_post_session
response = http_session.request(
method, url, verify=mlrun.config.config.httpdb.http.verify, **kwargs
)
if not response.ok:
Expand Down
Loading