diff --git a/CHANGELOG.md b/CHANGELOG.md index d0887afd0..84e9a1e71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - `DataCube.sar_backscatter()`: add corresponding band names to metadata when enabling "mask", "contributing_area", "local_incidence_angle" or "ellipsoid_incidence_angle" ([#804](https://github.com/Open-EO/openeo-python-client/issues/804)) +- Proactively refresh access/bearer token in `MultiBackendJobManager` before launching a job start thread ([#817](https://github.com/Open-EO/openeo-python-client/issues/817)) ## [0.45.0] - 2025-09-17 diff --git a/openeo/extra/job_management/__init__.py b/openeo/extra/job_management/__init__.py index fcc72f91b..983db7221 100644 --- a/openeo/extra/job_management/__init__.py +++ b/openeo/extra/job_management/__init__.py @@ -244,6 +244,8 @@ def __init__( ) self._thread = None self._worker_pool = None + # Generic cache + self._cache = {} def add_backend( self, @@ -650,6 +652,8 @@ def _launch_job(self, start_job, df, i, backend_name, stats: Optional[dict] = No # start job if not yet done by callback try: job_con = job.connection + # Proactively refresh bearer token (because task in thread will not be able to do that) + self._refresh_bearer_token(connection=job_con) task = _JobStartTask( root_url=job_con.root_url, bearer_token=job_con.auth.bearer if isinstance(job_con.auth, BearerAuth) else None, @@ -670,6 +674,21 @@ def _launch_job(self, start_job, df, i, backend_name, stats: Optional[dict] = No df.loc[i, "status"] = "skipped" stats["start_job skipped"] += 1 + def _refresh_bearer_token(self, connection: Connection, *, max_age: float = 60) -> None: + """ + Helper to proactively refresh the bearer (access) token of the connection + (but not too often, based on `max_age`). + """ + # TODO: be smarter about timing, e.g. by inspecting expiry of current token? + now = time.time() + key = f"connection:{id(connection)}:refresh-time" + if self._cache.get(key, 0) + max_age < now: + refreshed = connection.try_access_token_refresh() + if refreshed: + self._cache[key] = now + else: + _log.warning("Failed to proactively refresh bearer token") + def _process_threadworker_updates( self, worker_pool: _JobManagerWorkerThreadPool, diff --git a/openeo/rest/_testing.py b/openeo/rest/_testing.py index 8ef166fe0..998874551 100644 --- a/openeo/rest/_testing.py +++ b/openeo/rest/_testing.py @@ -132,6 +132,22 @@ def at_url(cls, root_url: str, *, requests_mock, capabilities: Optional[dict] = connection = Connection(root_url) return cls(requests_mock=requests_mock, connection=connection) + def setup_credentials_oidc(self, *, issuer: str = "https://oidc.test", id: str = "oi"): + self._requests_mock.get( + self.connection.build_url("/credentials/oidc"), + json={ + "providers": [ + { + "id": id, + "issuer": issuer, + "title": id, + "scopes": ["openid"], + } + ] + }, + ) + return self + def setup_collection( self, collection_id: str, diff --git a/openeo/rest/auth/testing.py b/openeo/rest/auth/testing.py index 0160f24db..930869f18 100644 --- a/openeo/rest/auth/testing.py +++ b/openeo/rest/auth/testing.py @@ -143,6 +143,11 @@ def token_callback_resource_owner_password_credentials(self, params: dict, conte assert params["scope"] == self.expected_fields["scope"] return self._build_token_response() + def token_callback_block_400(self, params: dict, context): + """Failing callback with 400 Bad Request""" + context.status_code = 400 + return "block_400" + def device_code_callback(self, request: requests_mock.request._RequestObjectProxy, context): params = self._get_query_params(query=request.text) assert params["client_id"] == self.expected_client_id diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 94133f46f..4a457c03b 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -342,28 +342,32 @@ def _authenticate_oidc( *, provider_id: str, store_refresh_token: bool = False, - fallback_refresh_token_to_store: Optional[str] = None, + auto_renew_from_refresh_token: bool = False, + fallback_refresh_token: Optional[str] = None, oidc_auth_renewer: Optional[OidcAuthenticator] = None, ) -> Connection: """ Authenticate through OIDC and set up bearer token (based on OIDC access_token) for further requests. """ - tokens = authenticator.get_tokens(request_refresh_token=store_refresh_token) + request_refresh_token = store_refresh_token or (not oidc_auth_renewer and auto_renew_from_refresh_token) + tokens = authenticator.get_tokens(request_refresh_token=request_refresh_token) _log.info("Obtained tokens: {t}".format(t=[k for k, v in tokens._asdict().items() if v])) + + refresh_token = tokens.refresh_token or fallback_refresh_token if store_refresh_token: - refresh_token = tokens.refresh_token or fallback_refresh_token_to_store if refresh_token: self._get_refresh_token_store().set_refresh_token( issuer=authenticator.provider_info.issuer, client_id=authenticator.client_id, refresh_token=refresh_token ) - if not oidc_auth_renewer: - oidc_auth_renewer = OidcRefreshTokenAuthenticator( - client_info=authenticator.client_info, refresh_token=refresh_token - ) else: _log.warning("No OIDC refresh token to store.") + if not oidc_auth_renewer and auto_renew_from_refresh_token and refresh_token: + oidc_auth_renewer = OidcRefreshTokenAuthenticator( + client_info=authenticator.client_info, refresh_token=refresh_token + ) + token = tokens.access_token self.auth = OidcBearerAuth(provider_id=provider_id, access_token=token) self._oidc_auth_renewer = oidc_auth_renewer @@ -452,7 +456,12 @@ def authenticate_oidc_resource_owner_password_credentials( authenticator = OidcResourceOwnerPasswordAuthenticator( client_info=client_info, username=username, password=password ) - return self._authenticate_oidc(authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token) + return self._authenticate_oidc( + authenticator, + provider_id=provider_id, + store_refresh_token=store_refresh_token, + oidc_auth_renewer=authenticator, + ) def authenticate_oidc_refresh_token( self, @@ -493,7 +502,7 @@ def authenticate_oidc_refresh_token( authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token, - fallback_refresh_token_to_store=refresh_token, + fallback_refresh_token=refresh_token, oidc_auth_renewer=authenticator, ) @@ -534,7 +543,13 @@ def authenticate_oidc_device( authenticator = OidcDeviceAuthenticator( client_info=client_info, use_pkce=use_pkce, max_poll_time=max_poll_time, **kwargs ) - return self._authenticate_oidc(authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token) + return self._authenticate_oidc( + authenticator, + provider_id=provider_id, + store_refresh_token=store_refresh_token, + # TODO: expose `auto_renew_from_refresh_token` directly as option instead of reusing `store_refresh_token` arg? + auto_renew_from_refresh_token=store_refresh_token, + ) def authenticate_oidc( self, @@ -604,7 +619,8 @@ def authenticate_oidc( authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token, - fallback_refresh_token_to_store=refresh_token, + fallback_refresh_token=refresh_token, + oidc_auth_renewer=authenticator, ) # TODO: pluggable/jupyter-aware display function? print("Authenticated using refresh token.") @@ -622,6 +638,8 @@ def authenticate_oidc( authenticator, provider_id=provider_id, store_refresh_token=store_refresh_token, + # TODO: expose `auto_renew_from_refresh_token` directly as option instead of reusing `store_refresh_token` arg? + auto_renew_from_refresh_token=store_refresh_token, ) print("Authenticated using device code flow.") return con @@ -665,6 +683,28 @@ def authenticate_bearer_token(self, bearer_token: str) -> Connection: self._oidc_auth_renewer = None return self + def try_access_token_refresh(self, *, reason: Optional[str] = None) -> bool: + """ + Try to get a fresh access token if possible. + Returns whether a new access token was obtained. + """ + reason = f" Reason: {reason}" if reason else "" + if isinstance(self.auth, OidcBearerAuth) and self._oidc_auth_renewer: + try: + self._authenticate_oidc( + authenticator=self._oidc_auth_renewer, + provider_id=self._oidc_auth_renewer.provider_info.id, + store_refresh_token=False, + oidc_auth_renewer=self._oidc_auth_renewer, + ) + _log.info(f"Obtained new access token (grant {self._oidc_auth_renewer.grant_type!r}).{reason}") + return True + except OpenEoClientException as auth_exc: + _log.error( + f"Failed to obtain new access token (grant {self._oidc_auth_renewer.grant_type!r}): {auth_exc!r}.{reason}" + ) + return False + def request( self, method: str, @@ -690,24 +730,11 @@ def _request(): api_exc.http_status_code in {HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN} and api_exc.code == "TokenInvalid" ): - # Auth token expired: can we refresh? - if isinstance(self.auth, OidcBearerAuth) and self._oidc_auth_renewer: - msg = f"OIDC access token expired ({api_exc.http_status_code} {api_exc.code})." - try: - self._authenticate_oidc( - authenticator=self._oidc_auth_renewer, - provider_id=self._oidc_auth_renewer.provider_info.id, - store_refresh_token=False, - oidc_auth_renewer=self._oidc_auth_renewer, - ) - _log.info(f"{msg} Obtained new access token (grant {self._oidc_auth_renewer.grant_type!r}).") - except OpenEoClientException as auth_exc: - _log.error( - f"{msg} Failed to obtain new access token (grant {self._oidc_auth_renewer.grant_type!r}): {auth_exc!r}." - ) - else: - # Retry request. - return _request() + # Retry if we can refresh the access token + if self.try_access_token_refresh( + reason=f"OIDC access token expired ({api_exc.http_status_code} {api_exc.code})." + ): + return _request() raise def describe_account(self) -> dict: diff --git a/tests/extra/job_management/test_job_management.py b/tests/extra/job_management/test_job_management.py index d1a147216..a75fe889a 100644 --- a/tests/extra/job_management/test_job_management.py +++ b/tests/extra/job_management/test_job_management.py @@ -47,6 +47,7 @@ _TaskResult, ) from openeo.rest._testing import OPENEO_BACKEND, DummyBackend, build_capabilities +from openeo.rest.auth.testing import OidcMock from openeo.util import rfc3339 from openeo.utils.version import ComparableVersion @@ -269,7 +270,7 @@ def test_create_job_db(self, tmp_path, job_manager, job_manager_root_dir, sleep_ assert set(result.status) == {"finished"} assert set(result.backend_name) == {"foo", "bar"} - def test_basic_threading(self, tmp_path, job_manager, job_manager_root_dir, sleep_mock): + def test_start_job_thread_basic(self, tmp_path, job_manager, job_manager_root_dir, sleep_mock): df = pd.DataFrame( { "year": [2018, 2019, 2020, 2021, 2022], @@ -868,6 +869,52 @@ def execute(self): assert any("Skipping invalid db_update" in msg for msg in caplog.messages) assert any("Skipping invalid stats_update" in msg for msg in caplog.messages) + def test_refresh_bearer_token_before_start( + self, + tmp_path, + job_manager, + dummy_backend_foo, + dummy_backend_bar, + job_manager_root_dir, + sleep_mock, + requests_mock, + ): + + client_id = "client123" + client_secret = "$3cr3t" + oidc_issuer = "https://oidc.test/" + oidc_mock = OidcMock( + requests_mock=requests_mock, + expected_grant_type="client_credentials", + expected_client_id=client_id, + expected_fields={"client_secret": client_secret, "scope": "openid"}, + oidc_issuer=oidc_issuer, + ) + dummy_backend_foo.setup_credentials_oidc(issuer=oidc_issuer) + dummy_backend_bar.setup_credentials_oidc(issuer=oidc_issuer) + dummy_backend_foo.connection.authenticate_oidc_client_credentials(client_id="client123", client_secret="$3cr3t") + dummy_backend_bar.connection.authenticate_oidc_client_credentials(client_id="client123", client_secret="$3cr3t") + + # After this setup, we have 2 client credential token requests (one for each backend) + assert len(oidc_mock.grant_request_history) == 2 + + df = pd.DataFrame({"year": [2020, 2021, 2022, 2023, 2024]}) + job_db_path = tmp_path / "jobs.csv" + job_db = CsvJobDatabase(job_db_path).initialize_from_df(df) + run_stats = job_manager.run_jobs(job_db=job_db, start_job=self._create_year_job) + + assert run_stats == dirty_equals.IsPartialDict( + { + "job_queued_for_start": 5, + "job started running": 5, + "job finished": 5, + } + ) + + # Because of proactive+throttled token refreshing, + # we should have 2 additional token requests now + assert len(oidc_mock.grant_request_history) == 4 + JOB_DB_DF_BASICS = pd.DataFrame( { diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index dfa12c7a9..bda3d50c7 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -2119,6 +2119,8 @@ def test_authenticate_oidc_auto_renew_expired_access_token_initial_refresh_token } + + @pytest.mark.parametrize( ["invalidate", "token_invalid_status_code"], [ @@ -2363,17 +2365,17 @@ def test_authenticate_oidc_auto_renew_expired_access_token_initial_client_creden requests_mock.get(API_URL, json={"api_version": "1.0.0"}) client_id = "myclient" client_secret = "$3cr3t" - issuer = "https://oidc.test" + oidc_issuer = "https://oidc.test" requests_mock.get( API_URL + "credentials/oidc", - json={"providers": [{"id": "oi", "issuer": issuer, "title": "example", "scopes": ["openid"]}]}, + json={"providers": [{"id": "oi", "issuer": oidc_issuer, "title": "example", "scopes": ["openid"]}]}, ) oidc_mock = OidcMock( requests_mock=requests_mock, expected_grant_type="client_credentials", expected_client_id=client_id, expected_fields={"client_secret": client_secret, "scope": "openid"}, - oidc_issuer=issuer, + oidc_issuer=oidc_issuer, ) _setup_get_me_handler( @@ -2381,12 +2383,13 @@ def test_authenticate_oidc_auto_renew_expired_access_token_initial_client_creden ) caplog.set_level(logging.INFO) - # Explicit authentication with `authenticate_oidc_refresh_token` + # Initial authentication with `authenticate_oidc_client_credentials` conn = Connection(API_URL, refresh_token_store=refresh_token_store) assert isinstance(conn.auth, NullAuth) conn.authenticate_oidc_client_credentials(client_id=client_id, client_secret=client_secret) assert isinstance(conn.auth, BearerAuth) assert conn.auth.bearer == "oidc/oi/" + oidc_mock.state["access_token"] + # Just one "client_credentials" auth request so far assert [h["grant_type"] for h in oidc_mock.grant_request_history] == ["client_credentials"] access_token1 = oidc_mock.state["access_token"] @@ -2458,7 +2461,7 @@ def test_authenticate_oidc_auto_renew_expired_access_token_initial_client_creden ) caplog.set_level(logging.INFO) - # Explicit authentication with `authenticate_oidc_refresh_token` + # Initial authentication with `authenticate_oidc_client_credentials` conn = Connection(API_URL, refresh_token_store=refresh_token_store) assert isinstance(conn.auth, NullAuth) conn.authenticate_oidc_client_credentials(client_id=client_id, client_secret=client_secret) @@ -2475,9 +2478,9 @@ def test_authenticate_oidc_auto_renew_expired_access_token_initial_client_creden "_used_access_token": access_token1, } - # Expire access token don't accept client credentials anymore + # Expire access token, and don't accept client credentials anymore oidc_mock.invalidate_access_token() - requests_mock.post(oidc_mock.token_endpoint, status_code=401, text="nope") + oidc_mock.token_callback_client_credentials = oidc_mock.token_callback_block_400 # Do request that requires auth headers and might trigger re-authentication assert f"{token_invalid_status_code} TokenInvalid" not in caplog.text with pytest.raises( @@ -2490,6 +2493,258 @@ def test_authenticate_oidc_auto_renew_expired_access_token_initial_client_creden assert "Failed to obtain new access token (grant 'client_credentials')" in caplog.text +@pytest.mark.parametrize( + ["invalid_refresh_token", "expect_refresh"], + [ + (False, True), + (True, False), + ], +) +def test_try_access_token_refresh_initial_refresh_token( + requests_mock, refresh_token_store, caplog, invalid_refresh_token, expect_refresh +): + requests_mock.get(API_URL, json={"api_version": "1.0.0"}) + client_id = "myclient" + initial_refresh_token = "r3fr35h!" + oidc_issuer = "https://oidc.test" + requests_mock.get( + API_URL + "credentials/oidc", + json={"providers": [{"id": "oi", "issuer": oidc_issuer, "title": "example", "scopes": ["openid"]}]}, + ) + oidc_mock = OidcMock( + requests_mock=requests_mock, + expected_grant_type="refresh_token", + expected_client_id=client_id, + oidc_issuer=oidc_issuer, + expected_fields={"refresh_token": initial_refresh_token}, + ) + _setup_get_me_handler(requests_mock=requests_mock, oidc_mock=oidc_mock) + caplog.set_level(logging.WARNING) + + # Initial authentication with `authenticate_oidc_refresh_token` + conn = Connection(API_URL, refresh_token_store=refresh_token_store) + assert isinstance(conn.auth, NullAuth) + conn.authenticate_oidc_refresh_token(refresh_token=initial_refresh_token, client_id=client_id) + assert isinstance(conn.auth, BearerAuth) and conn.auth.bearer == "oidc/oi/" + oidc_mock.state["access_token"] + + # Just one "refresh_token" auth request so far + assert [h["grant_type"] for h in oidc_mock.grant_request_history] == ["refresh_token"] + access_token1 = oidc_mock.state["access_token"] + + # Do request that requires auth headers + assert conn.describe_account() == { + "user_id": "john", + "_used_oidc_provider": "oi", + "_used_access_token": access_token1, + } + + # Prepare for refresh attempt + if invalid_refresh_token: + oidc_mock.expected_fields["refresh_token"] = "nope-not-accepting-that" + + # Trigger refresh + refreshed = conn.try_access_token_refresh(reason="Can I Haz Fresh") + + # Two "refresh_token" auth request attempts should have happened now + assert [h["grant_type"] for h in oidc_mock.grant_request_history] == ["refresh_token", "refresh_token"] + access_token2 = oidc_mock.state["access_token"] + + if expect_refresh: + assert refreshed == True + assert access_token2 != access_token1 + assert caplog.messages == [] + else: + assert refreshed == False + assert access_token2 == access_token1 + assert caplog.messages == [ + dirty_equals.IsStr( + regex="Failed to obtain new access token.*grant 'refresh_token'.*invalid refresh token.*Reason: Can I Haz Fresh" + ) + ] + + # New request with fresh token? + assert conn.describe_account() == { + "user_id": "john", + "_used_oidc_provider": "oi", + "_used_access_token": access_token2, + } + + +@pytest.mark.parametrize( + ["invalid_refresh_token", "expect_refresh"], + [ + (False, True), + (True, False), + ], +) +def test_try_access_token_refresh_initial_device_code( + requests_mock, refresh_token_store, caplog, invalid_refresh_token, expect_refresh, oidc_device_code_flow_checker +): + requests_mock.get(API_URL, json={"api_version": "1.0.0"}) + client_id = "myclient" + oidc_issuer = "https://oidc.test" + requests_mock.get( + API_URL + "credentials/oidc", + json={"providers": [{"id": "oi", "issuer": oidc_issuer, "title": "example", "scopes": ["openid"]}]}, + ) + oidc_mock = OidcMock( + requests_mock=requests_mock, + expected_grant_type="urn:ietf:params:oauth:grant-type:device_code", + expected_client_id=client_id, + oidc_issuer=oidc_issuer, + expected_fields={ + "scope": "openid", + "code_verifier": True, + "code_challenge": True, + }, + scopes_supported=["openid"], + ) + _setup_get_me_handler(requests_mock=requests_mock, oidc_mock=oidc_mock) + caplog.set_level(logging.WARNING) + + # Initial authentication with `authenticate_oidc_refresh_token` + conn = Connection(API_URL, refresh_token_store=refresh_token_store) + assert isinstance(conn.auth, NullAuth) + oidc_mock.state["device_code_callback_timeline"] = ["great success"] + with oidc_device_code_flow_checker(): + conn.authenticate_oidc_device(client_id=client_id, use_pkce=True, store_refresh_token=True) + assert isinstance(conn.auth, BearerAuth) and conn.auth.bearer == "oidc/oi/" + oidc_mock.state["access_token"] + + # Just one "refresh_token" auth request so far + assert [h["grant_type"] for h in oidc_mock.grant_request_history] == [ + "urn:ietf:params:oauth:grant-type:device_code" + ] + access_token1 = oidc_mock.state["access_token"] + refresh_token1 = oidc_mock.state["refresh_token"] + assert refresh_token1 is not None + + # Do request that requires auth headers + assert conn.describe_account() == { + "user_id": "john", + "_used_oidc_provider": "oi", + "_used_access_token": access_token1, + } + + # Prepare for refresh attempt + oidc_mock.expected_grant_type = "refresh_token" + if invalid_refresh_token: + oidc_mock.expected_fields["refresh_token"] = "nope-not-accepting-that" + else: + oidc_mock.expected_fields["refresh_token"] = refresh_token1 + + # Trigger refresh + refreshed = conn.try_access_token_refresh(reason="Can I Haz Fresh") + + # Two "refresh_token" auth request attempts should have happened now + assert [h["grant_type"] for h in oidc_mock.grant_request_history] == [ + "urn:ietf:params:oauth:grant-type:device_code", + "refresh_token", + ] + access_token2 = oidc_mock.state["access_token"] + + if expect_refresh: + assert refreshed == True + assert access_token2 != access_token1 + assert caplog.messages == [] + else: + assert refreshed == False + assert access_token2 == access_token1 + assert caplog.messages == [ + dirty_equals.IsStr( + regex="Failed to obtain new access token.*grant 'refresh_token'.*invalid refresh token.*Reason: Can I Haz Fresh" + ) + ] + + # New request with fresh token? + assert conn.describe_account() == { + "user_id": "john", + "_used_oidc_provider": "oi", + "_used_access_token": access_token2, + } + + +@pytest.mark.parametrize( + ["block_token_endpoint", "expect_refresh"], + [ + (False, True), + (True, False), + ], +) +def test_try_access_token_refresh_initial_client_credentials( + requests_mock, refresh_token_store, caplog, block_token_endpoint, expect_refresh, oidc_device_code_flow_checker +): + requests_mock.get(API_URL, json={"api_version": "1.0.0"}) + client_id = "myclient" + client_secret = "$3cr3t" + oidc_issuer = "https://oidc.test" + requests_mock.get( + API_URL + "credentials/oidc", + json={"providers": [{"id": "oi", "issuer": oidc_issuer, "title": "example", "scopes": ["openid"]}]}, + ) + oidc_mock = OidcMock( + requests_mock=requests_mock, + expected_grant_type="client_credentials", + expected_client_id=client_id, + expected_fields={"client_secret": client_secret, "scope": "openid"}, + oidc_issuer=oidc_issuer, + ) + _setup_get_me_handler(requests_mock=requests_mock, oidc_mock=oidc_mock) + caplog.set_level(logging.WARNING) + + # Initial authentication with `authenticate_oidc_client_credentials` + conn = Connection(API_URL, refresh_token_store=refresh_token_store) + assert isinstance(conn.auth, NullAuth) + conn.authenticate_oidc_client_credentials(client_id=client_id, client_secret=client_secret) + assert isinstance(conn.auth, BearerAuth) and conn.auth.bearer == "oidc/oi/" + oidc_mock.state["access_token"] + + # Just one "refresh_token" auth request so far + assert [h["grant_type"] for h in oidc_mock.grant_request_history] == [ + "client_credentials", + ] + access_token1 = oidc_mock.state["access_token"] + + # Do request that requires auth headers + assert conn.describe_account() == { + "user_id": "john", + "_used_oidc_provider": "oi", + "_used_access_token": access_token1, + } + + # Prepare for refresh attempt + if block_token_endpoint: + oidc_mock.token_callback_client_credentials = oidc_mock.token_callback_block_400 + + # Trigger refresh + refreshed = conn.try_access_token_refresh(reason="Can I Haz Fresh") + + # Two "refresh_token" auth request attempts should have happened now + assert [h["grant_type"] for h in oidc_mock.grant_request_history] == [ + "client_credentials", + "client_credentials", + ] + access_token2 = oidc_mock.state["access_token"] + + if expect_refresh: + assert refreshed == True + assert access_token2 != access_token1 + assert caplog.messages == [] + else: + assert refreshed == False + assert access_token2 == access_token1 + assert caplog.messages == [ + dirty_equals.IsStr( + regex="Failed to obtain new access token.*grant 'client_credentials'.*block_400.*Reason: Can I Haz Fresh" + ) + ] + + # New request with fresh token? + assert conn.describe_account() == { + "user_id": "john", + "_used_oidc_provider": "oi", + "_used_access_token": access_token2, + } + + class TestAuthenticateOidcAccessToken: @pytest.fixture(autouse=True) def _setup(self, requests_mock):