tests.system.aiplatform.test_experiments.TestExperiments: test_end_run failed #1663
Description
This test failed!
To configure my behavior, see the Flaky Bot documentation.
If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.
commit: a3cc5a3
buildURL: Build Status, Sponge
status: failed
Test output
args = (context {
name: "projects/580378083368/locations/us-central1/metadataStores/default/contexts/tmpvrtxsdk-e2e--a64ab3... }
}
}
fields {
key: "_state"
value {
string_value: "COMPLETE"
}
}
}
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'context.name=projects/580378083368/locations/us-central1/metadataStores/defau...d495453ba-run-1'), ('x-goog-api-client', 'model-builder/1.17.0 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.17.0')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
try:
return callable_(*args, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:
self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7fee62a9aa60>
request = context {
name: "projects/580378083368/locations/us-central1/metadataStores/default/contexts/tmpvrtxsdk-e2e--a64ab3d... }
}
}
fields {
key: "_state"
value {
string_value: "COMPLETE"
}
}
}
}
timeout = None
metadata = [('x-goog-request-params', 'context.name=projects/580378083368/locations/us-central1/metadataStores/default/contexts/t...7d495453ba-run-1'), ('x-goog-api-client', 'model-builder/1.17.0 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.17.0')]
credentials = None, wait_for_ready = None, compression = None
def __call__(self,
request,
timeout=None,
metadata=None,
credentials=None,
wait_for_ready=None,
compression=None):
state, call, = self._blocking(request, timeout, metadata, credentials,
wait_for_ready, compression)
return _end_unary_response_blocking(state, call, False, None)
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:
state = <grpc._channel._RPCState object at 0x7fee6017f7f0>
call = <grpc._cython.cygrpc.SegregatedCall object at 0x7fee600dc900>
with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
if state.code is grpc.StatusCode.OK:
if with_call:
rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
return state.response, rendezvous
else:
return state.response
else:
raise _InactiveRpcError(state)
E grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E status = StatusCode.ABORTED
E details = "Specified Context etag: 1663148213197 does not match server etag: 1663148180150"
E debug_error_string = "{"created":"@1663148213.581078660","description":"Error received from peer ipv4:74.125.20.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Specified Context etag: 1663148213197 does not match server etag: 1663148180150","grpc_status":10}"
E >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_experiments.TestExperiments object at 0x7fee69070910>
def test_end_run(self):
aiplatform.init(
project=e2e_base._PROJECT,
location=e2e_base._LOCATION,
experiment=self._experiment_name,
)
aiplatform.start_run(_RUN, resume=True)
aiplatform.end_run()
tests/system/aiplatform/test_experiments.py:246:
google/cloud/aiplatform/metadata/metadata.py:327: in end_run
self._experiment_run.end_run(state=state)
google/cloud/aiplatform/metadata/experiment_run_resource.py:1076: in end_run
self.update_state(state)
google/cloud/aiplatform/metadata/experiment_run_resource.py:348: in update_state
self._metadata_node.update(metadata={constants._STATE_KEY: state.name})
google/cloud/aiplatform/metadata/resource.py:307: in update
update_gca_resource = self._update_resource(
google/cloud/aiplatform/metadata/context.py:294: in _update_resource
return client.update_context(context=resource)
google/cloud/aiplatform_v1/services/metadata_service/client.py:2198: in update_context
response = rpc(
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call
return wrapped_func(*args, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable
six.raise_from(exceptions.from_grpc_error(exc), exc)
value = None
from_value = <_InactiveRpcError of RPC that terminated with:
status = StatusCode.ABORTED
details = "Specified Context etag: 16...message":"Specified Context etag: 1663148213197does not match serveretag: 1663148180150`","grpc_status":10}"
???
E google.api_core.exceptions.Aborted: 409 Specified Context etag: 1663148213197 does not match server etag: 1663148180150
:3: Aborted