tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring: test_create_endpoint failed

[flaky-bot]

This test failed!

To configure my behavior, see the Flaky Bot documentation.

If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.

---

commit: 921bfc0
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/580378083368/locations/us-central1/endpoints/5786681318473465856/operations/1231593966755905536"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/endpoints/5786681318473465856/operations/1231593966755905536'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')], 'timeout': 20.0}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f50e4bfd1c0>

request = name: "projects/580378083368/locations/us-central1/endpoints/5786681318473465856/operations/1231593966755905536"
timeout = 20.0

metadata = [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/endpoints/5786681318473465856/operations/1231593966755905536'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f50e4c11be0>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f50e4cb6800>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.DEADLINE_EXCEEDED

E           	details = "Deadline Exceeded"

E           	debug_error_string = "{"created":"@1683036611.128873581","description":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring object at 0x7f50e8307e50>

shared_state = {'resources': [<google.cloud.aiplatform.models.Endpoint object at 0x7f50e6042940>

resource name: projects/580378083368/locations/us-central1/endpoints/5786681318473465856]}
def test_create_endpoint(self, shared_state):
    # initial setup
    aiplatform.init(project=e2e_base._PROJECT, location=e2e_base._LOCATION)
    self.endpoint = aiplatform.Endpoint.create(self._make_display_name("endpoint"))
    shared_state["resources"] = [self.endpoint]
    self.model = aiplatform.Model(PERMANENT_CHURN_MODEL_ID)
    self.endpoint.deploy(
        self.model,
        deployed_model_display_name=self._make_display_name(key=JOB_NAME),
    )


  self.endpoint.deploy(


        self.model,
        deployed_model_display_name=self._make_display_name(key=JOB_NAME),
        traffic_percentage=50,
    )

tests/system/aiplatform/test_model_monitoring.py:134:

google/cloud/aiplatform/models.py:853: in deploy

self._deploy(

google/cloud/aiplatform/base.py:814: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/models.py:970: in _deploy

self._deploy_call(

google/cloud/aiplatform/models.py:1244: in _deploy_call

operation_future.result(timeout=None)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result

self._blocking_poll(timeout=timeout, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108: in blocking_poll

retry(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:86: in _done_or_raise

if not self.done(**kwargs):

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:170: in done

self._refresh_and_update(retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:158: in _refresh_and_update

self._operation = self._refresh(retry=retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operations_v1/operations_client.py:142: in get_operation

return self._get_operation(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/timeout.py:214: in func_with_timeout

return func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.DEADLINE_EXCEEDED

details = "Deadline Exceeded"

...ption":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"



???

E   google.api_core.exceptions.DeadlineExceeded: 504 Deadline Exceeded

:3: DeadlineExceeded

[flaky-bot]

Test passed for commit 43488fe (Build Status, Sponge)! Closing this issue.