tests.system.aiplatform.test_model_version_management.TestVersionManagement: test_upload_deploy_manage_versioned_model failed

[flaky-bot]

Note: #1957 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

---

commit: 921bfc0
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/580378083368/locations/us-central1/models/my_model_idafcb73f2a4b44685892899ba768d8bfb/operations/2911436627765100544"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/models/my_model_idafcb73f2a4...b/operations/2911436627765100544'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')], 'timeout': 20.0}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f474e0f4bb0>

request = name: "projects/580378083368/locations/us-central1/models/my_model_idafcb73f2a4b44685892899ba768d8bfb/operations/2911436627765100544"
timeout = 20.0

metadata = [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/models/my_model_idafcb73f2a4b44685892899ba768d8bfb/operations/2911436627765100544'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.47.0 gax/1.32.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f47adaa2400>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f47adae5140>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.DEADLINE_EXCEEDED

E           	details = "Deadline Exceeded"

E           	debug_error_string = "{"created":"@1683036474.917908239","description":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_model_version_management.TestVersionManagement object at 0x7f475047f7f0>

shared_state = {'bucket': <Bucket: ucaip-sample-tests-vertex-staging-us-central1>, 'resources': [<google.cloud.aiplatform.models.Mode...c8bb0>

resource name: projects/580378083368/locations/us-central1/models/my_model_idafcb73f2a4b44685892899ba768d8bfb]}
def test_upload_deploy_manage_versioned_model(self, shared_state):
    """Upload XGBoost model from local file and deploy it for prediction. Additionally, update model name, description and labels"""

    aiplatform.init(
        project=e2e_base._PROJECT,
        location=e2e_base._LOCATION,
    )

    storage_client = storage.Client(project=e2e_base._PROJECT)
    model_blob = storage.Blob.from_string(
        uri=test_model_upload._XGBOOST_MODEL_URI, client=storage_client
    )
    model_path = tempfile.mktemp() + ".my_model.xgb"
    model_blob.download_to_filename(filename=model_path)

    model_id = "my_model_id" + uuid.uuid4().hex
    version_description = "My description"
    version_aliases = ["system-test-model", "testing"]

    model = aiplatform.Model.upload_xgboost_model_file(
        model_file_path=model_path,
        version_aliases=version_aliases,
        model_id=model_id,
        version_description=version_description,
    )
    shared_state["resources"] = [model]

    staging_bucket = storage.Blob.from_string(
        uri=model.uri, client=storage_client
    ).bucket
    # Checking that the bucket is auto-generated
    assert "-vertex-staging-" in staging_bucket.name

    shared_state["bucket"] = staging_bucket

    assert model.version_description == version_description
    assert model.version_aliases == version_aliases
    assert "default" in model.version_aliases


  model2 = aiplatform.Model.upload_xgboost_model_file(


        model_file_path=model_path, parent_model=model_id, is_default_version=False
    )

tests/system/aiplatform/test_model_version_management.py:75:

google/cloud/aiplatform/base.py:814: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/models.py:4194: in upload_xgboost_model_file

return cls.upload(

google/cloud/aiplatform/base.py:814: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/models.py:3162: in upload

model_upload_response = lro.result()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result

self._blocking_poll(timeout=timeout, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108: in blocking_poll

retry(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:86: in _done_or_raise

if not self.done(**kwargs):

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:170: in done

self._refresh_and_update(retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operation.py:158: in _refresh_and_update

self._operation = self._refresh(retry=retry)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/operations_v1/operations_client.py:142: in get_operation

return self._get_operation(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/timeout.py:214: in func_with_timeout

return func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.DEADLINE_EXCEEDED

details = "Deadline Exceeded"

...ption":"Deadline Exceeded","file":"src/core/ext/filters/deadline/deadline_filter.cc","file_line":81,"grpc_status":4}"



???

E   google.api_core.exceptions.DeadlineExceeded: 504 Deadline Exceeded

:3: DeadlineExceeded

[flaky-bot]

Test passed for commit 43488fe (Build Status, Sponge)! Closing this issue.