404 while deleting worksapce-level group

[mwojtyczka]

While replacing groups the tool is trying to attempt to delete groups that were already deleted. It seems there is a race condition in the _replace_group method.

Error:
07:26 INFO [d.l.ucx.workspace_access.groups] Deleting the workspace-level group idm2bcd_dssi03prod_smg_super with id 439677076072037 07:26 DEBUG [d.s] DELETE /api/2.0/preview/scim/v2/Groups/439677076072037 < 404 Not Found < { < "detail": "Group with id 439677076072037 not found.", < "schemas": [ < "urn:ietf:params:scim:api:messages:2.0:Error" < ], < "status": "404" < }

the following code in the _replace_group is not thread-safe:
self._workspace_groups = [g for g in self._workspace_groups if g.id != ws_group.id]

[mwojtyczka]

Might be a root cause for #383 as well.

[nfx]

This is very likely an internal state corruption caused by sloppy implementation of multithreading.

Can we measure the total time of cleanup step without the thread pool?

[mwojtyczka]

Got the following timeout after the end of the execution of _replace_group method. So same issue as in #383.
Probably some tasks timeouted out trying to delete groups that were already deleted and it was manifested at the end when the results were collected.

`TimeoutError: Timed out after 0:20:00

DatabricksError Traceback (most recent call last)
File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/sdk/retries.py:29, in retried..decorator..wrapper(*args, **kwargs)
28 try:
---> 29 return func(*args, **kwargs)
30 except Exception as err:

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/mixins/hardening.py:57, in rate_limited..decorator..wrapper(*args, **kwargs)
56 rate_limiter.throttle()
---> 57 return func(*args, **kwargs)

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/workspace_access/groups.py:150, in GroupManager._delete_workspace_group(self, ws_group)
148 logger.info(f"Deleting the workspace-level group {ws_group.display_name} with id {ws_group.id}")
--> 150 self._ws.groups.delete(id=ws_group.id)
152 logger.info(f"Workspace-level group {ws_group.display_name} with id {ws_group.id} was deleted")

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/sdk/service/iam.py:1731, in GroupsAPI.delete(self, id)
1730 headers = {}
-> 1731 self._api.do('DELETE', f'/api/2.0/preview/scim/v2/Groups/{id}', headers=headers)

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/sdk/core.py:1061, in ApiClient.do(self, method, path, query, headers, body, raw, files, data)
1059 retryable = retried(timeout=timedelta(seconds=self._retry_timeout_seconds),
1060 is_retryable=self._is_retryable)
-> 1061 return retryable(self._perform)(method,
1062 path,
1063 query=query,
1064 headers=headers,
1065 body=body,
1066 raw=raw,
1067 files=files,
1068 data=data)

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/sdk/retries.py:47, in retried..decorator..wrapper(*args, **kwargs)
45 if retry_reason is None:
46 # raise if exception is not retryable
---> 47 raise err
49 logger.debug(f'Retrying: {retry_reason} (sleeping ~{sleep}s)')

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/sdk/retries.py:29, in retried..decorator..wrapper(*args, **kwargs)
28 try:
---> 29 return func(*args, **kwargs)
30 except Exception as err:

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/sdk/core.py:1150, in ApiClient._perform(self, method, path, query, headers, body, raw, files, data)
1149 payload = response.json()
-> 1150 raise self._make_nicer_error(response=response, **payload) from None
1151 if raw:

DatabricksError: None Group with id 498910685893996 not found.

The above exception was the direct cause of the following exception:

TimeoutError Traceback (most recent call last)
File ~/.ipykernel/1024/command--1-1764370551:18
15 entry = [ep for ep in metadata.distribution("databricks_labs_ucx").entry_points if ep.name == "runtime"]
16 if entry:
17 # Load and execute the entrypoint, assumes no parameters
---> 18 entry[0].load()()
19 else:
20 import databricks_labs_ucx

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/runtime.py:215, in main()
214 def main():
--> 215 trigger(*sys.argv)

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/framework/tasks.py:93, in trigger(*argv)
90 cfg = WorkspaceConfig.from_file(Path(args["config"]))
91 logging.getLogger("databricks").setLevel(cfg.log_level)
---> 93 current_task.fn(cfg)

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/runtime.py:195, in migrate_permissions(cfg)
193 if toolkit.has_groups():
194 toolkit.apply_permissions_to_backup_groups()
--> 195 toolkit.replace_workspace_groups_with_account_groups()
196 toolkit.apply_permissions_to_account_groups()
197 else:

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/workspace_access/migration.py:138, in GroupMigrationToolkit.replace_workspace_groups_with_account_groups(self)
137 def replace_workspace_groups_with_account_groups(self):
--> 138 self._group_manager.replace_workspace_groups_with_account_groups()

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/workspace_access/groups.py:238, in GroupManager.replace_workspace_groups_with_account_groups(self)
236 logger.info("No groups were loaded or initialized, nothing to do")
237 return
--> 238 ThreadedExecution.gather(
239 "groups: workspace -> account",
240 [partial(self._replace_group, migration_info) for migration_info in self.migration_groups_provider.groups],
241 )
242 logger.info("Workspace groups were successfully replaced with account-level groups")

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/framework/parallel.py:48, in ThreadedExecution.gather(cls, name, tasks)
45 @classmethod
46 def gather(cls, name: str, tasks: list[ExecutableFunction]) -> list[ExecutableResult]:
47 reporter = ProgressReporter(len(tasks), f"{name}: ")
---> 48 return cls(tasks, num_threads=4, progress_reporter=reporter).run()

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/framework/parallel.py:63, in ThreadedExecution.run(self)
60 results = concurrent.futures.wait(self._futures, return_when=ALL_COMPLETED)
62 logger.debug("Collecting the results from threaded execution")
---> 63 collected = [future.result() for future in results.done]
64 return collected

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/framework/parallel.py:63, in (.0)
60 results = concurrent.futures.wait(self._futures, return_when=ALL_COMPLETED)
62 logger.debug("Collecting the results from threaded execution")
---> 63 collected = [future.result() for future in results.done]
64 return collected

File /usr/lib/python3.10/concurrent/futures/_base.py:451, in Future.result(self, timeout)
449 raise CancelledError()
450 elif self._state == FINISHED:
--> 451 return self.__get_result()
453 self._condition.wait(timeout)
455 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

File /usr/lib/python3.10/concurrent/futures/_base.py:403, in Future.__get_result(self)
401 if self._exception:
402 try:
--> 403 raise self._exception
404 finally:
405 # Break a reference cycle with the exception in self._exception
406 self = None

File /usr/lib/python3.10/concurrent/futures/thread.py:58, in _WorkItem.run(self)
55 return
57 try:
---> 58 result = self.fn(*self.args, **self.kwargs)
59 except BaseException as exc:
60 self.future.set_exception(exc)

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/labs/ucx/workspace_access/groups.py:138, in GroupManager._replace_group(self, migration_info)
135 def _replace_group(self, migration_info: MigrationGroupInfo):
136 ws_group = migration_info.workspace
--> 138 self._delete_workspace_group(ws_group)
140 # delete ws_group from the list of workspace groups
141 self._workspace_groups = [g for g in self._workspace_groups if g.id != ws_group.id]

File /local_disk0/.ephemeral_nfs/cluster_libraries/python/lib/python3.10/site-packages/databricks/sdk/retries.py:52, in retried..decorator..wrapper(*args, **kwargs)
50 time.sleep(sleep + random())
51 attempt += 1
---> 52 raise TimeoutError(f'Timed out after {timeout}') from last_err

TimeoutError: Timed out after 0:20:00`