[Bug]: Internal Server Error when deleting a user #3254
Description
Steps to reproduce
- Create a user.
- Add it to a project.
- Start a run as this user and wait until it is
running. - Wait a few more seconds.
- Stop the run.
- Delete the user.
Actual behaviour
/api/users/delete returns Internal Server Error, status code 500. User not deleted.
Expected behaviour
/api/users/delete returns status code 200. User is deleted.
dstack version
0.19.35
Server logs
Traceback (most recent call last):
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/base.py", line 1964, in _exec_single_context
self.dialect.do_execute(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/default.py", line 945, in do_execute
cursor.execute(statement, parameters)
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 172, in execute
self._adapt_connection._handle_exception(error)
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 323, in _handle_exception
raise error
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 154, in execute
self.await_(_cursor.execute(operation, parameters))
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 132, in await_only
return current.parent.switch(awaitable) # type: ignore[no-any-return,attr-defined] # noqa: E501
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 196, in greenlet_spawn
value = await result
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/cursor.py", line 40, in execute
await self._execute(self._cursor.execute, sql, parameters)
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/cursor.py", line 32, in _execute
return await self._conn._execute(fn, *args, **kwargs)
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/core.py", line 122, in _execute
return await future
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/core.py", line 105, in run
result = function()
sqlite3.IntegrityError: FOREIGN KEY constraint failed
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/dstack/venv/lib64/python3.10/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/dstack/venv/lib64/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/errors.py", line 187, in __call__
raise exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/errors.py", line 165, in __call__
await self.app(scope, receive, _send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/lib64/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
File "/dstack/src/dstack/_internal/server/app.py", line 340, in check_client_version
return await call_next(request)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/lib64/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
File "/dstack/src/dstack/_internal/server/app.py", line 299, in log_http_metrics
response: Response = await call_next(request)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/lib64/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
File "/dstack/src/dstack/_internal/server/app.py", line 256, in log_request
response: Response = await call_next(request)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/routing.py", line 714, in __call__
await self.middleware_stack(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/routing.py", line 734, in app
await route.handle(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/routing.py", line 288, in handle
await self.app(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/routing.py", line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/dstack/venv/lib64/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/dstack/venv/lib64/python3.10/site-packages/starlette/routing.py", line 73, in app
response = await f(request)
File "/dstack/venv/lib64/python3.10/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
File "/dstack/venv/lib64/python3.10/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
return await dependant.call(**values)
File "/dstack/src/dstack/_internal/server/routers/users.py", line 131, in delete_users
await users.delete_users(
File "/dstack/src/dstack/_internal/server/services/users.py", line 176, in delete_users
await session.execute(delete(UserModel).where(UserModel.name.in_(usernames)))
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/ext/asyncio/session.py", line 463, in execute
result = await greenlet_spawn(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 201, in greenlet_spawn
result = context.throw(*sys.exc_info())
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/orm/session.py", line 2365, in execute
return self._execute_internal(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/orm/session.py", line 2251, in _execute_internal
result: Result[Any] = compile_state_cls.orm_execute_statement(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/orm/bulk_persistence.py", line 2021, in orm_execute_statement
return super().orm_execute_statement(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/orm/context.py", line 306, in orm_execute_statement
result = conn.execute(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/base.py", line 1416, in execute
return meth(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/sql/elements.py", line 523, in _execute_on_connection
return connection._execute_clauseelement(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/base.py", line 1638, in _execute_clauseelement
ret = self._execute_context(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/base.py", line 1843, in _execute_context
return self._exec_single_context(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/base.py", line 1983, in _exec_single_context
self._handle_dbapi_exception(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/base.py", line 2352, in _handle_dbapi_exception
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/base.py", line 1964, in _exec_single_context
self.dialect.do_execute(
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/engine/default.py", line 945, in do_execute
cursor.execute(statement, parameters)
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 172, in execute
self._adapt_connection._handle_exception(error)
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 323, in _handle_exception
raise error
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/dialects/sqlite/aiosqlite.py", line 154, in execute
self.await_(_cursor.execute(operation, parameters))
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 132, in await_only
return current.parent.switch(awaitable) # type: ignore[no-any-return,attr-defined] # noqa: E501
File "/dstack/venv/lib64/python3.10/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 196, in greenlet_spawn
value = await result
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/cursor.py", line 40, in execute
await self._execute(self._cursor.execute, sql, parameters)
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/cursor.py", line 32, in _execute
return await self._conn._execute(fn, *args, **kwargs)
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/core.py", line 122, in _execute
return await future
File "/dstack/venv/lib64/python3.10/site-packages/aiosqlite/core.py", line 105, in run
result = function()
sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) FOREIGN KEY constraint failed
[SQL: DELETE FROM users WHERE users.name IN (?)]
[parameters: ('temp',)]
(Background on this error at: https://sqlalche.me/e/20/gkpj)Additional information
The current implementation suggests a cascade delete of all user's resources, including runs and jobs. However, jobs' child resources such as probes and metrics do not set the ON DELETE CASCADE constraint, so their FOREIGN KEY constraint fails.
Overall, using cascade deletes for resources doesn't seem to be the right solution, since it may lead to data loss and inconsistencies.
Alternative solutions:
- Soft deletes for users (
deleted = true). ON DELETE SET NULLfor resource to user foreign keys.