diff --git a/xllm/core/runtime/worker_impl.cpp b/xllm/core/runtime/worker_impl.cpp
index 3fad5fc7..47a78165 100644
--- a/xllm/core/runtime/worker_impl.cpp
+++ b/xllm/core/runtime/worker_impl.cpp
@@ -66,6 +66,7 @@ WorkerImpl::WorkerImpl(const ParallelArgs& parallel_args,
 
   device_.set_device();
   device_.init_device_context();
+  threadpool_.schedule([this]() mutable { device_.set_device(); });
   general_threadpool_.schedule([this]() mutable { device_.set_device(); });
 
   prepare_stream_ = device_.get_stream_from_pool();