[BugFix] Fix return type of executor execute_model methods (#4402)

vllm-project · Apr 27, 2024 · ba4be44 · ba4be44
1 parent d6e520e
commit ba4be44
Show file tree

Hide file tree

Showing 6 changed files with 9 additions and 8 deletions.
diff --git a/vllm/executor/cpu_executor.py b/vllm/executor/cpu_executor.py
@@ -109,7 +109,7 @@ async def execute_model_async(
         blocks_to_swap_in: Dict[int, int],
         blocks_to_swap_out: Dict[int, int],
         blocks_to_copy: Dict[int, List[int]],
-    ) -> SamplerOutput:
+    ) -> List[SamplerOutput]:
         output = await make_async(self.driver_worker.execute_model)(
             seq_group_metadata_list=seq_group_metadata_list,
             blocks_to_swap_in=blocks_to_swap_in,

diff --git a/vllm/executor/distributed_gpu_executor.py b/vllm/executor/distributed_gpu_executor.py
@@ -1,5 +1,5 @@
 from abc import abstractmethod
-from typing import Any, Dict, Optional, Set, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple
 
 from vllm.executor.executor_base import ExecutorAsyncBase
 from vllm.executor.gpu_executor import GPUExecutor
@@ -52,7 +52,7 @@ def initialize_cache(self, num_gpu_blocks: int,
                           num_gpu_blocks=num_gpu_blocks,
                           num_cpu_blocks=num_cpu_blocks)
 
-    def execute_model(self, *args, **kwargs) -> SamplerOutput:
+    def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
         all_outputs = self._run_workers("execute_model",
                                         driver_args=args,
                                         driver_kwargs=kwargs)
@@ -105,7 +105,8 @@ async def _run_workers_async(
         """Runs the given method on all workers."""
         raise NotImplementedError
 
-    async def execute_model_async(self, *args, **kwargs) -> SamplerOutput:
+    async def execute_model_async(self, *args,
+                                  **kwargs) -> List[SamplerOutput]:
         all_outputs = await self._run_workers_async("execute_model",
                                                     driver_args=args,
                                                     driver_kwargs=kwargs)

diff --git a/vllm/executor/executor_base.py b/vllm/executor/executor_base.py
@@ -112,7 +112,7 @@ async def execute_model_async(
         blocks_to_swap_in: Dict[int, int],
         blocks_to_swap_out: Dict[int, int],
         blocks_to_copy: Dict[int, List[int]],
-    ) -> SamplerOutput:
+    ) -> List[SamplerOutput]:
         """Executes one model step on the given sequences."""
         raise NotImplementedError
 

diff --git a/vllm/executor/gpu_executor.py b/vllm/executor/gpu_executor.py
@@ -163,7 +163,7 @@ async def execute_model_async(
         blocks_to_swap_in: Dict[int, int],
         blocks_to_swap_out: Dict[int, int],
         blocks_to_copy: Dict[int, List[int]],
-    ) -> SamplerOutput:
+    ) -> List[SamplerOutput]:
         output = await make_async(self.driver_worker.execute_model)(
             seq_group_metadata_list=seq_group_metadata_list,
             blocks_to_swap_in=blocks_to_swap_in,

diff --git a/vllm/executor/neuron_executor.py b/vllm/executor/neuron_executor.py
@@ -84,7 +84,7 @@ async def execute_model_async(
         blocks_to_swap_in: Dict[int, int],
         blocks_to_swap_out: Dict[int, int],
         blocks_to_copy: Dict[int, List[int]],
-    ) -> SamplerOutput:
+    ) -> List[SamplerOutput]:
         output = await make_async(self.driver_worker.execute_model)(
             seq_group_metadata_list=seq_group_metadata_list, )
         return output

diff --git a/vllm/executor/ray_gpu_executor.py b/vllm/executor/ray_gpu_executor.py
@@ -188,7 +188,7 @@ def execute_model(self,
                       blocks_to_swap_in: Dict[int, int],
                       blocks_to_swap_out: Dict[int, int],
                       blocks_to_copy: Dict[int, List[int]],
-                      num_lookahead_slots: int = 0) -> SamplerOutput:
+                      num_lookahead_slots: int = 0) -> List[SamplerOutput]:
         all_outputs = self._run_workers(
             "execute_model",
             driver_kwargs={