trivial - None generation config

hpcaitech · Apr 23, 2024 · d4d6d1b · d4d6d1b
1 parent 747109c
commit d4d6d1b
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/colossalai/inference/core/engine.py b/colossalai/inference/core/engine.py
@@ -518,11 +518,12 @@ def generate(
         """
         with torch.inference_mode():
             if prompts is not None or prompts_token_ids is not None:
+                gen_config_dict = generation_config.to_dict() if generation_config is not None else {}
                 self.add_request(
                     request_ids=request_ids,
                     prompts=prompts,
                     prompts_token_ids=prompts_token_ids,
-                    **generation_config.to_dict(),
+                    **gen_config_dict,
                 )
 
             output_seqs_list = []