From 9c2e525158ceb5a5c4e3fb4068a2195fdca6746d Mon Sep 17 00:00:00 2001 From: Tim Kellogg Date: Tue, 1 Oct 2024 09:20:33 -0400 Subject: [PATCH 1/2] Add `num_retries` to signature_opt_typed This adds a `num_retries` option to `signature_opt_typed.optimize_signature()`. This is passed through to both `TypedChainOfThought` and `TypedPredictor`. **Why?** Because I was stuck. I kept getting this error message: > Value error, Never propose a signature already in the list above.: proposed_signature (error type: value_error) I can get around this error by raising the model's temperature as well as setting this to `num_retries=10`. Note: I see comments about not supporting temperature because it breaks parsing. While that's true, having temp be zero also leads to getting completely stuck because there's no simple solution for common errors like this. With high enough temperature and enough retries, I should be able to unstick myself from any situation. --- dspy/teleprompt/signature_opt_typed.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dspy/teleprompt/signature_opt_typed.py b/dspy/teleprompt/signature_opt_typed.py index 9da50374f5..38428596a1 100644 --- a/dspy/teleprompt/signature_opt_typed.py +++ b/dspy/teleprompt/signature_opt_typed.py @@ -150,6 +150,7 @@ def optimize_signature( prompt_model=None, initial_prompts=2, verbose=False, + num_retries: int = 3, ) -> dspy.Program: """Create a new program that is optimized for the given task. @@ -177,6 +178,8 @@ def optimize_signature( Note that we also use the "plain" signature as a prompt, so the total number of prompts is initial_prompts + 1. verbose : bool, optional Whether to print debug information, by default False + num_retries : int + The number of retries to use when generating new signatures, by default 3. Notes: ----- @@ -216,7 +219,7 @@ def optimize_signature( if verbose: print(f"Generating {initial_prompts} initial signatures for {name}...") info = candidates[name][0] # Use initial info, to make sure types are identical - generator = TypedChainOfThought(MyGenerateInstructionInitial[type(info)]) + generator = TypedChainOfThought(MyGenerateInstructionInitial[type(info)], num_retries=num_retries) candidates[name] += generator( basic_signature=info, ).proposed_signatures @@ -267,7 +270,7 @@ def optimize_signature( # We can only tell the LM to avoid the signatures we are actually giving it as demos. avoid = [ex.proposed_signature for ex in demos] - generator = TypedPredictor(generate_with_avoidance(avoid)[SignatureInfo]) + generator = TypedPredictor(generate_with_avoidance(avoid)[SignatureInfo], num_retries=num_retries) generator.predictor.demos = demos if verbose: From 1972a563d91033115ca859dff22a107ab6c35a65 Mon Sep 17 00:00:00 2001 From: Tim Kellogg Date: Tue, 1 Oct 2024 09:44:53 -0400 Subject: [PATCH 2/2] Add `num_retries` to signature_opt_typed This adds a `num_retries` option to `signature_opt_typed.optimize_signature()`. This is passed through to both `TypedChainOfThought` and `TypedPredictor`. **Why?** Because I was stuck. I kept getting this error message: > Value error, Never propose a signature already in the list above.: proposed_signature (error type: value_error) I can get around this error by raising the model's temperature as well as setting this to `num_retries=10`. Note: I see comments about not supporting temperature because it breaks parsing. While that's true, having temp be zero also leads to getting completely stuck because there's no simple solution for common errors like this. With high enough temperature and enough retries, I should be able to unstick myself from any situation. --- dspy/teleprompt/signature_opt_typed.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dspy/teleprompt/signature_opt_typed.py b/dspy/teleprompt/signature_opt_typed.py index 38428596a1..3e5c7587f9 100644 --- a/dspy/teleprompt/signature_opt_typed.py +++ b/dspy/teleprompt/signature_opt_typed.py @@ -150,7 +150,7 @@ def optimize_signature( prompt_model=None, initial_prompts=2, verbose=False, - num_retries: int = 3, + max_retries: int = 3, ) -> dspy.Program: """Create a new program that is optimized for the given task. @@ -178,7 +178,7 @@ def optimize_signature( Note that we also use the "plain" signature as a prompt, so the total number of prompts is initial_prompts + 1. verbose : bool, optional Whether to print debug information, by default False - num_retries : int + max_retries : int The number of retries to use when generating new signatures, by default 3. Notes: @@ -219,7 +219,7 @@ def optimize_signature( if verbose: print(f"Generating {initial_prompts} initial signatures for {name}...") info = candidates[name][0] # Use initial info, to make sure types are identical - generator = TypedChainOfThought(MyGenerateInstructionInitial[type(info)], num_retries=num_retries) + generator = TypedChainOfThought(MyGenerateInstructionInitial[type(info)], max_retries=max_retries) candidates[name] += generator( basic_signature=info, ).proposed_signatures @@ -270,7 +270,7 @@ def optimize_signature( # We can only tell the LM to avoid the signatures we are actually giving it as demos. avoid = [ex.proposed_signature for ex in demos] - generator = TypedPredictor(generate_with_avoidance(avoid)[SignatureInfo], num_retries=num_retries) + generator = TypedPredictor(generate_with_avoidance(avoid)[SignatureInfo], max_retries=max_retries) generator.predictor.demos = demos if verbose: