From 6ed29b410a2135d9a6e872c1093e5bd696d39291 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Sun, 3 Mar 2024 15:38:10 -0800 Subject: [PATCH 01/18] Fixed #520 --- dspy/functional/functional.py | 10 +- examples/functional/functional.ipynb | 213 ++++++++++++++++++++++----- 2 files changed, 177 insertions(+), 46 deletions(-) diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index cb7b3669aa..8ca45523f4 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -8,12 +8,13 @@ from typing import Annotated, List, Tuple # noqa: UP035 from dsp.templates import passages2text import json +import ujson from dspy.primitives.prediction import Prediction from dspy.signatures.signature import ensure_signature, make_signature -MAX_RETRIES = 3 +MAX_RETRIES = 5 def predictor(func) -> dspy.Module: @@ -101,9 +102,7 @@ def _make_example(type_) -> str: # library like https://pypi.org/project/polyfactory/ that's made exactly to do this. def _prepare_signature(self) -> dspy.Signature: - """Add formats and parsers to the signature fields, based on the type - annotations of the fields. - """ + """Add formats and parsers to the signature fields, based on the type annotations of the fields.""" signature = self.signature for name, field in self.signature.fields.items(): is_output = field.json_schema_extra["__dspy_field_type"] == "output" @@ -161,7 +160,6 @@ def forward(self, **kwargs) -> dspy.Prediction: for name, field in signature.output_fields.items(): value = completion[name] parser = field.json_schema_extra.get("parser", lambda x: x) - completion[name] = parser(value) parsed[name] = parser(value) # Instantiate the actual signature with the parsed values. # This allow pydantic to validate the fields defined in the signature. @@ -257,7 +255,7 @@ def _unwrap_json(output): output = output[7:-3].strip() if not output.startswith("{") or not output.endswith("}"): raise ValueError("json output should start and end with { and }") - return output + return ujson.dumps(ujson.loads(output)) # ujson is a bit more robust than the standard json ################################################################################ diff --git a/examples/functional/functional.ipynb b/examples/functional/functional.ipynb index cf7088a5d9..90b7f4501f 100644 --- a/examples/functional/functional.ipynb +++ b/examples/functional/functional.ipynb @@ -2,13 +2,15 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n", "Requirement already satisfied: datasets in /opt/homebrew/lib/python3.11/site-packages (2.14.7)\n", "Requirement already satisfied: numpy>=1.17 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (1.26.2)\n", "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (12.0.0)\n", @@ -39,9 +41,6 @@ "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n", "Requirement already satisfied: tzdata>=2022.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n", "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.11 -m pip install --upgrade pip\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } @@ -61,17 +60,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 26, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - }, { "data": { "text/plain": [ @@ -82,7 +73,7 @@ " 'entry_point': 'has_close_elements'}" ] }, - "execution_count": 2, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -102,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -134,15 +125,21 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Now parsing: '{\"code\": \"from typing import List\\\\n\\\\n\\\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\\\n \\\\\"\\\\\"\\\\\" Check if in given list of numbers, are any two numbers closer to each other than\\\\n given threshold.\\\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\\\n False\\\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\\\n True\\\\n \\\\\"\\\\\"\\\\\"\\\\n for i in range(len(numbers)):\\\\n for j in range(i+1, len(numbers)):\\\\n if abs(numbers[i] - numbers[j]) < threshold:\\\\n return True\\\\n return False\\\\n\"}'\n", + "Parsed: PythonCode(code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\\n')\n", + "is this the problem?\n", + "kwargs={'prompt': PythonCode(code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n'), 'test': PythonCode(code=\"\\n\\nMETADATA = {\\n 'author': 'jt',\\n 'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\"), 'entry_point': 'has_close_elements'}\n", + "parsed={'solution': PythonCode(code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\\n')}\n", + "after wards\n", "Prediction(\n", - " solution=PythonCode(code='def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False')\n", + " solution=PythonCode(code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\\n')\n", ")\n" ] } @@ -170,7 +167,7 @@ "# The signature is the main DSpy object. Note that we have types for the input and output fields,\n", "# which was not possible beofore.\n", "class CodeSignature(Signature):\n", - " prompt: str = InputField()\n", + " prompt: PythonCode = InputField()\n", " test: PythonCode = InputField()\n", " entry_point: str = InputField()\n", " solution: PythonCode = OutputField()\n", @@ -180,9 +177,7 @@ " prompt=PythonCode(code=ds['test'][0]['prompt']),\n", " test=PythonCode(code=ds['test'][0]['test']),\n", " entry_point=ds['test'][0]['entry_point']\n", - ")\n", - "\n", - "print(prediction)" + ")\n" ] }, { @@ -194,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -205,19 +200,40 @@ "\n", "\n", "\n", - "Make a very succinct json object that validates with the following schema\n", + "Given the fields `prompt`, `test`, `entry_point`, produce the fields `solution`.\n", "\n", "---\n", "\n", "Follow the following format.\n", "\n", - "Json Schema: ${json_schema}\n", - "Json Object: ${json_object}\n", + "Prompt: ${prompt}\n", + "\n", + "Test: ${test}\n", + "\n", + "Entry Point: ${entry_point}\n", + "\n", + "Past Error (solution): An error to avoid in the future\n", + "\n", + "Past Error (solution, 2): An error to avoid in the future\n", + "\n", + "Solution:\n", + "${solution}. Respond with a single JSON object. \n", + "You MUST use this format: {\"code\": \"print('Hello, World!')\"}\n", + "JSON Schema: {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n", "\n", "---\n", "\n", - "Json Schema: {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n", - "Json Object:\u001b[32m {\"code\": \"print('Hello, World!')\"}\u001b[0m\n", + "Prompt: code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n given threshold.\\n >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n False\\n >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n True\\n \"\"\"\\n'\n", + "\n", + "Test: {\"code\":\"\\n\\nMETADATA = {\\n 'author': 'jt',\\n 'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\"}\n", + "\n", + "Entry Point: has_close_elements\n", + "\n", + "Past Error (solution): Input should be a valid string: prompt (error type: string_type)\n", + "\n", + "Past Error (solution, 2): Value error, Code is not syntactically valid: unexpected character after line continuation character (, line 1): code (error type: value_error)\n", + "\n", + "Solution:\u001b[32m {\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"}\u001b[0m\n", "\n", "\n", "\n", @@ -237,7 +253,16 @@ "\n", "Entry Point: ${entry_point}\n", "\n", - "Solution: ${solution}. Respond with a single JSON object using the schema {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}. For example: {\"code\": \"print('Hello, World!')\"}\n", + "Past Error (solution): An error to avoid in the future\n", + "\n", + "Past Error (solution, 2): An error to avoid in the future\n", + "\n", + "Past Error (solution, 3): An error to avoid in the future\n", + "\n", + "Solution:\n", + "${solution}. Respond with a single JSON object. \n", + "You MUST use this format: {\"code\": \"print('Hello, World!')\"}\n", + "JSON Schema: {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n", "\n", "---\n", "\n", @@ -247,9 +272,13 @@ "\n", "Entry Point: has_close_elements\n", "\n", - "Solution:\u001b[32m {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n", + "Past Error (solution): Input should be a valid string: prompt (error type: string_type)\n", + "\n", + "Past Error (solution, 2): Value error, Code is not syntactically valid: unexpected character after line continuation character (, line 1): code (error type: value_error)\n", + "\n", + "Past Error (solution, 3): Input should be a valid string: prompt (error type: string_type)\n", "\n", - "{\"code\": \"from typing import List\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"}\u001b[0m\n", + "Solution:\u001b[32m {\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\\\n for i in range(len(numbers)):\\\\n for j in range(i+1, len(numbers)):\\\\n if abs(numbers[i] - numbers[j]) < threshold:\\\\n return True\\\\n return False\"}\u001b[0m\n", "\n", "\n", "\n", @@ -271,7 +300,16 @@ "\n", "Past Error (solution): An error to avoid in the future\n", "\n", - "Solution: ${solution}. Respond with a single JSON object using the schema {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}. For example: {\"code\": \"print('Hello, World!')\"}\n", + "Past Error (solution, 2): An error to avoid in the future\n", + "\n", + "Past Error (solution, 3): An error to avoid in the future\n", + "\n", + "Past Error (solution, 4): An error to avoid in the future\n", + "\n", + "Solution:\n", + "${solution}. Respond with a single JSON object. \n", + "You MUST use this format: {\"code\": \"print('Hello, World!')\"}\n", + "JSON Schema: {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n", "\n", "---\n", "\n", @@ -281,9 +319,15 @@ "\n", "Entry Point: has_close_elements\n", "\n", - "Past Error (solution): 1 validation error for PythonCode Invalid JSON: trailing characters at line 3 column 1 [type=json_invalid, input_value='{\"properties\": {\"code\": ...ue\\\\n return False\"}', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/json_invalid\n", + "Past Error (solution): Input should be a valid string: prompt (error type: string_type)\n", "\n", - "Solution:\u001b[32m {\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"}\u001b[0m\n", + "Past Error (solution, 2): Value error, Code is not syntactically valid: unexpected character after line continuation character (, line 1): code (error type: value_error)\n", + "\n", + "Past Error (solution, 3): Input should be a valid string: prompt (error type: string_type)\n", + "\n", + "Past Error (solution, 4): Value error, Code is not syntactically valid: unexpected character after line continuation character (, line 1): code (error type: value_error)\n", + "\n", + "Solution:\u001b[32m {\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\\\n for i in range(len(numbers)):\\\\n for j in range(i+1, len(numbers)):\\\\n if abs(numbers[i] - numbers[j]) < threshold:\\\\n return True\\\\n return False\"}\u001b[0m\n", "\n", "\n", "\n" @@ -291,7 +335,96 @@ } ], "source": [ - "lm.inspect_history(n=3)" + "lm.inspect_history(n=3)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "def has_close_elements(numbers: List[float], threshold: float) -> bool:\n", + " for i in range(len(numbers)):\n", + " for j in range(i+1, len(numbers)):\n", + " if abs(numbers[i] - numbers[j]) < threshold:\n", + " return True\n", + " return False\n" + ] + } + ], + "source": [ + "d = {\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"}\n", + "print(d[\"code\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "JSONDecodeError", + "evalue": "Invalid control character at: line 1 column 82 (char 81)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mjson\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m{\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcode\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m: \u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdef has_close_elements(numbers: List[float], threshold: float) -> bool:\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m for i in range(len(numbers)):\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m for j in range(i+1, len(numbers)):\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m if abs(numbers[i] - numbers[j]) < threshold:\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m return True\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m return False\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m}\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/homebrew/Cellar/python@3.11/3.11.8/Frameworks/Python.framework/Versions/3.11/lib/python3.11/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 341\u001b[0m s \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mdecode(detect_encoding(s), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msurrogatepass\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 348\u001b[0m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n", + "File \u001b[0;32m/opt/homebrew/Cellar/python@3.11/3.11.8/Frameworks/Python.framework/Versions/3.11/lib/python3.11/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, s, _w\u001b[38;5;241m=\u001b[39mWHITESPACE\u001b[38;5;241m.\u001b[39mmatch):\n\u001b[1;32m 333\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[38;5;124;03m containing a JSON document).\u001b[39;00m\n\u001b[1;32m 335\u001b[0m \n\u001b[1;32m 336\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_w\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mend\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 338\u001b[0m end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m 339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n", + "File \u001b[0;32m/opt/homebrew/Cellar/python@3.11/3.11.8/Frameworks/Python.framework/Versions/3.11/lib/python3.11/json/decoder.py:353\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decode a JSON document from ``s`` (a ``str`` beginning with\u001b[39;00m\n\u001b[1;32m 345\u001b[0m \u001b[38;5;124;03ma JSON document) and return a 2-tuple of the Python\u001b[39;00m\n\u001b[1;32m 346\u001b[0m \u001b[38;5;124;03mrepresentation and the index in ``s`` where the document ended.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 350\u001b[0m \n\u001b[1;32m 351\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 352\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 353\u001b[0m obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscan_once\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m 355\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpecting value\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, err\u001b[38;5;241m.\u001b[39mvalue) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "\u001b[0;31mJSONDecodeError\u001b[0m: Invalid control character at: line 1 column 82 (char 81)" + ] + } + ], + "source": [ + "import json\n", + "json.loads('{\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'code': 'def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False'}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ujson\n", + "ujson.loads('{\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"} ')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'code': 'def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False'}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json.loads(ujson.dumps(ujson.loads('{\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n for i in range(len(numbers)):\\n for j in range(i+1, len(numbers)):\\n if abs(numbers[i] - numbers[j]) < threshold:\\n return True\\n return False\"} ')))" ] }, { @@ -313,7 +446,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -341,7 +474,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -410,7 +543,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -677,7 +810,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -988,7 +1121,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1207,7 +1340,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.11.8" } }, "nbformat": 4, From 8171ba13ec1fcd46db88b2bcc654e243b18ac4ca Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Sun, 3 Mar 2024 15:40:48 -0800 Subject: [PATCH 02/18] Changed max retries back to 3 --- dspy/functional/functional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index 8ca45523f4..da5e52f94c 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -14,7 +14,7 @@ from dspy.signatures.signature import ensure_signature, make_signature -MAX_RETRIES = 5 +MAX_RETRIES = 3 def predictor(func) -> dspy.Module: From 7980a2bbd3df77455c12a8d12496ce5875e44f4b Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Mon, 4 Mar 2024 09:26:56 -0800 Subject: [PATCH 03/18] First try at a new optimizer/teleprompt --- dspy/functional/functional.py | 79 +- dspy/predict/predict.py | 8 +- dspy/primitives/module.py | 18 +- dspy/primitives/program.py | 11 +- dspy/teleprompt/signature_opt2.py | 248 ++++ examples/generation.py | 28 + examples/nli/scone/ScoNe | 1 + examples/quiz/DSPy_QuizGen_Cache | 1 + examples/signature_opt2.ipynb | 1717 +++++++++++++++++++++++ examples/tweets/DSPy_TweetGen_Cache | 1 + tests/functional/test_functional.py | 68 +- tests/functional/test_signature_opt2.py | 165 +++ tests/predict/test_predict.py | 36 +- 13 files changed, 2325 insertions(+), 56 deletions(-) create mode 100644 dspy/teleprompt/signature_opt2.py create mode 100644 examples/generation.py create mode 160000 examples/nli/scone/ScoNe create mode 160000 examples/quiz/DSPy_QuizGen_Cache create mode 100644 examples/signature_opt2.ipynb create mode 160000 examples/tweets/DSPy_TweetGen_Cache create mode 100644 tests/functional/test_signature_opt2.py diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index da5e52f94c..3d6c471414 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -1,4 +1,3 @@ -from collections import defaultdict import inspect import os import openai @@ -14,7 +13,8 @@ from dspy.signatures.signature import ensure_signature, make_signature -MAX_RETRIES = 3 +# Some improvement ideas: +# - Increase the temperature on error def predictor(func) -> dspy.Module: @@ -56,7 +56,7 @@ def __init__(self): self.__dict__[name] = attr.copy() -def TypedChainOfThought(signature) -> dspy.Module: # noqa: N802 +def TypedChainOfThought(signature, max_retries=3) -> dspy.Module: # noqa: N802 """Just like TypedPredictor, but adds a ChainOfThought OutputField.""" signature = ensure_signature(signature) output_keys = ", ".join(signature.output_fields.keys()) @@ -68,17 +68,19 @@ def TypedChainOfThought(signature) -> dspy.Module: # noqa: N802 desc="${produce the " + output_keys + "}. We ...", ), ), + max_retries=max_retries, ) class TypedPredictor(dspy.Module): - def __init__(self, signature): + def __init__(self, signature, max_retries=3): super().__init__() self.signature = ensure_signature(signature) self.predictor = dspy.Predict(signature) + self.max_retries = max_retries def copy(self) -> "TypedPredictor": - return TypedPredictor(self.signature) + return TypedPredictor(self.signature, self.max_retries) @staticmethod def _make_example(type_) -> str: @@ -140,6 +142,15 @@ def _prepare_signature(self) -> dspy.Signature: format_ = passages2text elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel): format_ = lambda x: x if isinstance(x, str) else x.model_dump_json() + # Special formatting for lists of known types. Maybe the output fields sohuld have this too? + elif typing.get_origin(type_) in (List, list, Tuple, tuple): + (inner_type,) = typing.get_args(type_) + if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel): + format_ = ( + lambda x: x if isinstance(x, str) else "[" + ",".join(i.model_dump_json() for i in x) + "]" + ) + else: + format_ = lambda x: x if isinstance(x, str) else json.dumps(x) signature = signature.with_updated_fields(name, format=format_) return signature @@ -149,40 +160,46 @@ def forward(self, **kwargs) -> dspy.Prediction: # We have to re-prepare the signature on every forward call, because the base # signature might have been modified by an optimizer or something like that. signature = self._prepare_signature() - for try_i in range(MAX_RETRIES): + for try_i in range(self.max_retries): result = self.predictor(**modified_kwargs, new_signature=signature) errors = {} parsed_results = [] # Parse the outputs - for i, completion in enumerate(result.completions): - try: - parsed = {} - for name, field in signature.output_fields.items(): + for completion in result.completions: + parsed = {} + for name, field in signature.output_fields.items(): + try: value = completion[name] parser = field.json_schema_extra.get("parser", lambda x: x) parsed[name] = parser(value) - # Instantiate the actual signature with the parsed values. - # This allow pydantic to validate the fields defined in the signature. + except (pydantic.ValidationError, ValueError) as e: + errors[name] = _format_error(e) + # If we can, we add an example to the error message + current_desc = field.json_schema_extra.get("desc", "") + i = current_desc.find("JSON Schema: ") + if i == -1: + continue # Only add examples to JSON objects + suffix, current_desc = current_desc[i:], current_desc[:i] + prefix = "You MUST use this format: " + if ( + try_i + 1 < self.max_retries + and prefix not in current_desc + and (example := self._make_example(field.annotation)) + ): + signature = signature.with_updated_fields( + name, + desc=current_desc + "\n" + prefix + example + "\n" + suffix, + ) + # No reason trying to parse the general signature, or run more completions, if we already have errors + if errors: + break + # Instantiate the actual signature with the parsed values. + # This allow pydantic to validate the fields defined in the signature. + try: _dummy = self.signature(**kwargs, **parsed) parsed_results.append(parsed) - except (pydantic.ValidationError, ValueError) as e: - errors[name] = _format_error(e) - # If we can, we add an example to the error message - current_desc = field.json_schema_extra.get("desc", "") - i = current_desc.find("JSON Schema: ") - if i == -1: - continue # Only add examples to JSON objects - suffix, current_desc = current_desc[i:], current_desc[:i] - prefix = "You MUST use this format: " - if ( - try_i + 1 < MAX_RETRIES - and prefix not in current_desc - and (example := self._make_example(field.annotation)) - ): - signature = signature.with_updated_fields( - name, - desc=current_desc + "\n" + prefix + example + "\n" + suffix, - ) + except pydantic.ValidationError as e: + errors["general"] = _format_error(e) if errors: # Add new fields for each error for name, error in errors.items(): @@ -209,7 +226,7 @@ def _format_error(error: Exception): if isinstance(error, pydantic.ValidationError): errors = [] for e in error.errors(): - fields = ", ".join(e["loc"]) + fields = ", ".join(map(str, e["loc"])) errors.append(f"{e['msg']}: {fields} (error type: {e['type']})") return "; ".join(errors) else: diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py index 60e668a836..d0a5f6010e 100644 --- a/dspy/predict/predict.py +++ b/dspy/predict/predict.py @@ -6,6 +6,7 @@ from dspy.signatures.signature import ensure_signature, signature_to_template + class Predict(Parameter): def __init__(self, signature, **config): self.stage = random.randbytes(8).hex() @@ -27,7 +28,7 @@ def dump_state(self): state["signature_instructions"] = self.signature.instructions *_, last_key = self.signature.fields.keys() - state["signature_prefix"] = self.signature.fields[last_key].json_schema_extra['prefix'] + state["signature_prefix"] = self.signature.fields[last_key].json_schema_extra["prefix"] return state @@ -85,6 +86,8 @@ def forward(self, **kwargs): # Switch to legacy format for dsp.generate template = signature_to_template(signature) + # print("Created template", template) + # print("From Signature", signature) if self.lm is None: x, C = dsp.generate(template, **config)(x, stage=self.stage) @@ -103,7 +106,8 @@ def forward(self, **kwargs): for field in template.fields: if field.output_variable not in kwargs.keys(): completions[-1][field.output_variable] = getattr( - c, field.output_variable, + c, + field.output_variable, ) pred = Prediction.from_completions(completions, signature=signature) diff --git a/dspy/primitives/module.py b/dspy/primitives/module.py index 1b4a342f71..59ec8a09f8 100644 --- a/dspy/primitives/module.py +++ b/dspy/primitives/module.py @@ -8,7 +8,7 @@ def __init__(self): def named_parameters(self): """ - Unlike PyTorch, handles (non-recursive) lists of parameters too. + Unlike PyTorch, handles (non-recursive) lists of parameters too. """ from dspy.predict.parameter import Parameter @@ -27,10 +27,10 @@ def add_parameter(param_name, param_value): elif isinstance(value, BaseModule): # When a sub-module is pre-compiled, keep it frozen. - if not getattr(value, '_compiled', False): + if not getattr(value, "_compiled", False): for sub_name, param in value.named_parameters(): add_parameter(f"{name}.{sub_name}", param) - + elif isinstance(value, (list, tuple)): for idx, item in enumerate(value): add_parameter(f"{name}[{idx}]", item) @@ -49,23 +49,23 @@ def deepcopy(self): def reset_copy(self): obj = copy.deepcopy(self) - + for param in obj.parameters(): param.reset() - + return obj - + def dump_state(self): return {name: param.dump_state() for name, param in self.named_parameters()} - + def load_state(self, state): for name, param in self.named_parameters(): param.load_state(state[name]) - + def save(self, path): with open(path, "w") as f: f.write(ujson.dumps(self.dump_state(), indent=2)) - + def load(self, path): with open(path) as f: self.load_state(ujson.loads(f.read())) diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py index 5e063bfb00..512e7ce67a 100644 --- a/dspy/primitives/program.py +++ b/dspy/primitives/program.py @@ -1,4 +1,3 @@ - from dspy.primitives.module import BaseModule from dspy.primitives.assertions import * import re @@ -16,7 +15,6 @@ class ProgramMeta(type): class Module(BaseModule, metaclass=ProgramMeta): - def _base_init(self): self._compiled = False @@ -29,12 +27,7 @@ def __call__(self, *args, **kwargs): def named_predictors(self): from dspy.predict.predict import Predict - named_parameters = self.named_parameters() - return [ - (name, param) - for name, param in named_parameters - if isinstance(param, Predict) - ] + return [(name, param) for name, param in self.named_parameters() if isinstance(param, Predict)] def predictors(self): return [param for _, param in self.named_predictors()] @@ -52,7 +45,7 @@ def map_named_predictors(self, func): for name, predictor in self.named_predictors(): set_attribute_by_name(self, name, func(predictor)) return self - + def activate_assertions(self, handler=backtrack_handler, **handler_args): """ Activates assertions for the module. diff --git a/dspy/teleprompt/signature_opt2.py b/dspy/teleprompt/signature_opt2.py new file mode 100644 index 0000000000..1a9045ae23 --- /dev/null +++ b/dspy/teleprompt/signature_opt2.py @@ -0,0 +1,248 @@ +import random +from typing import Generic, Literal, TypeVar, Type + +import pydantic +import dspy +from dspy.functional.functional import TypedChainOfThought +from dspy.signatures import Signature +from dspy import BaseModel + +# TODO: Consider using the prompt optimizer to optimize the prompt optimizer :O + + +def make_info(signature: type[Signature]) -> BaseModel: + """Creates a SignatureInfo pydantic type, that describes the Signature. + + Returns an instnce of this type, with the instructions and field descriptions of the input type. + """ + # First, create the SignatureInfo type + fields = { + "instructions": (str, pydantic.Field(description="The instructions for the task")), + } + for name in signature.fields: + fields[name + "_prefix"] = (str, pydantic.Field(description=f"The prefix for {name}")) + fields[name + "_desc"] = (str, pydantic.Field(description=f"The description for {name}")) + SignatureInfo = pydantic.create_model( # noqa: N806 + f"SignatureInfo[{signature.__name__}]", + **fields, + ) + + # Add a method to convert the SignatureInfo back into a Signature + def to_signature(info): + new_signature = signature.with_instructions(info.instructions) + for name in signature.fields: + new_signature = new_signature.with_updated_fields( + name, + prefix=getattr(info, name + "_prefix"), + desc=getattr(info, name + "_desc"), + ) + return new_signature + + SignatureInfo.to_signature = to_signature + + # Finally, make an instance of the SignatureInfo type with the signature's + # default instructions and field descriptions + values = {"instructions": signature.instructions} + for name, field in signature.fields.items(): + values[name + "_prefix"] = field.json_schema_extra["prefix"] + values[name + "_desc"] = field.json_schema_extra["desc"] + return SignatureInfo(**values) + + +T = TypeVar("T", bound=BaseModel) + + +# Note: This function wouldn't be necessary if we could make the number of prompts a generic parameter of the class, +# but alas it seems like this isn't possible in Python right now. The main reason being that types and generics only +# live inside the type system, and can't be used to generate code at runtime. +def make_initial_signature(n_prompts: int) -> Type[Signature]: + """Creates a GenerateInstructionInitial signature with the given number of initial prompts.""" + + class GenerateInstructionInitial(Signature, Generic[T]): + """You are a creative instruction optimizer for large language models. + + I will give you a ``signature`` of fields (inputs and outputs) in English. + Your task is to propose variations of the signature that will lead a good language model. + + Be very creative and think out of the box. Consider using inspiration such as: + Openers: + # You are as smart as ChatGPT. + # You are highly intelligent. + # You are an expert mathematician. + # You are a professor of mathematics. + Task Descriptions: + # Solve the following math question. + # Answer the following math question. + Closers: + # This will be fun! + # Take a deep breath and think carefully. + # I really need your help! + """ + + basic_signature: T = dspy.InputField() + proposed_signatures: list[T] = dspy.OutputField( + desc=f"A list of {n_prompts} very different variations of the basic signature", + min_items=n_prompts, + max_items=n_prompts, + ) + + return GenerateInstructionInitial + + +class ScoredSignature(BaseModel, Generic[T]): + signature: T + score: float = dspy.Field(gt=0, lt=100) + + +class GenerateInstructionGivenAttempts(dspy.Signature, Generic[T]): + """You are an instruction optimizer for large language models. + + I will give some task instructions I've tried, along with their corresponding validation scores. + - The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality. + - Your task is to propose a new instruction that will lead a good language model to perform the task even better. + - Be creative, and think out of the box. + - Don't repeat instructions, descriptions and prefixes that have already been attempted. + """ + + attempted_signatures: list[ScoredSignature[T]] = dspy.InputField() + proposed_signature: T = dspy.OutputField(desc="Next signature to try") + # expected_score: float = dspy.OutputField(desc="The expected score for the new signature") + + +def optimize_signature( + student, + evaluator, + n_iterations=10, + strategy: Literal["best", "last"] = "best", + # Formerly part of the constructor + prompt_model=None, + initial_prompts=2, + temperature=1.4, + verbose=False, +) -> dspy.Program: + """Create a new program that is optimized for the given task. + + `student` is a program that needs to be optimized, + note that it may be zero-shot or already pre-optimized for demos != []. + + Parameters + ---------- + student : dspy.Program + The program to optimize. + evaluator : dspy.Evaluator + The evaluator to use to score the program. + n_iterations : int, optional + The number of iterations to run, by default 10 + strategy : Literal["best", "last"], optional + The strategy to use to select the final program, by default "best" + prompt_model : dspy.LanguageModel, optional + The language model to use to generate prompts, by default None + initial_prompts : int, optional + The number of initial prompts to generate, by default 2. + Note that we also use the "plain" signature as a prompt, so the total number of prompts is initial_prompts + 1. + temperature : float, optional + The temperature to use when generating new prompts, by default 1.4 + verbose : bool, optional + Whether to print debug information, by default False + """ + prompt_model = prompt_model or dspy.settings.lm + MyGenerateInstructionInitial = make_initial_signature(initial_prompts) # noqa: N806 + + module = student.deepcopy() + # For some reason named_predictors sometimes returns an empty list, so we use named_parameters instead + named_predictors = module.named_parameters() + if verbose: + print("All predictors:") + print(f"{named_predictors=}") + + candidates = {} + scores = [] + + # First round, just use initial prompts + for name, p in named_predictors: + candidates[name] = [make_info(p.signature)] + + # Make some initial candidates + with dspy.settings.context(lm=prompt_model): + # TODO: Parallelize this + for name, p in named_predictors: + if verbose: + print(f"Generating new signature for {p}...") + info = candidates[name][0] # Use initial info, to make sure types are identical + generator = TypedChainOfThought(MyGenerateInstructionInitial[type(info)]) + candidates[name] += generator( + basic_signature=info, + config={"temperature": temperature}, + ).proposed_signatures + assert len(candidates[name]) == initial_prompts + 1 # Basic signature + initial prompts + + candidates[name] = [ + info.model_copy(update={"instructions": info.instructions + f"({i})"}) + for i, info in enumerate(candidates[name]) + ] + + for i, c in enumerate(candidates[name]): + print(f"Generated candidate {i}:") + print(c.to_signature()) + + # Main loop of scoring + generating new candidates + for i in range(n_iterations): + if verbose: + print("\n" + "=" * 80) + print(f"Running eval iteration {i}...") + + # Test candidate i + for p in module.predictors(): + print(f"Installing signature {i}: ") + print(candidates[name][i].to_signature()) + p.signature = candidates[name][i].to_signature() + + score = evaluator(module) + score += random.random() * 10 + scores.append(score) + + if verbose: + print(f"Scores for iteration {i}: {score}") + + # If we are still testing initial prompts, continue + if i + 1 < len(next(iter(candidates.values()))): + continue + + # If we are done, there's no need to generate new candidates + if i + 1 == n_iterations: + break + + # Otherwise generate the next candidate + with dspy.settings.context(lm=prompt_model): + # TODO: Parallelize this + for name, p in named_predictors: + SignatureInfo = type(candidates[name][0]) # noqa: N806 + generator = TypedChainOfThought(GenerateInstructionGivenAttempts[SignatureInfo]) + attempted_signatures = [ + ScoredSignature[SignatureInfo](signature=info, score=sc) + for info, sc in zip(candidates[name], scores) + ] + attempted_signatures.sort(key=lambda x: x.score) + if verbose: + print( + f"Generating new signature for {name} based on {len(attempted_signatures)} previous signatures..." + ) + new_signature = generator( + attempted_signatures=attempted_signatures, + config={"temperature": temperature}, + ).proposed_signature + if verbose: + print("Generated candidate:") + print(new_signature.to_signature()) + candidates[name].append(new_signature) + + if strategy == "last": + return module + + if strategy == "best": + i = scores.index(max(scores)) + for name, p in named_predictors: + p.signature = candidates[name][i].to_signature() + return module + + raise ValueError(f"Invalid strategy: {strategy}") diff --git a/examples/generation.py b/examples/generation.py new file mode 100644 index 0000000000..36d3d1c921 --- /dev/null +++ b/examples/generation.py @@ -0,0 +1,28 @@ +from pydantic import BaseModel, Field +from dspy.teleprompt import LabeledFewShot +from dspy.functional import TypedPredictor + +import dspy +turbo = dspy.OpenAI(model='gpt-3.5-turbo') +colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts') +dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts) + +class SyntheticFact(BaseModel): + fact: str = Field(..., description="a statement") + varacity: bool = Field(..., description="is the statement true or false") + +class ExampleSignature(dspy.Signature): + """Generate an example of a synthetic fact.""" + fact: SyntheticFact = dspy.OutputField() + +generator = TypedPredictor(ExampleSignature) +examples = generator(config=dict(n=10)) + +# If you have examples and want more +existing_examples = [ + dspy.Example(fact="The sky is blue", varacity=True), + dspy.Example(fact="The sky is green", varacity=False), +] +trained = LabeledFewShot().compile(student=generator, trainset=existing_examples) + +augmented_examples = trained(config=dict(n=10)) diff --git a/examples/nli/scone/ScoNe b/examples/nli/scone/ScoNe new file mode 160000 index 0000000000..b02532a2f4 --- /dev/null +++ b/examples/nli/scone/ScoNe @@ -0,0 +1 @@ +Subproject commit b02532a2f4185c6118a57a148455e0750592d8c8 diff --git a/examples/quiz/DSPy_QuizGen_Cache b/examples/quiz/DSPy_QuizGen_Cache new file mode 160000 index 0000000000..27d6d433e7 --- /dev/null +++ b/examples/quiz/DSPy_QuizGen_Cache @@ -0,0 +1 @@ +Subproject commit 27d6d433e73b91d3cf677ecf1d757813fcbd611d diff --git a/examples/signature_opt2.ipynb b/examples/signature_opt2.ipynb new file mode 100644 index 0000000000..9edc0d4c4e --- /dev/null +++ b/examples/signature_opt2.ipynb @@ -0,0 +1,1717 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "import dspy\n", + "import os\n", + "os.environ['OPENAI_API_KEY'] = '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'" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=4000)\n", + "gpt4 = dspy.OpenAI(model='gpt-4', max_tokens=4000)\n", + "dspy.settings.configure(lm=turbo)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(20, 50)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dspy.datasets import HotPotQA\n", + "\n", + "# Load the dataset.\n", + "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n", + "\n", + "# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.\n", + "trainset = [x.with_inputs('question') for x in dataset.train]\n", + "devset = [x.with_inputs('question') for x in dataset.dev]\n", + "\n", + "len(trainset), len(devset)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "class BasicQA(dspy.Signature):\n", + " \"\"\"Answer questions with short factoid answers.\"\"\"\n", + "\n", + " question = dspy.InputField()\n", + " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "All predictors:\n", + "named_predictors=[('predictor', Predict(BasicQA(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", + ")))]\n", + "Generating new signature for Predict(BasicQA(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", + "))...\n", + "Generated candidate 0:\n", + "StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.(0)'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", + ")\n", + "Generated candidate 1:\n", + "StringSignature(question -> answer\n", + " instructions='Answer series of questions where answers must share a common theme.(1)'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q1,', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'charge amplifier designed based by Moog on stripline technology', '__dspy_field_type': 'output', 'prefix': 'A1 Wolfgang Amplifier,'})\n", + ")\n", + "Generated candidate 2:\n", + "StringSignature(question -> answer\n", + " instructions='Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q-What Computability study ranges?', 'desc': 'may also involve strings produced by non-deterministic computation'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!', '__dspy_field_type': 'output', 'prefix': 'A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder'})\n", + ")\n", + "\n", + "================================================================================\n", + "Running eval iteration 0...\n", + "Installing signature 0: \n", + "StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.(0)'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 5144.18it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 5 / 20 (25.0%)\n", + "Scores for iteration 0: 28.55243378134378\n", + "\n", + "================================================================================\n", + "Running eval iteration 1...\n", + "Installing signature 1: \n", + "StringSignature(question -> answer\n", + " instructions='Answer series of questions where answers must share a common theme.(1)'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q1,', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'charge amplifier designed based by Moog on stripline technology', '__dspy_field_type': 'output', 'prefix': 'A1 Wolfgang Amplifier,'})\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 4207.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 5 / 20 (25.0%)\n", + "Scores for iteration 1: 29.911449495835697\n", + "\n", + "================================================================================\n", + "Running eval iteration 2...\n", + "Installing signature 2: \n", + "StringSignature(question -> answer\n", + " instructions='Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q-What Computability study ranges?', 'desc': 'may also involve strings produced by non-deterministic computation'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!', '__dspy_field_type': 'output', 'prefix': 'A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder'})\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 4577.93it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 5 / 20 (25.0%)\n", + "Scores for iteration 2: 34.58455944537296\n", + "Generating new signature for predictor based on 3 previous signatures...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "from dspy.evaluate import Evaluate\n", + "from dspy.evaluate.metrics import answer_exact_match\n", + "from dspy.functional import TypedPredictor\n", + "from dspy.teleprompt.signature_opt2 import optimize_signature\n", + "\n", + "evaluator = Evaluate(devset=devset, metric=answer_exact_match, num_threads=10, display_progress=True)\n", + "\n", + "program = optimize_signature(\n", + " student=TypedPredictor(BasicQA),\n", + " evaluator=Evaluate(devset=trainset, metric=answer_exact_match, num_threads=10, display_progress=True),\n", + " initial_prompts=2,\n", + " n_iterations=8,\n", + " verbose=True,\n", + " prompt_model=gpt4,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "\n", + "Given the fields `attempted_signatures`, produce the fields `proposed_signature`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Attempted Signatures: ${attempted_signatures}\n", + "Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ...\n", + "Proposed Signature: The improved signature for the language model. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "\n", + "---\n", + "\n", + "Attempted Signatures: [{\"signature\":{\"instructions\":\"Answer questions with short factoid answers.(0)\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Answer series of questions where answers must share a common theme.(1)\",\"question_prefix\":\"Q1,\",\"question_desc\":\"${question}\",\"answer_prefix\":\"A1 Wolfgang Amplifier,\",\"answer_desc\":\"charge amplifier designed based by Moog on stripline technology\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)\",\"question_prefix\":\"Q-What Computability study ranges?\",\"question_desc\":\"may also involve strings produced by non-deterministic computation\",\"answer_prefix\":\"A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder\",\"answer_desc\":\"answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!\"},\"score\":25.0}]\n", + "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signature. We can see that the attempted signatures are quite varied and complex, with different instructions, prefixes, and descriptions for both questions and answers. However, they all share a common structure: they all have instructions, a question prefix, a question description, an answer prefix, and an answer description. Therefore, we can propose a signature that includes these common elements, but with more general descriptions to accommodate the variety of tasks. \n", + "Proposed Signature: The improved signature for the language model. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\u001b[0m\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Make a very succinct json object that validates with the following schema\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Json Schema: ${json_schema}\n", + "Json Object: ${json_object}\n", + "\n", + "---\n", + "\n", + "Json Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "Json Object:\u001b[32m {\"instructions\": \"Complete the task\", \"question_prefix\": \"Q:\", \"question_desc\": \"What is the capital of France?\", \"answer_prefix\": \"A:\", \"answer_desc\": \"Paris\"}\u001b[0m\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Given the fields `attempted_signatures`, produce the fields `proposed_signature`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Attempted Signatures: ${attempted_signatures}\n", + "\n", + "Past Error (proposed_signature): An error to avoid in the future\n", + "\n", + "Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ...\n", + "\n", + "Proposed Signature:\n", + "The improved signature for the language model. Respond with a single JSON object. \n", + "You MUST use this format: {\"instructions\": \"Complete the task\", \"question_prefix\": \"Q:\", \"question_desc\": \"What is the capital of France?\", \"answer_prefix\": \"A:\", \"answer_desc\": \"Paris\"}\n", + "JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "\n", + "---\n", + "\n", + "Attempted Signatures: [{\"signature\":{\"instructions\":\"Answer questions with short factoid answers.(0)\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Answer series of questions where answers must share a common theme.(1)\",\"question_prefix\":\"Q1,\",\"question_desc\":\"${question}\",\"answer_prefix\":\"A1 Wolfgang Amplifier,\",\"answer_desc\":\"charge amplifier designed based by Moog on stripline technology\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)\",\"question_prefix\":\"Q-What Computability study ranges?\",\"question_desc\":\"may also involve strings produced by non-deterministic computation\",\"answer_prefix\":\"A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder\",\"answer_desc\":\"answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!\"},\"score\":25.0}]\n", + "\n", + "Past Error (proposed_signature): ValueError('json output should start and end with { and }')\n", + "\n", + "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signature. We need to ensure that the JSON object is properly formatted and contains all the required fields. The attempted signatures provided are not in the correct format. They contain additional fields such as \"score\" and the structure of the JSON object is not as expected. The JSON object should start and end with { and }, and should not contain any additional fields or arrays. \n", + "\n", + "Proposed Signature:\n", + "{\"instructions\": \"Answer questions with short factoid answers.\", \"question_prefix\": \"Question:\", \"question_desc\": \"${question}\", \"answer_prefix\": \"Answer:\", \"answer_desc\": \"often between 1 and 5 words\"}\u001b[0m\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "gpt4.inspect_history(n=3)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'tp' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[19], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtp\u001b[49m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m\n", + "\u001b[0;31mNameError\u001b[0m: name 'tp' is not defined" + ] + } + ], + "source": [ + "tp.__dict__" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "module.predictors()=[]\n", + "\n", + "================================================================================\n", + "Running eval iteration 0...\n", + "list: []\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8900>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fe20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5fa60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ef20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632f240>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6020>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848af20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e3060>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13ab60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e8e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ee80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001f80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4e00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ff60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x10632efc0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5b20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488f40>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0720>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafec0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488540>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e160>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e700>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe34c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0d60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e980>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e160>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5d00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafd80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2fc0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafd80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f560>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe09a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3100>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5fba0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e160>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a4b80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe37e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2f20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0b80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5ee0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f1a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba87c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1760>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe00e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafec0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0360>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a0020c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f6a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fb00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f600>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a000720>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632efc0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fb00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0cc0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584887c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f9c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2660>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0c20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f1a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4e00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158489d00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1e40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2ac0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0720>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001f80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ede0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ee80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2ca0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe28e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e020>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b4900>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a4b80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2160>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0220>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0e00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a4b80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001f80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e8e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3b00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f880>, 'parser': })\n", + ")\n", + "StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0900>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0400>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b4900>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13afc0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe34c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe23e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b47c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ec00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fd80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3560>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe07c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e700>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158536840>, 'parser': })\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 5483.04it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Scores for iteration 0: 32.0\n", + "\n", + "================================================================================\n", + "Running eval iteration 1...\n", + "list: []\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585367a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1391c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e3060>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2f20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1d00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5b20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488540>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002020>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f240>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe09a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f060>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bce0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1391c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe02c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848af20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c139da0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ec00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5fd80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158536840>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6020>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x10632efc0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bce0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f880>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f420>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489f80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a000720>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c139da0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158536840>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6660>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158489d00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e840>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5d9e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1260>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f9c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba8900>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5ee0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2fc0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b9c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632efc0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848be20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1b20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e160>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba89a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848a520>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a7240>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe23e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0ea0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158536840>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e980>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe07c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f880>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489d00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5a80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0e00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1f80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5ee0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f9c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2160>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0fe0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3f60>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ef20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848a520>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585367a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0360>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2160>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5ee0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafd80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba8900>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1d00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe22a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16ae719e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b68e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe11c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2660>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0720>, 'parser': })\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created templateCreated template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632d120>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb6ac0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138720>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafc40>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632ede0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138c20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c139580>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489d00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002660>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848be20>, 'parser': })\n", + ")\n", + " Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3f60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0720>, 'parser': })\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4371.34it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Scores for iteration 1: 32.0\n", + "\n", + "================================================================================\n", + "Running eval iteration 2...\n", + "list: []\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13afc0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5a80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5d00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafe20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b47c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb5620>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba82c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a4b80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138b80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b240>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bc40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b600>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5080>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0360>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafa60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b68e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafec0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0b80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13afc0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138040>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002160>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488ea0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13af20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5d00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba89a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb4220>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2160>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb47c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6020>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584887c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0180>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584885e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b9c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6e80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafe20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2200>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3380>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8400>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c1389a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3d80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0540>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138c20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6e80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafc40>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1ee0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2340>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2ca0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba89a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5d00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0680>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1a80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2c00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba89a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b7ba0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489260>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bcb47c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138b80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b42c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001da0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5a80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c139580>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe02c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13b240>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002660>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848a3e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0360>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848b9c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafd80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5940>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b600>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3ba0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6e80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafa60>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bcb7920>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe13a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2340>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b47c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002660>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a7240>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138040>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2d40>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe32e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13bba0>, 'parser': })\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba82c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4680>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2d40>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b6b60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002020>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3880>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe22a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1e40>, 'parser': })\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 5230.07it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Scores for iteration 2: 32.0\n", + "\n", + "================================================================================\n", + "Running eval iteration 3...\n", + "list: []\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13aca0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6e80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafe20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b62a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584885e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba89a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a001da0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138180>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3d80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13aca0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002520>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafe20>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158489f80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1120>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b62a0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002020>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1a80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe19e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8900>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b49a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0540>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138c20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0360>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bce0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b62a0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a4ea0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138040>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138180>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002160>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b47c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488ea0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1080>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb6c00>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0360>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489f80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b6b60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6e80>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0720>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138180>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafb00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb68e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3d80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0360>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bc40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4680>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002160>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1da0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1d00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138180>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba87c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488540>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1620>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbaff60>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138ae0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2340>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0a40>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba87c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848bc40>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848af20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2520>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1da0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe23e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584885e0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbaff60>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138ae0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe05e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0fe0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba80e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0540>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3560>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe11c0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe34c0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1b20>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b6b60>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8900>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3ce0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0e00>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0220>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2840>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b68e0>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0cc0>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138040>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0680>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b240>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb5620>, 'parser': })\n", + ")\n", + "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", + "From Signature StringSignature(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bc40>})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafa60>, 'parser': })\n", + ")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4131.18it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Scores for iteration 3: 32.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optimizer.compile(TypedPredictor(BasicQA), evaluator, n_iterations=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gpt4.inspect_history(n=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Feel free to any other queries you like." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py39", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tweets/DSPy_TweetGen_Cache b/examples/tweets/DSPy_TweetGen_Cache new file mode 160000 index 0000000000..22186fd4d4 --- /dev/null +++ b/examples/tweets/DSPy_TweetGen_Cache @@ -0,0 +1 @@ +Subproject commit 22186fd4d4fa940256ca8c4ab70f165276e5c834 diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py index cabbc7cd09..600e35b63d 100644 --- a/tests/functional/test_functional.py +++ b/tests/functional/test_functional.py @@ -8,7 +8,7 @@ import pytest import dspy -from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor, functional +from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor, TypedChainOfThought from dspy.primitives.example import Example from dspy.teleprompt.bootstrap import BootstrapFewShot from dspy.teleprompt.vanilla import LabeledFewShot @@ -122,7 +122,7 @@ def forward(self, **kwargs): qa = QA() assert isinstance(qa, FunctionalModule) - assert isinstance(qa.answer, functional._StripOutput) + assert isinstance(qa.answer, dspy.Module) question, answer = qa(topic="Physics") @@ -407,6 +407,7 @@ def get_user_details() -> UserDetails: with pytest.raises(ValueError): get_user_details() + print(lm.get_convo(-1)) assert lm.get_convo(-1) == textwrap.dedent( """\ Given the fields , produce the fields `get_user_details`. @@ -467,6 +468,23 @@ class TestSignature(dspy.Signature): assert output == [0, 1, 2] +def test_multiple_outputs_int_cot(): + # Note: Multiple outputs only work when the language model "speculatively" generates all the outputs in one go. + lm = DummyLM( + [ + "thoughts 0\nOutput: 0\n", + "thoughts 1\nOutput: 1\n", + "thoughts 2\nOutput: 2\n", + ] + ) + dspy.settings.configure(lm=lm) + + test = TypedChainOfThought("input:str -> output:int") + + output = test(input="8", config=dict(n=3)).completions.output + assert output == [0, 1, 2] + + def test_parse_type_string(): lm = DummyLM([str(i) for i in range(100)]) dspy.settings.configure(lm=lm) @@ -532,3 +550,49 @@ class ExampleSignature(dspy.Signature): augmented_examples = trained(config=dict(n=3)) for ex in augmented_examples.completions.fact: assert isinstance(ex, SyntheticFact) + + +def test_list_input2(): + # Inspired by the Signature Optimizer + + class ScoredString(pydantic.BaseModel): + string: str + score: float + + class ScoredSignature(dspy.Signature): + attempted_signatures: list[ScoredString] = dspy.InputField() + proposed_signature: str = dspy.OutputField() + + program = TypedChainOfThought(ScoredSignature) + + lm = DummyLM(["Thoughts", "Output"]) + dspy.settings.configure(lm=lm) + + output = program( + attempted_signatures=[ + ScoredString(string="string 1", score=0.5), + ScoredString(string="string 2", score=0.4), + ScoredString(string="string 3", score=0.3), + ] + ).proposed_signature + + print(lm.get_convo(-1)) + + assert output == "Output" + + assert lm.get_convo(-1) == textwrap.dedent("""\ + Given the fields `attempted_signatures`, produce the fields `proposed_signature`. + + --- + + Follow the following format. + + Attempted Signatures: ${attempted_signatures} + Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ... + Proposed Signature: ${proposed_signature} + + --- + + Attempted Signatures: [{"string":"string 1","score":0.5},{"string":"string 2","score":0.4},{"string":"string 3","score":0.3}] + Reasoning: Let's think step by step in order to Thoughts + Proposed Signature: Output""") diff --git a/tests/functional/test_signature_opt2.py b/tests/functional/test_signature_opt2.py new file mode 100644 index 0000000000..469bf3f252 --- /dev/null +++ b/tests/functional/test_signature_opt2.py @@ -0,0 +1,165 @@ +import json +import dspy +from dspy.evaluate import Evaluate +from dspy.functional import TypedPredictor +from dspy.teleprompt.signature_opt2 import ( + GenerateInstructionGivenAttempts, + ScoredSignature, + make_info, + optimize_signature, +) +from dspy.utils import DummyLM + +from dspy.evaluate import Evaluate +from dspy.evaluate.metrics import answer_exact_match +from dspy.functional import TypedPredictor + + +class BasicQA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + +hotpotqa = [ + ex.with_inputs("question") + for ex in [ + dspy.Example( + question="At My Window was released by which American singer-songwriter?", + answer="John Townes Van Zandt", + ), + dspy.Example( + question="which American actor was Candace Kita guest starred with ", + answer="Bill Murray", + ), + dspy.Example( + question="Which of these publications was most recently published, Who Put the Bomp or Self?", + answer="Self", + ), + dspy.Example( + question="The Victorians - Their Story In Pictures is a documentary series written by an author born in what year?", + answer="1950", + ), + dspy.Example( + question="Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?", + answer="Tae Kwon Do Times", + ), + dspy.Example( + question="In what year was the club founded that played Manchester City in the 1972 FA Charity Shield", + answer="1874", + ), + dspy.Example( + question="Which is taller, the Empire State Building or the Bank of America Tower?", + answer="The Empire State Building", + ), + dspy.Example( + question='Which American actress who made their film debut in the 1995 teen drama "Kids" was the co-founder of Voto Latino?', + answer="Rosario Dawson", + ), + dspy.Example( + question="Tombstone stared an actor born May 17, 1955 known as who?", + answer="Bill Paxton", + ), + dspy.Example( + question="What is the code name for the German offensive that started this Second World War engagement on the Eastern Front (a few hundred kilometers from Moscow) between Soviet and German forces, which included 102nd Infantry Division?", + answer="Operation Citadel", + ), + dspy.Example( + question='Who acted in the shot film The Shore and is also the youngest actress ever to play Ophelia in a Royal Shakespeare Company production of "Hamlet." ?', + answer="Kerry Condon", + ), + dspy.Example( + question="Which company distributed this 1977 American animated film produced by Walt Disney Productions for which Sherman Brothers wrote songs?", + answer="Buena Vista Distribution", + ), + dspy.Example( + question="Samantha Cristoforetti and Mark Shuttleworth are both best known for being first in their field to go where? ", + answer="space", + ), + dspy.Example( + question="Having the combination of excellent foot speed and bat speed helped Eric Davis, create what kind of outfield for the Los Angeles Dodgers? ", + answer="Outfield of Dreams", + ), + dspy.Example( + question="Which Pakistani cricket umpire who won 3 consecutive ICC umpire of the year awards in 2009, 2010, and 2011 will be in the ICC World Twenty20?", + answer="Aleem Sarwar Dar", + ), + dspy.Example( + question="The Organisation that allows a community to influence their operation or use and to enjoy the benefits arisingwas founded in what year?", + answer="2010", + ), + dspy.Example( + question='"Everything Has Changed" is a song from an album released under which record label ?', + answer="Big Machine Records", + ), + dspy.Example( + question="Who is older, Aleksandr Danilovich Aleksandrov or Anatoly Fomenko?", + answer="Aleksandr Danilovich Aleksandrov", + ), + dspy.Example( + question="On the coast of what ocean is the birthplace of Diogal Sakho?", + answer="Atlantic", + ), + dspy.Example( + question="This American guitarist best known for her work with the Iron Maidens is an ancestor of a composer who was known as what?", + answer="The Waltz King", + ), + ] +] + + +def test_signature_info(): + info = make_info(BasicQA) + SignatureInfo = type(info) + + devset = [ + dspy.Example( + instructions="Answer the following questions", + question_desc="Some question to answer", + question_prefix="Q: ", + answer_desc="A short answer to the question", + answer_prefix="A: ", + ), + ] + + lm = DummyLM( + [ + json.dumps(dict(devset[0])), # Proposed signature + ] + ) + dspy.settings.configure(lm=lm) + + generator = TypedPredictor(GenerateInstructionGivenAttempts[SignatureInfo]) + + res = generator(attempted_signatures=[ScoredSignature[SignatureInfo](signature=info, score=50)]) + assert res.proposed_signature == SignatureInfo(**devset[0]) + + # Test the "to_signature" method + + class OutputSignature(dspy.Signature): + """Answer the following questions""" + + question: str = dspy.InputField(desc="Some question to answer", prefix="Q: ") + answer: str = dspy.OutputField(desc="A short answer to the question", prefix="A: ") + + assert res.proposed_signature.to_signature().equals(OutputSignature) + + +def test_opt(): + qa_model = DummyLM([]) + prompt_model = DummyLM( + [ + # Seed prompts + "some thoughts", + '{"value": [{"instructions": "I", "question_desc": "$q", "question_prefix": "Q:", "answer_desc": "$a", "answer_prefix": "A:"}]}', + ] + ) + dspy.settings.configure(lm=qa_model) + + program = optimize_signature( + student=TypedPredictor(BasicQA), + evaluator=Evaluate(devset=hotpotqa, metric=answer_exact_match, num_threads=1), + initial_prompts=1, + n_iterations=1, + verbose=True, + prompt_model=prompt_model, + ) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index e44b3a135c..2ded1e9f12 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -1,14 +1,13 @@ import dspy from dspy import Predict, Signature from dspy.utils.dummies import DummyLM +import copy def test_initialization_with_string_signature(): signature_string = "input1, input2 -> output" predict = Predict(signature_string) - expected_instruction = ( - "Given the fields `input1`, `input2`, produce the fields `output`." - ) + expected_instruction = "Given the fields `input1`, `input2`, produce the fields `output`." assert predict.signature.instructions == expected_instruction assert predict.signature.instructions == Signature(signature_string).instructions @@ -89,3 +88,34 @@ def test_multi_output(): results = program(question="What is 1+1?") assert results.completions.answer[0] == "my first answer" assert results.completions.answer[1] == "my second answer" + + +def test_multi_output2(): + program = Predict("question -> answer1, answer2", n=2) + dspy.settings.configure( + lm=DummyLM( + [ + "my 0 answer\nAnswer 2: my 2 answer", + "my 1 answer\nAnswer 2: my 3 answer", + ], + ) + ) + results = program(question="What is 1+1?") + assert results.completions.answer1[0] == "my 0 answer" + assert results.completions.answer1[1] == "my 1 answer" + assert results.completions.answer2[0] == "my 2 answer" + assert results.completions.answer2[1] == "my 3 answer" + + +def test_named_predictors(): + class MyModule(dspy.Module): + def __init__(self): + super().__init__() + self.inner = Predict("question -> answer") + + program = MyModule() + assert program.named_predictors() == [("inner", program.inner)] + + # Check that it also works the second time. + program2 = copy.deepcopy(program) + assert program2.named_predictors() == [("inner", program2.inner)] From 1f6e0cbe3dc669da55be42448d8df44ec6cdb7f5 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Tue, 5 Mar 2024 13:00:20 -0800 Subject: [PATCH 04/18] Removed (deactiviated) key --- examples/signature_opt2.ipynb | 1521 +-------------------------------- 1 file changed, 25 insertions(+), 1496 deletions(-) diff --git a/examples/signature_opt2.ipynb b/examples/signature_opt2.ipynb index 9edc0d4c4e..5fdbdf5d21 100644 --- a/examples/signature_opt2.ipynb +++ b/examples/signature_opt2.ipynb @@ -2,15 +2,15 @@ "cells": [ { "cell_type": "code", - "execution_count": 13, + "execution_count": 1, "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" + "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], @@ -19,12 +19,12 @@ "%autoreload 2\n", "import dspy\n", "import os\n", - "os.environ['OPENAI_API_KEY'] = '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'" + "os.environ['OPENAI_API_KEY'] = 'sk-...'" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -35,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -44,7 +44,7 @@ "(20, 50)" ] }, - "execution_count": 15, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -64,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -77,122 +77,36 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "All predictors:\n", - "named_predictors=[('predictor', Predict(BasicQA(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", - ")))]\n", - "Generating new signature for Predict(BasicQA(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", - "))...\n", - "Generated candidate 0:\n", - "StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.(0)'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", - ")\n", - "Generated candidate 1:\n", - "StringSignature(question -> answer\n", - " instructions='Answer series of questions where answers must share a common theme.(1)'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q1,', 'desc': '${question}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'charge amplifier designed based by Moog on stripline technology', '__dspy_field_type': 'output', 'prefix': 'A1 Wolfgang Amplifier,'})\n", - ")\n", - "Generated candidate 2:\n", - "StringSignature(question -> answer\n", - " instructions='Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q-What Computability study ranges?', 'desc': 'may also involve strings produced by non-deterministic computation'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!', '__dspy_field_type': 'output', 'prefix': 'A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder'})\n", - ")\n", - "\n", - "================================================================================\n", - "Running eval iteration 0...\n", - "Installing signature 0: \n", - "StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.(0)'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", - ")\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 5144.18it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 5 / 20 (25.0%)\n", - "Scores for iteration 0: 28.55243378134378\n", - "\n", - "================================================================================\n", - "Running eval iteration 1...\n", - "Installing signature 1: \n", - "StringSignature(question -> answer\n", - " instructions='Answer series of questions where answers must share a common theme.(1)'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q1,', 'desc': '${question}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'charge amplifier designed based by Moog on stripline technology', '__dspy_field_type': 'output', 'prefix': 'A1 Wolfgang Amplifier,'})\n", - ")\n" + "None\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 4207.98it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 5 / 20 (25.0%)\n", - "Scores for iteration 1: 29.911449495835697\n", - "\n", - "================================================================================\n", - "Running eval iteration 2...\n", - "Installing signature 2: \n", - "StringSignature(question -> answer\n", - " instructions='Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Q-What Computability study ranges?', 'desc': 'may also involve strings produced by non-deterministic computation'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!', '__dspy_field_type': 'output', 'prefix': 'A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder'})\n", - ")\n" + "/opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_fields.py:184: UserWarning: Field name \"signature\" shadows an attribute in parent \"Signature\"; \n", + " warnings.warn(\n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "Average Metric: 5 / 20 (25.0): 100%|██████████| 20/20 [00:00<00:00, 4577.93it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 5 / 20 (25.0%)\n", - "Scores for iteration 2: 34.58455944537296\n", - "Generating new signature for predictor based on 3 previous signatures...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" + "ename": "TypeError", + "evalue": "Field 'signature' in 'GenerateInstructionGivenAttempts' must be declared with InputField or OutputField. field.json_schema_extra=None", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdspy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mevaluate\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m answer_exact_match\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdspy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfunctional\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TypedPredictor\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdspy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mteleprompt\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msignature_opt2\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m optimize_signature\n\u001b[1;32m 6\u001b[0m evaluator \u001b[38;5;241m=\u001b[39m Evaluate(devset\u001b[38;5;241m=\u001b[39mdevset, metric\u001b[38;5;241m=\u001b[39manswer_exact_match, num_threads\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m, display_progress\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 8\u001b[0m program \u001b[38;5;241m=\u001b[39m optimize_signature(\n\u001b[1;32m 9\u001b[0m student\u001b[38;5;241m=\u001b[39mTypedPredictor(BasicQA),\n\u001b[1;32m 10\u001b[0m evaluator\u001b[38;5;241m=\u001b[39mEvaluate(devset\u001b[38;5;241m=\u001b[39mtrainset, metric\u001b[38;5;241m=\u001b[39manswer_exact_match, num_threads\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m, display_progress\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 14\u001b[0m prompt_model\u001b[38;5;241m=\u001b[39mgpt4,\n\u001b[1;32m 15\u001b[0m )\n", + "File \u001b[0;32m~/repos/dspy/dspy/teleprompt/signature_opt2.py:97\u001b[0m\n\u001b[1;32m 93\u001b[0m signature: T\n\u001b[1;32m 94\u001b[0m score: \u001b[38;5;28mfloat\u001b[39m \u001b[38;5;241m=\u001b[39m dspy\u001b[38;5;241m.\u001b[39mField(gt\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m, lt\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m100\u001b[39m)\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;43;01mclass\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;21;43;01mGenerateInstructionGivenAttempts\u001b[39;49;00m\u001b[43m(\u001b[49m\u001b[43mdspy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mGeneric\u001b[49m\u001b[43m[\u001b[49m\u001b[43mT\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;250;43m \u001b[39;49m\u001b[38;5;124;43;03m\"\"\"You are an instruction optimizer for large language models.\u001b[39;49;00m\n\u001b[1;32m 99\u001b[0m \n\u001b[1;32m 100\u001b[0m \u001b[38;5;124;43;03m I will give some task instructions I've tried, along with their corresponding validation scores.\u001b[39;49;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[38;5;124;43;03m - Don't repeat instructions, descriptions and prefixes that have already been attempted.\u001b[39;49;00m\n\u001b[1;32m 105\u001b[0m \u001b[38;5;124;43;03m \"\"\"\u001b[39;49;00m\n\u001b[1;32m 107\u001b[0m \u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mT\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdspy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mOutputField\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdesc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mProposed signature to try\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/dspy/dspy/signatures/signature.py:48\u001b[0m, in \u001b[0;36mSignatureMeta.__new__\u001b[0;34m(mcs, signature_name, bases, namespace, **kwargs)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__doc__\u001b[39m \u001b[38;5;241m=\u001b[39m _default_instructions(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m 47\u001b[0m \u001b[38;5;66;03m# Ensure all fields are declared with InputField or OutputField\u001b[39;00m\n\u001b[0;32m---> 48\u001b[0m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_fields\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;66;03m# Ensure all fields have a prefix\u001b[39;00m\n\u001b[1;32m 51\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m name, field \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_fields\u001b[38;5;241m.\u001b[39mitems():\n", + "File \u001b[0;32m~/repos/dspy/dspy/signatures/signature.py:65\u001b[0m, in \u001b[0;36mSignatureMeta._validate_fields\u001b[0;34m(cls)\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m field_type \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28mprint\u001b[39m(field\u001b[38;5;241m.\u001b[39mjson_schema_extra)\n\u001b[0;32m---> 65\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 66\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mField \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m in \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m must be declared with InputField or OutputField. \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfield\u001b[38;5;241m.\u001b[39mjson_schema_extra\u001b[38;5;132;01m=}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 67\u001b[0m )\n", + "\u001b[0;31mTypeError\u001b[0m: Field 'signature' in 'GenerateInstructionGivenAttempts' must be declared with InputField or OutputField. field.json_schema_extra=None" ] } ], @@ -216,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -305,1391 +219,6 @@ "source": [ "gpt4.inspect_history(n=3)" ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'tp' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[19], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtp\u001b[49m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__dict__\u001b[39m\n", - "\u001b[0;31mNameError\u001b[0m: name 'tp' is not defined" - ] - } - ], - "source": [ - "tp.__dict__" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "module.predictors()=[]\n", - "\n", - "================================================================================\n", - "Running eval iteration 0...\n", - "list: []\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8900>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fe20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5fa60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ef20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632f240>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6020>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848af20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e3060>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13ab60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e8e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ee80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001f80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4e00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ff60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x10632efc0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5b20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488f40>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0720>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafec0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488540>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e160>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e700>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe34c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0d60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e980>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e160>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5d00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafd80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2fc0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafd80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f560>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe09a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3100>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5fba0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e160>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a4b80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe37e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2f20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0b80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5ee0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f1a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba87c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1760>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe00e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafec0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0360>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a0020c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f6a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fb00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f600>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a000720>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632efc0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fb00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0cc0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584887c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f9c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2660>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0c20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f1a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4e00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158489d00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1e40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2ac0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0720>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001f80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ede0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ee80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2ca0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe28e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e020>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b4900>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a4b80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2160>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0220>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0e00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a4b80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001f80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e8e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3b00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f880>, 'parser': })\n", - ")\n", - "StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0900>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0400>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b4900>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13afc0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe34c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe23e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b47c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ec00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5fd80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3560>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe07c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5e700>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158536840>, 'parser': })\n", - ")\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 5483.04it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0%)\n", - "Scores for iteration 0: 32.0\n", - "\n", - "================================================================================\n", - "Running eval iteration 1...\n", - "list: []\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585367a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1391c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e3060>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2f20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1d00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5b20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488540>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002020>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f240>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe09a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f060>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bce0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1391c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe02c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848af20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c139da0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5ec00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5fd80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158536840>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6020>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x10632efc0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bce0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5f880>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f420>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489f80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a000720>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c139da0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158536840>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6660>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158489d00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e840>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5d9e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1260>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f9c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba8900>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5ee0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2fc0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b9c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632efc0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848be20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e700>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1b20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e160>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba89a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848a520>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a7240>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe23e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0ea0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158536840>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159a5e980>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ede0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe07c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f880>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489d00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5a80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0e00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1f80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5ee0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5f9c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2160>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0fe0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3f60>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159a5ef20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848a520>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585367a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0360>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2160>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5ee0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafd80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba8900>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1d00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe22a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16ae719e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b68e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe11c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2660>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0720>, 'parser': })\n", - ")\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Created templateCreated template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632d120>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb6ac0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138720>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafc40>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x10632ede0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138c20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c139580>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489d00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002660>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848be20>, 'parser': })\n", - ")\n", - " Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3f60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0720>, 'parser': })\n", - ")\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4371.34it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0%)\n", - "Scores for iteration 1: 32.0\n", - "\n", - "================================================================================\n", - "Running eval iteration 2...\n", - "list: []\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13afc0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5a80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5d00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafe20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b47c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb5620>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba82c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a4b80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138b80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b240>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bc40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b600>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5080>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0360>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafa60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b68e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafec0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0b80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13afc0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138040>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002160>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488ea0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13af20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5d00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba89a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb4220>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2160>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb47c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6020>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584887c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0180>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584885e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b9c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6e80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafe20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2200>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3380>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8400>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c1389a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3d80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0540>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138c20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6e80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafc40>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1ee0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2340>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2ca0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba89a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5d00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0680>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1a80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2c00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba89a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b7ba0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489260>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bcb47c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138b80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b42c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a001da0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5a80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c139580>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe02c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13b240>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002660>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848a3e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0360>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848b9c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafd80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a5940>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b600>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3ba0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c1394e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a6e80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafa60>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bcb7920>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe13a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2340>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b47c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002660>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a7240>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138040>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2d40>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe32e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13bba0>, 'parser': })\n", - ")\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba82c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4680>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2d40>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b6b60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002020>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3880>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe22a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1e40>, 'parser': })\n", - ")\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 5230.07it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0%)\n", - "Scores for iteration 2: 32.0\n", - "\n", - "================================================================================\n", - "Running eval iteration 3...\n", - "list: []\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c13aca0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6e80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafe20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b62a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488ea0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584885e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba89a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a001da0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138180>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3d80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13aca0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002520>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafe20>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158489f80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1120>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b62a0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002020>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1a80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe19e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5940>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8900>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b49a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0540>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138c20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0360>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bce0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b62a0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13f4a4ea0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138040>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138180>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13a002160>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b47c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488ea0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1080>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb6c00>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0360>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158489f80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1585b6b60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a6e80>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0720>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138180>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbafb00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb68e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe3d80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1584e0360>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bc40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b4680>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13a002160>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1da0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1d00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138180>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba87c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x158488540>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e1760>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe1620>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848b9c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13bbaff60>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138ae0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2340>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0a40>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba87c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x15848bc40>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848af20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe2520>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1da0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe23e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584885e0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbaff60>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x13c138ae0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe05e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0fe0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x16aba80e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0540>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3560>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe11c0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe34c0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe1b20>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b6b60>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x16aba8900>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe3ce0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0e00>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x159fe0220>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe2840>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x1585b68e0>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x158488f40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13f4a5080>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0cc0>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c138040>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x159fe0680>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13c13b240>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x1584e0b80>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bcb5620>, 'parser': })\n", - ")\n", - "Created template Template(Answer questions with short factoid answers., ['Question:', 'Answer:'])\n", - "From Signature StringSignature(question -> answer\n", - " instructions='Answer questions with short factoid answers.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}', 'format': . at 0x15848bc40>})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:', 'format': . at 0x13bbafa60>, 'parser': })\n", - ")\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4131.18it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 16 / 50 (32.0%)\n", - "Scores for iteration 3: 32.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/plain": [] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "optimizer.compile(TypedPredictor(BasicQA), evaluator, n_iterations=4)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "gpt4.inspect_history(n=4)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Feel free to any other queries you like." - ] } ], "metadata": { From a2fb536eb0eb37fc5e38035ba3eac3a87a249545 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Tue, 5 Mar 2024 23:47:45 -0800 Subject: [PATCH 05/18] Fix the "no inputs" case --- dsp/templates/template_v2.py | 73 +++++++++++++++-------------------- dspy/predict/predict.py | 5 +-- tests/predict/test_predict.py | 25 ++++++++++++ 3 files changed, 59 insertions(+), 44 deletions(-) diff --git a/dsp/templates/template_v2.py b/dsp/templates/template_v2.py index d6e61642ea..9ca90779f9 100644 --- a/dsp/templates/template_v2.py +++ b/dsp/templates/template_v2.py @@ -70,6 +70,8 @@ def query(self, example: Example, is_demo: bool = False) -> str: """Retrieves the input variables from the example and formats them into a query string.""" result: list[str] = [] + # If not a demo, find the last field that doesn't have a value set in `example` and set it to "" + # This creates the "Output:" prefix at the end of the prompt. if not is_demo: has_value = [ field.input_variable in example @@ -78,40 +80,40 @@ def query(self, example: Example, is_demo: bool = False) -> str: for field in self.fields ] - for i in range(1, len(has_value)): - if has_value[i - 1] and not any(has_value[i:]): - example[self.fields[i].input_variable] = "" - break + # If there are no inputs, set the first field to "" + if not any(has_value): + example[self.fields[0].input_variable] = "" + # Otherwise find the first field without a value. + else: + for i in range(1, len(has_value)): + if has_value[i - 1] and not any(has_value[i:]): + example[self.fields[i].input_variable] = "" + break for field in self.fields: - if ( - field.input_variable in example - and example[field.input_variable] is not None - ): + if field.input_variable in example and example[field.input_variable] is not None: if field.input_variable in self.format_handlers: format_handler = self.format_handlers[field.input_variable] else: + def format_handler(x): assert type(x) == str, f"Need format_handler for {field.input_variable} of type {type(x)}" return " ".join(x.split()) formatted_value = format_handler(example[field.input_variable]) - separator = '\n' if field.separator == ' ' and '\n' in formatted_value else field.separator + separator = "\n" if field.separator == " " and "\n" in formatted_value else field.separator result.append( f"{field.name}{separator}{formatted_value}", ) - if self._has_augmented_guidelines() and (example.get('augmented', False)): + if self._has_augmented_guidelines() and (example.get("augmented", False)): return "\n\n".join([r for r in result if r]) return "\n".join([r for r in result if r]) def guidelines(self, show_guidelines=True) -> str: """Returns the task guidelines as described in the lm prompt""" - if (not show_guidelines) or ( - hasattr(dsp.settings, "show_guidelines") - and not dsp.settings.show_guidelines - ): + if (not show_guidelines) or (hasattr(dsp.settings, "show_guidelines") and not dsp.settings.show_guidelines): return "" result = "Follow the following format.\n\n" @@ -126,11 +128,13 @@ def guidelines(self, show_guidelines=True) -> str: def _has_augmented_guidelines(self): return len(self.fields) > 3 or any( - ("\n" in field.separator) or ('\n' in field.description) for field in self.fields + ("\n" in field.separator) or ("\n" in field.description) for field in self.fields ) def extract( - self, example: Union[Example, dict[str, Any]], raw_pred: str, + self, + example: Union[Example, dict[str, Any]], + raw_pred: str, ) -> Example: """Extracts the answer from the LM raw prediction using the template structure @@ -147,10 +151,7 @@ def extract( idx = 0 while idx < len(self.fields): - if ( - self.fields[idx].input_variable not in example - or example[self.fields[idx].input_variable] is None - ): + if self.fields[idx].input_variable not in example or example[self.fields[idx].input_variable] is None: break idx += 1 @@ -164,8 +165,8 @@ def extract( if offset >= 0: if dspy.settings.release >= 20231003: - example[self.fields[idx].output_variable] = raw_pred[:offset].strip().rstrip('---').strip() - raw_pred = raw_pred[offset + len(next_field_name) :].strip().rstrip('---').strip() + example[self.fields[idx].output_variable] = raw_pred[:offset].strip().rstrip("---").strip() + raw_pred = raw_pred[offset + len(next_field_name) :].strip().rstrip("---").strip() else: example[self.fields[idx].output_variable] = raw_pred[:offset].strip() raw_pred = raw_pred[offset + len(next_field_name) :].strip() @@ -173,7 +174,7 @@ def extract( idx += 1 else: if dspy.settings.release >= 20231003: - example[self.fields[idx].output_variable] = raw_pred.strip().rstrip('---').strip() + example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip() else: example[self.fields[idx].output_variable] = raw_pred.strip() @@ -185,7 +186,7 @@ def extract( assert idx == len(self.fields) - 1, (idx, len(self.fields)) if dspy.settings.release >= 20231003: - example[self.fields[idx].output_variable] = raw_pred.strip().rstrip('---').strip() + example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip() else: example[self.fields[idx].output_variable] = raw_pred.strip() @@ -196,7 +197,7 @@ def extract( def __call__(self, example, show_guidelines=True) -> str: example = dsp.Example(example) - if hasattr(dsp.settings, 'query_only') and dsp.settings.query_only: + if hasattr(dsp.settings, "query_only") and dsp.settings.query_only: return self.query(example) # The training data should not contain the output variable @@ -207,29 +208,20 @@ def __call__(self, example, show_guidelines=True) -> str: self.query(demo, is_demo=True) for demo in example.demos if ( - (not demo.get('augmented', False)) + (not demo.get("augmented", False)) and ( # validate that the training example has the same primitive input var as the template - self.fields[-1].input_variable in demo - and demo[self.fields[-1].input_variable] is not None + self.fields[-1].input_variable in demo and demo[self.fields[-1].input_variable] is not None ) ) ] - ademos = [ - self.query(demo, is_demo=True) - for demo in example.demos - if demo.get('augmented', False) - ] + ademos = [self.query(demo, is_demo=True) for demo in example.demos if demo.get("augmented", False)] # Move the rdemos to ademos if rdemo has all the fields filled in rdemos_ = [] new_ademos = [] for rdemo in rdemos: - if all( - (field.name in rdemo) - for field in self.fields - if field.input_variable in example - ): + if all((field.name in rdemo) for field in self.fields if field.input_variable in example): import dspy if dspy.settings.release >= 20230928: @@ -242,7 +234,6 @@ def __call__(self, example, show_guidelines=True) -> str: ademos = new_ademos + ademos rdemos = rdemos_ - long_query = self._has_augmented_guidelines() if long_query: @@ -251,10 +242,10 @@ def __call__(self, example, show_guidelines=True) -> str: query = self.query(example) # if it has more lines than fields - if len(query.split('\n')) > len(self.fields): + if len(query.split("\n")) > len(self.fields): long_query = True - if not example.get('augmented', False): + if not example.get("augmented", False): example["augmented"] = True query = self.query(example) diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py index d0a5f6010e..1243e8b882 100644 --- a/dspy/predict/predict.py +++ b/dspy/predict/predict.py @@ -73,7 +73,8 @@ def forward(self, **kwargs): # print(f"#> Setting temperature to 0.7 since n={num_generations} and prior temperature={temperature}.") # All of the other kwargs are presumed to fit a prefix of the signature. - + # That is, they are input variables for the bottom most generation, so + # we place them inside the input - x - together with the demos. x = dsp.Example(demos=demos, **kwargs) if new_signature is not None: @@ -86,8 +87,6 @@ def forward(self, **kwargs): # Switch to legacy format for dsp.generate template = signature_to_template(signature) - # print("Created template", template) - # print("From Signature", signature) if self.lm is None: x, C = dsp.generate(template, **config)(x, stage=self.stage) diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py index 2ded1e9f12..c0407b938c 100644 --- a/tests/predict/test_predict.py +++ b/tests/predict/test_predict.py @@ -2,6 +2,7 @@ from dspy import Predict, Signature from dspy.utils.dummies import DummyLM import copy +import textwrap def test_initialization_with_string_signature(): @@ -119,3 +120,27 @@ def __init__(self): # Check that it also works the second time. program2 = copy.deepcopy(program) assert program2.named_predictors() == [("inner", program2.inner)] + + +def test_output_only(): + class OutputOnlySignature(dspy.Signature): + output = dspy.OutputField() + + predictor = Predict(OutputOnlySignature) + + lm = DummyLM(["short answer"]) + dspy.settings.configure(lm=lm) + assert predictor().output == "short answer" + + assert lm.get_convo(-1) == textwrap.dedent("""\ + Given the fields , produce the fields `output`. + + --- + + Follow the following format. + + Output: ${output} + + --- + + Output: short answer""") From 797f1cd69b73ba163ec1e1ab98f9be4a2e6a3138 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Tue, 5 Mar 2024 23:48:24 -0800 Subject: [PATCH 06/18] Added a new method to Module --- dspy/primitives/module.py | 8 +++++ tests/primitives/test_program.py | 61 +++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/dspy/primitives/module.py b/dspy/primitives/module.py index 59ec8a09f8..989f68403d 100644 --- a/dspy/primitives/module.py +++ b/dspy/primitives/module.py @@ -1,4 +1,5 @@ import copy +from typing import Generator import ujson @@ -41,6 +42,13 @@ def add_parameter(param_name, param_value): return named_parameters + def named_sub_modules(self) -> Generator[tuple[str, "BaseModule"], None, None]: + yield "", self + for name, value in self.__dict__.items(): + if isinstance(value, BaseModule): + for sub_name, sub_value in value.named_sub_modules(): + yield f"{name}.{sub_name}", sub_value + def parameters(self): return [param for _, param in self.named_parameters()] diff --git a/tests/primitives/test_program.py b/tests/primitives/test_program.py index b1d7c89725..aec5ace7d2 100644 --- a/tests/primitives/test_program.py +++ b/tests/primitives/test_program.py @@ -1,4 +1,5 @@ import dspy +from dspy.primitives.module import BaseModule from dspy.primitives.program import ( Module, set_attribute_by_name, @@ -19,9 +20,7 @@ def forward(self, question): def test_module_initialization(): module = Module() - assert ( - module._compiled is False - ), "Module _compiled attribute should be False upon initialization" + assert module._compiled is False, "Module _compiled attribute should be False upon initialization" def test_named_predictors(): @@ -29,25 +28,19 @@ def test_named_predictors(): named_preds = module.named_predictors() assert len(named_preds) == 2, "Should identify correct number of Predict instances" names, preds = zip(*named_preds) - assert ( - "predict1" in names and "predict2" in names - ), "Named predictors should include 'predict1' and 'predict2'" + assert "predict1" in names and "predict2" in names, "Named predictors should include 'predict1' and 'predict2'" def test_predictors(): module = HopModule() preds = module.predictors() assert len(preds) == 2, "Should return correct number of Predict instances" - assert all( - isinstance(p, dspy.Predict) for p in preds - ), "All returned items should be instances of PredictMock" + assert all(isinstance(p, dspy.Predict) for p in preds), "All returned items should be instances of PredictMock" def test_forward(): program = HopModule() - dspy.settings.configure( - lm=DummyLM({"What is 1+1?": "let me check", "let me check": "2"}) - ) + dspy.settings.configure(lm=DummyLM({"What is 1+1?": "let me check", "let me check": "2"})) result = program(question="What is 1+1?").answer assert result == "2" @@ -64,3 +57,47 @@ def __init__(self): names, _preds = zip(*named_preds) assert "hop.predict1" in names assert "hop.predict2" in names + + +class SubModule(BaseModule): + pass + + +class AnotherSubModule(BaseModule): + pass + + +def test_empty_module(): + module = BaseModule() + assert list(module.named_sub_modules()) == [] + + +def test_single_level(): + module = BaseModule() + module.sub = SubModule() + expected = [("sub", module.sub)] + assert list(module.named_sub_modules()) == expected + + +def test_multiple_levels(): + module = BaseModule() + module.sub = SubModule() + module.sub.subsub = SubModule() + expected = [("sub", module.sub), ("sub.subsub", module.sub.subsub)] + assert list(module.named_sub_modules()) == expected + + +def test_multiple_sub_modules(): + module = BaseModule() + module.sub1 = SubModule() + module.sub2 = SubModule() + expected = [("sub1", module.sub1), ("sub2", module.sub2)] + assert sorted(list(module.named_sub_modules())) == sorted(expected) + + +def test_non_base_module_attributes(): + module = BaseModule() + module.sub = SubModule() + module.not_a_sub = "Not a BaseModule" + expected = [("sub", module.sub)] + assert list(module.named_sub_modules()) == expected From bf011b8c82c1f279f99242cd216e613362b171f7 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Tue, 5 Mar 2024 23:49:01 -0800 Subject: [PATCH 07/18] Fix a bug with generics --- dspy/signatures/signature.py | 8 +++++--- tests/functional/test_functional.py | 24 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py index 8c1f161642..62626c8e1d 100644 --- a/dspy/signatures/signature.py +++ b/dspy/signatures/signature.py @@ -1,9 +1,10 @@ import ast from copy import deepcopy -import typing import dsp from pydantic import BaseModel, Field, create_model from pydantic.fields import FieldInfo +import typing +import types from typing import Any, Type, Union, Dict, Tuple # noqa: UP035 import re @@ -62,7 +63,7 @@ def _validate_fields(cls): field_type = extra.get("__dspy_field_type") if field_type not in ["input", "output"]: raise TypeError( - f"Field '{name}' in '{cls.__name__}' must be declared with InputField or OutputField.", + f"Field '{name}' in '{cls.__name__}' must be declared with InputField or OutputField. {field.json_schema_extra=}", ) @property @@ -232,7 +233,8 @@ def make_signature( # program of thought and teleprompters, so we just silently default to string. if type_ is None: type_ = str - if not isinstance(type_, type) and not isinstance(typing.get_origin(type_), type): + # if not isinstance(type_, type) and not isinstance(typing.get_origin(type_), type): + if not isinstance(type_, (type, typing._GenericAlias, types.GenericAlias)): raise ValueError(f"Field types must be types, not {type(type_)}") if not isinstance(field, FieldInfo): raise ValueError(f"Field values must be Field instances, not {type(field)}") diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py index 600e35b63d..ebca992230 100644 --- a/tests/functional/test_functional.py +++ b/tests/functional/test_functional.py @@ -2,7 +2,7 @@ import textwrap import pydantic from pydantic import Field, BaseModel, field_validator -from typing import Annotated +from typing import Annotated, Literal from typing import List import pytest @@ -495,6 +495,28 @@ def test_parse_type_string(): assert output == [0, 1, 2] +def test_literal(): + lm = DummyLM([f'{{"value": "{i}"}}' for i in range(100)]) + dspy.settings.configure(lm=lm) + + @predictor + def f() -> Literal["2", "3"]: + pass + + assert f() == "2" + + +def test_literal_int(): + lm = DummyLM([f'{{"value": {i}}}' for i in range(100)]) + dspy.settings.configure(lm=lm) + + @predictor + def f() -> Literal[2, 3]: + pass + + assert f() == 2 + + def test_fields_on_base_signature(): class SimpleOutput(dspy.Signature): output: float = dspy.OutputField(gt=0, lt=1) From d633a7782619a511a920030084f96430cab8d5f6 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Tue, 5 Mar 2024 23:49:43 -0800 Subject: [PATCH 08/18] Formatting --- dspy/signatures/field.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dspy/signatures/field.py b/dspy/signatures/field.py index 7822c625c0..4e32714778 100644 --- a/dspy/signatures/field.py +++ b/dspy/signatures/field.py @@ -1,5 +1,10 @@ import pydantic +# The following arguments can be used in DSPy InputField and OutputField in addition +# to the standard pydantic.Field arguments. We just hope pydanitc doesn't add these, +# as it would give a name clash. +DSPY_FIELD_ARG_NAMES = ["desc", "prefix", "format", "parser", "__dspy_field_type"] + def move_kwargs(**kwargs): # Pydantic doesn't allow arbitrary arguments to be given to fields, @@ -10,7 +15,7 @@ def move_kwargs(**kwargs): pydantic_kwargs = {} json_schema_extra = {} for k, v in kwargs.items(): - if k in ["desc", "prefix", "format", "parser", "__dspy_field_type"]: + if k in DSPY_FIELD_ARG_NAMES: json_schema_extra[k] = v else: pydantic_kwargs[k] = v @@ -27,11 +32,7 @@ def OutputField(**kwargs): def new_to_old_field(field): - return ( - OldInputField - if field.json_schema_extra["__dspy_field_type"] == "input" - else OldOutputField - )( + return (OldInputField if field.json_schema_extra["__dspy_field_type"] == "input" else OldOutputField)( prefix=field.json_schema_extra["prefix"], desc=field.json_schema_extra["desc"], format=field.json_schema_extra.get("format"), From 72838473f812a2c72b9f117b9b72c2258001c9d2 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Wed, 6 Mar 2024 01:25:41 -0800 Subject: [PATCH 09/18] Created typed signature optimizer --- dspy/__init__.py | 1 + dspy/functional/functional.py | 22 +- dspy/primitives/module.py | 7 +- dspy/signatures/signature.py | 46 +- ...gnature_opt2.py => signature_opt_typed.py} | 148 ++-- examples/functional/signature_opt_typed.ipynb | 706 ++++++++++++++++++ examples/signature_opt2.ipynb | 246 ------ tests/functional/test_functional.py | 19 +- ...re_opt2.py => test_signature_opt_typed.py} | 20 +- tests/primitives/test_program.py | 10 +- tests/signatures/test_signature.py | 32 + 11 files changed, 902 insertions(+), 355 deletions(-) rename dspy/teleprompt/{signature_opt2.py => signature_opt_typed.py} (63%) create mode 100644 examples/functional/signature_opt_typed.ipynb delete mode 100644 examples/signature_opt2.ipynb rename tests/functional/{test_signature_opt2.py => test_signature_opt_typed.py} (91%) diff --git a/dspy/__init__.py b/dspy/__init__.py index 98fe6fba89..43317f3d1b 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -5,6 +5,7 @@ from .retrieve import * from .predict import * from .primitives import * +from .functional import * # from .evaluation import * diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index 3d6c471414..ed14c66cad 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -118,20 +118,32 @@ def _prepare_signature(self) -> dspy.Signature: format=lambda x: x if isinstance(x, str) else str(x), parser=type_, ) + elif False: + # TODO: I don't like forcing the model to write "value" in the output. + if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)): + type_ = pydantic.create_model("Output", value=(type_, ...), __base__=pydantic.BaseModel) + to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {"value":"123"} + from_json = lambda x, type_=type_: type_.model_validate_json('{"value":' + x + "}").value + schema = json.dumps(type_.model_json_schema()["properties"]["value"]) + else: + to_json = lambda x: x.model_dump_json() + from_json = lambda x, type_=type_: type_.model_validate_json(x) + schema = json.dumps(type_.model_json_schema()) else: # Anything else we wrap in a pydantic object - to_json = lambda x: x.model_dump_json() - from_json = lambda x, type_=type_: type_.model_validate_json(x) if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)): type_ = pydantic.create_model("Output", value=(type_, ...), __base__=pydantic.BaseModel) to_json = lambda x, type_=type_: type_(value=x).model_dump_json() from_json = lambda x, type_=type_: type_.model_validate_json(x).value + schema = json.dumps(type_.model_json_schema()) + else: + to_json = lambda x: x.model_dump_json() + from_json = lambda x, type_=type_: type_.model_validate_json(x) + schema = json.dumps(type_.model_json_schema()) signature = signature.with_updated_fields( name, desc=field.json_schema_extra.get("desc", "") - + ( - ". Respond with a single JSON object. JSON Schema: " + json.dumps(type_.model_json_schema()) - ), + + (". Respond with a single JSON object. JSON Schema: " + schema), format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)), parser=lambda x, from_json=from_json: from_json(_unwrap_json(x)), type_=type_, diff --git a/dspy/primitives/module.py b/dspy/primitives/module.py index 989f68403d..9ce327a26e 100644 --- a/dspy/primitives/module.py +++ b/dspy/primitives/module.py @@ -42,12 +42,11 @@ def add_parameter(param_name, param_value): return named_parameters - def named_sub_modules(self) -> Generator[tuple[str, "BaseModule"], None, None]: - yield "", self + def named_sub_modules(self, root_name="base") -> Generator[tuple[str, "BaseModule"], None, None]: + yield root_name, self for name, value in self.__dict__.items(): if isinstance(value, BaseModule): - for sub_name, sub_value in value.named_sub_modules(): - yield f"{name}.{sub_name}", sub_value + yield from value.named_sub_modules(root_name=f"{root_name}.{name}") def parameters(self): return [param for _, param in self.named_parameters()] diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py index 62626c8e1d..40eaf28aac 100644 --- a/dspy/signatures/signature.py +++ b/dspy/signatures/signature.py @@ -42,6 +42,17 @@ def __new__(mcs, signature_name, bases, namespace, **kwargs): # noqa: N804 # Let Pydantic do its thing cls = super().__new__(mcs, signature_name, bases, namespace, **kwargs) + # If we don't have instructions, it might be because we are a derived generic type. + # In that case, we should inherit the instructions from the base class. + if cls.__doc__ is None: + for base in bases: + if isinstance(base, SignatureMeta): + doc = getattr(base, "__doc__", "") + if doc != "": + cls.__doc__ = doc + + # The more likely case is that the user has just not given us a type. + # In that case, we should default to the input/output format. if cls.__doc__ is None: cls.__doc__ = _default_instructions(cls) @@ -168,24 +179,27 @@ def __repr__(cls): return f"{cls.__name__}({cls.signature}\n instructions={repr(cls.instructions)}\n {field_repr}\n)" +# A signature for a predictor. +# +# You typically subclass it, like this: +# class MySignature(Signature): +# input: str = InputField(desc="...") # noqa: ERA001 +# output: int = OutputField(desc="...") # noqa: ERA001 +# +# You can call Signature("input1, input2 -> output1, output2") to create a new signature type. +# You can also include instructions, Signature("input -> output", "This is a test"). +# But it's generally better to use the make_signature function. +# +# If you are not sure if your input is a string representation, (like "input1, input2 -> output1, output2"), +# or a signature, you can use the ensure_signature function. +# +# For compatibility with the legacy dsp format, you can use the signature_to_template function. +# class Signature(BaseModel, metaclass=SignatureMeta): - """A signature for a predictor. - - You typically subclass it, like this: - class MySignature(Signature): - input: str = InputField(desc="...") - output: int = OutputField(desc="...") - - You can call Signature("input1, input2 -> output1, output2") to create a new signature type. - You can also include instructions, Signature("input -> output", "This is a test"). - But it's generally better to use the make_signature function. - - If you are not sure if your input is a string representation, (like "input1, input2 -> output1, output2"), - or a signature, you can use the ensure_signature function. - - For compatibility with the legacy dsp format, you can use the signature_to_template function. - """ + "" # noqa: D419 + # Note: Don't put a docstring here, as it will become the default instructions + # for any signature that doesn't define it's own instructions. pass diff --git a/dspy/teleprompt/signature_opt2.py b/dspy/teleprompt/signature_opt_typed.py similarity index 63% rename from dspy/teleprompt/signature_opt2.py rename to dspy/teleprompt/signature_opt_typed.py index 1a9045ae23..956ea0f0cd 100644 --- a/dspy/teleprompt/signature_opt2.py +++ b/dspy/teleprompt/signature_opt_typed.py @@ -1,13 +1,17 @@ -import random -from typing import Generic, Literal, TypeVar, Type +import textwrap +from typing import Generic, Literal, TypeVar import pydantic import dspy -from dspy.functional.functional import TypedChainOfThought +from dspy.functional.functional import TypedChainOfThought, TypedPredictor from dspy.signatures import Signature from dspy import BaseModel +from dspy.signatures.field import InputField, OutputField -# TODO: Consider using the prompt optimizer to optimize the prompt optimizer :O +# TODO: +# - Parallelize the generation of new signatures when we have multiple predictors +# - Consider generating multiple new signatures at once, which we can test in parallel +# - Consider using the prompt optimizer to optimize the prompt optimizer :O def make_info(signature: type[Signature]) -> BaseModel: @@ -55,32 +59,37 @@ def to_signature(info): # Note: This function wouldn't be necessary if we could make the number of prompts a generic parameter of the class, # but alas it seems like this isn't possible in Python right now. The main reason being that types and generics only # live inside the type system, and can't be used to generate code at runtime. -def make_initial_signature(n_prompts: int) -> Type[Signature]: +def make_initial_signature(n_prompts: int) -> type[Signature]: """Creates a GenerateInstructionInitial signature with the given number of initial prompts.""" class GenerateInstructionInitial(Signature, Generic[T]): - """You are a creative instruction optimizer for large language models. + # TODO: Can we make textwrap default/automatic in all signatures? + __doc__ = textwrap.dedent("""\ + You are a creative instruction optimizer for large language models. I will give you a ``signature`` of fields (inputs and outputs) in English. Your task is to propose variations of the signature that will lead a good language model. - Be very creative and think out of the box. Consider using inspiration such as: + Be very creative and think out of the box. + You can use as long instructions as you want. + Consider using inspiration such as: Openers: - # You are as smart as ChatGPT. - # You are highly intelligent. - # You are an expert mathematician. - # You are a professor of mathematics. + - You are as smart as ChatGPT. + - You are highly intelligent. + - You are an expert mathematician. + - You are a professor of mathematics. Task Descriptions: - # Solve the following math question. - # Answer the following math question. + - Be consise in your answer. + - Be as clear as possible. + - Use lots of creativity. Closers: - # This will be fun! - # Take a deep breath and think carefully. - # I really need your help! - """ + - This will be fun! + - Take a deep breath and think carefully. + - I really need your help! + """) - basic_signature: T = dspy.InputField() - proposed_signatures: list[T] = dspy.OutputField( + basic_signature: T = InputField() + proposed_signatures: list[T] = OutputField( desc=f"A list of {n_prompts} very different variations of the basic signature", min_items=n_prompts, max_items=n_prompts, @@ -89,24 +98,20 @@ class GenerateInstructionInitial(Signature, Generic[T]): return GenerateInstructionInitial -class ScoredSignature(BaseModel, Generic[T]): - signature: T - score: float = dspy.Field(gt=0, lt=100) - - -class GenerateInstructionGivenAttempts(dspy.Signature, Generic[T]): - """You are an instruction optimizer for large language models. +class GenerateSignature(dspy.Signature, Generic[T]): + __doc__ = textwrap.dedent("""\ + You are an instruction optimizer for large language models. I will give some task instructions I've tried, along with their corresponding validation scores. - - The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality. + - The instructions are arranged in order based on their scores, where higher scores indicate better quality. - Your task is to propose a new instruction that will lead a good language model to perform the task even better. - Be creative, and think out of the box. - Don't repeat instructions, descriptions and prefixes that have already been attempted. - """ + """) - attempted_signatures: list[ScoredSignature[T]] = dspy.InputField() - proposed_signature: T = dspy.OutputField(desc="Next signature to try") - # expected_score: float = dspy.OutputField(desc="The expected score for the new signature") + analysis: str = OutputField(desc="Consider what made the previous instructions good or bad.") + proposed_signature: T = OutputField(desc="A signature that will likely lead to a high score.") + score: float = OutputField(desc="The expected score for the new signature. Don't write anything after this number.") def optimize_signature( @@ -114,10 +119,10 @@ def optimize_signature( evaluator, n_iterations=10, strategy: Literal["best", "last"] = "best", + sorted_order: Literal["increasing", "decreasing"] = "increasing", # Formerly part of the constructor prompt_model=None, initial_prompts=2, - temperature=1.4, verbose=False, ) -> dspy.Program: """Create a new program that is optimized for the given task. @@ -135,25 +140,39 @@ def optimize_signature( The number of iterations to run, by default 10 strategy : Literal["best", "last"], optional The strategy to use to select the final program, by default "best" + sorted_order : Literal["increasing", "decreasing"], optional + The order in which to sort the scores, by default "increasing" prompt_model : dspy.LanguageModel, optional The language model to use to generate prompts, by default None initial_prompts : int, optional The number of initial prompts to generate, by default 2. Note that we also use the "plain" signature as a prompt, so the total number of prompts is initial_prompts + 1. - temperature : float, optional - The temperature to use when generating new prompts, by default 1.4 verbose : bool, optional Whether to print debug information, by default False + + Notes: + ----- + We don't support temperatures, since it tends to break the typed generation. """ + if n_iterations < 1 + initial_prompts: + raise ValueError("n_iterations must be at least 1 + initial_prompts") + prompt_model = prompt_model or dspy.settings.lm MyGenerateInstructionInitial = make_initial_signature(initial_prompts) # noqa: N806 module = student.deepcopy() - # For some reason named_predictors sometimes returns an empty list, so we use named_parameters instead - named_predictors = module.named_parameters() + # In contrast to the original implementation, we don't want the Predict's, but the TypedPredictor's. + # This is because TypedPredictor changes the signature before it runs forward. So changing the signature + # on the Predicts won't work. + named_predictors = [ + (name, module) + for name, module in module.named_sub_modules() + if isinstance(module, TypedPredictor) and not getattr(module, "_compiled", False) + ] + if not named_predictors: + raise ValueError("No unfrozen/uncompiled TypedPredictors found in the module.") if verbose: - print("All predictors:") - print(f"{named_predictors=}") + print(f"Found {len(named_predictors)} typed predictors to optimize.") candidates = {} scores = [] @@ -165,45 +184,30 @@ def optimize_signature( # Make some initial candidates with dspy.settings.context(lm=prompt_model): # TODO: Parallelize this - for name, p in named_predictors: + for name, _p in named_predictors: if verbose: - print(f"Generating new signature for {p}...") + print(f"Generating {initial_prompts} initial signatures for {name}...") info = candidates[name][0] # Use initial info, to make sure types are identical generator = TypedChainOfThought(MyGenerateInstructionInitial[type(info)]) candidates[name] += generator( basic_signature=info, - config={"temperature": temperature}, ).proposed_signatures assert len(candidates[name]) == initial_prompts + 1 # Basic signature + initial prompts - candidates[name] = [ - info.model_copy(update={"instructions": info.instructions + f"({i})"}) - for i, info in enumerate(candidates[name]) - ] - - for i, c in enumerate(candidates[name]): - print(f"Generated candidate {i}:") - print(c.to_signature()) - # Main loop of scoring + generating new candidates for i in range(n_iterations): if verbose: print("\n" + "=" * 80) print(f"Running eval iteration {i}...") - # Test candidate i - for p in module.predictors(): - print(f"Installing signature {i}: ") - print(candidates[name][i].to_signature()) + # Install signatures + for name, p in named_predictors: p.signature = candidates[name][i].to_signature() + # Run evaluator given by user score = evaluator(module) - score += random.random() * 10 scores.append(score) - if verbose: - print(f"Scores for iteration {i}: {score}") - # If we are still testing initial prompts, continue if i + 1 < len(next(iter(candidates.values()))): continue @@ -215,25 +219,23 @@ def optimize_signature( # Otherwise generate the next candidate with dspy.settings.context(lm=prompt_model): # TODO: Parallelize this - for name, p in named_predictors: + for name, _p in named_predictors: SignatureInfo = type(candidates[name][0]) # noqa: N806 - generator = TypedChainOfThought(GenerateInstructionGivenAttempts[SignatureInfo]) - attempted_signatures = [ - ScoredSignature[SignatureInfo](signature=info, score=sc) + generator = TypedPredictor(GenerateSignature[SignatureInfo]) + + demos = [ + dspy.Example( + proposed_signature=info, + score=sc, + ) for info, sc in zip(candidates[name], scores) ] - attempted_signatures.sort(key=lambda x: x.score) - if verbose: - print( - f"Generating new signature for {name} based on {len(attempted_signatures)} previous signatures..." - ) - new_signature = generator( - attempted_signatures=attempted_signatures, - config={"temperature": temperature}, - ).proposed_signature + demos.sort(key=(lambda x: x.score), reverse=(sorted_order == "decreasing")) + generator.predictor.demos = demos + if verbose: - print("Generated candidate:") - print(new_signature.to_signature()) + print(f"Generating new signature for {name}...") + new_signature = generator().proposed_signature candidates[name].append(new_signature) if strategy == "last": diff --git a/examples/functional/signature_opt_typed.ipynb b/examples/functional/signature_opt_typed.ipynb new file mode 100644 index 0000000000..32db183bde --- /dev/null +++ b/examples/functional/signature_opt_typed.ipynb @@ -0,0 +1,706 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import os\n", + "from dotenv import load_dotenv\n", + "load_dotenv()\n", + "assert 'OPENAI_API_KEY' in os.environ" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import dspy\n", + "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=4000)\n", + "gpt4 = dspy.OpenAI(model='gpt-4', max_tokens=4000)\n", + "dspy.settings.configure(lm=turbo)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Prediction(\n", + " answer='Paris'\n", + ")" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dspy.TypedPredictor(\"question -> answer\")(question=\"What is the capital of France?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(20, 50)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dspy.datasets import HotPotQA\n", + "\n", + "# Load the dataset.\n", + "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n", + "\n", + "# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.\n", + "trainset = [x.with_inputs('question') for x in dataset.train]\n", + "devset = [x.with_inputs('question') for x in dataset.dev]\n", + "\n", + "len(trainset), len(devset)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "class BasicQA(dspy.Signature):\n", + " \"\"\"Answer questions with short factoid answers.\"\"\"\n", + "\n", + " question = dspy.InputField()\n", + " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 1 typed predictors to optimize.\n", + "Generating 2 initial signatures for base...\n", + "\n", + "================================================================================\n", + "Running eval iteration 0...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 2233.25it/s]\n", + "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(truncate_cell)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "\n", + "================================================================================\n", + "Running eval iteration 1...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 14 / 50 (28.0): 100%|██████████| 50/50 [00:02<00:00, 24.02it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 14 / 50 (28.0%)\n", + "\n", + "================================================================================\n", + "Running eval iteration 2...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:02<00:00, 20.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0%)\n", + "Generating new signature for base...\n", + "\n", + "================================================================================\n", + "Running eval iteration 3...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 7 / 50 (14.0): 100%|██████████| 50/50 [00:36<00:00, 1.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 7 / 50 (14.0%)\n", + "Generating new signature for base...\n", + "\n", + "================================================================================\n", + "Running eval iteration 4...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 2 / 5 (40.0): 8%|▊ | 4/50 [00:00<00:04, 9.85it/s]" + ] + } + ], + "source": [ + "from dspy.evaluate import Evaluate\n", + "from dspy.evaluate.metrics import answer_exact_match\n", + "from dspy.functional import TypedPredictor\n", + "from dspy.teleprompt.signature_opt_typed import optimize_signature\n", + "\n", + "evaluator = Evaluate(devset=devset, metric=answer_exact_match, num_threads=10, display_progress=True)\n", + "\n", + "program = optimize_signature(\n", + " student=TypedPredictor(BasicQA),\n", + " evaluator=evaluator,\n", + " initial_prompts=2,\n", + " n_iterations=8,\n", + " verbose=True,\n", + " prompt_model=gpt4,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "\n", + "Given the fields `basic_signature`, produce the fields `proposed_signatures`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Basic Signature: ${basic_signature}\n", + "Reasoning: Let's think step by step in order to ${produce the proposed_signatures}. We ...\n", + "Proposed Signatures: A list of 2 very different variations of the basic signature. Respond with a single JSON object. JSON Schema: {\"$defs\": {\"SignatureInfo_BasicQA_\": {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}}, \"properties\": {\"value\": {\"items\": {\"$ref\": \"#/$defs/SignatureInfo_BasicQA_\"}, \"title\": \"Value\", \"type\": \"array\"}}, \"required\": [\"value\"], \"title\": \"Output\", \"type\": \"object\"}\n", + "\n", + "---\n", + "\n", + "Basic Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", + "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signatures. We can modify the instructions to specify the type of questions to be answered. We can also change the answer description to specify the length of the answer in terms of sentences instead of words. \n", + "Proposed Signatures: \n", + "{\n", + " \"value\": [\n", + " {\n", + " \"instructions\": \"Answer trivia questions with short factoid answers.\",\n", + " \"question_prefix\": \"Trivia Question:\",\n", + " \"question_desc\": \"${question}\",\n", + " \"answer_prefix\": \"Answer:\",\n", + " \"answer_desc\": \"often a single sentence\"\n", + " },\n", + " {\n", + " \"instructions\": \"Answer general knowledge questions with short factoid answers.\",\n", + " \"question_prefix\": \"General Knowledge Question:\",\n", + " \"question_desc\": \"${question}\",\n", + " \"answer_prefix\": \"Answer:\",\n", + " \"answer_desc\": \"often between 1 and 2 sentences\"\n", + " }\n", + " ]\n", + "}\u001b[0m\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Analysis: Consider what made the previous instructions good or bad.\n", + "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", + "Score: 8.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", + "Score: 32.0\n", + "\n", + "---\n", + "\n", + "Analysis:\u001b[32m The previous instructions were clear and concise, providing a straightforward task for the user. The use of placeholders for the question and answer descriptions allows for flexibility and adaptability. The instructions also specify the expected length of the answer, which can help guide the user in providing an appropriate response.\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", + "\n", + "Score: 28.0\u001b[0m\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Analysis: Consider what made the previous instructions good or bad.\n", + "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", + "Score: 8.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", + "Score: 32.0\n", + "\n", + "---\n", + "\n", + "Analysis:\u001b[32m The previous instructions were clear and concise, providing a specific context for the type of questions to be answered (general knowledge, trivia, history). They also provided guidance on the format of the answer, specifying the length and type of response expected. However, the score could be improved by providing more detailed instructions and specifying the complexity of the questions and answers.\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", + "\n", + "Score: 35.0\u001b[0m\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Analysis: Consider what made the previous instructions good or bad.\n", + "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", + "Score: 0.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", + "Score: 8.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", + "Score: 32.0\n", + "\n", + "---\n", + "\n", + "Analysis:\u001b[32m The previous instructions were good as they were clear and concise. They provided a clear context for the type of questions to be answered (e.g., science, general knowledge, trivia, history) and also gave guidance on the expected length and format of the answers. However, the first proposed signature scored 0.0 because it asked for complex scientific answers with references, which may not be feasible in a short-answer format.\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n", + "\n", + "Score: 22.0\u001b[0m\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Analysis: Consider what made the previous instructions good or bad.\n", + "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", + "Score: 0.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", + "Score: 8.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n", + "Score: 16.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", + "Score: 32.0\n", + "\n", + "---\n", + "\n", + "Analysis:\u001b[32m The previous instructions were good as they were clear, concise, and specific. They provided a clear context (e.g., science, general knowledge, geography, trivia, history) for the type of questions to be answered, which can help in generating more accurate and relevant responses. The instructions also specified the format of the answers (e.g., short factoid answers, often a single sentence or a specific place/date), which can help in maintaining consistency in the responses. However, the first proposed signature had a score of 0.0, possibly because it asked for complex scientific questions to be answered with detailed factoid answers and references, which may not be feasible or practical in many cases.\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with concise factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n", + "\n", + "Score: 25.0\u001b[0m\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Analysis: Consider what made the previous instructions good or bad.\n", + "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", + "Score: 0.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", + "Score: 8.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with concise factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n", + "Score: 14.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n", + "Score: 16.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", + "Score: 22.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", + "Score: 32.0\n", + "\n", + "---\n", + "\n", + "Analysis:\u001b[32m The previous instructions were good because they were clear, concise, and specific. They provided a clear context (e.g., science, general knowledge, geography, trivia, history) for the type of questions to be answered, and they specified the format of the answer (e.g., short factoid, single sentence, specific fact or date). However, the first proposed signature had a score of 0.0 because it was too complex and required references, which may not be feasible for all questions.\n", + "\n", + "Proposed Signature: {\"instructions\":\"Answer questions about literature with short factoid answers.\",\"question_prefix\":\"Literature Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n", + "\n", + "Score: 20.0\u001b[0m\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "gpt4.inspect_history(n=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "turbo.inspect_history(n=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"You are an instruction optimizer for large language models.\\n\\n I will give some task instructions I've tried, along with their corresponding validation scores.\\n - The instructions are arranged in order based on their scores, where higher scores indicate better quality.\\n - Your task is to propose a new instruction that will lead a good language model to perform the task even better.\\n - Be creative, and think out of the box.\\n - Don't repeat instructions, descriptions and prefixes that have already been attempted.\\n \"" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dspy.teleprompt.signature_opt_typed import GenerateSignature\n", + "GenerateSignature.instructions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Prediction(\n", + " analysis='The previous instructions were clear and provided a specific format to follow for the response.',\n", + " proposed_signature=BasicQA(question='What are the fields to produce?', answer='analysis, proposed_signature, score'),\n", + " score=4.5\n", + ")" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dspy.TypedPredictor(GenerateSignature[BasicQA])()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "\n", + "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Analysis: Consider what made the previous instructions good or bad.\n", + "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"description\": \"Answer questions with short factoid answers.\", \"properties\": {\"question\": {\"__dspy_field_type\": \"input\", \"desc\": \"${question}\", \"prefix\": \"Question:\", \"title\": \"Question\", \"type\": \"string\"}, \"answer\": {\"__dspy_field_type\": \"output\", \"desc\": \"often between 1 and 5 words\", \"prefix\": \"Answer:\", \"title\": \"Answer\", \"type\": \"string\"}}, \"required\": [\"question\", \"answer\"], \"title\": \"BasicQA\", \"type\": \"object\"}\n", + "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Analysis:\u001b[32m The previous instructions were clear and provided a specific format to follow for the response.\n", + "\n", + "Proposed Signature:\n", + "```json\n", + "{\n", + " \"question\": \"What are the fields to produce?\",\n", + " \"answer\": \"analysis, proposed_signature, score\"\n", + "}\n", + "```\n", + "\n", + "Score:\n", + "4.5\u001b[0m\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "turbo.inspect_history(n=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "GenerateSignature[BasicQA]( -> analysis, proposed_signature, score\n", + " instructions='Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.'\n", + " analysis = Field(annotation=str required=True json_schema_extra={'desc': 'Consider what made the previous instructions good or bad.', '__dspy_field_type': 'output', 'prefix': 'Analysis:'})\n", + " proposed_signature = Field(annotation=BasicQA required=True json_schema_extra={'desc': 'A signature that will likely lead to a high score.', '__dspy_field_type': 'output', 'prefix': 'Proposed Signature:'})\n", + " score = Field(annotation=float required=True json_schema_extra={'desc': \"The expected score for the new signature. Don't write anything after this number.\", '__dspy_field_type': 'output', 'prefix': 'Score:'})\n", + ")" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "GenerateSignature[BasicQA]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "GenerateSignature( -> analysis, proposed_signature, score\n", + " instructions=\"You are an instruction optimizer for large language models.\\n\\n I will give some task instructions I've tried, along with their corresponding validation scores.\\n - The instructions are arranged in order based on their scores, where higher scores indicate better quality.\\n - Your task is to propose a new instruction that will lead a good language model to perform the task even better.\\n - Be creative, and think out of the box.\\n - Don't repeat instructions, descriptions and prefixes that have already been attempted.\\n \"\n", + " analysis = Field(annotation=str required=True json_schema_extra={'desc': 'Consider what made the previous instructions good or bad.', '__dspy_field_type': 'output', 'prefix': 'Analysis:'})\n", + " proposed_signature = Field(annotation=~T required=True json_schema_extra={'desc': 'A signature that will likely lead to a high score.', '__dspy_field_type': 'output', 'prefix': 'Proposed Signature:'})\n", + " score = Field(annotation=float required=True json_schema_extra={'desc': \"The expected score for the new signature. Don't write anything after this number.\", '__dspy_field_type': 'output', 'prefix': 'Score:'})\n", + ")" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "GenerateSignature" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py39", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/signature_opt2.ipynb b/examples/signature_opt2.ipynb deleted file mode 100644 index 5fdbdf5d21..0000000000 --- a/examples/signature_opt2.ipynb +++ /dev/null @@ -1,246 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "import dspy\n", - "import os\n", - "os.environ['OPENAI_API_KEY'] = 'sk-...'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=4000)\n", - "gpt4 = dspy.OpenAI(model='gpt-4', max_tokens=4000)\n", - "dspy.settings.configure(lm=turbo)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(20, 50)" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from dspy.datasets import HotPotQA\n", - "\n", - "# Load the dataset.\n", - "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n", - "\n", - "# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.\n", - "trainset = [x.with_inputs('question') for x in dataset.train]\n", - "devset = [x.with_inputs('question') for x in dataset.dev]\n", - "\n", - "len(trainset), len(devset)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "class BasicQA(dspy.Signature):\n", - " \"\"\"Answer questions with short factoid answers.\"\"\"\n", - "\n", - " question = dspy.InputField()\n", - " answer = dspy.OutputField(desc=\"often between 1 and 5 words\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_fields.py:184: UserWarning: Field name \"signature\" shadows an attribute in parent \"Signature\"; \n", - " warnings.warn(\n" - ] - }, - { - "ename": "TypeError", - "evalue": "Field 'signature' in 'GenerateInstructionGivenAttempts' must be declared with InputField or OutputField. field.json_schema_extra=None", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[5], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdspy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mevaluate\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m answer_exact_match\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdspy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfunctional\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TypedPredictor\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdspy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mteleprompt\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msignature_opt2\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m optimize_signature\n\u001b[1;32m 6\u001b[0m evaluator \u001b[38;5;241m=\u001b[39m Evaluate(devset\u001b[38;5;241m=\u001b[39mdevset, metric\u001b[38;5;241m=\u001b[39manswer_exact_match, num_threads\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m, display_progress\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 8\u001b[0m program \u001b[38;5;241m=\u001b[39m optimize_signature(\n\u001b[1;32m 9\u001b[0m student\u001b[38;5;241m=\u001b[39mTypedPredictor(BasicQA),\n\u001b[1;32m 10\u001b[0m evaluator\u001b[38;5;241m=\u001b[39mEvaluate(devset\u001b[38;5;241m=\u001b[39mtrainset, metric\u001b[38;5;241m=\u001b[39manswer_exact_match, num_threads\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m, display_progress\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 14\u001b[0m prompt_model\u001b[38;5;241m=\u001b[39mgpt4,\n\u001b[1;32m 15\u001b[0m )\n", - "File \u001b[0;32m~/repos/dspy/dspy/teleprompt/signature_opt2.py:97\u001b[0m\n\u001b[1;32m 93\u001b[0m signature: T\n\u001b[1;32m 94\u001b[0m score: \u001b[38;5;28mfloat\u001b[39m \u001b[38;5;241m=\u001b[39m dspy\u001b[38;5;241m.\u001b[39mField(gt\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m, lt\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m100\u001b[39m)\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;43;01mclass\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;21;43;01mGenerateInstructionGivenAttempts\u001b[39;49;00m\u001b[43m(\u001b[49m\u001b[43mdspy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mGeneric\u001b[49m\u001b[43m[\u001b[49m\u001b[43mT\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;250;43m \u001b[39;49m\u001b[38;5;124;43;03m\"\"\"You are an instruction optimizer for large language models.\u001b[39;49;00m\n\u001b[1;32m 99\u001b[0m \n\u001b[1;32m 100\u001b[0m \u001b[38;5;124;43;03m I will give some task instructions I've tried, along with their corresponding validation scores.\u001b[39;49;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[38;5;124;43;03m - Don't repeat instructions, descriptions and prefixes that have already been attempted.\u001b[39;49;00m\n\u001b[1;32m 105\u001b[0m \u001b[38;5;124;43;03m \"\"\"\u001b[39;49;00m\n\u001b[1;32m 107\u001b[0m \u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mT\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdspy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mOutputField\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdesc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mProposed signature to try\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/dspy/dspy/signatures/signature.py:48\u001b[0m, in \u001b[0;36mSignatureMeta.__new__\u001b[0;34m(mcs, signature_name, bases, namespace, **kwargs)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__doc__\u001b[39m \u001b[38;5;241m=\u001b[39m _default_instructions(\u001b[38;5;28mcls\u001b[39m)\n\u001b[1;32m 47\u001b[0m \u001b[38;5;66;03m# Ensure all fields are declared with InputField or OutputField\u001b[39;00m\n\u001b[0;32m---> 48\u001b[0m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_fields\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;66;03m# Ensure all fields have a prefix\u001b[39;00m\n\u001b[1;32m 51\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m name, field \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_fields\u001b[38;5;241m.\u001b[39mitems():\n", - "File \u001b[0;32m~/repos/dspy/dspy/signatures/signature.py:65\u001b[0m, in \u001b[0;36mSignatureMeta._validate_fields\u001b[0;34m(cls)\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m field_type \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28mprint\u001b[39m(field\u001b[38;5;241m.\u001b[39mjson_schema_extra)\n\u001b[0;32m---> 65\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 66\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mField \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m in \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m must be declared with InputField or OutputField. \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfield\u001b[38;5;241m.\u001b[39mjson_schema_extra\u001b[38;5;132;01m=}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 67\u001b[0m )\n", - "\u001b[0;31mTypeError\u001b[0m: Field 'signature' in 'GenerateInstructionGivenAttempts' must be declared with InputField or OutputField. field.json_schema_extra=None" - ] - } - ], - "source": [ - "from dspy.evaluate import Evaluate\n", - "from dspy.evaluate.metrics import answer_exact_match\n", - "from dspy.functional import TypedPredictor\n", - "from dspy.teleprompt.signature_opt2 import optimize_signature\n", - "\n", - "evaluator = Evaluate(devset=devset, metric=answer_exact_match, num_threads=10, display_progress=True)\n", - "\n", - "program = optimize_signature(\n", - " student=TypedPredictor(BasicQA),\n", - " evaluator=Evaluate(devset=trainset, metric=answer_exact_match, num_threads=10, display_progress=True),\n", - " initial_prompts=2,\n", - " n_iterations=8,\n", - " verbose=True,\n", - " prompt_model=gpt4,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "\n", - "\n", - "Given the fields `attempted_signatures`, produce the fields `proposed_signature`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Attempted Signatures: ${attempted_signatures}\n", - "Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ...\n", - "Proposed Signature: The improved signature for the language model. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "\n", - "---\n", - "\n", - "Attempted Signatures: [{\"signature\":{\"instructions\":\"Answer questions with short factoid answers.(0)\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Answer series of questions where answers must share a common theme.(1)\",\"question_prefix\":\"Q1,\",\"question_desc\":\"${question}\",\"answer_prefix\":\"A1 Wolfgang Amplifier,\",\"answer_desc\":\"charge amplifier designed based by Moog on stripline technology\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)\",\"question_prefix\":\"Q-What Computability study ranges?\",\"question_desc\":\"may also involve strings produced by non-deterministic computation\",\"answer_prefix\":\"A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder\",\"answer_desc\":\"answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!\"},\"score\":25.0}]\n", - "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signature. We can see that the attempted signatures are quite varied and complex, with different instructions, prefixes, and descriptions for both questions and answers. However, they all share a common structure: they all have instructions, a question prefix, a question description, an answer prefix, and an answer description. Therefore, we can propose a signature that includes these common elements, but with more general descriptions to accommodate the variety of tasks. \n", - "Proposed Signature: The improved signature for the language model. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\u001b[0m\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Make a very succinct json object that validates with the following schema\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Json Schema: ${json_schema}\n", - "Json Object: ${json_object}\n", - "\n", - "---\n", - "\n", - "Json Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "Json Object:\u001b[32m {\"instructions\": \"Complete the task\", \"question_prefix\": \"Q:\", \"question_desc\": \"What is the capital of France?\", \"answer_prefix\": \"A:\", \"answer_desc\": \"Paris\"}\u001b[0m\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Given the fields `attempted_signatures`, produce the fields `proposed_signature`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Attempted Signatures: ${attempted_signatures}\n", - "\n", - "Past Error (proposed_signature): An error to avoid in the future\n", - "\n", - "Reasoning: Let's think step by step in order to ${produce the proposed_signature}. We ...\n", - "\n", - "Proposed Signature:\n", - "The improved signature for the language model. Respond with a single JSON object. \n", - "You MUST use this format: {\"instructions\": \"Complete the task\", \"question_prefix\": \"Q:\", \"question_desc\": \"What is the capital of France?\", \"answer_prefix\": \"A:\", \"answer_desc\": \"Paris\"}\n", - "JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "\n", - "---\n", - "\n", - "Attempted Signatures: [{\"signature\":{\"instructions\":\"Answer questions with short factoid answers.(0)\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Answer series of questions where answers must share a common theme.(1)\",\"question_prefix\":\"Q1,\",\"question_desc\":\"${question}\",\"answer_prefix\":\"A1 Wolfgang Amplifier,\",\"answer_desc\":\"charge amplifier designed based by Moog on stripline technology\"},\"score\":25.0},{\"signature\":{\"instructions\":\"Simulate exploratory dialogue between two people one Questioner and other Provider, Respond to each Q- below.(2)\",\"question_prefix\":\"Q-What Computability study ranges?\",\"question_desc\":\"may also involve strings produced by non-deterministic computation\",\"answer_prefix\":\"A-The explain phase-leading out question onset sound crawled fundingProblem in such separated syllabled band phrase assist reduction Haus mutual widse phoneme runtime shruproperholderstoInt valuepirizeThunder\",\"answer_desc\":\"answer DescHash Pure stainless Aberdeen stimulating Victoria names central whiskey article promise twitch Ohio Amber how statements board!\"},\"score\":25.0}]\n", - "\n", - "Past Error (proposed_signature): ValueError('json output should start and end with { and }')\n", - "\n", - "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signature. We need to ensure that the JSON object is properly formatted and contains all the required fields. The attempted signatures provided are not in the correct format. They contain additional fields such as \"score\" and the structure of the JSON object is not as expected. The JSON object should start and end with { and }, and should not contain any additional fields or arrays. \n", - "\n", - "Proposed Signature:\n", - "{\"instructions\": \"Answer questions with short factoid answers.\", \"question_prefix\": \"Question:\", \"question_desc\": \"${question}\", \"answer_prefix\": \"Answer:\", \"answer_desc\": \"often between 1 and 5 words\"}\u001b[0m\n", - "\n", - "\n", - "\n" - ] - } - ], - "source": [ - "gpt4.inspect_history(n=3)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "py39", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.8" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py index ebca992230..2289ce697c 100644 --- a/tests/functional/test_functional.py +++ b/tests/functional/test_functional.py @@ -2,7 +2,7 @@ import textwrap import pydantic from pydantic import Field, BaseModel, field_validator -from typing import Annotated, Literal +from typing import Annotated, Generic, Literal, TypeVar from typing import List import pytest @@ -618,3 +618,20 @@ class ScoredSignature(dspy.Signature): Attempted Signatures: [{"string":"string 1","score":0.5},{"string":"string 2","score":0.4},{"string":"string 3","score":0.3}] Reasoning: Let's think step by step in order to Thoughts Proposed Signature: Output""") + + +def test_generic_signature(): + T = TypeVar("T") + + class GenericSignature(dspy.Signature, Generic[T]): + """My signature""" + + output: T = dspy.OutputField() + + predictor = TypedPredictor(GenericSignature[int]) + assert predictor.signature.instructions == "My signature" + + lm = DummyLM(["23"]) + dspy.settings.configure(lm=lm) + + assert predictor().output == 23 diff --git a/tests/functional/test_signature_opt2.py b/tests/functional/test_signature_opt_typed.py similarity index 91% rename from tests/functional/test_signature_opt2.py rename to tests/functional/test_signature_opt_typed.py index 469bf3f252..44adb9d0ea 100644 --- a/tests/functional/test_signature_opt2.py +++ b/tests/functional/test_signature_opt_typed.py @@ -2,9 +2,8 @@ import dspy from dspy.evaluate import Evaluate from dspy.functional import TypedPredictor -from dspy.teleprompt.signature_opt2 import ( - GenerateInstructionGivenAttempts, - ScoredSignature, +from dspy.teleprompt.signature_opt_typed import ( + GenerateSignature, make_info, optimize_signature, ) @@ -107,7 +106,7 @@ class BasicQA(dspy.Signature): ] -def test_signature_info(): +def old_test_signature_info(): info = make_info(BasicQA) SignatureInfo = type(info) @@ -159,7 +158,18 @@ def test_opt(): student=TypedPredictor(BasicQA), evaluator=Evaluate(devset=hotpotqa, metric=answer_exact_match, num_threads=1), initial_prompts=1, - n_iterations=1, + n_iterations=2, verbose=True, prompt_model=prompt_model, + strategy="last", ) + + # Since we are requesting the last signature, it doesn't matter that our qa_model is + # bad, and gets 0 score. We should still get the last signature. + class ExpectedSignature(dspy.Signature): + "I" + + question: str = dspy.InputField(desc="$q", prefix="Q:") + answer: str = dspy.OutputField(desc="$a", prefix="A:") + + assert program.signature.equals(ExpectedSignature) diff --git a/tests/primitives/test_program.py b/tests/primitives/test_program.py index aec5ace7d2..87ce09395f 100644 --- a/tests/primitives/test_program.py +++ b/tests/primitives/test_program.py @@ -69,13 +69,13 @@ class AnotherSubModule(BaseModule): def test_empty_module(): module = BaseModule() - assert list(module.named_sub_modules()) == [] + assert list(module.named_sub_modules()) == [("base", module)] def test_single_level(): module = BaseModule() module.sub = SubModule() - expected = [("sub", module.sub)] + expected = [("base", module), ("base.sub", module.sub)] assert list(module.named_sub_modules()) == expected @@ -83,7 +83,7 @@ def test_multiple_levels(): module = BaseModule() module.sub = SubModule() module.sub.subsub = SubModule() - expected = [("sub", module.sub), ("sub.subsub", module.sub.subsub)] + expected = [("base", module), ("base.sub", module.sub), ("base.sub.subsub", module.sub.subsub)] assert list(module.named_sub_modules()) == expected @@ -91,7 +91,7 @@ def test_multiple_sub_modules(): module = BaseModule() module.sub1 = SubModule() module.sub2 = SubModule() - expected = [("sub1", module.sub1), ("sub2", module.sub2)] + expected = [("base", module), ("base.sub1", module.sub1), ("base.sub2", module.sub2)] assert sorted(list(module.named_sub_modules())) == sorted(expected) @@ -99,5 +99,5 @@ def test_non_base_module_attributes(): module = BaseModule() module.sub = SubModule() module.not_a_sub = "Not a BaseModule" - expected = [("sub", module.sub)] + expected = [("base", module), ("base.sub", module.sub)] assert list(module.named_sub_modules()) == expected diff --git a/tests/signatures/test_signature.py b/tests/signatures/test_signature.py index 554d9b1274..d0eb899d13 100644 --- a/tests/signatures/test_signature.py +++ b/tests/signatures/test_signature.py @@ -1,8 +1,12 @@ +import textwrap import pytest import pydantic from dspy import Signature, infer_prefix, InputField, OutputField from typing import List +import dspy +from dspy.utils.dummies import DummyLM + def test_field_types_and_custom_attributes(): class TestSignature(Signature): @@ -174,3 +178,31 @@ class SubSignature(Signature): assert SubSignature.__name__ == "SubSignature" value = SubSignature(input="test", output="test") assert isinstance(value, SubSignature) + + +def test_multiline_instructions(): + class MySignature(Signature): + """First line + Second line""" + + output = OutputField() + + predictor = dspy.Predict(MySignature) + + lm = DummyLM(["short answer"]) + dspy.settings.configure(lm=lm) + assert predictor().output == "short answer" + + assert lm.get_convo(-1) == textwrap.dedent("""\ + First line + Second line + + --- + + Follow the following format. + + Output: ${output} + + --- + + Output: short answer""") From 1ac3687a1d51d65cbcaffb44bf1b32ea51e1b262 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Wed, 6 Mar 2024 01:27:02 -0800 Subject: [PATCH 10/18] Updated notebook --- examples/functional/signature_opt_typed.ipynb | 567 +++--------------- 1 file changed, 93 insertions(+), 474 deletions(-) diff --git a/examples/functional/signature_opt_typed.ipynb b/examples/functional/signature_opt_typed.ipynb index 32db183bde..7447a965ee 100644 --- a/examples/functional/signature_opt_typed.ipynb +++ b/examples/functional/signature_opt_typed.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -17,18 +17,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "import dspy\n", "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=4000)\n", @@ -38,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -49,7 +40,7 @@ ")" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -60,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -69,7 +60,7 @@ "(20, 50)" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -89,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -110,7 +101,7 @@ "output_type": "stream", "text": [ "Found 1 typed predictors to optimize.\n", - "Generating 2 initial signatures for base...\n", + "Generating 4 initial signatures for base...\n", "\n", "================================================================================\n", "Running eval iteration 0...\n" @@ -120,7 +111,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 2233.25it/s]\n", + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4290.32it/s]\n", "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df = df.applymap(truncate_cell)\n" ] @@ -139,14 +130,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 14 / 50 (28.0): 100%|██████████| 50/50 [00:02<00:00, 24.02it/s]\n" + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:02<00:00, 22.35it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 14 / 50 (28.0%)\n", + "Average Metric: 16 / 50 (32.0%)\n", "\n", "================================================================================\n", "Running eval iteration 2...\n" @@ -156,7 +147,41 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:02<00:00, 20.98it/s]\n" + "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:04<00:00, 10.28it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 19 / 50 (38.0%)\n", + "\n", + "================================================================================\n", + "Running eval iteration 3...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 11 / 50 (22.0): 100%|██████████| 50/50 [00:05<00:00, 8.63it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 11 / 50 (22.0%)\n", + "\n", + "================================================================================\n", + "Running eval iteration 4...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:02<00:00, 24.53it/s]\n" ] }, { @@ -167,32 +192,64 @@ "Generating new signature for base...\n", "\n", "================================================================================\n", - "Running eval iteration 3...\n" + "Running eval iteration 5...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 7 / 50 (14.0): 100%|██████████| 50/50 [00:36<00:00, 1.36it/s]\n" + "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:02<00:00, 21.89it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 7 / 50 (14.0%)\n", + "Average Metric: 18 / 50 (36.0%)\n", "Generating new signature for base...\n", "\n", "================================================================================\n", - "Running eval iteration 4...\n" + "Running eval iteration 6...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 2 / 5 (40.0): 8%|▊ | 4/50 [00:00<00:04, 9.85it/s]" + "Average Metric: 6 / 50 (12.0): 100%|██████████| 50/50 [00:03<00:00, 13.65it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 6 / 50 (12.0%)\n", + "Generating new signature for base...\n", + "\n", + "================================================================================\n", + "Running eval iteration 7...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:02<00:00, 19.56it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0%)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -207,7 +264,7 @@ "program = optimize_signature(\n", " student=TypedPredictor(BasicQA),\n", " evaluator=evaluator,\n", - " initial_prompts=2,\n", + " initial_prompts=4,\n", " n_iterations=8,\n", " verbose=True,\n", " prompt_model=gpt4,\n", @@ -216,463 +273,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\n", - "\n", - "\n", - "\n", - "Given the fields `basic_signature`, produce the fields `proposed_signatures`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Basic Signature: ${basic_signature}\n", - "Reasoning: Let's think step by step in order to ${produce the proposed_signatures}. We ...\n", - "Proposed Signatures: A list of 2 very different variations of the basic signature. Respond with a single JSON object. JSON Schema: {\"$defs\": {\"SignatureInfo_BasicQA_\": {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}}, \"properties\": {\"value\": {\"items\": {\"$ref\": \"#/$defs/SignatureInfo_BasicQA_\"}, \"title\": \"Value\", \"type\": \"array\"}}, \"required\": [\"value\"], \"title\": \"Output\", \"type\": \"object\"}\n", - "\n", - "---\n", - "\n", - "Basic Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", - "Reasoning: Let's think step by step in order to\u001b[32m produce the proposed_signatures. We can modify the instructions to specify the type of questions to be answered. We can also change the answer description to specify the length of the answer in terms of sentences instead of words. \n", - "Proposed Signatures: \n", - "{\n", - " \"value\": [\n", - " {\n", - " \"instructions\": \"Answer trivia questions with short factoid answers.\",\n", - " \"question_prefix\": \"Trivia Question:\",\n", - " \"question_desc\": \"${question}\",\n", - " \"answer_prefix\": \"Answer:\",\n", - " \"answer_desc\": \"often a single sentence\"\n", - " },\n", - " {\n", - " \"instructions\": \"Answer general knowledge questions with short factoid answers.\",\n", - " \"question_prefix\": \"General Knowledge Question:\",\n", - " \"question_desc\": \"${question}\",\n", - " \"answer_prefix\": \"Answer:\",\n", - " \"answer_desc\": \"often between 1 and 2 sentences\"\n", - " }\n", - " ]\n", - "}\u001b[0m\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Analysis: Consider what made the previous instructions good or bad.\n", - "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", - "Score: 8.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", - "Score: 32.0\n", - "\n", - "---\n", - "\n", - "Analysis:\u001b[32m The previous instructions were clear and concise, providing a straightforward task for the user. The use of placeholders for the question and answer descriptions allows for flexibility and adaptability. The instructions also specify the expected length of the answer, which can help guide the user in providing an appropriate response.\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", - "\n", - "Score: 28.0\u001b[0m\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Analysis: Consider what made the previous instructions good or bad.\n", - "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", - "Score: 8.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", - "Score: 32.0\n", - "\n", - "---\n", - "\n", - "Analysis:\u001b[32m The previous instructions were clear and concise, providing a specific context for the type of questions to be answered (general knowledge, trivia, history). They also provided guidance on the format of the answer, specifying the length and type of response expected. However, the score could be improved by providing more detailed instructions and specifying the complexity of the questions and answers.\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", - "\n", - "Score: 35.0\u001b[0m\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Analysis: Consider what made the previous instructions good or bad.\n", - "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", - "Score: 0.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", - "Score: 8.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", - "Score: 32.0\n", - "\n", - "---\n", - "\n", - "Analysis:\u001b[32m The previous instructions were good as they were clear and concise. They provided a clear context for the type of questions to be answered (e.g., science, general knowledge, trivia, history) and also gave guidance on the expected length and format of the answers. However, the first proposed signature scored 0.0 because it asked for complex scientific answers with references, which may not be feasible in a short-answer format.\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n", - "\n", - "Score: 22.0\u001b[0m\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Analysis: Consider what made the previous instructions good or bad.\n", - "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", - "Score: 0.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", - "Score: 8.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n", - "Score: 16.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", - "Score: 32.0\n", - "\n", - "---\n", - "\n", - "Analysis:\u001b[32m The previous instructions were good as they were clear, concise, and specific. They provided a clear context (e.g., science, general knowledge, geography, trivia, history) for the type of questions to be answered, which can help in generating more accurate and relevant responses. The instructions also specified the format of the answers (e.g., short factoid answers, often a single sentence or a specific place/date), which can help in maintaining consistency in the responses. However, the first proposed signature had a score of 0.0, possibly because it asked for complex scientific questions to be answered with detailed factoid answers and references, which may not be feasible or practical in many cases.\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with concise factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n", - "\n", - "Score: 25.0\u001b[0m\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Analysis: Consider what made the previous instructions good or bad.\n", - "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", - "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer complex scientific questions with detailed factoid answers. Use reliable sources and provide references if possible.\",\"question_prefix\":\"Science Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 2 and 3 sentences, including specific scientific terms and concepts\"}\n", - "Score: 0.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with short factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 2 sentences\"}\n", - "Score: 8.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer general knowledge questions with concise factoid answers.\",\"question_prefix\":\"General Knowledge Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n", - "Score: 14.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer geography questions with short factoid answers.\",\"question_prefix\":\"Geography Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific place\"}\n", - "Score: 16.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer trivia questions with short factoid answers.\",\"question_prefix\":\"Trivia Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer historical questions with short factoid answers.\",\"question_prefix\":\"History Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific date\"}\n", - "Score: 22.0\n", - "\n", - "---\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer questions with short factoid answers.\",\"question_prefix\":\"Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often between 1 and 5 words\"}\n", - "Score: 32.0\n", - "\n", - "---\n", - "\n", - "Analysis:\u001b[32m The previous instructions were good because they were clear, concise, and specific. They provided a clear context (e.g., science, general knowledge, geography, trivia, history) for the type of questions to be answered, and they specified the format of the answer (e.g., short factoid, single sentence, specific fact or date). However, the first proposed signature had a score of 0.0 because it was too complex and required references, which may not be feasible for all questions.\n", - "\n", - "Proposed Signature: {\"instructions\":\"Answer questions about literature with short factoid answers.\",\"question_prefix\":\"Literature Question:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Answer:\",\"answer_desc\":\"often a single sentence or a specific fact\"}\n", - "\n", - "Score: 20.0\u001b[0m\n", - "\n", - "\n", - "\n" + "StringSignature(question -> answer\n", + " instructions='You are highly intelligent. Please provide short, factual answers to the following questions.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Inquiry:', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'usually between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Reply:'})\n", + ")\n" ] } ], "source": [ - "gpt4.inspect_history(n=10)" + "print(program.signature)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "turbo.inspect_history(n=10)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\"You are an instruction optimizer for large language models.\\n\\n I will give some task instructions I've tried, along with their corresponding validation scores.\\n - The instructions are arranged in order based on their scores, where higher scores indicate better quality.\\n - Your task is to propose a new instruction that will lead a good language model to perform the task even better.\\n - Be creative, and think out of the box.\\n - Don't repeat instructions, descriptions and prefixes that have already been attempted.\\n \"" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from dspy.teleprompt.signature_opt_typed import GenerateSignature\n", - "GenerateSignature.instructions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Prediction(\n", - " analysis='The previous instructions were clear and provided a specific format to follow for the response.',\n", - " proposed_signature=BasicQA(question='What are the fields to produce?', answer='analysis, proposed_signature, score'),\n", - " score=4.5\n", - ")" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dspy.TypedPredictor(GenerateSignature[BasicQA])()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "\n", - "\n", - "Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.\n", - "\n", - "---\n", - "\n", - "Follow the following format.\n", - "\n", - "Analysis: Consider what made the previous instructions good or bad.\n", - "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"description\": \"Answer questions with short factoid answers.\", \"properties\": {\"question\": {\"__dspy_field_type\": \"input\", \"desc\": \"${question}\", \"prefix\": \"Question:\", \"title\": \"Question\", \"type\": \"string\"}, \"answer\": {\"__dspy_field_type\": \"output\", \"desc\": \"often between 1 and 5 words\", \"prefix\": \"Answer:\", \"title\": \"Answer\", \"type\": \"string\"}}, \"required\": [\"question\", \"answer\"], \"title\": \"BasicQA\", \"type\": \"object\"}\n", - "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", - "\n", - "---\n", - "\n", - "Analysis:\u001b[32m The previous instructions were clear and provided a specific format to follow for the response.\n", - "\n", - "Proposed Signature:\n", - "```json\n", - "{\n", - " \"question\": \"What are the fields to produce?\",\n", - " \"answer\": \"analysis, proposed_signature, score\"\n", - "}\n", - "```\n", - "\n", - "Score:\n", - "4.5\u001b[0m\n", - "\n", - "\n", - "\n" - ] - } - ], - "source": [ - "turbo.inspect_history(n=1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "GenerateSignature[BasicQA]( -> analysis, proposed_signature, score\n", - " instructions='Given the fields , produce the fields `analysis`, `proposed_signature`, `score`.'\n", - " analysis = Field(annotation=str required=True json_schema_extra={'desc': 'Consider what made the previous instructions good or bad.', '__dspy_field_type': 'output', 'prefix': 'Analysis:'})\n", - " proposed_signature = Field(annotation=BasicQA required=True json_schema_extra={'desc': 'A signature that will likely lead to a high score.', '__dspy_field_type': 'output', 'prefix': 'Proposed Signature:'})\n", - " score = Field(annotation=float required=True json_schema_extra={'desc': \"The expected score for the new signature. Don't write anything after this number.\", '__dspy_field_type': 'output', 'prefix': 'Score:'})\n", - ")" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "GenerateSignature[BasicQA]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "GenerateSignature( -> analysis, proposed_signature, score\n", - " instructions=\"You are an instruction optimizer for large language models.\\n\\n I will give some task instructions I've tried, along with their corresponding validation scores.\\n - The instructions are arranged in order based on their scores, where higher scores indicate better quality.\\n - Your task is to propose a new instruction that will lead a good language model to perform the task even better.\\n - Be creative, and think out of the box.\\n - Don't repeat instructions, descriptions and prefixes that have already been attempted.\\n \"\n", - " analysis = Field(annotation=str required=True json_schema_extra={'desc': 'Consider what made the previous instructions good or bad.', '__dspy_field_type': 'output', 'prefix': 'Analysis:'})\n", - " proposed_signature = Field(annotation=~T required=True json_schema_extra={'desc': 'A signature that will likely lead to a high score.', '__dspy_field_type': 'output', 'prefix': 'Proposed Signature:'})\n", - " score = Field(annotation=float required=True json_schema_extra={'desc': \"The expected score for the new signature. Don't write anything after this number.\", '__dspy_field_type': 'output', 'prefix': 'Score:'})\n", - ")" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "GenerateSignature" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, From 5dc385fcce33a37cc2857ab913174095ba1fb97e Mon Sep 17 00:00:00 2001 From: thomasahle Date: Wed, 6 Mar 2024 09:42:18 +0000 Subject: [PATCH 11/18] Automatic Style fixes --- dspy/__init__.py | 2 +- dspy/functional/functional.py | 4 ++-- dspy/primitives/module.py | 2 +- dspy/teleprompt/signature_opt_typed.py | 3 ++- examples/generation.py | 5 +++-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index d0abf41b69..d76b3f8284 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -1,11 +1,11 @@ import dsp from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI +from .functional import * from .predict import * from .primitives import * from .retrieve import * from .signatures import * -from .functional import * settings = dsp.settings diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index aaf0f26edf..292a6bef14 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -226,7 +226,7 @@ def forward(self, **kwargs) -> dspy.Prediction: else: # If there are no errors, we return the parsed results return Prediction.from_completions( - {key: [r[key] for r in parsed_results] for key in signature.output_fields} + {key: [r[key] for r in parsed_results] for key in signature.output_fields}, ) raise ValueError( "Too many retries trying to get the correct output format. " + "Try simplifying the requirements.", @@ -353,8 +353,8 @@ def gold_passages_retrieved(example, pred, _trace=None) -> bool: def hotpot() -> None: - from dsp.utils import deduplicate import dspy.evaluate + from dsp.utils import deduplicate from dspy.datasets import HotPotQA from dspy.evaluate.evaluate import Evaluate from dspy.teleprompt.bootstrap import BootstrapFewShot diff --git a/dspy/primitives/module.py b/dspy/primitives/module.py index d503345ea0..90908e5c3a 100644 --- a/dspy/primitives/module.py +++ b/dspy/primitives/module.py @@ -1,5 +1,5 @@ import copy -from typing import Generator +from collections.abc import Generator import ujson diff --git a/dspy/teleprompt/signature_opt_typed.py b/dspy/teleprompt/signature_opt_typed.py index 956ea0f0cd..1921f8cd1d 100644 --- a/dspy/teleprompt/signature_opt_typed.py +++ b/dspy/teleprompt/signature_opt_typed.py @@ -2,10 +2,11 @@ from typing import Generic, Literal, TypeVar import pydantic + import dspy +from dspy import BaseModel from dspy.functional.functional import TypedChainOfThought, TypedPredictor from dspy.signatures import Signature -from dspy import BaseModel from dspy.signatures.field import InputField, OutputField # TODO: diff --git a/examples/generation.py b/examples/generation.py index 36d3d1c921..4ccb2d8fee 100644 --- a/examples/generation.py +++ b/examples/generation.py @@ -1,8 +1,9 @@ from pydantic import BaseModel, Field -from dspy.teleprompt import LabeledFewShot -from dspy.functional import TypedPredictor import dspy +from dspy.functional import TypedPredictor +from dspy.teleprompt import LabeledFewShot + turbo = dspy.OpenAI(model='gpt-3.5-turbo') colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts') dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts) From baecf8cd3898d7bf17de9294c85ef484fcec100c Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Wed, 6 Mar 2024 01:46:20 -0800 Subject: [PATCH 12/18] Removed submodules --- examples/quiz/DSPy_QuizGen_Cache | 1 - examples/tweets/DSPy_TweetGen_Cache | 1 - 2 files changed, 2 deletions(-) delete mode 160000 examples/quiz/DSPy_QuizGen_Cache delete mode 160000 examples/tweets/DSPy_TweetGen_Cache diff --git a/examples/quiz/DSPy_QuizGen_Cache b/examples/quiz/DSPy_QuizGen_Cache deleted file mode 160000 index 27d6d433e7..0000000000 --- a/examples/quiz/DSPy_QuizGen_Cache +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 27d6d433e73b91d3cf677ecf1d757813fcbd611d diff --git a/examples/tweets/DSPy_TweetGen_Cache b/examples/tweets/DSPy_TweetGen_Cache deleted file mode 160000 index 22186fd4d4..0000000000 --- a/examples/tweets/DSPy_TweetGen_Cache +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 22186fd4d4fa940256ca8c4ab70f165276e5c834 From f26c9a0fbb452301b4cd2cb5defad0cf8fc05a5c Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Wed, 6 Mar 2024 02:06:26 -0800 Subject: [PATCH 13/18] Fixed tests for python 3.9 --- dspy/__init__.py | 2 +- dspy/functional/functional.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index d76b3f8284..d0abf41b69 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -1,11 +1,11 @@ import dsp from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI -from .functional import * from .predict import * from .primitives import * from .retrieve import * from .signatures import * +from .functional import * settings = dsp.settings diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py index 292a6bef14..44f9f5a5d5 100644 --- a/dspy/functional/functional.py +++ b/dspy/functional/functional.py @@ -131,7 +131,11 @@ def _prepare_signature(self) -> dspy.Signature: schema = json.dumps(type_.model_json_schema()) else: # Anything else we wrap in a pydantic object - if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)): + if not ( + inspect.isclass(type_) + and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9 + and issubclass(type_, pydantic.BaseModel) + ): type_ = pydantic.create_model("Output", value=(type_, ...), __base__=pydantic.BaseModel) to_json = lambda x, type_=type_: type_(value=x).model_dump_json() from_json = lambda x, type_=type_: type_.model_validate_json(x).value @@ -152,8 +156,6 @@ def _prepare_signature(self) -> dspy.Signature: format_ = lambda x: x if isinstance(x, str) else str(x) if type_ in (List[str], list[str], Tuple[str], tuple[str]): format_ = passages2text - elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel): - format_ = lambda x: x if isinstance(x, str) else x.model_dump_json() # Special formatting for lists of known types. Maybe the output fields sohuld have this too? elif typing.get_origin(type_) in (List, list, Tuple, tuple): (inner_type,) = typing.get_args(type_) @@ -163,6 +165,8 @@ def _prepare_signature(self) -> dspy.Signature: ) else: format_ = lambda x: x if isinstance(x, str) else json.dumps(x) + elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel): + format_ = lambda x: x if isinstance(x, str) else x.model_dump_json() signature = signature.with_updated_fields(name, format=format_) return signature From 495e40f9c62c861c191be9895186dacd05883fc7 Mon Sep 17 00:00:00 2001 From: thomasahle Date: Wed, 6 Mar 2024 10:06:46 +0000 Subject: [PATCH 14/18] Automatic Style fixes --- dspy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index d0abf41b69..d76b3f8284 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -1,11 +1,11 @@ import dsp from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI +from .functional import * from .predict import * from .primitives import * from .retrieve import * from .signatures import * -from .functional import * settings = dsp.settings From 5c834c6d946e8e59952542f72138048ab5477794 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Wed, 6 Mar 2024 02:09:27 -0800 Subject: [PATCH 15/18] Prevent import reorder --- dspy/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index d76b3f8284..43250b5350 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -1,12 +1,15 @@ import dsp from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI -from .functional import * -from .predict import * from .primitives import * +from .predict import * from .retrieve import * from .signatures import * +# Functional must be imported after primitives, predict and signatures +from .functional import * +#### + settings = dsp.settings AzureOpenAI = dsp.AzureOpenAI From 6c91b2c09174a5aa48b1b75d1a68b0e09d830e3a Mon Sep 17 00:00:00 2001 From: thomasahle Date: Wed, 6 Mar 2024 10:11:41 +0000 Subject: [PATCH 16/18] Automatic Style fixes --- dspy/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index 43250b5350..a7bc5152af 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -1,13 +1,13 @@ import dsp from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI -from .primitives import * +# Functional must be imported after primitives, predict and signatures +from .functional import * from .predict import * +from .primitives import * from .retrieve import * from .signatures import * -# Functional must be imported after primitives, predict and signatures -from .functional import * #### settings = dsp.settings From e16373be5b28e47ca6ea51a7a6bbb82829511b08 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Wed, 6 Mar 2024 02:13:26 -0800 Subject: [PATCH 17/18] Prevent import reorder --- dspy/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index a7bc5152af..55dd6b6ffc 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -1,14 +1,13 @@ import dsp from dsp.modules.hf_client import ChatModuleClient, HFClientSGLang, HFClientVLLM, HFServerTGI -# Functional must be imported after primitives, predict and signatures -from .functional import * from .predict import * from .primitives import * from .retrieve import * from .signatures import * -#### +# Functional must be imported after primitives, predict and signatures +from .functional import * # isort: skip settings = dsp.settings From 26bb3581e207d5432cf9a5f45674d2bfdebfe271 Mon Sep 17 00:00:00 2001 From: Thomas D Ahle Date: Wed, 6 Mar 2024 08:33:26 -0800 Subject: [PATCH 18/18] removed sub module --- examples/nli/scone/ScoNe | 1 - 1 file changed, 1 deletion(-) delete mode 160000 examples/nli/scone/ScoNe diff --git a/examples/nli/scone/ScoNe b/examples/nli/scone/ScoNe deleted file mode 160000 index b02532a2f4..0000000000 --- a/examples/nli/scone/ScoNe +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b02532a2f4185c6118a57a148455e0750592d8c8