diff --git a/examples/requirements/email.pdl b/examples/requirements/email.pdl index c949b33d3..0c6b26f3e 100644 --- a/examples/requirements/email.pdl +++ b/examples/requirements/email.pdl @@ -12,7 +12,9 @@ defs: Does the following email end with Kind regards. Answer with a JSON object and a result field with value True or False only. Email: ${ response } parser: json - - ${ result.result } + - if: ${ result.result } + then: 0 + else: -1000000 fix: function: @@ -36,7 +38,7 @@ text: - "Write an email to ${ name } using the notes following: ${ notes }" - model: ollama_chat/granite3.2:2b requirements: - - description: The email should end with Kind regards + - expect: The email should end with Kind regards evaluate: ${ eval } transformContext: ${ fix } retry: 5 diff --git a/examples/requirements/gsm8k.pdl b/examples/requirements/gsm8k.pdl new file mode 100644 index 000000000..4a1dd5b14 --- /dev/null +++ b/examples/requirements/gsm8k.pdl @@ -0,0 +1,52 @@ +defs: + model: ollama_chat/granite3.3:8b + llm_as_judge: ollama_chat/gpt-oss:20b + + problem: | + Carla is downloading a 200 GB file. Normally she can download 2 GB/minute, + but 40% of the way through the download, Windows forces a restart to install updates, + which takes 20 minutes. Then Carla has to restart the download from the beginning. + How load does it take to download the file? + + extract_answer: + function: + solution: string + return: + lastOf: + - ${ solution } + - Extract the result from the above solution into a JSON object with field "answer" and a float as value. Remove any dollar signs or other symbols. + - model: ${ model } + parser: json + spec: { "answer": number } + + solve: + function: + problem: string + return: + defs: + solution: + text: + - ${ problem } + - "\n\n" + - model: ${ model } + parameters: + temperature: 0.1 + requirements: + - expect: "The solution to this problem should be correct. Problem: ${ problem }" + retry: 1 + answer_obj: + call: ${ extract_answer } + args: + solution: ${ solution } + pdl_context: [] + debug: + lang: python + code: | + print(answer_obj) + result = "" + data: ${ answer_obj.answer } + +call: ${ solve } +args: + problem: ${ problem } + pdl_context: [] diff --git a/examples/requirements/gsm8k_short.pdl b/examples/requirements/gsm8k_short.pdl new file mode 100644 index 000000000..c4bebc874 --- /dev/null +++ b/examples/requirements/gsm8k_short.pdl @@ -0,0 +1,21 @@ +defs: + problem: |+ + Carla is downloading a 200 GB file. Normally she can download 2 GB/minute, + but 40% of the way through the download, Windows forces a restart to install updates, + which takes 20 minutes. Then Carla has to restart the download from the beginning. + How load does it take to download the file? + + +lastOf: +- ${ problem } +- model: ollama_chat/granite3.3:8b + parameters: + temperature: 0.2 +- Extract the result from the above solution into a JSON object with field "answer" and a float as value. Remove any dollar signs or other symbols. +- model: ollama_chat/granite3.3:8b + def: result + parser: json + spec: { "answer": number } + requirements: + - expect: "This solution to the following math problem is correct: ${ problem }" +- ${ result.answer } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b107dd749..242216fde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ version_file = "src/pdl/_version.py" where = ["src"] [tool.setuptools.package-data] -pdl = ["pdl-schema.json"] +pdl = ["pdl-schema.json", "pdl_stlib.pdl"] [tool.pyright] include = ["src", "tests", "examples", "docs"] diff --git a/src/pdl/pdl-schema.json b/src/pdl/pdl-schema.json index 872549646..aa12df222 100644 --- a/src/pdl/pdl-schema.json +++ b/src/pdl/pdl-schema.json @@ -4504,7 +4504,7 @@ "additionalProperties": false, "description": "Single requirement definition.", "properties": { - "description": { + "expect": { "anyOf": [ { "$ref": "#/$defs/LocalizedExpression_TypeVar_" @@ -4514,7 +4514,7 @@ "type": "string" } ], - "title": "Description" + "title": "Expect" }, "evaluate": { "anyOf": [ @@ -4531,6 +4531,7 @@ "type": "null" } ], + "default": null, "title": "Evaluate" }, "transformContext": { @@ -4548,13 +4549,12 @@ "type": "null" } ], + "default": null, "title": "Transformcontext" } }, "required": [ - "description", - "evaluate", - "transformContext" + "expect" ], "title": "RequirementType", "type": "object" diff --git a/src/pdl/pdl_ast.py b/src/pdl/pdl_ast.py index 48959da53..9f13293eb 100644 --- a/src/pdl/pdl_ast.py +++ b/src/pdl/pdl_ast.py @@ -338,13 +338,13 @@ class RequirementType(BaseModel): model_config = ConfigDict(extra="forbid") - description: ExpressionType + expect: ExpressionType """English description of the requirement""" - evaluate: Optional[ExpressionType["FunctionBlock"]] + evaluate: Optional[ExpressionType["FunctionBlock"]] = None """Evaluation function for the requirement""" - transformContext: Optional[ExpressionType["FunctionBlock"]] + transformContext: Optional[ExpressionType["FunctionBlock"]] = None """Function to transform the context for the requirement""" diff --git a/src/pdl/pdl_dumper.py b/src/pdl/pdl_dumper.py index 59b9885b9..0eb8b108a 100644 --- a/src/pdl/pdl_dumper.py +++ b/src/pdl/pdl_dumper.py @@ -412,9 +412,11 @@ def usage_to_dict(usage: PdlUsage) -> dict: def requirement_to_dict(req: RequirementType, json_compatible: bool) -> dict: d: dict = {} - d["description"] = req.description - d["evaluate"] = expr_to_dict(req.evaluate, json_compatible) - d["transformContext"] = expr_to_dict(req.transformContext, json_compatible) + d["expect"] = req.expect + if req.evaluate is not None: + d["evaluate"] = expr_to_dict(req.evaluate, json_compatible) + if req.transformContext is not None: + d["transformContext"] = expr_to_dict(req.transformContext, json_compatible) return d diff --git a/src/pdl/pdl_interpreter.py b/src/pdl/pdl_interpreter.py index 168e781eb..4bba1677a 100644 --- a/src/pdl/pdl_interpreter.py +++ b/src/pdl/pdl_interpreter.py @@ -304,8 +304,21 @@ def process_prog( PDLRuntimeError: If the program raises an error. """ scope = empty_scope | scope + + # Process stdlib + stdlib_file = Path(__file__).parent / "pdl_stdlib.pdl" + stdlib, _ = parse_file(stdlib_file) + _, _, stdlib_dict, _ = process_block( + state.with_yield_background(False).with_yield_result(False), + empty_scope, + stdlib.root, + loc, + ) + + stdlib_scope = scope | PdlDict({"stdlib": stdlib_dict}) + result, document, final_scope, trace = process_block( - state, scope, block=prog.root, loc=loc + state, stdlib_scope, block=prog.root, loc=loc ) return result, document, final_scope, trace @@ -441,7 +454,7 @@ def set_error_to_scope_for_retry( return scope -def process_advanced_block( +def process_advanced_block( # noqa: C901 state: InterpreterState, scope: ScopeType, block: AdvancedBlockType, @@ -471,7 +484,7 @@ def process_advanced_block( return result, background, new_scope, trace -def process_advance_block_retry( +def process_advance_block_retry( # noqa: C901 state: InterpreterState, scope: ScopeType, block: AdvancedBlockType, @@ -494,7 +507,7 @@ def process_advance_block_retry( max_retry = block.retry if block.retry else 0 trial_total = max_retry + 1 - for trial_idx in range(trial_total): + for trial_idx in range(trial_total): # pylint: disable=too-many-nested-blocks try: result, background, new_scope, trace = process_block_body( state, scope, block, loc @@ -502,23 +515,31 @@ def process_advance_block_retry( if block.requirements != []: requirements_satisfied = True for req in block.requirements: - evalfn, _ = process_expr(scope, getattr(req, "evaluate"), loc) - evaluation = evalfn( - requirement=getattr(req, "description"), response=result - ) - if evaluation.result() is False: + evaluate = getattr(req, "evaluate", None) + stdlib_dict: Any = scope["stdlib"] + if evaluate is None: + evaluate = stdlib_dict["requirements"]["evaluation"] + evalfn: Any + evalfn, _ = process_expr(scope, evaluate, loc) + requirement, _ = process_expr(scope, getattr(req, "expect"), loc) + evaluation = evalfn(requirement=requirement, response=result) + if evaluation < -0.3: requirements_satisfied = False - transfn, _ = process_expr( - scope, getattr(req, "transformContext"), loc - ) + transform_context = getattr(req, "transformContext", None) + if transform_context is None: + transform_context = stdlib_dict["requirements"][ + "transformContext" + ] + transfn: Any + transfn, _ = process_expr(scope, transform_context, loc) new_context = transfn( pdl_context=scope["pdl_context"], - requirement=getattr(req, "description"), + requirement=requirement, response=result, ) - scope = scope | {"pdl_context": new_context} + if trial_idx < max_retry: + scope = scope | {"pdl_context": new_context} if requirements_satisfied is False: - print("\nTrying again!") continue result = lazy_apply(id_with_set_first_use_nanos(block.pdl__timing), result) @@ -680,7 +701,7 @@ def process_block_body( yield_result(result.result(), block.kind) if state.yield_background: yield_background(background) - case TextBlock(): # HERE + case TextBlock(): result, background, scope, trace = process_blocks_of( block, "text", diff --git a/src/pdl/pdl_stdlib.pdl b/src/pdl/pdl_stdlib.pdl new file mode 100644 index 000000000..95e881d55 --- /dev/null +++ b/src/pdl/pdl_stdlib.pdl @@ -0,0 +1,69 @@ + +defs: + reward: + function: + response: + return: + defs: + top_logprobs: ${ response.choices[0].logprobs.content[0].top_logprobs} + lastOf: + - for: + tp: ${ top_logprobs } + repeat: + match: ${ tp.token } + with: + - case: "Yes" + then: + data: ${ tp.logprob } + def: lp_y + - case: "No" + then: + data: ${ tp.logprob } + def: lp_n + - lang: python + code: | + import math + result = math.log(math.exp(lp_y) / (math.exp(lp_y) + math.exp(lp_n))) + + requirements: + object: + evaluation: + function: + requirement: string + response: string + llm_as_judge: {optional: string} + return: + lastOf: + - model: ${ llm_as_judge | default('watsonx/meta-llama/llama-3-3-70b-instruct') } + def: evaluation + input: | + Is the following requirement satisfied in the solution below? Requirement: ${ requirement } + ${ response } + + Respond with only 'Yes' or 'No'. + modelResponse: out + parameters: + temperature: 0 + logprobs: true + top_logprobs: 5 + - ${ reward(out) } + + + transformContext: + function: + requirement: string + response: string + model: {optional: string} + return: + lastOf: + - model: ${ model | default('ollama_chat/granite3.3:8b') } + input: | + The following requirement is not satisfied, what instruction can be added to get the correct answer? + Requirement: ${ requirement } + Answer with only the instruction. + - ${ pdl_context } + + + + + diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 697112c7e..653f229fe 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -34,8 +34,11 @@ skip: - examples/optimizer/optimized_grammar_correction.pdl - examples/optimizer/eval_levenshtein.pdl - examples/requirements/email.pdl + - examples/requirements/gsm8k.pdl + - examples/requirements/gsm8k_short.pdl - examples/skeleton-of-thought/tips.pdl - examples/tutorial/sdk/lib.pdl + - src/pdl/pdl_stdlib.pdl with_inputs: examples/tutorial/programs/chatbot.pdl: stdin: |