From 278a5ce0e39669dba1dc821cbe2663dabd802519 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Thu, 23 Jul 2026 17:51:12 +0100 Subject: [PATCH 1/2] Add file-based transport `ServeFile` for evaluation and preview commands --- Kernel/EvaluationFunctionToolkit.wl | 82 +++++++++++++++++++++++++++++ PacletInfo.wl | 2 +- README.md | 42 +++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/Kernel/EvaluationFunctionToolkit.wl b/Kernel/EvaluationFunctionToolkit.wl index 97f53dc..6a87073 100644 --- a/Kernel/EvaluationFunctionToolkit.wl +++ b/Kernel/EvaluationFunctionToolkit.wl @@ -5,6 +5,7 @@ BeginPackage["LambdaFeedback`EvaluationFunctionToolkit`"] (* Export public symbols *) Serve +ServeFile Begin["`Private`"] @@ -146,6 +147,87 @@ Serve[eval_] := Module[{}, Close[socket]; ]; +(* ---- File-based transport ---- *) + +buildErrorResult[msg_] := <| "message" -> ToString[msg] |>; + +(* Catches Wolfram Messages raised by user code so a crash still produces a + JSON response instead of leaving the response file unwritten. *) +safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed]; + +processEvalRequest[evalFn_, requestData_] := Module[ + {params, answer, response, evalParams, result, errorMsg}, + params = requestData["params"]; + answer = params["answer"]; + response = params["response"]; + evalParams = params["params"]; + + Print["Running eval"]; + result = safeCall[evalFn, answer, response, evalParams]; + + If[result === $Failed, + Return[<| "command" -> "eval", "error" -> buildErrorResult["Evaluation function raised an error"] |>] + ]; + + errorMsg = Lookup[result, "error", Null]; + If[errorMsg =!= Null, + Return[<| "command" -> "eval", "error" -> buildErrorResult[errorMsg] |>] + ]; + + <| + "command" -> "eval", + "result" -> <| + "is_correct" -> result["is_correct"], + "feedback" -> result["feedback"] + |> + |> +]; + +processPreviewRequest[previewFn_, requestData_] := Module[ + {params, response, previewParams, result}, + params = requestData["params"]; + response = params["response"]; + previewParams = Lookup[params, "params", <||>]; + + Print["Running preview"]; + result = safeCall[previewFn, response, previewParams]; + + If[result === $Failed, + Return[<| "command" -> "preview", "error" -> buildErrorResult["Preview function raised an error"] |>] + ]; + + <| "command" -> "preview", "result" -> <| "preview" -> result |> |> +]; + +processFileRequest[evalFn_, previewFn_, requestData_] := Module[{command}, + command = Lookup[requestData, "command", "unknown"]; + Which[ + command === "eval", processEvalRequest[evalFn, requestData], + command === "preview", processPreviewRequest[previewFn, requestData], + True, <| "command" -> command, "error" -> buildErrorResult["Unknown command: " <> ToString[command]] |> + ] +]; + +ServeFile[evalFn_, previewFn_, requestPath_String, responsePath_String] := Module[ + {requestData, responseData}, + requestData = Import[requestPath, "JSON"] //. List :> Association; + + Print["Input"]; + Print[requestData]; + + responseData = processFileRequest[evalFn, previewFn, requestData]; + + Print["Output"]; + Print[responseData]; + + Export[responsePath, responseData, "JSON", "Compact" -> True]; +]; + +ServeFile[evalFn_, previewFn_] := Module[{argv}, + argv = Rest[$ScriptCommandLine]; + ServeFile[evalFn, previewFn, argv[[1]], argv[[2]]] +]; + End[] (* End `Private` *) EndPackage[] diff --git a/PacletInfo.wl b/PacletInfo.wl index 71b7805..ba09660 100644 --- a/PacletInfo.wl +++ b/PacletInfo.wl @@ -7,7 +7,7 @@ PacletObject[ "Creator" -> "Andreas Pfurtscheller", "License" -> "MIT", "PublisherID" -> "LambdaFeedback", - "Version" -> "1.0.0", + "Version" -> "1.1.0", "WolframVersion" -> "13.+", "PrimaryContext" -> "LambdaFeedback`EvaluationFunctionToolkit`", "Extensions" -> { diff --git a/README.md b/README.md index 57ef328..ab66290 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,48 @@ A collection of utilities for creating Lambda Feedback evaluation functions for the Wolfram Language. +## Usage + +The toolkit exposes one function per Shimmy comms transport. Currently: + +- `ServeFile[EvaluationFunction, PreviewFunction]` — the file-based transport + (`FUNCTION_INTERFACE="file"`): reads a request JSON file and writes a + response JSON file, as invoked by `wolframscript -f evaluation_function.wl + request.json response.json`. +- `Serve[EvaluationFunction]` — a socket/JSON-RPC transport for the `eval` + method only. + +More Shimmy transports (stdio, ipc) are expected to be added over time, +mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s +`lf_toolkit/io/`. + +```wolfram +Needs["LambdaFeedback`EvaluationFunctionToolkit`"] + +EvaluationFunction[answer_, response_, params_] := <| + "is_correct" -> answer == response, + "feedback" -> If[answer == response, "Correct!", "Incorrect!"], + "error" -> Null +|>; + +PreviewFunction[response_, params_] := <| + "latex" -> ToString[ToExpression[response], TeXForm], + "sympy" -> ToString[ToExpression[response], InputForm] +|>; + +ServeFile[EvaluationFunction, PreviewFunction] +``` + +`EvaluationFunction` must return an association with `is_correct`, +`feedback`, and `error` (`Null` on success, an error message otherwise). +`PreviewFunction`'s return value is passed straight through under +`result.preview` — it is not inspected for an `"error"` key, so preview +functions are free to embed their own inline error/unavailable state. + +If `EvaluationFunction` or `PreviewFunction` raises a Wolfram error/message +(not a `Throw`/`Abort`), `ServeFile` catches it and returns a normal +`{"command", "error"}` JSON response rather than crashing. + ## Development ### Publishing From 1eec4a3073c91d768c77b3ec6450c3ce03acfd1c Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Thu, 23 Jul 2026 17:51:23 +0100 Subject: [PATCH 2/2] Add test suite for `ServeFile` to validate evaluation and preview handling --- Tests/ServeFile.wlt | 101 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 Tests/ServeFile.wlt diff --git a/Tests/ServeFile.wlt b/Tests/ServeFile.wlt new file mode 100644 index 0000000..32e4904 --- /dev/null +++ b/Tests/ServeFile.wlt @@ -0,0 +1,101 @@ +(* ::Package:: *) + +Needs["LambdaFeedback`EvaluationFunctionToolkit`"] + +evalOk[answer_, response_, params_] := <| + "is_correct" -> True, "feedback" -> "Correct!", "error" -> Null +|>; + +evalFail[answer_, response_, params_] := <| + "is_correct" -> False, "feedback" -> "", "error" -> "bad answer" +|>; + +evalCrash[answer_, response_, params_] := 1/0; + +previewOk[response_, params_] := <|"latex" -> "x^2", "sympy" -> "x**2"|>; + +previewNestedError[response_, params_] := <| + "error" -> <|"message" -> "Failed to parse"|> +|>; + +withRequestResponse[requestAssoc_, testFn_] := Module[ + {requestPath, responsePath, result}, + requestPath = FileNameJoin[{$TemporaryDirectory, "servefile-test-request-" <> ToString[RandomInteger[10^9]] <> ".json"}]; + responsePath = FileNameJoin[{$TemporaryDirectory, "servefile-test-response-" <> ToString[RandomInteger[10^9]] <> ".json"}]; + Export[requestPath, requestAssoc, "JSON", "Compact" -> True]; + result = testFn[requestPath, responsePath]; + Quiet[DeleteFile[{requestPath, responsePath}]]; + result +]; + +VerificationTest[ + withRequestResponse[ + <|"command" -> "eval", "params" -> <|"answer" -> "x", "response" -> "x", "params" -> <||>|>|>, + Function[{req, resp}, + ServeFile[evalOk, previewOk, req, resp]; + Import[resp, "RawJSON"] + ] + ], + <|"command" -> "eval", "result" -> <|"is_correct" -> True, "feedback" -> "Correct!"|>|>, + TestID -> "ServeFile-eval-success" +] + +VerificationTest[ + withRequestResponse[ + <|"command" -> "eval", "params" -> <|"answer" -> "x", "response" -> "y", "params" -> <||>|>|>, + Function[{req, resp}, + ServeFile[evalFail, previewOk, req, resp]; + Import[resp, "RawJSON"] + ] + ], + <|"command" -> "eval", "error" -> <|"message" -> "bad answer"|>|>, + TestID -> "ServeFile-eval-error" +] + +VerificationTest[ + withRequestResponse[ + <|"command" -> "eval", "params" -> <|"answer" -> "x", "response" -> "y", "params" -> <||>|>|>, + Function[{req, resp}, + ServeFile[evalCrash, previewOk, req, resp]; + Import[resp, "RawJSON"] + ] + ], + <|"command" -> "eval", "error" -> <|"message" -> "Evaluation function raised an error"|>|>, + TestID -> "ServeFile-eval-crash-is-caught" +] + +VerificationTest[ + withRequestResponse[ + <|"command" -> "preview", "params" -> <|"response" -> "x^2"|>|>, + Function[{req, resp}, + ServeFile[evalOk, previewOk, req, resp]; + Import[resp, "RawJSON"] + ] + ], + <|"command" -> "preview", "result" -> <|"preview" -> <|"latex" -> "x^2", "sympy" -> "x**2"|>|>|>, + TestID -> "ServeFile-preview-success" +] + +VerificationTest[ + withRequestResponse[ + <|"command" -> "preview", "params" -> <|"response" -> "x^2"|>|>, + Function[{req, resp}, + ServeFile[evalOk, previewNestedError, req, resp]; + Import[resp, "RawJSON"] + ] + ], + <|"command" -> "preview", "result" -> <|"preview" -> <|"error" -> <|"message" -> "Failed to parse"|>|>|>|>, + TestID -> "ServeFile-preview-nested-error-is-passthrough" +] + +VerificationTest[ + withRequestResponse[ + <|"command" -> "frobnicate", "params" -> <||>|>, + Function[{req, resp}, + ServeFile[evalOk, previewOk, req, resp]; + Import[resp, "RawJSON"] + ] + ], + <|"command" -> "frobnicate", "error" -> <|"message" -> "Unknown command: frobnicate"|>|>, + TestID -> "ServeFile-unknown-command" +]