Feature/file io

Kernel/EvaluationFunctionToolkit.wl

@@ -5,6 +5,7 @@ BeginPackage["LambdaFeedback`EvaluationFunctionToolkit`"]
 (* Export public symbols *)
 
 Serve
+ServeFile
 
 Begin["`Private`"]
 
@@ -146,6 +147,87 @@ Serve[eval_] := Module[{},
   Close[socket];
 ];
 
+(* ---- File-based transport ---- *)
+
+buildErrorResult[msg_] := <| "message" -> ToString[msg] |>;
+
+(* Catches Wolfram Messages raised by user code so a crash still produces a
+   JSON response instead of leaving the response file unwritten. *)
+safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed];
+
+processEvalRequest[evalFn_, requestData_] := Module[
+  {params, answer, response, evalParams, result, errorMsg},
+  params = requestData["params"];
+  answer = params["answer"];
+  response = params["response"];
+  evalParams = params["params"];
+
+  Print["Running eval"];
+  result = safeCall[evalFn, answer, response, evalParams];
+
+  If[result === $Failed,
+    Return[<| "command" -> "eval", "error" -> buildErrorResult["Evaluation function raised an error"] |>]
+  ];
+
+  errorMsg = Lookup[result, "error", Null];
+  If[errorMsg =!= Null,
+    Return[<| "command" -> "eval", "error" -> buildErrorResult[errorMsg] |>]
+  ];
+
+  <|
+    "command" -> "eval",
+    "result" -> <|
+      "is_correct" -> result["is_correct"],
+      "feedback" -> result["feedback"]
+    |>
+  |>
+];
+
+processPreviewRequest[previewFn_, requestData_] := Module[
+  {params, response, previewParams, result},
+  params = requestData["params"];
+  response = params["response"];
+  previewParams = Lookup[params, "params", <||>];
+
+  Print["Running preview"];
+  result = safeCall[previewFn, response, previewParams];
+
+  If[result === $Failed,
+    Return[<| "command" -> "preview", "error" -> buildErrorResult["Preview function raised an error"] |>]
+  ];
+
+  <| "command" -> "preview", "result" -> <| "preview" -> result |> |>
+];
+
+processFileRequest[evalFn_, previewFn_, requestData_] := Module[{command},
+  command = Lookup[requestData, "command", "unknown"];
+  Which[
+    command === "eval", processEvalRequest[evalFn, requestData],
+    command === "preview", processPreviewRequest[previewFn, requestData],
+    True, <| "command" -> command, "error" -> buildErrorResult["Unknown command: " <> ToString[command]] |>
+  ]
+];
+
+ServeFile[evalFn_, previewFn_, requestPath_String, responsePath_String] := Module[
+  {requestData, responseData},
+  requestData = Import[requestPath, "JSON"] //. List :> Association;
+
+  Print["Input"];
+  Print[requestData];
+
+  responseData = processFileRequest[evalFn, previewFn, requestData];
+
+  Print["Output"];
+  Print[responseData];
+
+  Export[responsePath, responseData, "JSON", "Compact" -> True];
+];
+
+ServeFile[evalFn_, previewFn_] := Module[{argv},
+  argv = Rest[$ScriptCommandLine];
+  ServeFile[evalFn, previewFn, argv[[1]], argv[[2]]]
+];
+
 End[] (* End `Private` *)
 
 EndPackage[]

PacletInfo.wl

@@ -7,7 +7,7 @@ PacletObject[
     "Creator" -> "Andreas Pfurtscheller",
     "License" -> "MIT",
     "PublisherID" -> "LambdaFeedback",
-    "Version" -> "1.0.0",
+    "Version" -> "1.1.0",
     "WolframVersion" -> "13.+",
     "PrimaryContext" -> "LambdaFeedback`EvaluationFunctionToolkit`",
     "Extensions" -> {

README.md

@@ -4,6 +4,48 @@
 
 A collection of utilities for creating Lambda Feedback evaluation functions for the Wolfram Language.
 
+## Usage
+
+The toolkit exposes one function per Shimmy comms transport. Currently:
+
+- `ServeFile[EvaluationFunction, PreviewFunction]` — the file-based transport
+  (`FUNCTION_INTERFACE="file"`): reads a request JSON file and writes a
+  response JSON file, as invoked by `wolframscript -f evaluation_function.wl
+  request.json response.json`.
+- `Serve[EvaluationFunction]` — a socket/JSON-RPC transport for the `eval`
+  method only.
+
+More Shimmy transports (stdio, ipc) are expected to be added over time,
+mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s
+`lf_toolkit/io/`.
+
+```wolfram
+Needs["LambdaFeedback`EvaluationFunctionToolkit`"]
+
+EvaluationFunction[answer_, response_, params_] := <|
+  "is_correct" -> answer == response,
+  "feedback" -> If[answer == response, "Correct!", "Incorrect!"],
+  "error" -> Null
+|>;
+
+PreviewFunction[response_, params_] := <|
+  "latex" -> ToString[ToExpression[response], TeXForm],
+  "sympy" -> ToString[ToExpression[response], InputForm]
+|>;
+
+ServeFile[EvaluationFunction, PreviewFunction]
+```
+
+`EvaluationFunction` must return an association with `is_correct`,
+`feedback`, and `error` (`Null` on success, an error message otherwise).
+`PreviewFunction`'s return value is passed straight through under
+`result.preview` — it is not inspected for an `"error"` key, so preview
+functions are free to embed their own inline error/unavailable state.
+
+If `EvaluationFunction` or `PreviewFunction` raises a Wolfram error/message
+(not a `Throw`/`Abort`), `ServeFile` catches it and returns a normal
+`{"command", "error"}` JSON response rather than crashing.
+
 ## Development
 
 ### Publishing

Tests/ServeFile.wlt

@@ -0,0 +1,101 @@
+(* ::Package:: *)
+
+Needs["LambdaFeedback`EvaluationFunctionToolkit`"]
+
+evalOk[answer_, response_, params_] := <|
+  "is_correct" -> True, "feedback" -> "Correct!", "error" -> Null
+|>;
+
+evalFail[answer_, response_, params_] := <|
+  "is_correct" -> False, "feedback" -> "", "error" -> "bad answer"
+|>;
+
+evalCrash[answer_, response_, params_] := 1/0;
+
+previewOk[response_, params_] := <|"latex" -> "x^2", "sympy" -> "x**2"|>;
+
+previewNestedError[response_, params_] := <|
+  "error" -> <|"message" -> "Failed to parse"|>
+|>;
+
+withRequestResponse[requestAssoc_, testFn_] := Module[
+  {requestPath, responsePath, result},
+  requestPath = FileNameJoin[{$TemporaryDirectory, "servefile-test-request-" <> ToString[RandomInteger[10^9]] <> ".json"}];
+  responsePath = FileNameJoin[{$TemporaryDirectory, "servefile-test-response-" <> ToString[RandomInteger[10^9]] <> ".json"}];
+  Export[requestPath, requestAssoc, "JSON", "Compact" -> True];
+  result = testFn[requestPath, responsePath];
+  Quiet[DeleteFile[{requestPath, responsePath}]];
+  result
+];
+
+VerificationTest[
+  withRequestResponse[
+    <|"command" -> "eval", "params" -> <|"answer" -> "x", "response" -> "x", "params" -> <||>|>|>,
+    Function[{req, resp},
+      ServeFile[evalOk, previewOk, req, resp];
+      Import[resp, "RawJSON"]
+    ]
+  ],
+  <|"command" -> "eval", "result" -> <|"is_correct" -> True, "feedback" -> "Correct!"|>|>,
+  TestID -> "ServeFile-eval-success"
+]
+
+VerificationTest[
+  withRequestResponse[
+    <|"command" -> "eval", "params" -> <|"answer" -> "x", "response" -> "y", "params" -> <||>|>|>,
+    Function[{req, resp},
+      ServeFile[evalFail, previewOk, req, resp];
+      Import[resp, "RawJSON"]
+    ]
+  ],
+  <|"command" -> "eval", "error" -> <|"message" -> "bad answer"|>|>,
+  TestID -> "ServeFile-eval-error"
+]
+
+VerificationTest[
+  withRequestResponse[
+    <|"command" -> "eval", "params" -> <|"answer" -> "x", "response" -> "y", "params" -> <||>|>|>,
+    Function[{req, resp},
+      ServeFile[evalCrash, previewOk, req, resp];
+      Import[resp, "RawJSON"]
+    ]
+  ],
+  <|"command" -> "eval", "error" -> <|"message" -> "Evaluation function raised an error"|>|>,
+  TestID -> "ServeFile-eval-crash-is-caught"
+]
+
+VerificationTest[
+  withRequestResponse[
+    <|"command" -> "preview", "params" -> <|"response" -> "x^2"|>|>,
+    Function[{req, resp},
+      ServeFile[evalOk, previewOk, req, resp];
+      Import[resp, "RawJSON"]
+    ]
+  ],
+  <|"command" -> "preview", "result" -> <|"preview" -> <|"latex" -> "x^2", "sympy" -> "x**2"|>|>|>,
+  TestID -> "ServeFile-preview-success"
+]
+
+VerificationTest[
+  withRequestResponse[
+    <|"command" -> "preview", "params" -> <|"response" -> "x^2"|>|>,
+    Function[{req, resp},
+      ServeFile[evalOk, previewNestedError, req, resp];
+      Import[resp, "RawJSON"]
+    ]
+  ],
+  <|"command" -> "preview", "result" -> <|"preview" -> <|"error" -> <|"message" -> "Failed to parse"|>|>|>|>,
+  TestID -> "ServeFile-preview-nested-error-is-passthrough"
+]
+
+VerificationTest[
+  withRequestResponse[
+    <|"command" -> "frobnicate", "params" -> <||>|>,
+    Function[{req, resp},
+      ServeFile[evalOk, previewOk, req, resp];
+      Import[resp, "RawJSON"]
+    ]
+  ],
+  <|"command" -> "frobnicate", "error" -> <|"message" -> "Unknown command: frobnicate"|>|>,
+  TestID -> "ServeFile-unknown-command"
+]