Merge pull request #5403 from langchain-ai/dqbd/run-on-dataset-criter…

…ia-fix

fix[runOnDataset]: accept custom criteria in config factory

langchain/src/smith/config.ts

@@ -213,9 +213,24 @@ const getSingleStringifiedValue = (value: unknown) => {
  * @example
  * ```ts
  * const evalConfig = {
+ *   evaluators: [Criteria("helpfulness")],
+ * };
+ * @example
+ * ```ts
+ * const evalConfig = {
+ *   evaluators: [
+ *     Criteria({
+ *       "isCompliant": "Does the submission comply with the requirements of XYZ"
+ *     })
+ *   ],
+ * };
+ * @example
+ * ```ts
+ * const evalConfig = {
  *   evaluators: [{
  *     evaluatorType: "criteria",
  *     criteria: "helpfulness"
+ *     formatEvaluatorInputs: ...
  *   }]
  * };
  * ```
@@ -224,7 +239,8 @@ const getSingleStringifiedValue = (value: unknown) => {
  * const evalConfig = {
  *   evaluators: [{
  *     evaluatorType: "criteria",
- *     criteria: { "isCompliant": "Does the submission comply with the requirements of XYZ"
+ *     criteria: { "isCompliant": "Does the submission comply with the requirements of XYZ" },
+ *     formatEvaluatorInputs: ...
  *   }]
  * };
  */
@@ -249,7 +265,7 @@ export type Criteria = EvalConfig & {
 export type CriteriaEvalChainConfig = Criteria;
 
 export function Criteria(
-  criteria: CriteriaType,
+  criteria: CriteriaType | Record<string, string>,
   config?: Pick<
     Partial<LabeledCriteria>,
     "formatEvaluatorInputs" | "llm" | "feedbackKey"
@@ -262,10 +278,19 @@ export function Criteria(
       input: getSingleStringifiedValue(payload.rawInput),
     }));
 
+  if (typeof criteria !== "string" && Object.keys(criteria).length !== 1) {
+    throw new Error(
+      "Only one criteria key is allowed when specifying custom criteria."
+    );
+  }
+
+  const criteriaKey =
+    typeof criteria === "string" ? criteria : Object.keys(criteria)[0];
+
   return {
     evaluatorType: "criteria",
     criteria,
-    feedbackKey: config?.feedbackKey ?? criteria,
+    feedbackKey: config?.feedbackKey ?? criteriaKey,
     llm: config?.llm,
     formatEvaluatorInputs,
   };
@@ -283,9 +308,24 @@ export function Criteria(
  * @example
  * ```ts
  * const evalConfig = {
+ *   evaluators: [LabeledCriteria("correctness")],
+ * };
+ * @example
+ * ```ts
+ * const evalConfig = {
+ *   evaluators: [
+ *     LabeledCriteria({
+ *       "mentionsAllFacts": "Does the include all facts provided in the reference?"
+ *     })
+ *   ],
+ * };
+ * @example
+ * ```ts
+ * const evalConfig = {
  *   evaluators: [{
  *     evaluatorType: "labeled_criteria",
- *     criteria: "correctness"
+ *     criteria: "correctness",
+ *     formatEvaluatorInputs: ...
  *   }],
  * };
  * ```
@@ -294,7 +334,8 @@ export function Criteria(
  * const evalConfig = {
  *   evaluators: [{
  *     evaluatorType: "labeled_criteria",
- *     criteria: { "mentionsAllFacts": "Does the include all facts provided in the reference?" }
+ *     criteria: { "mentionsAllFacts": "Does the include all facts provided in the reference?" },
+ *     formatEvaluatorInputs: ...
  *   }],
  * };
  */
@@ -316,7 +357,7 @@ export type LabeledCriteria = EvalConfig & {
 };
 
 export function LabeledCriteria(
-  criteria: CriteriaType,
+  criteria: CriteriaType | Record<string, string>,
   config?: Pick<
     Partial<LabeledCriteria>,
     "formatEvaluatorInputs" | "llm" | "feedbackKey"
@@ -330,10 +371,19 @@ export function LabeledCriteria(
       reference: getSingleStringifiedValue(payload.rawReferenceOutput),
     }));
 
+  if (typeof criteria !== "string" && Object.keys(criteria).length !== 1) {
+    throw new Error(
+      "Only one labeled criteria key is allowed when specifying custom criteria."
+    );
+  }
+
+  const criteriaKey =
+    typeof criteria === "string" ? criteria : Object.keys(criteria)[0];
+
   return {
     evaluatorType: "labeled_criteria",
     criteria,
-    feedbackKey: config?.feedbackKey ?? criteria,
+    feedbackKey: config?.feedbackKey ?? criteriaKey,
     llm: config?.llm,
     formatEvaluatorInputs,
   };