From e310b7273cb3fc36a74fed9a68f498d8a4275a62 Mon Sep 17 00:00:00 2001 From: Olzhas Nurpeisov Date: Mon, 26 May 2025 15:05:17 +0100 Subject: [PATCH] feat: add metadata support --- .gitignore | 1 + src/lmnr/sdk/client/asynchronous/resources/evals.py | 7 ++++--- src/lmnr/sdk/client/synchronous/resources/evals.py | 7 ++++--- src/lmnr/sdk/evaluations.py | 2 ++ src/lmnr/sdk/types.py | 4 ++++ 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d36cbe80..b2bff549 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ uv.lock lmnr_engine/ .vscode .venv +.idea \ No newline at end of file diff --git a/src/lmnr/sdk/client/asynchronous/resources/evals.py b/src/lmnr/sdk/client/asynchronous/resources/evals.py index c7c1f4f0..7aacb3e8 100644 --- a/src/lmnr/sdk/client/asynchronous/resources/evals.py +++ b/src/lmnr/sdk/client/asynchronous/resources/evals.py @@ -1,12 +1,13 @@ """Evals resource for interacting with Laminar evaluations API.""" import uuid -from typing import Optional +from typing import Optional, Union from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource from lmnr.sdk.types import ( InitEvaluationResponse, EvaluationResultDatapoint, + PartialEvaluationDatapoint, ) @@ -39,14 +40,14 @@ async def init( async def save_datapoints( self, eval_id: uuid.UUID, - datapoints: list[EvaluationResultDatapoint], + datapoints: list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]], group_name: Optional[str] = None, ): """Save evaluation datapoints. Args: eval_id (uuid.UUID): The evaluation ID. - datapoints (list[EvaluationResultDatapoint]): The datapoints to save. + datapoints (list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]]): The datapoints to save. group_name (Optional[str], optional): Group name for the datapoints. Defaults to None. Raises: diff --git a/src/lmnr/sdk/client/synchronous/resources/evals.py b/src/lmnr/sdk/client/synchronous/resources/evals.py index 3fe6f66c..7efcb8d8 100644 --- a/src/lmnr/sdk/client/synchronous/resources/evals.py +++ b/src/lmnr/sdk/client/synchronous/resources/evals.py @@ -2,12 +2,13 @@ import uuid import urllib.parse -from typing import Optional +from typing import Optional, Union from lmnr.sdk.client.synchronous.resources.base import BaseResource from lmnr.sdk.types import ( InitEvaluationResponse, EvaluationResultDatapoint, + PartialEvaluationDatapoint, GetDatapointsResponse, ) @@ -41,14 +42,14 @@ def init( def save_datapoints( self, eval_id: uuid.UUID, - datapoints: list[EvaluationResultDatapoint], + datapoints: list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]], group_name: Optional[str] = None, ): """Save evaluation datapoints. Args: eval_id (uuid.UUID): The evaluation ID. - datapoints (list[EvaluationResultDatapoint]): The datapoints to save. + datapoints (list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]]): The datapoints to save. group_name (Optional[str], optional): Group name for the datapoints. Defaults to None. Raises: diff --git a/src/lmnr/sdk/evaluations.py b/src/lmnr/sdk/evaluations.py index d78f56fa..26ee0292 100644 --- a/src/lmnr/sdk/evaluations.py +++ b/src/lmnr/sdk/evaluations.py @@ -312,6 +312,7 @@ async def _evaluate_datapoint( index=index, trace_id=trace_id, executor_span_id=executor_span_id, + metadata=datapoint.metadata, ) # First, create datapoint with trace_id so that we can show the dp in the UI await self.client._evals.save_datapoints( @@ -367,6 +368,7 @@ async def _evaluate_datapoint( human_evaluators=self.human_evaluators, executor_span_id=executor_span_id, index=index, + metadata=datapoint.metadata, ) # Create background upload task without awaiting it diff --git a/src/lmnr/sdk/types.py b/src/lmnr/sdk/types.py index b901158c..21e2e101 100644 --- a/src/lmnr/sdk/types.py +++ b/src/lmnr/sdk/types.py @@ -66,6 +66,7 @@ class PartialEvaluationDatapoint(pydantic.BaseModel): index: int trace_id: uuid.UUID executor_span_id: uuid.UUID + metadata: EvaluationDatapointMetadata = pydantic.Field(default=None) # uuid is not serializable by default, so we need to convert it to a string def to_dict(self): @@ -77,6 +78,7 @@ def to_dict(self): "index": self.index, "traceId": str(self.trace_id), "executorSpanId": str(self.executor_span_id), + "metadata": serialize(self.metadata) if self.metadata is not None else None, } except Exception as e: raise ValueError(f"Error serializing PartialEvaluationDatapoint: {e}") @@ -92,6 +94,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel): human_evaluators: list[HumanEvaluator] = pydantic.Field(default_factory=list) trace_id: uuid.UUID executor_span_id: uuid.UUID + metadata: EvaluationDatapointMetadata = pydantic.Field(default=None) # uuid is not serializable by default, so we need to convert it to a string def to_dict(self): @@ -115,6 +118,7 @@ def to_dict(self): ], "executorSpanId": str(self.executor_span_id), "index": self.index, + "metadata": serialize(self.metadata) if self.metadata is not None else None, } except Exception as e: raise ValueError(f"Error serializing EvaluationResultDatapoint: {e}")