Skip to content

Commit

Permalink
[Python] Set example creation time to experiment start time (#765)
Browse files Browse the repository at this point in the history
For unit testing.

Also fix issue where test suite name isn't being honored
  • Loading branch information
hinthornw committed Jun 5, 2024
1 parent e7f46d0 commit 8bb007c
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 5 deletions.
12 changes: 9 additions & 3 deletions python/langsmith/_testing.py
Original file line number Diff line number Diff line change
Expand Up @@ -409,10 +409,13 @@ def experiment(self):

@classmethod
def from_test(
cls, client: Optional[ls_client.Client], func: Callable
cls,
client: Optional[ls_client.Client],
func: Callable,
test_suite_name: Optional[str] = None,
) -> _LangSmithTestSuite:
client = client or ls_client.Client()
test_suite_name = _get_test_suite_name(func)
test_suite_name = test_suite_name or _get_test_suite_name(func)
with cls._lock:
if not cls._instances:
cls._instances = {}
Expand Down Expand Up @@ -496,6 +499,7 @@ def _sync_example(
outputs=outputs_,
dataset_id=self.id,
metadata=metadata,
created_at=self._experiment.start_time,
)
if example.modified_at:
self.update_version(example.modified_at)
Expand Down Expand Up @@ -531,7 +535,9 @@ def _ensure_example(
if output_keys:
for k in output_keys:
outputs[k] = inputs.pop(k, None)
test_suite = _LangSmithTestSuite.from_test(client, func)
test_suite = _LangSmithTestSuite.from_test(
client, func, langtest_extra.get("test_suite_name")
)
example_id, example_name = _get_id(func, inputs, test_suite.id)
example_id = langtest_extra["id"] or example_id
test_suite.sync_example(
Expand Down
2 changes: 1 addition & 1 deletion python/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "langsmith"
version = "0.1.72"
version = "0.1.73"
description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
authors = ["LangChain <support@langchain.dev>"]
license = "MIT"
Expand Down
2 changes: 1 addition & 1 deletion python/tests/evaluation/test_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def test_bar_parametrized(x, y, z):
return {"z": x + y}


@unit
@unit(test_suite_name="tests.evaluation.test_evaluation::test_foo_async_parametrized")
@pytest.mark.parametrize("x, y", [(1, 2), (2, 3)])
async def test_foo_async_parametrized(x, y):
await asyncio.sleep(0.1)
Expand Down

0 comments on commit 8bb007c

Please sign in to comment.