From 9d5a187d9b6759394482cba0a23abfcc533255c1 Mon Sep 17 00:00:00 2001 From: Amadeusz Abramowski Date: Fri, 2 Aug 2024 21:13:59 +0200 Subject: [PATCH 1/6] Allow Metric.score to work within an existing asyncio loop --- src/ragas/metrics/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ragas/metrics/base.py b/src/ragas/metrics/base.py index bd6a12fc5..d63e02317 100644 --- a/src/ragas/metrics/base.py +++ b/src/ragas/metrics/base.py @@ -96,7 +96,11 @@ def score(self: t.Self, row: t.Dict, callbacks: Callbacks = None) -> float: callbacks = callbacks or [] rm, group_cm = new_group(self.name, inputs=row, callbacks=callbacks) try: - score = asyncio.run(self._ascore(row=row, callbacks=group_cm)) + loop = asyncio.get_event_loop() + if not loop.is_running(): + score = asyncio.run(self._ascore(row=row, callbacks=group_cm)) + else: + score = loop.run_until_complete(self._ascore(row=row, callbacks=group_cm)) except Exception as e: if not group_cm.ended: rm.on_chain_error(e) From dcfa3104e96e9bb65a78b9d7d7f074c0f13c4dbf Mon Sep 17 00:00:00 2001 From: Amadeusz Abramowski Date: Fri, 2 Aug 2024 21:48:47 +0200 Subject: [PATCH 2/6] Allow Metric.score to work within an existing asyncio loop --- src/ragas/metrics/base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/ragas/metrics/base.py b/src/ragas/metrics/base.py index d63e02317..5587a3656 100644 --- a/src/ragas/metrics/base.py +++ b/src/ragas/metrics/base.py @@ -97,10 +97,7 @@ def score(self: t.Self, row: t.Dict, callbacks: Callbacks = None) -> float: rm, group_cm = new_group(self.name, inputs=row, callbacks=callbacks) try: loop = asyncio.get_event_loop() - if not loop.is_running(): - score = asyncio.run(self._ascore(row=row, callbacks=group_cm)) - else: - score = loop.run_until_complete(self._ascore(row=row, callbacks=group_cm)) + score = loop.run_until_complete(self._ascore(row=row, callbacks=group_cm)) except Exception as e: if not group_cm.ended: rm.on_chain_error(e) From 0459b91969c4f0ed63d79edac6177865e52ba048 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Mon, 5 Aug 2024 20:09:37 +0530 Subject: [PATCH 3/6] added some tests in jupyter --- tests/e2e/test_evaluation_in_jupyter.ipynb | 7 --- tests/unit/test_executor_in_jupyter.ipynb | 63 ++++++++++++++++++++-- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/tests/e2e/test_evaluation_in_jupyter.ipynb b/tests/e2e/test_evaluation_in_jupyter.ipynb index 32078f43e..6b018a8d2 100644 --- a/tests/e2e/test_evaluation_in_jupyter.ipynb +++ b/tests/e2e/test_evaluation_in_jupyter.ipynb @@ -103,13 +103,6 @@ "\n", "result" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/tests/unit/test_executor_in_jupyter.ipynb b/tests/unit/test_executor_in_jupyter.ipynb index f1dba1e42..539d54346 100644 --- a/tests/unit/test_executor_in_jupyter.ipynb +++ b/tests/unit/test_executor_in_jupyter.ipynb @@ -1,13 +1,10 @@ { "cells": [ { - "cell_type": "code", - "execution_count": 1, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "%load_ext autoreload\n", - "%autoreload 2" + "# Test Executor " ] }, { @@ -128,6 +125,62 @@ "assert exec.results(), \"didn't get anything from results\"" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test Metric" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.metrics.base import Metric, EvaluationMode\n", + "\n", + "class FakeMetric(Metric):\n", + " name = \"fake_metric\"\n", + " evaluation_mode = EvaluationMode.qa\n", + "\n", + " def init(self):\n", + " pass\n", + "\n", + " def _ascore(self, row, callbacks)->float:\n", + " return 0\n", + "\n", + "fm = FakeMetric()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "ename": "RuntimeError", + "evalue": "This event loop is already running", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mfm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscore\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mrow\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquestion\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mq\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43manswer\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43ma\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/jjmachan/explodinggradients/ragas/src/ragas/metrics/base.py:104\u001b[0m, in \u001b[0;36mMetric.score\u001b[0;34m(self, row, callbacks)\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m group_cm\u001b[38;5;241m.\u001b[39mended:\n\u001b[1;32m 103\u001b[0m rm\u001b[38;5;241m.\u001b[39mon_chain_error(e)\n\u001b[0;32m--> 104\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m 105\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 106\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m group_cm\u001b[38;5;241m.\u001b[39mended:\n", + "File \u001b[0;32m~/jjmachan/explodinggradients/ragas/src/ragas/metrics/base.py:100\u001b[0m, in \u001b[0;36mMetric.score\u001b[0;34m(self, row, callbacks)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 99\u001b[0m loop \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mget_event_loop()\n\u001b[0;32m--> 100\u001b[0m score \u001b[38;5;241m=\u001b[39m \u001b[43mloop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_until_complete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ascore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrow\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrow\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup_cm\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m group_cm\u001b[38;5;241m.\u001b[39mended:\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.12/lib/python3.10/asyncio/base_events.py:625\u001b[0m, in \u001b[0;36mBaseEventLoop.run_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m 614\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Run until the Future is done.\u001b[39;00m\n\u001b[1;32m 615\u001b[0m \n\u001b[1;32m 616\u001b[0m \u001b[38;5;124;03mIf the argument is a coroutine, it is wrapped in a Task.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 622\u001b[0m \u001b[38;5;124;03mReturn the Future's result, or raise its exception.\u001b[39;00m\n\u001b[1;32m 623\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 624\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_closed()\n\u001b[0;32m--> 625\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_running\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 627\u001b[0m new_task \u001b[38;5;241m=\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m futures\u001b[38;5;241m.\u001b[39misfuture(future)\n\u001b[1;32m 628\u001b[0m future \u001b[38;5;241m=\u001b[39m tasks\u001b[38;5;241m.\u001b[39mensure_future(future, loop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.12/lib/python3.10/asyncio/base_events.py:584\u001b[0m, in \u001b[0;36mBaseEventLoop._check_running\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_check_running\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 583\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_running():\n\u001b[0;32m--> 584\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThis event loop is already running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 585\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m events\u001b[38;5;241m.\u001b[39m_get_running_loop() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 586\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 587\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCannot run the event loop while another loop is running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "\u001b[0;31mRuntimeError\u001b[0m: This event loop is already running" + ] + } + ], + "source": [ + "fm.score(\n", + " row={\"question\": [\"q\"], \"answer\": [\"a\"]},\n", + " callbacks=None,\n", + ")" + ] + }, { "cell_type": "code", "execution_count": null, From 787885ebcecbdbbe15c0d8ceff07a7bfb3da7282 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Mon, 5 Aug 2024 20:12:12 +0530 Subject: [PATCH 4/6] added tests in just python --- tests/unit/test_metric.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_metric.py b/tests/unit/test_metric.py index 6f1fcf516..311914972 100644 --- a/tests/unit/test_metric.py +++ b/tests/unit/test_metric.py @@ -27,3 +27,20 @@ def test_get_available_metrics(): for metric in get_available_metrics(ds) ] ), "All metrics should have evaluation mode qa" + + +def test_metric(): + from ragas.metrics.base import Metric + + class FakeMetric(Metric): + name = "fake_metric" # type: ignore + evaluation_mode = EvaluationMode.qa # type: ignore + + def init(self, run_config): + pass + + async def _ascore(self, row, callbacks) -> float: + return 0 + + fm = FakeMetric() + assert fm.score({"question": "a", "answer": "b"}) == 0 From 8c01454ecda3dec58d2f71dac1492dc44420cc36 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Mon, 5 Aug 2024 20:21:15 +0530 Subject: [PATCH 5/6] fix test in notebook --- tests/unit/test_executor_in_jupyter.ipynb | 56 +++++++---------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/tests/unit/test_executor_in_jupyter.ipynb b/tests/unit/test_executor_in_jupyter.ipynb index 539d54346..7fc0dca67 100644 --- a/tests/unit/test_executor_in_jupyter.ipynb +++ b/tests/unit/test_executor_in_jupyter.ipynb @@ -9,13 +9,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b78a418208b84895b03c93c54f1d1d61", + "model_id": "ebb0705d6a05459a89f4ae87cbbbfd84", "version_major": 2, "version_minor": 0 }, @@ -33,14 +33,14 @@ "\n", "exec = Executor(raise_exceptions=True)\n", "for i in range(10):\n", - " exec.submit(sleep, i)\n", + " exec.submit(sleep, i/10)\n", "\n", "assert exec.results(), \"didn't get anything from results\"" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -80,13 +80,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9bb608f8b2de42628fb525581d496d3a", + "model_id": "985b8a189c9047c29d6ccebf7c5a938b", "version_major": 2, "version_minor": 0 }, @@ -100,7 +100,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "128ca1d600b3457c863ddf376d24c44e", + "model_id": "ff3097e24dc249fbab6e610e59ccc9b6", "version_major": 2, "version_minor": 0 }, @@ -115,12 +115,12 @@ "source": [ "exec = Executor(raise_exceptions=True)\n", "for i in range(1000):\n", - " exec.submit(sleep, 1)\n", + " exec.submit(sleep, 0.01)\n", "\n", "assert exec.results(), \"didn't get anything from results\"\n", "\n", "for i in range(1000):\n", - " exec.submit(sleep, 1)\n", + " exec.submit(sleep, 0.01)\n", "\n", "assert exec.results(), \"didn't get anything from results\"" ] @@ -134,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -147,7 +147,7 @@ " def init(self):\n", " pass\n", "\n", - " def _ascore(self, row, callbacks)->float:\n", + " async def _ascore(self, row, callbacks)->float:\n", " return 0\n", "\n", "fm = FakeMetric()" @@ -155,38 +155,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 24, "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "This event loop is already running", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mfm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscore\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mrow\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mquestion\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mq\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43manswer\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43ma\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/jjmachan/explodinggradients/ragas/src/ragas/metrics/base.py:104\u001b[0m, in \u001b[0;36mMetric.score\u001b[0;34m(self, row, callbacks)\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m group_cm\u001b[38;5;241m.\u001b[39mended:\n\u001b[1;32m 103\u001b[0m rm\u001b[38;5;241m.\u001b[39mon_chain_error(e)\n\u001b[0;32m--> 104\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m 105\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 106\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m group_cm\u001b[38;5;241m.\u001b[39mended:\n", - "File \u001b[0;32m~/jjmachan/explodinggradients/ragas/src/ragas/metrics/base.py:100\u001b[0m, in \u001b[0;36mMetric.score\u001b[0;34m(self, row, callbacks)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 99\u001b[0m loop \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mget_event_loop()\n\u001b[0;32m--> 100\u001b[0m score \u001b[38;5;241m=\u001b[39m \u001b[43mloop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_until_complete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ascore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrow\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrow\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup_cm\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m group_cm\u001b[38;5;241m.\u001b[39mended:\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.12/lib/python3.10/asyncio/base_events.py:625\u001b[0m, in \u001b[0;36mBaseEventLoop.run_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m 614\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Run until the Future is done.\u001b[39;00m\n\u001b[1;32m 615\u001b[0m \n\u001b[1;32m 616\u001b[0m \u001b[38;5;124;03mIf the argument is a coroutine, it is wrapped in a Task.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 622\u001b[0m \u001b[38;5;124;03mReturn the Future's result, or raise its exception.\u001b[39;00m\n\u001b[1;32m 623\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 624\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_closed()\n\u001b[0;32m--> 625\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_running\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 627\u001b[0m new_task \u001b[38;5;241m=\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m futures\u001b[38;5;241m.\u001b[39misfuture(future)\n\u001b[1;32m 628\u001b[0m future \u001b[38;5;241m=\u001b[39m tasks\u001b[38;5;241m.\u001b[39mensure_future(future, loop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.12/lib/python3.10/asyncio/base_events.py:584\u001b[0m, in \u001b[0;36mBaseEventLoop._check_running\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_check_running\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 583\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_running():\n\u001b[0;32m--> 584\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mThis event loop is already running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 585\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m events\u001b[38;5;241m.\u001b[39m_get_running_loop() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 586\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 587\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCannot run the event loop while another loop is running\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", - "\u001b[0;31mRuntimeError\u001b[0m: This event loop is already running" - ] - } - ], + "outputs": [], "source": [ - "fm.score(\n", + "score = fm.score(\n", " row={\"question\": [\"q\"], \"answer\": [\"a\"]},\n", " callbacks=None,\n", - ")" + ")\n", + "assert score == 0" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From a5cb7fe019f6bfb4571aeff017517d3822af6d5e Mon Sep 17 00:00:00 2001 From: jjmachan Date: Mon, 5 Aug 2024 20:21:25 +0530 Subject: [PATCH 6/6] added nbmake to test --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4802cb8ac..89db6533b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,7 +97,7 @@ jobs: OPTS=(--dist loadfile -n auto) fi # Now run the unit tests - pytest tests/unit "${OPTS[@]}" + pytest --nbmake tests/unit "${OPTS[@]}" env: __RAGAS_DEBUG_TRACKING: true RAGAS_DO_NOT_TRACK: true