From 7be3c2820e2a38fc5deee0f3328ca5315c71381b Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 13:14:18 +0530 Subject: [PATCH 01/11] fix: field issue --- src/ragas/_analytics.py | 3 +-- tests/unit/test_analytics.py | 7 +++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index cdd9eda62..135a91c79 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -10,8 +10,7 @@ import requests from appdirs import user_data_dir -from langchain_core.pydantic_v1 import BaseModel -from pydantic import Field +from langchain_core.pydantic_v1 import BaseModel, Field from ragas.utils import get_debug_mode diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 87f608fd8..416a55bc5 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -7,8 +7,8 @@ def test_add_userid(): evaluation_event = EvaluationEvent( event_type="evaluation", metrics=["harmfulness"], num_rows=1, evaluation_mode="" ) - payload = evaluation_event.__dict__ - assert payload.get("user_id") is not None + payload = evaluation_event.dict() + assert isinstance(payload.get("user_id"), str) def setup_user_id_filepath(tmp_path, monkeypatch): @@ -33,6 +33,9 @@ def test_write_to_file(tmp_path, monkeypatch): from ragas._analytics import get_userid import json + # clear LRU cache since its created in setup for the above test + get_userid.cache_clear() + userid = get_userid() assert userid_filepath.exists() with open(userid_filepath, "r") as f: From cdc1732d6bf2d883f72ed6a8ae9237b7f92d2b98 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 13:14:53 +0530 Subject: [PATCH 02/11] fmt issues --- src/ragas/_analytics.py | 1 - src/ragas/llms/base.py | 5 +---- src/ragas/metrics/_answer_relevance.py | 2 -- tests/unit/test_analytics.py | 3 ++- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index 135a91c79..2ecdc6592 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -5,7 +5,6 @@ import os import typing as t import uuid -from dataclasses import asdict from functools import lru_cache, wraps import requests diff --git a/src/ragas/llms/base.py b/src/ragas/llms/base.py index 2f33981cc..dde4aea88 100644 --- a/src/ragas/llms/base.py +++ b/src/ragas/llms/base.py @@ -9,13 +9,11 @@ from langchain_core.language_models import BaseLanguageModel from langchain_core.outputs import LLMResult - if t.TYPE_CHECKING: from langchain_core.callbacks import Callbacks from langchain_core.prompts import ChatPromptTemplate - - from ragas.llms.prompt import PromptValue + from ragas.llms.prompt import PromptValue MULTIPLE_COMPLETION_SUPPORTED = [ @@ -69,7 +67,6 @@ def generate_text_with_hmpt( stop: t.Optional[t.List[str]] = None, callbacks: Callbacks = [], ) -> LLMResult: - from ragas.llms.prompt import PromptValue prompt = PromptValue(prompt_str=prompts[0].format()) diff --git a/src/ragas/metrics/_answer_relevance.py b/src/ragas/metrics/_answer_relevance.py index 23f6caf3c..deb09bbd7 100644 --- a/src/ragas/metrics/_answer_relevance.py +++ b/src/ragas/metrics/_answer_relevance.py @@ -5,10 +5,8 @@ from dataclasses import dataclass, field import numpy as np -from langchain.embeddings import OpenAIEmbeddings from ragas.embeddings.base import embedding_factory -from ragas.exceptions import OpenAIKeyNotFound from ragas.llms.json_load import json_loader from ragas.llms.prompt import Prompt from ragas.metrics.base import EvaluationMode, MetricWithLLM diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 416a55bc5..22d594cf2 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -30,9 +30,10 @@ def test_write_to_file(tmp_path, monkeypatch): # check if file created if not existing assert not userid_filepath.exists() - from ragas._analytics import get_userid import json + from ragas._analytics import get_userid + # clear LRU cache since its created in setup for the above test get_userid.cache_clear() From 597e9419e7cab0691a1f4ed096648c5b507ff1a0 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 13:18:42 +0530 Subject: [PATCH 03/11] silence user_data --- src/ragas/_analytics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index 2ecdc6592..af4381e03 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -62,14 +62,15 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any: @lru_cache(maxsize=1) +@silent def get_userid() -> str: user_id_path = user_data_dir(USER_DATA_DIR_NAME) uuid_filepath = os.path.join(user_id_path, "uuid.json") if os.path.exists(uuid_filepath): user_id = json.load(open(uuid_filepath))["userid"] else: - os.mkdir(user_id_path) user_id = "a-" + uuid.uuid4().hex + os.mkdir(user_id_path) with open(uuid_filepath, "w") as f: json.dump({"userid": user_id}, f) return user_id From c655bbb35cf05f6fff0a453418abb244efa223a5 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:13:07 +0530 Subject: [PATCH 04/11] fix dict? --- src/ragas/_analytics.py | 2 +- tests/unit/test_analytics.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index af4381e03..d518afa2f 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -92,7 +92,7 @@ def track(event_properties: BaseEvent): if do_not_track(): return - payload = event_properties.dict() + payload = dict(event_properties) if _usage_event_debugging(): # For internal debugging purpose diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 22d594cf2..54808a55e 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,14 +1,25 @@ from __future__ import annotations -def test_add_userid(): +def test_base_event(): + from ragas._analytics import BaseEvent + + be = BaseEvent(event_type="evaluation") + assert isinstance(dict(be).get("event_type"), str) + assert isinstance(dict(be).get("user_id"), str) + + +def test_evaluation_event(): from ragas._analytics import EvaluationEvent evaluation_event = EvaluationEvent( event_type="evaluation", metrics=["harmfulness"], num_rows=1, evaluation_mode="" ) - payload = evaluation_event.dict() + + payload = dict(evaluation_event) assert isinstance(payload.get("user_id"), str) + assert isinstance(payload.get("evaluation_mode"), str) + assert isinstance(payload.get("metrics"), list) def setup_user_id_filepath(tmp_path, monkeypatch): From 5198d8b3ded67eebb3c39b3c43ab4777195c8559 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:18:20 +0530 Subject: [PATCH 05/11] fix in windows --- src/ragas/_analytics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index d518afa2f..2159b9805 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -64,7 +64,9 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any: @lru_cache(maxsize=1) @silent def get_userid() -> str: - user_id_path = user_data_dir(USER_DATA_DIR_NAME) + user_id_path = user_data_dir( + appname=USER_DATA_DIR_NAME, appauthor=False, roaming=True + ) uuid_filepath = os.path.join(user_id_path, "uuid.json") if os.path.exists(uuid_filepath): user_id = json.load(open(uuid_filepath))["userid"] @@ -93,7 +95,6 @@ def track(event_properties: BaseEvent): return payload = dict(event_properties) - if _usage_event_debugging(): # For internal debugging purpose logger.info("Tracking Payload: %s", payload) From c087f752f6d86c5638c5adae4ee4c3a1bfb288be Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:24:49 +0530 Subject: [PATCH 06/11] maybe fix --- .github/workflows/ci.yaml | 2 ++ src/ragas/_analytics.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8bfdbba7f..f98b0460a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -94,6 +94,8 @@ jobs: # we will use pytest-xdist to improve tests run-time. OPTS=(--dist loadfile -n auto) fi + # ENV Vars + export RAGAS_DEBUG_TRACKING=true # Now run the unit tests pytest tests/unit "${OPTS[@]}" diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index 2159b9805..5864afc7a 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -62,10 +62,9 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any: @lru_cache(maxsize=1) -@silent def get_userid() -> str: user_id_path = user_data_dir( - appname=USER_DATA_DIR_NAME, appauthor=False, roaming=True + appname=USER_DATA_DIR_NAME ) uuid_filepath = os.path.join(user_id_path, "uuid.json") if os.path.exists(uuid_filepath): From 185bf2c3b682e4a78cd047b74197cec25d571fc4 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:28:28 +0530 Subject: [PATCH 07/11] silent error --- src/ragas/_analytics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index 5864afc7a..44705343f 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -62,6 +62,7 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any: @lru_cache(maxsize=1) +@silent def get_userid() -> str: user_id_path = user_data_dir( appname=USER_DATA_DIR_NAME From 7c9b2db78eb9a630ddad21c5d4d7b4b951df0d30 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:29:28 +0530 Subject: [PATCH 08/11] raise error --- src/ragas/_analytics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index 44705343f..ff7a4aa41 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -53,6 +53,7 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any: logger.error( "Tracking Error: %s", err, stack_info=True, stacklevel=3 ) + raise err else: logger.info("Tracking Error: %s", err) else: From fcc647f09c3b12f54dc26a0c5b383d312cdfb8c4 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:34:06 +0530 Subject: [PATCH 09/11] check debug_tracking --- tests/unit/test_analytics.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 54808a55e..04289e989 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -1,6 +1,13 @@ from __future__ import annotations +def test_debug_tracking_flag(): + import os + from ragas._analytics import RAGAS_DEBUG_TRACKING + + assert os.environ.get(RAGAS_DEBUG_TRACKING, "").lower() == "true" + + def test_base_event(): from ragas._analytics import BaseEvent From 4c335d19b34cc5f76a525ed2aaf4c2090091ee6e Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:37:57 +0530 Subject: [PATCH 10/11] ci envvar --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f98b0460a..923426d27 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -94,10 +94,11 @@ jobs: # we will use pytest-xdist to improve tests run-time. OPTS=(--dist loadfile -n auto) fi - # ENV Vars - export RAGAS_DEBUG_TRACKING=true # Now run the unit tests pytest tests/unit "${OPTS[@]}" + env: + __RAGAS_DEBUG_TRACKING: true + RAGAS_DO_NOT_TRACK: true codestyle_check: runs-on: ubuntu-latest From b59e54236eab023656997bec438d331a8363cd04 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 7 Jan 2024 15:43:34 +0530 Subject: [PATCH 11/11] recursively create dir --- src/ragas/_analytics.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ragas/_analytics.py b/src/ragas/_analytics.py index ff7a4aa41..7f6a1ad96 100644 --- a/src/ragas/_analytics.py +++ b/src/ragas/_analytics.py @@ -22,10 +22,11 @@ USAGE_TRACKING_URL = "https://t.explodinggradients.com" -RAGAS_DO_NOT_TRACK = "RAGAS_DO_NOT_TRACK" -RAGAS_DEBUG_TRACKING = "__RAGAS_DEBUG_TRACKING" USAGE_REQUESTS_TIMEOUT_SEC = 1 USER_DATA_DIR_NAME = "ragas" +# Any chance you chance this also change the variable in our ci.yaml file +RAGAS_DO_NOT_TRACK = "RAGAS_DO_NOT_TRACK" +RAGAS_DEBUG_TRACKING = "__RAGAS_DEBUG_TRACKING" @lru_cache(maxsize=1) @@ -65,15 +66,13 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any: @lru_cache(maxsize=1) @silent def get_userid() -> str: - user_id_path = user_data_dir( - appname=USER_DATA_DIR_NAME - ) + user_id_path = user_data_dir(appname=USER_DATA_DIR_NAME) uuid_filepath = os.path.join(user_id_path, "uuid.json") if os.path.exists(uuid_filepath): user_id = json.load(open(uuid_filepath))["userid"] else: user_id = "a-" + uuid.uuid4().hex - os.mkdir(user_id_path) + os.makedirs(user_id_path) with open(uuid_filepath, "w") as f: json.dump({"userid": user_id}, f) return user_id