fix: tests for analytics

.github/workflows/ci.yaml

@@ -96,6 +96,9 @@ jobs:
           fi
           # Now run the unit tests
           pytest tests/unit "${OPTS[@]}"
+        env:
+          __RAGAS_DEBUG_TRACKING: true
+          RAGAS_DO_NOT_TRACK: true
 
   codestyle_check:
     runs-on: ubuntu-latest

src/ragas/_analytics.py

@@ -5,13 +5,11 @@
 import os
 import typing as t
 import uuid
-from dataclasses import asdict
 from functools import lru_cache, wraps
 
 import requests
 from appdirs import user_data_dir
-from langchain_core.pydantic_v1 import BaseModel
-from pydantic import Field
+from langchain_core.pydantic_v1 import BaseModel, Field
 
 from ragas.utils import get_debug_mode
 
@@ -24,10 +22,11 @@
 
 
 USAGE_TRACKING_URL = "https://t.explodinggradients.com"
-RAGAS_DO_NOT_TRACK = "RAGAS_DO_NOT_TRACK"
-RAGAS_DEBUG_TRACKING = "__RAGAS_DEBUG_TRACKING"
 USAGE_REQUESTS_TIMEOUT_SEC = 1
 USER_DATA_DIR_NAME = "ragas"
+# Any chance you chance this also change the variable in our ci.yaml file
+RAGAS_DO_NOT_TRACK = "RAGAS_DO_NOT_TRACK"
+RAGAS_DEBUG_TRACKING = "__RAGAS_DEBUG_TRACKING"
 
 
 @lru_cache(maxsize=1)
@@ -55,6 +54,7 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any:
                     logger.error(
                         "Tracking Error: %s", err, stack_info=True, stacklevel=3
                     )
+                    raise err
                 else:
                     logger.info("Tracking Error: %s", err)
             else:
@@ -64,14 +64,15 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any:
 
 
 @lru_cache(maxsize=1)
+@silent
 def get_userid() -> str:
-    user_id_path = user_data_dir(USER_DATA_DIR_NAME)
+    user_id_path = user_data_dir(appname=USER_DATA_DIR_NAME)
     uuid_filepath = os.path.join(user_id_path, "uuid.json")
     if os.path.exists(uuid_filepath):
         user_id = json.load(open(uuid_filepath))["userid"]
     else:
-        os.mkdir(user_id_path)
         user_id = "a-" + uuid.uuid4().hex
+        os.makedirs(user_id_path)
         with open(uuid_filepath, "w") as f:
             json.dump({"userid": user_id}, f)
     return user_id
@@ -93,8 +94,7 @@ def track(event_properties: BaseEvent):
     if do_not_track():
         return
 
-    payload = event_properties.dict()
-
+    payload = dict(event_properties)
     if _usage_event_debugging():
         # For internal debugging purpose
         logger.info("Tracking Payload: %s", payload)

src/ragas/llms/base.py

@@ -9,13 +9,11 @@
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.outputs import LLMResult
 
-
 if t.TYPE_CHECKING:
     from langchain_core.callbacks import Callbacks
     from langchain_core.prompts import ChatPromptTemplate
-
-    from ragas.llms.prompt import PromptValue
 
+    from ragas.llms.prompt import PromptValue
 
 
 MULTIPLE_COMPLETION_SUPPORTED = [
@@ -69,7 +67,6 @@ def generate_text_with_hmpt(
         stop: t.Optional[t.List[str]] = None,
         callbacks: Callbacks = [],
     ) -> LLMResult:
-
         from ragas.llms.prompt import PromptValue
 
         prompt = PromptValue(prompt_str=prompts[0].format())

src/ragas/metrics/_answer_relevance.py

@@ -5,10 +5,8 @@
 from dataclasses import dataclass, field
 
 import numpy as np
-from langchain.embeddings import OpenAIEmbeddings
 
 from ragas.embeddings.base import embedding_factory
-from ragas.exceptions import OpenAIKeyNotFound
 from ragas.llms.json_load import json_loader
 from ragas.llms.prompt import Prompt
 from ragas.metrics.base import EvaluationMode, MetricWithLLM

tests/unit/test_analytics.py

@@ -1,14 +1,32 @@
 from __future__ import annotations
 
 
-def test_add_userid():
+def test_debug_tracking_flag():
+    import os
+    from ragas._analytics import RAGAS_DEBUG_TRACKING
+
+    assert os.environ.get(RAGAS_DEBUG_TRACKING, "").lower() == "true"
+
+
+def test_base_event():
+    from ragas._analytics import BaseEvent
+
+    be = BaseEvent(event_type="evaluation")
+    assert isinstance(dict(be).get("event_type"), str)
+    assert isinstance(dict(be).get("user_id"), str)
+
+
+def test_evaluation_event():
     from ragas._analytics import EvaluationEvent
 
     evaluation_event = EvaluationEvent(
         event_type="evaluation", metrics=["harmfulness"], num_rows=1, evaluation_mode=""
     )
-    payload = evaluation_event.__dict__
-    assert payload.get("user_id") is not None
+
+    payload = dict(evaluation_event)
+    assert isinstance(payload.get("user_id"), str)
+    assert isinstance(payload.get("evaluation_mode"), str)
+    assert isinstance(payload.get("metrics"), list)
 
 
 def setup_user_id_filepath(tmp_path, monkeypatch):
@@ -30,9 +48,13 @@ def test_write_to_file(tmp_path, monkeypatch):
 
     # check if file created if not existing
     assert not userid_filepath.exists()
-    from ragas._analytics import get_userid
     import json
 
+    from ragas._analytics import get_userid
+
+    # clear LRU cache since its created in setup for the above test
+    get_userid.cache_clear()
+
     userid = get_userid()
     assert userid_filepath.exists()
     with open(userid_filepath, "r") as f: