diff --git a/src/examples/inspect_ai_example/basic_eval.py b/src/examples/inspect_ai_example/basic_eval.py index d2d1fe95..6cc9e7ab 100644 --- a/src/examples/inspect_ai_example/basic_eval.py +++ b/src/examples/inspect_ai_example/basic_eval.py @@ -1,10 +1,12 @@ import fsspec +from dotenv import find_dotenv, load_dotenv from inspect_ai import Task, task from inspect_ai.dataset import csv_dataset, Sample from inspect_ai.scorer import model_graded_qa from inspect_ai.solver import chain_of_thought, self_critique from langtrace_python_sdk.extensions.langtrace_filesystem import LangTraceFileSystem +_ = load_dotenv(find_dotenv()) # Manually register the filesystem with fsspec # Note: This is only necessary because the filesystem is not registered. @@ -24,9 +26,9 @@ def hydrate_with_question(record): @task -def pricing_question(): +def basic_eval(): return Task( - dataset=csv_dataset("langtracefs://clyythmcs0001145cuvi426zi", hydrate_with_question), + dataset=csv_dataset("langtracefs://clz0p4i1t000fwv0xjtlvkxyx"), plan=[chain_of_thought(), self_critique()], scorer=model_graded_qa(), ) diff --git a/src/examples/openai_example/send_user_feedback.py b/src/examples/openai_example/send_user_feedback.py new file mode 100644 index 00000000..a29198b8 --- /dev/null +++ b/src/examples/openai_example/send_user_feedback.py @@ -0,0 +1,40 @@ +from dotenv import find_dotenv, load_dotenv +from openai import OpenAI +from langtrace_python_sdk import langtrace, with_langtrace_root_span, SendUserFeedback + +_ = load_dotenv(find_dotenv()) + +# Initialize Langtrace SDK +langtrace.init() +client = OpenAI() + + +def api(span_id, trace_id): + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": "What is the best place to live in the US?"}, + ], + stream=False, + ) + + # Collect user feedback and send it to Langtrace + user_score = 1 # Example user score + user_id = 'user_1234' # Example user ID + data = { + "userScore": user_score, + "userId": user_id, + "spanId": span_id, + "traceId": trace_id + } + SendUserFeedback().evaluate(data=data) + + # Return the response + return response.choices[0].message.content + + +# wrap the API call with the Langtrace root span +wrapped_api = with_langtrace_root_span()(api) + +# Call the wrapped API +wrapped_api() diff --git a/src/examples/routellm_example/basic.py b/src/examples/routellm_example/basic.py new file mode 100644 index 00000000..9f908fb0 --- /dev/null +++ b/src/examples/routellm_example/basic.py @@ -0,0 +1,41 @@ +import sys + +sys.path.insert(0, "/Users/karthikkalyanaraman/work/langtrace/langtrace-python-sdk/src") + +from langtrace_python_sdk import langtrace +from langtrace_python_sdk.utils.with_root_span import with_langtrace_root_span +from routellm.controller import Controller +from dotenv import load_dotenv + +load_dotenv() + +langtrace.init() + +# litellm.set_verbose=True +client = Controller( + routers=["mf"], + strong_model="claude-3-opus-20240229", + weak_model="claude-3-opus-20240229", +) + + +@with_langtrace_root_span("Routellm") +def Routellm(prompt): + try: + + response = client.chat.completions.create( + model="router-mf-0.11593", messages=[{"role": "user", "content": prompt}] + ) + + for chunk in response: + if hasattr(chunk, "choices"): + print(chunk.choices[0].delta.content or "", end="") + else: + print(chunk) + + except Exception as e: + print(f"An error occurred: {e}") + + +Routellm("what is the square root of 12182382932.99") +Routellm("Write me a short story") diff --git a/src/langtrace_python_sdk/extensions/langtrace_filesystem.py b/src/langtrace_python_sdk/extensions/langtrace_filesystem.py index 6506be40..c89f75aa 100644 --- a/src/langtrace_python_sdk/extensions/langtrace_filesystem.py +++ b/src/langtrace_python_sdk/extensions/langtrace_filesystem.py @@ -27,13 +27,25 @@ def __new__(cls, value): class LangTraceFile(io.BytesIO): - _host: str = os.environ.get("LANGTRACE_API_HOST", None) or LANGTRACE_REMOTE_URL def __init__(self, fs: "LangTraceFileSystem", path: str, mode: OpenMode): super().__init__() self.fs = fs self.path = path self.mode = mode + self._host: str = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL) + self._api_key: str = os.environ.get("LANGTRACE_API_KEY", None) + if self._host.endswith("/api/trace"): + self._host = self._host.replace("/api/trace", "") + + if self._api_key is None: + print(Fore.RED) + print( + f"Missing Langtrace API key, proceed to {self._host} to create one" + ) + print("Set the API key as an environment variable LANGTRACE_API_KEY") + print(Fore.RESET) + return def close(self) -> None: if not self.closed: @@ -71,7 +83,7 @@ def upload_to_server(self, file_data: bytes) -> None: data=json.dumps(data), headers={ "Content-Type": "application/json", - "x-api-key": os.environ.get("LANGTRACE_API_KEY"), + "x-api-key": self._api_key, }, timeout=20, ) @@ -82,7 +94,6 @@ def upload_to_server(self, file_data: bytes) -> None: class LangTraceFileSystem(AbstractFileSystem): - _host: str = os.environ.get("LANGTRACE_API_HOST", None) or LANGTRACE_REMOTE_URL protocol = "langtracefs" sep = "/" @@ -90,6 +101,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.files = {} self.dirs = set() + self._host: str = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL) + self._api_key: str = os.environ.get("LANGTRACE_API_KEY", None) + if self._host.endswith("/api/trace"): + self._host = self._host.replace("/api/trace", "") + + if self._api_key is None: + print(Fore.RED) + print( + f"Missing Langtrace API key, proceed to {self._host} to create one" + ) + print("Set the API key as an environment variable LANGTRACE_API_KEY") + print(Fore.RESET) + return def open( self, @@ -118,7 +142,7 @@ def fetch_file_from_api(self, dataset_id: str) -> bytes: url=f"{self._host}/api/dataset/download?id={dataset_id}", headers={ "Content-Type": "application/json", - "x-api-key": os.environ.get("LANGTRACE_API_KEY"), + "x-api-key": self._api_key, }, timeout=20, ) diff --git a/src/langtrace_python_sdk/utils/with_root_span.py b/src/langtrace_python_sdk/utils/with_root_span.py index 79d0bf81..6fcb0279 100644 --- a/src/langtrace_python_sdk/utils/with_root_span.py +++ b/src/langtrace_python_sdk/utils/with_root_span.py @@ -25,6 +25,9 @@ from opentelemetry.trace import SpanKind from opentelemetry.trace.propagation import set_span_in_context +from langtrace_python_sdk.constants.exporter.langtrace_exporter import ( + LANGTRACE_REMOTE_URL, +) from langtrace_python_sdk.constants.instrumentation.common import ( LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY, ) @@ -142,7 +145,10 @@ class SendUserFeedback: _langtrace_api_key: str def __init__(self): - self._langtrace_host = os.environ["LANGTRACE_API_HOST"] + self._langtrace_host = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL) + # When the host is set to /api/trace, remove the /api/trace + if self._langtrace_host.endswith("/api/trace"): + self._langtrace_host = self._langtrace_host.replace("/api/trace", "") self._langtrace_api_key = os.environ.get("LANGTRACE_API_KEY", None) def evaluate(self, data: EvaluationAPIData) -> None: @@ -155,6 +161,16 @@ def evaluate(self, data: EvaluationAPIData) -> None: print("Set the API key as an environment variable LANGTRACE_API_KEY") print(Fore.RESET) return + + # convert spanId and traceId to hexadecimals + span_hex_number = hex(int(data["spanId"], 10))[2:] # Convert to hex and remove the '0x' prefix + formatted_span_hex_number = span_hex_number.zfill(16) # Pad with zeros to 16 characters + data["spanId"] = f"0x{formatted_span_hex_number}" + + trace_hex_number = hex(int(data["traceId"], 10))[2:] # Convert to hex and remove the '0x' prefix + formatted_trace_hex_number = trace_hex_number.zfill(32) # Pad with zeros to 32 characters + data["traceId"] = f"0x{formatted_trace_hex_number}" + evaluation = self.get_evaluation(data["spanId"]) headers = {"x-api-key": self._langtrace_api_key} if evaluation is not None: diff --git a/src/langtrace_python_sdk/version.py b/src/langtrace_python_sdk/version.py index 90a1f38f..23bc6ef5 100644 --- a/src/langtrace_python_sdk/version.py +++ b/src/langtrace_python_sdk/version.py @@ -1 +1 @@ -__version__ = "2.2.7" +__version__ = "2.2.8"