From 3897fa36de12338ffa735a8257c55559062ad65c Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman <105607645+karthikscale3@users.noreply.github.com> Date: Wed, 24 Jul 2024 20:39:13 -0700 Subject: [PATCH 1/4] Bugfixes to sending user feedback and consolidating env var (#261) * ENV var bugfixes, send user feedback fixes * Slack template * Bump version --- src/examples/inspect_ai_example/basic_eval.py | 6 ++- .../openai_example/send_user_feedback.py | 40 ++++++++++++++++++ src/examples/routellm_example/basic.py | 41 +++++++++++++++++++ .../extensions/langtrace_filesystem.py | 32 +++++++++++++-- .../utils/with_root_span.py | 18 +++++++- src/langtrace_python_sdk/version.py | 2 +- 6 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 src/examples/openai_example/send_user_feedback.py create mode 100644 src/examples/routellm_example/basic.py diff --git a/src/examples/inspect_ai_example/basic_eval.py b/src/examples/inspect_ai_example/basic_eval.py index d2d1fe95..6cc9e7ab 100644 --- a/src/examples/inspect_ai_example/basic_eval.py +++ b/src/examples/inspect_ai_example/basic_eval.py @@ -1,10 +1,12 @@ import fsspec +from dotenv import find_dotenv, load_dotenv from inspect_ai import Task, task from inspect_ai.dataset import csv_dataset, Sample from inspect_ai.scorer import model_graded_qa from inspect_ai.solver import chain_of_thought, self_critique from langtrace_python_sdk.extensions.langtrace_filesystem import LangTraceFileSystem +_ = load_dotenv(find_dotenv()) # Manually register the filesystem with fsspec # Note: This is only necessary because the filesystem is not registered. @@ -24,9 +26,9 @@ def hydrate_with_question(record): @task -def pricing_question(): +def basic_eval(): return Task( - dataset=csv_dataset("langtracefs://clyythmcs0001145cuvi426zi", hydrate_with_question), + dataset=csv_dataset("langtracefs://clz0p4i1t000fwv0xjtlvkxyx"), plan=[chain_of_thought(), self_critique()], scorer=model_graded_qa(), ) diff --git a/src/examples/openai_example/send_user_feedback.py b/src/examples/openai_example/send_user_feedback.py new file mode 100644 index 00000000..a29198b8 --- /dev/null +++ b/src/examples/openai_example/send_user_feedback.py @@ -0,0 +1,40 @@ +from dotenv import find_dotenv, load_dotenv +from openai import OpenAI +from langtrace_python_sdk import langtrace, with_langtrace_root_span, SendUserFeedback + +_ = load_dotenv(find_dotenv()) + +# Initialize Langtrace SDK +langtrace.init() +client = OpenAI() + + +def api(span_id, trace_id): + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": "What is the best place to live in the US?"}, + ], + stream=False, + ) + + # Collect user feedback and send it to Langtrace + user_score = 1 # Example user score + user_id = 'user_1234' # Example user ID + data = { + "userScore": user_score, + "userId": user_id, + "spanId": span_id, + "traceId": trace_id + } + SendUserFeedback().evaluate(data=data) + + # Return the response + return response.choices[0].message.content + + +# wrap the API call with the Langtrace root span +wrapped_api = with_langtrace_root_span()(api) + +# Call the wrapped API +wrapped_api() diff --git a/src/examples/routellm_example/basic.py b/src/examples/routellm_example/basic.py new file mode 100644 index 00000000..9f908fb0 --- /dev/null +++ b/src/examples/routellm_example/basic.py @@ -0,0 +1,41 @@ +import sys + +sys.path.insert(0, "/Users/karthikkalyanaraman/work/langtrace/langtrace-python-sdk/src") + +from langtrace_python_sdk import langtrace +from langtrace_python_sdk.utils.with_root_span import with_langtrace_root_span +from routellm.controller import Controller +from dotenv import load_dotenv + +load_dotenv() + +langtrace.init() + +# litellm.set_verbose=True +client = Controller( + routers=["mf"], + strong_model="claude-3-opus-20240229", + weak_model="claude-3-opus-20240229", +) + + +@with_langtrace_root_span("Routellm") +def Routellm(prompt): + try: + + response = client.chat.completions.create( + model="router-mf-0.11593", messages=[{"role": "user", "content": prompt}] + ) + + for chunk in response: + if hasattr(chunk, "choices"): + print(chunk.choices[0].delta.content or "", end="") + else: + print(chunk) + + except Exception as e: + print(f"An error occurred: {e}") + + +Routellm("what is the square root of 12182382932.99") +Routellm("Write me a short story") diff --git a/src/langtrace_python_sdk/extensions/langtrace_filesystem.py b/src/langtrace_python_sdk/extensions/langtrace_filesystem.py index 6506be40..c89f75aa 100644 --- a/src/langtrace_python_sdk/extensions/langtrace_filesystem.py +++ b/src/langtrace_python_sdk/extensions/langtrace_filesystem.py @@ -27,13 +27,25 @@ def __new__(cls, value): class LangTraceFile(io.BytesIO): - _host: str = os.environ.get("LANGTRACE_API_HOST", None) or LANGTRACE_REMOTE_URL def __init__(self, fs: "LangTraceFileSystem", path: str, mode: OpenMode): super().__init__() self.fs = fs self.path = path self.mode = mode + self._host: str = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL) + self._api_key: str = os.environ.get("LANGTRACE_API_KEY", None) + if self._host.endswith("/api/trace"): + self._host = self._host.replace("/api/trace", "") + + if self._api_key is None: + print(Fore.RED) + print( + f"Missing Langtrace API key, proceed to {self._host} to create one" + ) + print("Set the API key as an environment variable LANGTRACE_API_KEY") + print(Fore.RESET) + return def close(self) -> None: if not self.closed: @@ -71,7 +83,7 @@ def upload_to_server(self, file_data: bytes) -> None: data=json.dumps(data), headers={ "Content-Type": "application/json", - "x-api-key": os.environ.get("LANGTRACE_API_KEY"), + "x-api-key": self._api_key, }, timeout=20, ) @@ -82,7 +94,6 @@ def upload_to_server(self, file_data: bytes) -> None: class LangTraceFileSystem(AbstractFileSystem): - _host: str = os.environ.get("LANGTRACE_API_HOST", None) or LANGTRACE_REMOTE_URL protocol = "langtracefs" sep = "/" @@ -90,6 +101,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.files = {} self.dirs = set() + self._host: str = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL) + self._api_key: str = os.environ.get("LANGTRACE_API_KEY", None) + if self._host.endswith("/api/trace"): + self._host = self._host.replace("/api/trace", "") + + if self._api_key is None: + print(Fore.RED) + print( + f"Missing Langtrace API key, proceed to {self._host} to create one" + ) + print("Set the API key as an environment variable LANGTRACE_API_KEY") + print(Fore.RESET) + return def open( self, @@ -118,7 +142,7 @@ def fetch_file_from_api(self, dataset_id: str) -> bytes: url=f"{self._host}/api/dataset/download?id={dataset_id}", headers={ "Content-Type": "application/json", - "x-api-key": os.environ.get("LANGTRACE_API_KEY"), + "x-api-key": self._api_key, }, timeout=20, ) diff --git a/src/langtrace_python_sdk/utils/with_root_span.py b/src/langtrace_python_sdk/utils/with_root_span.py index 79d0bf81..6fcb0279 100644 --- a/src/langtrace_python_sdk/utils/with_root_span.py +++ b/src/langtrace_python_sdk/utils/with_root_span.py @@ -25,6 +25,9 @@ from opentelemetry.trace import SpanKind from opentelemetry.trace.propagation import set_span_in_context +from langtrace_python_sdk.constants.exporter.langtrace_exporter import ( + LANGTRACE_REMOTE_URL, +) from langtrace_python_sdk.constants.instrumentation.common import ( LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY, ) @@ -142,7 +145,10 @@ class SendUserFeedback: _langtrace_api_key: str def __init__(self): - self._langtrace_host = os.environ["LANGTRACE_API_HOST"] + self._langtrace_host = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL) + # When the host is set to /api/trace, remove the /api/trace + if self._langtrace_host.endswith("/api/trace"): + self._langtrace_host = self._langtrace_host.replace("/api/trace", "") self._langtrace_api_key = os.environ.get("LANGTRACE_API_KEY", None) def evaluate(self, data: EvaluationAPIData) -> None: @@ -155,6 +161,16 @@ def evaluate(self, data: EvaluationAPIData) -> None: print("Set the API key as an environment variable LANGTRACE_API_KEY") print(Fore.RESET) return + + # convert spanId and traceId to hexadecimals + span_hex_number = hex(int(data["spanId"], 10))[2:] # Convert to hex and remove the '0x' prefix + formatted_span_hex_number = span_hex_number.zfill(16) # Pad with zeros to 16 characters + data["spanId"] = f"0x{formatted_span_hex_number}" + + trace_hex_number = hex(int(data["traceId"], 10))[2:] # Convert to hex and remove the '0x' prefix + formatted_trace_hex_number = trace_hex_number.zfill(32) # Pad with zeros to 32 characters + data["traceId"] = f"0x{formatted_trace_hex_number}" + evaluation = self.get_evaluation(data["spanId"]) headers = {"x-api-key": self._langtrace_api_key} if evaluation is not None: diff --git a/src/langtrace_python_sdk/version.py b/src/langtrace_python_sdk/version.py index 90a1f38f..23bc6ef5 100644 --- a/src/langtrace_python_sdk/version.py +++ b/src/langtrace_python_sdk/version.py @@ -1 +1 @@ -__version__ = "2.2.7" +__version__ = "2.2.8" From 655fe76d3ec59e19346809b4677dff6827c613f0 Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Thu, 25 Jul 2024 07:34:43 +0300 Subject: [PATCH 2/4] disable completions and prompts (#258) * disable completions and prompts * fix import * Bump version --------- Co-authored-by: Karthik Kalyanaraman --- README.md | 5 +++ .../instrumentation/anthropic/patch.py | 6 ++-- .../instrumentation/cohere/patch.py | 6 ++-- .../instrumentation/groq/patch.py | 18 +++++------ .../instrumentation/ollama/patch.py | 16 +++------- src/langtrace_python_sdk/utils/__init__.py | 21 ++++++++---- src/langtrace_python_sdk/utils/llm.py | 32 ++++++++----------- src/langtrace_python_sdk/version.py | 2 +- 8 files changed, 52 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 4c0af2b8..b6068619 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,11 @@ from langtrace_python_sdk import get_prompt_from_registry prompt = get_prompt_from_registry(, options={"prompt_version": 1, "variables": {"foo": "bar"} }) ``` +### Opt out of tracing prompt and completion data +By default, prompt and completion data are captured. If you would like to opt out of it, set the following env var, + +`TRACE_PROMPT_COMPLETION_DATA=false` + ## Supported integrations Langtrace automatically captures traces from the following vendors: diff --git a/src/langtrace_python_sdk/instrumentation/anthropic/patch.py b/src/langtrace_python_sdk/instrumentation/anthropic/patch.py index dbaade3e..69618667 100644 --- a/src/langtrace_python_sdk/instrumentation/anthropic/patch.py +++ b/src/langtrace_python_sdk/instrumentation/anthropic/patch.py @@ -25,6 +25,7 @@ get_llm_url, is_streaming, set_event_completion, + set_event_completion_chunk, set_usage_attributes, ) from opentelemetry.trace import SpanKind @@ -119,10 +120,7 @@ def handle_streaming_response(result, span): # Assuming span.add_event is part of a larger logging or event system # Add event for each chunk of content if content: - span.add_event( - Event.STREAM_OUTPUT.value, - {SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: "".join(content)}, - ) + set_event_completion_chunk(span, "".join(content)) # Assuming this is part of a generator, yield chunk or aggregated content yield content diff --git a/src/langtrace_python_sdk/instrumentation/cohere/patch.py b/src/langtrace_python_sdk/instrumentation/cohere/patch.py index e3e26dc1..560ff631 100644 --- a/src/langtrace_python_sdk/instrumentation/cohere/patch.py +++ b/src/langtrace_python_sdk/instrumentation/cohere/patch.py @@ -22,6 +22,7 @@ get_extra_attributes, get_llm_url, set_event_completion, + set_event_completion_chunk, set_usage_attributes, ) from langtrace.trace_attributes import Event, LLMSpanAttributes @@ -403,10 +404,7 @@ def traced_method(wrapped, instance, args, kwargs): content = event.text else: content = "" - span.add_event( - Event.STREAM_OUTPUT.value, - {SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: "".join(content)}, - ) + set_event_completion_chunk(span, "".join(content)) if ( hasattr(event, "finish_reason") diff --git a/src/langtrace_python_sdk/instrumentation/groq/patch.py b/src/langtrace_python_sdk/instrumentation/groq/patch.py index 9e19e51e..11c89a88 100644 --- a/src/langtrace_python_sdk/instrumentation/groq/patch.py +++ b/src/langtrace_python_sdk/instrumentation/groq/patch.py @@ -30,6 +30,7 @@ get_llm_url, get_langtrace_attributes, set_event_completion, + set_event_completion_chunk, set_usage_attributes, ) from langtrace_python_sdk.constants.instrumentation.common import ( @@ -242,15 +243,14 @@ def handle_streaming_response( content = content + [] else: content = [] - span.add_event( - Event.STREAM_OUTPUT.value, - { - SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: ( - "".join(content) - if len(content) > 0 and content[0] is not None - else "" - ) - }, + + set_event_completion_chunk( + span, + ( + "".join(content) + if len(content) > 0 and content[0] is not None + else "" + ), ) result_content.append(content[0] if len(content) > 0 else "") yield chunk diff --git a/src/langtrace_python_sdk/instrumentation/ollama/patch.py b/src/langtrace_python_sdk/instrumentation/ollama/patch.py index 584320e3..9c13073a 100644 --- a/src/langtrace_python_sdk/instrumentation/ollama/patch.py +++ b/src/langtrace_python_sdk/instrumentation/ollama/patch.py @@ -6,6 +6,7 @@ get_llm_request_attributes, get_llm_url, set_event_completion, + set_event_completion_chunk, ) from langtrace_python_sdk.utils.silently_fail import silently_fail from langtrace_python_sdk.constants.instrumentation.common import SERVICE_PROVIDERS @@ -177,12 +178,8 @@ def _handle_streaming_response(span, response, api): if api == "generate": accumulated_tokens["response"] += chunk["response"] - span.add_event( - Event.STREAM_OUTPUT.value, - { - SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: chunk.get("response") - or chunk.get("message").get("content"), - }, + set_event_completion_chunk( + span, chunk.get("response") or chunk.get("message").get("content") ) _set_response_attributes(span, chunk | accumulated_tokens) @@ -211,12 +208,7 @@ async def _ahandle_streaming_response(span, response, api): if api == "generate": accumulated_tokens["response"] += chunk["response"] - span.add_event( - Event.STREAM_OUTPUT.value, - { - SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: json.dumps(chunk), - }, - ) + set_event_completion_chunk(span, chunk) _set_response_attributes(span, chunk | accumulated_tokens) finally: # Finalize span after processing all chunks diff --git a/src/langtrace_python_sdk/utils/__init__.py b/src/langtrace_python_sdk/utils/__init__.py index 5f23d227..bcda19ce 100644 --- a/src/langtrace_python_sdk/utils/__init__.py +++ b/src/langtrace_python_sdk/utils/__init__.py @@ -2,23 +2,32 @@ from .sdk_version_checker import SDKVersionChecker from opentelemetry.trace import Span from langtrace.trace_attributes import SpanAttributes +import os def set_span_attribute(span: Span, name, value): if value is not None: if value != "" or value != NOT_GIVEN: if name == SpanAttributes.LLM_PROMPTS: - span.add_event( - name=SpanAttributes.LLM_CONTENT_PROMPT, - attributes={ - SpanAttributes.LLM_PROMPTS: value, - }, - ) + set_event_prompt(span, value) else: span.set_attribute(name, value) return +def set_event_prompt(span: Span, prompt): + enabled = os.environ.get("TRACE_PROMPT_COMPLETION_DATA", "true") + if enabled.lower() == "false": + return + + span.add_event( + name=SpanAttributes.LLM_CONTENT_PROMPT, + attributes={ + SpanAttributes.LLM_PROMPTS: prompt, + }, + ) + + def check_if_sdk_is_outdated(): SDKVersionChecker().check() return diff --git a/src/langtrace_python_sdk/utils/llm.py b/src/langtrace_python_sdk/utils/llm.py index bb00d18f..49352ede 100644 --- a/src/langtrace_python_sdk/utils/llm.py +++ b/src/langtrace_python_sdk/utils/llm.py @@ -30,6 +30,7 @@ from opentelemetry import baggage from opentelemetry.trace import Span from opentelemetry.trace.status import StatusCode +import os def estimate_tokens(prompt): @@ -42,6 +43,9 @@ def estimate_tokens(prompt): def set_event_completion_chunk(span: Span, chunk): + enabled = os.environ.get("TRACE_PROMPT_COMPLETION_DATA", "true") + if enabled.lower() == "false": + return span.add_event( name=SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK, attributes={ @@ -203,6 +207,9 @@ def get_tool_calls(item): def set_event_completion(span: Span, result_content): + enabled = os.environ.get("TRACE_PROMPT_COMPLETION_DATA", "true") + if enabled.lower() == "false": + return span.add_event( name=SpanAttributes.LLM_CONTENT_COMPLETION, @@ -352,15 +359,9 @@ def process_chunk(self, chunk): ) self.completion_tokens += token_counts content.append(tool_call.function.arguments) - self.span.add_event( - Event.STREAM_OUTPUT.value, - { - SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: ( - "".join(content) - if len(content) > 0 and content[0] is not None - else "" - ) - }, + set_event_completion_chunk( + self.span, + "".join(content) if len(content) > 0 and content[0] is not None else "", ) if content: self.result_content.append(content[0]) @@ -369,16 +370,11 @@ def process_chunk(self, chunk): token_counts = estimate_tokens(chunk.text) self.completion_tokens += token_counts content = [chunk.text] - self.span.add_event( - Event.STREAM_OUTPUT.value, - { - SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: ( - "".join(content) - if len(content) > 0 and content[0] is not None - else "" - ) - }, + set_event_completion_chunk( + self.span, + "".join(content) if len(content) > 0 and content[0] is not None else "", ) + if content: self.result_content.append(content[0]) diff --git a/src/langtrace_python_sdk/version.py b/src/langtrace_python_sdk/version.py index 23bc6ef5..73b4b053 100644 --- a/src/langtrace_python_sdk/version.py +++ b/src/langtrace_python_sdk/version.py @@ -1 +1 @@ -__version__ = "2.2.8" +__version__ = "2.2.9" From a58a24c0dbb59da9111950abf20922c76faac231 Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:27:40 +0300 Subject: [PATCH 3/4] lower case `gen_ai.system` --- src/langtrace_python_sdk/utils/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/langtrace_python_sdk/utils/llm.py b/src/langtrace_python_sdk/utils/llm.py index 49352ede..748d415f 100644 --- a/src/langtrace_python_sdk/utils/llm.py +++ b/src/langtrace_python_sdk/utils/llm.py @@ -92,7 +92,7 @@ def get_langtrace_attributes(version, service_provider, vendor_type="llm"): SpanAttributes.LANGTRACE_SERVICE_VERSION: version, SpanAttributes.LANGTRACE_SERVICE_NAME: service_provider, SpanAttributes.LANGTRACE_SERVICE_TYPE: vendor_type, - SpanAttributes.LLM_SYSTEM: service_provider, + SpanAttributes.LLM_SYSTEM: service_provider.lower(), } From 4d9b125f31badf2b97abb172c0ad5cf6120d7bdc Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:28:26 +0300 Subject: [PATCH 4/4] bump version --- src/langtrace_python_sdk/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/langtrace_python_sdk/version.py b/src/langtrace_python_sdk/version.py index 73b4b053..519574c8 100644 --- a/src/langtrace_python_sdk/version.py +++ b/src/langtrace_python_sdk/version.py @@ -1 +1 @@ -__version__ = "2.2.9" +__version__ = "2.2.10"