From a20edd9d95731824e0ef21576879004c0eace0fe Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman <105607645+karthikscale3@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:07:47 -0700 Subject: [PATCH] Release 2.1.21 (#211) * remove logs * remove requirements * DSPy instrumentation support (#210) * DSPy instrumentation basic * Fix * Fix * remove hardcodings * Bump version --- .../dspy_example/math_problems_cot.py | 42 ++++ .../dspy_example/program_of_thought_basic.py | 35 +++ src/examples/dspy_example/quiz_gen.py | 113 ++++++++++ src/examples/dspy_example/react.py | 39 ++++ .../constants/instrumentation/common.py | 1 + .../instrumentation/__init__.py | 2 + .../instrumentation/dspy/__init__.py | 3 + .../instrumentation/dspy/instrumentation.py | 85 ++++++++ .../instrumentation/dspy/patch.py | 206 ++++++++++++++++++ src/langtrace_python_sdk/langtrace.py | 2 + src/langtrace_python_sdk/version.py | 2 +- src/run_example.py | 2 +- 12 files changed, 530 insertions(+), 2 deletions(-) create mode 100644 src/examples/dspy_example/math_problems_cot.py create mode 100644 src/examples/dspy_example/program_of_thought_basic.py create mode 100644 src/examples/dspy_example/quiz_gen.py create mode 100644 src/examples/dspy_example/react.py create mode 100644 src/langtrace_python_sdk/instrumentation/dspy/__init__.py create mode 100644 src/langtrace_python_sdk/instrumentation/dspy/instrumentation.py create mode 100644 src/langtrace_python_sdk/instrumentation/dspy/patch.py diff --git a/src/examples/dspy_example/math_problems_cot.py b/src/examples/dspy_example/math_problems_cot.py new file mode 100644 index 0000000..48c8083 --- /dev/null +++ b/src/examples/dspy_example/math_problems_cot.py @@ -0,0 +1,42 @@ +import dspy +from dspy.datasets.gsm8k import GSM8K, gsm8k_metric +from dspy.teleprompt import BootstrapFewShot + +# flake8: noqa +from langtrace_python_sdk import langtrace, with_langtrace_root_span + +langtrace.init() + +turbo = dspy.OpenAI(model="gpt-3.5-turbo", max_tokens=250) +dspy.settings.configure(lm=turbo) + +# Load math questions from the GSM8K dataset +gsm8k = GSM8K() +gsm8k_trainset, gsm8k_devset = gsm8k.train[:10], gsm8k.dev[:10] + + +class CoT(dspy.Module): + def __init__(self): + super().__init__() + self.prog = dspy.ChainOfThought("question -> answer") + + def forward(self, question): + return self.prog(question=question) + + +@with_langtrace_root_span(name="math_problems_cot_example") +def example(): + + # Set up the optimizer: we want to "bootstrap" (i.e., self-generate) 4-shot examples of our CoT program. + config = dict(max_bootstrapped_demos=4, max_labeled_demos=4) + + # Optimize! Use the `gsm8k_metric` here. In general, the metric is going to tell the optimizer how well it's doing. + teleprompter = BootstrapFewShot(metric=gsm8k_metric, **config) + optimized_cot = teleprompter.compile(CoT(), trainset=gsm8k_trainset) + + ans = optimized_cot(question="What is the sqrt of 345?") + print(ans) + + +if __name__ == "__main__": + example() diff --git a/src/examples/dspy_example/program_of_thought_basic.py b/src/examples/dspy_example/program_of_thought_basic.py new file mode 100644 index 0000000..a81f885 --- /dev/null +++ b/src/examples/dspy_example/program_of_thought_basic.py @@ -0,0 +1,35 @@ +import dspy + +# flake8: noqa +from langtrace_python_sdk import langtrace, with_langtrace_root_span + +langtrace.init() + +turbo = dspy.OpenAI(model="gpt-3.5-turbo", max_tokens=250) +dspy.settings.configure(lm=turbo) + + +# Define a simple signature for basic question answering +class BasicQA(dspy.Signature): + """Answer questions with short factoid answers.""" + + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +@with_langtrace_root_span(name="pot_example") +def example(): + + # Pass signature to ProgramOfThought Module + pot = dspy.ProgramOfThought(BasicQA) + + # Call the ProgramOfThought module on a particular input + question = "Sarah has 5 apples. She buys 7 more apples from the store. How many apples does Sarah have now?" + result = pot(question=question) + + print(f"Question: {question}") + print(f"Final Predicted Answer (after ProgramOfThought process): {result.answer}") + + +if __name__ == "__main__": + example() diff --git a/src/examples/dspy_example/quiz_gen.py b/src/examples/dspy_example/quiz_gen.py new file mode 100644 index 0000000..51f1634 --- /dev/null +++ b/src/examples/dspy_example/quiz_gen.py @@ -0,0 +1,113 @@ +import dspy +import json +from dspy.datasets import HotPotQA +from dspy.teleprompt import BootstrapFewShot +from dspy.evaluate.evaluate import Evaluate + +# flake8: noqa +from langtrace_python_sdk import langtrace, with_langtrace_root_span + +langtrace.init() + + +colbertv2_wiki17_abstracts = dspy.ColBERTv2( + url="http://20.102.90.50:2017/wiki17_abstracts" +) +dspy.settings.configure(rm=colbertv2_wiki17_abstracts) +turbo = dspy.OpenAI(model="gpt-3.5-turbo-0613", max_tokens=500) +dspy.settings.configure(lm=turbo, trace=[], temperature=0.7) + +dataset = HotPotQA( + train_seed=1, + train_size=300, + eval_seed=2023, + dev_size=300, + test_size=0, + keep_details=True, +) +trainset = [x.with_inputs("question", "answer") for x in dataset.train] +devset = [x.with_inputs("question", "answer") for x in dataset.dev] + + +class GenerateAnswerChoices(dspy.Signature): + """Generate answer choices in JSON format that include the correct answer and plausible distractors for the specified question.""" + + question = dspy.InputField() + correct_answer = dspy.InputField() + number_of_choices = dspy.InputField() + answer_choices = dspy.OutputField(desc="JSON key-value pairs") + + +class QuizAnswerGenerator(dspy.Module): + def __init__(self): + super().__init__() + self.prog = dspy.ChainOfThought(GenerateAnswerChoices) + + def forward(self, question, answer): + choices = self.prog( + question=question, correct_answer=answer, number_of_choices="4" + ).answer_choices + # dspy.Suggest( + # format_checker(choices), + # "The format of the answer choices should be in JSON format. Please revise accordingly.", + # target_module=GenerateAnswerChoices, + # ) + return dspy.Prediction(choices=choices) + + +def format_checker(choice_string): + try: + choices = json.loads(choice_string) + if isinstance(choices, dict) and all( + isinstance(key, str) and isinstance(value, str) + for key, value in choices.items() + ): + return True + except json.JSONDecodeError: + return False + + return False + + +def format_valid_metric(gold, pred, trace=None): + generated_choices = pred.choices + format_valid = format_checker(generated_choices) + score = format_valid + return score + + +@with_langtrace_root_span(name="quiz_generator_1") +def quiz_generator_1(): + quiz_generator = QuizAnswerGenerator() + + example = devset[67] + print("Example Question: ", example.question) + print("Example Answer: ", example.answer) + # quiz_choices = quiz_generator(question=example.question, answer=example.answer) + # print("Generated Quiz Choices: ", quiz_choices.choices) + + optimizer = BootstrapFewShot( + metric=format_valid_metric, max_bootstrapped_demos=4, max_labeled_demos=4 + ) + compiled_quiz_generator = optimizer.compile( + quiz_generator, + trainset=trainset, + ) + quiz_choices = compiled_quiz_generator( + question=example.question, answer=example.answer + ) + print("Generated Quiz Choices: ", quiz_choices.choices) + + # Evaluate + evaluate = Evaluate( + metric=format_valid_metric, + devset=devset[67:70], + num_threads=1, + display_progress=True, + display_table=5, + ) + evaluate(quiz_generator) + + +if __name__ == "__main__": + quiz_generator_1() diff --git a/src/examples/dspy_example/react.py b/src/examples/dspy_example/react.py new file mode 100644 index 0000000..b4adbda --- /dev/null +++ b/src/examples/dspy_example/react.py @@ -0,0 +1,39 @@ +import sys +import os +import dspy + +# Add the local src folder to the Python path +sys.path.insert(0, os.path.abspath('/Users/karthikkalyanaraman/work/langtrace/langtrace-python-sdk/src')) + +# flake8: noqa +from langtrace_python_sdk import langtrace, with_langtrace_root_span +langtrace.init() + +turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=250) +dspy.settings.configure(lm=turbo) + +colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts') +dspy.settings.configure(rm=colbertv2_wiki17_abstracts) +retriever = dspy.Retrieve(k=3) + +# Define a simple signature for basic question answering +class BasicQA(dspy.Signature): + """Answer questions with short factoid answers.""" + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + +@with_langtrace_root_span(name="react_example") +def example(): + + # Pass signature to ReAct module + react_module = dspy.ReAct(BasicQA) + + # Call the ReAct module on a particular input + question = 'Aside from the Apple Remote, what other devices can control the program Apple Remote was originally designed to interact with?' + result = react_module(question=question) + + print(f"Question: {question}") + print(f"Final Predicted Answer (after ReAct process): {result.answer}") + +if __name__ == '__main__': + example() diff --git a/src/langtrace_python_sdk/constants/instrumentation/common.py b/src/langtrace_python_sdk/constants/instrumentation/common.py index 1f29ef9..579eeed 100644 --- a/src/langtrace_python_sdk/constants/instrumentation/common.py +++ b/src/langtrace_python_sdk/constants/instrumentation/common.py @@ -11,6 +11,7 @@ "ANTHROPIC": "Anthropic", "AZURE": "Azure", "CHROMA": "Chroma", + "DSPY": "DSPy", "GROQ": "Groq", "LANGCHAIN": "Langchain", "LANGCHAIN_COMMUNITY": "Langchain Community", diff --git a/src/langtrace_python_sdk/instrumentation/__init__.py b/src/langtrace_python_sdk/instrumentation/__init__.py index c08c31e..cc52148 100644 --- a/src/langtrace_python_sdk/instrumentation/__init__.py +++ b/src/langtrace_python_sdk/instrumentation/__init__.py @@ -12,6 +12,7 @@ from .qdrant import QdrantInstrumentation from .weaviate import WeaviateInstrumentation from .ollama import OllamaInstrumentor +from .dspy import DspyInstrumentor __all__ = [ "AnthropicInstrumentation", @@ -28,4 +29,5 @@ "QdrantInstrumentation", "WeaviateInstrumentation", "OllamaInstrumentor", + "DspyInstrumentor", ] diff --git a/src/langtrace_python_sdk/instrumentation/dspy/__init__.py b/src/langtrace_python_sdk/instrumentation/dspy/__init__.py new file mode 100644 index 0000000..c471918 --- /dev/null +++ b/src/langtrace_python_sdk/instrumentation/dspy/__init__.py @@ -0,0 +1,3 @@ +from .instrumentation import DspyInstrumentor + +__all__ = ["DspyInstrumentor"] diff --git a/src/langtrace_python_sdk/instrumentation/dspy/instrumentation.py b/src/langtrace_python_sdk/instrumentation/dspy/instrumentation.py new file mode 100644 index 0000000..515427c --- /dev/null +++ b/src/langtrace_python_sdk/instrumentation/dspy/instrumentation.py @@ -0,0 +1,85 @@ +""" +Copyright (c) 2024 Scale3 Labs + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.trace import get_tracer +from wrapt import wrap_function_wrapper as _W +from typing import Collection +from importlib_metadata import version as v +from .patch import patch_bootstrapfewshot_optimizer, patch_signature, patch_evaluate + + +class DspyInstrumentor(BaseInstrumentor): + """ + The DspyInstrumentor class represents the DSPy instrumentation""" + + def instrumentation_dependencies(self) -> Collection[str]: + return ["dspy >= 0.1.5"] + + def _instrument(self, **kwargs): + tracer_provider = kwargs.get("tracer_provider") + tracer = get_tracer(__name__, "", tracer_provider) + version = v("dspy") + _W( + "dspy.teleprompt.bootstrap", + "BootstrapFewShot.compile", + patch_bootstrapfewshot_optimizer( + "BootstrapFewShot.compile", version, tracer + ), + ) + _W( + "dspy.predict.predict", + "Predict.forward", + patch_signature("Predict.forward", version, tracer), + ) + _W( + "dspy.predict.chain_of_thought", + "ChainOfThought.forward", + patch_signature("ChainOfThought.forward", version, tracer), + ) + _W( + "dspy.predict.chain_of_thought_with_hint", + "ChainOfThoughtWithHint.forward", + patch_signature("ChainOfThoughtWithHint.forward", version, tracer), + ) + _W( + "dspy.predict.react", + "ReAct.forward", + patch_signature("ReAct.forward", version, tracer), + ) + _W( + "dspy.predict.program_of_thought", + "ProgramOfThought.forward", + patch_signature("ProgramOfThought.forward", version, tracer), + ) + _W( + "dspy.predict.multi_chain_comparison", + "MultiChainComparison.forward", + patch_signature("MultiChainComparison.forward", version, tracer), + ) + _W( + "dspy.predict.retry", + "Retry.forward", + patch_signature("Retry.forward", version, tracer), + ) + _W( + "dspy.evaluate.evaluate", + "Evaluate.__call__", + patch_evaluate("Evaluate", version, tracer), + ) + + def _uninstrument(self, **kwargs): + pass diff --git a/src/langtrace_python_sdk/instrumentation/dspy/patch.py b/src/langtrace_python_sdk/instrumentation/dspy/patch.py new file mode 100644 index 0000000..ef215eb --- /dev/null +++ b/src/langtrace_python_sdk/instrumentation/dspy/patch.py @@ -0,0 +1,206 @@ +import json +from importlib_metadata import version as v +from langtrace_python_sdk.constants import LANGTRACE_SDK_NAME +from langtrace_python_sdk.utils import set_span_attribute +from langtrace_python_sdk.utils.silently_fail import silently_fail +from langtrace_python_sdk.constants.instrumentation.common import ( + LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY, + SERVICE_PROVIDERS, +) +from opentelemetry import baggage +from langtrace.trace_attributes import FrameworkSpanAttributes +from opentelemetry.trace import SpanKind +from opentelemetry.trace.status import Status, StatusCode + + +def patch_bootstrapfewshot_optimizer(operation_name, version, tracer): + def traced_method(wrapped, instance, args, kwargs): + + service_provider = SERVICE_PROVIDERS["DSPY"] + extra_attributes = baggage.get_baggage(LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY) + span_attributes = { + "langtrace.sdk.name": "langtrace-python-sdk", + "langtrace.service.name": service_provider, + "langtrace.service.type": "framework", + "langtrace.service.version": version, + "langtrace.version": v(LANGTRACE_SDK_NAME), + **(extra_attributes if extra_attributes is not None else {}), + } + + if instance.__class__.__name__: + span_attributes["dspy.optimizer"] = instance.__class__.__name__ + if len(args) > 0: + span_attributes["dspy.optimizer.module"] = args[0].__class__.__name__ + if args[0].prog: + prog = { + "name": args[0].prog.__class__.__name__, + "signature": str(args[0].prog.signature) if args[0].prog.signature else None, + } + span_attributes["dspy.optimizer.module.prog"] = json.dumps(prog) + if instance.metric: + span_attributes["dspy.optimizer.metric"] = instance.metric.__name__ + if kwargs.get("trainset") and len(kwargs.get("trainset")) > 0: + span_attributes["dspy.optimizer.trainset"] = str(kwargs.get("trainset")) + config = {} + if instance.metric_threshold: + config["metric_threshold"] = instance.metric_threshold + if instance.teacher_settings: + config["teacher_settings"] = instance.teacher_settings + if instance.max_bootstrapped_demos: + config["max_bootstrapped_demos"] = instance.max_bootstrapped_demos + if instance.max_labeled_demos: + config["max_labeled_demos"] = instance.max_labeled_demos + if instance.max_rounds: + config["max_rounds"] = instance.max_rounds + if instance.max_errors: + config["max_errors"] = instance.max_errors + if instance.error_count: + config["error_count"] = instance.error_count + if config and len(config) > 0: + span_attributes["dspy.optimizer.config"] = json.dumps(config) + + attributes = FrameworkSpanAttributes(**span_attributes) + with tracer.start_as_current_span( + operation_name, kind=SpanKind.CLIENT + ) as span: + _set_input_attributes(span, kwargs, attributes) + + try: + result = wrapped(*args, **kwargs) + if result: + span.set_status(Status(StatusCode.OK)) + + span.end() + return result + + except Exception as err: + # Record the exception in the span + span.record_exception(err) + + # Set the span status to indicate an error + span.set_status(Status(StatusCode.ERROR, str(err))) + + # Reraise the exception to ensure it's not swallowed + raise + + return traced_method + + +def patch_signature(operation_name, version, tracer): + def traced_method(wrapped, instance, args, kwargs): + + service_provider = SERVICE_PROVIDERS["DSPY"] + extra_attributes = baggage.get_baggage(LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY) + span_attributes = { + "langtrace.sdk.name": "langtrace-python-sdk", + "langtrace.service.name": service_provider, + "langtrace.service.type": "framework", + "langtrace.service.version": version, + "langtrace.version": v(LANGTRACE_SDK_NAME), + **(extra_attributes if extra_attributes is not None else {}), + } + + if instance.__class__.__name__: + span_attributes["dspy.signature.name"] = instance.__class__.__name__ + span_attributes["dspy.signature"] = str(instance) + + if kwargs and len(kwargs) > 0: + span_attributes["dspy.signature.args"] = str(kwargs) + + attributes = FrameworkSpanAttributes(**span_attributes) + with tracer.start_as_current_span( + operation_name, kind=SpanKind.CLIENT + ) as span: + _set_input_attributes(span, kwargs, attributes) + + try: + result = wrapped(*args, **kwargs) + if result: + set_span_attribute(span, "dspy.signature.result", str(result)) + span.set_status(Status(StatusCode.OK)) + + span.end() + return result + + except Exception as err: + # Record the exception in the span + span.record_exception(err) + + # Set the span status to indicate an error + span.set_status(Status(StatusCode.ERROR, str(err))) + + # Reraise the exception to ensure it's not swallowed + raise + + return traced_method + + +def patch_evaluate(operation_name, version, tracer): + def traced_method(wrapped, instance, args, kwargs): + + service_provider = SERVICE_PROVIDERS["DSPY"] + extra_attributes = baggage.get_baggage(LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY) + span_attributes = { + "langtrace.sdk.name": "langtrace-python-sdk", + "langtrace.service.name": service_provider, + "langtrace.service.type": "framework", + "langtrace.service.version": version, + "langtrace.version": v(LANGTRACE_SDK_NAME), + **(extra_attributes if extra_attributes is not None else {}), + } + + if instance.devset is not None: + span_attributes["dspy.evaluate.devset"] = str(instance.devset) + if instance.display is not None: + span_attributes["dspy.evaluate.display"] = str(instance.display) + if instance.num_threads is not None: + span_attributes["dspy.evaluate.num_threads"] = str(instance.num_threads) + if instance.return_outputs is not None: + span_attributes["dspy.evaluate.return_outputs"] = str(instance.return_outputs) + if instance.display_table is not None: + span_attributes["dspy.evaluate.display_table"] = str(instance.display_table) + if instance.display_progress is not None: + span_attributes["dspy.evaluate.display_progress"] = str(instance.display_progress) + if instance.metric is not None: + span_attributes["dspy.evaluate.metric"] = instance.metric.__name__ + if instance.error_count is not None: + span_attributes["dspy.evaluate.error_count"] = str(instance.error_count) + if instance.error_lock is not None: + span_attributes["dspy.evaluate.error_lock"] = str(instance.error_lock) + if instance.max_errors is not None: + span_attributes["dspy.evaluate.max_errors"] = str(instance.max_errors) + if args and len(args) > 0: + span_attributes["dspy.evaluate.args"] = str(args) + + attributes = FrameworkSpanAttributes(**span_attributes) + with tracer.start_as_current_span( + operation_name, kind=SpanKind.CLIENT + ) as span: + _set_input_attributes(span, kwargs, attributes) + + try: + result = wrapped(*args, **kwargs) + if result is not None: + set_span_attribute(span, "dspy.evaluate.result", str(result)) + span.set_status(Status(StatusCode.OK)) + + span.end() + return result + + except Exception as err: + # Record the exception in the span + span.record_exception(err) + + # Set the span status to indicate an error + span.set_status(Status(StatusCode.ERROR, str(err))) + + # Reraise the exception to ensure it's not swallowed + raise + + return traced_method + + +@silently_fail +def _set_input_attributes(span, kwargs, attributes): + for field, value in attributes.model_dump(by_alias=True).items(): + set_span_attribute(span, field, value) diff --git a/src/langtrace_python_sdk/langtrace.py b/src/langtrace_python_sdk/langtrace.py index fc63352..28a34c6 100644 --- a/src/langtrace_python_sdk/langtrace.py +++ b/src/langtrace_python_sdk/langtrace.py @@ -51,6 +51,7 @@ QdrantInstrumentation, WeaviateInstrumentation, OllamaInstrumentor, + DspyInstrumentor, ) from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from colorama import Fore @@ -108,6 +109,7 @@ def init( "weaviate": WeaviateInstrumentation(), "sqlalchemy": SQLAlchemyInstrumentor(), "ollama": OllamaInstrumentor(), + "dspy": DspyInstrumentor(), } init_instrumentations(disable_instrumentations, all_instrumentations) diff --git a/src/langtrace_python_sdk/version.py b/src/langtrace_python_sdk/version.py index 201035e..c427aa7 100644 --- a/src/langtrace_python_sdk/version.py +++ b/src/langtrace_python_sdk/version.py @@ -1 +1 @@ -__version__ = "2.1.20" +__version__ = "2.1.21" diff --git a/src/run_example.py b/src/run_example.py index 10afa59..2de23e5 100644 --- a/src/run_example.py +++ b/src/run_example.py @@ -13,7 +13,7 @@ "pinecone": False, "qdrant": False, "weaviate": False, - "ollama": True, + "ollama": False, "groq": False, }