From eba76a6265da9c858f090b78028b2ed9dc065455 Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Wed, 17 Jan 2024 17:28:46 -0500 Subject: [PATCH 01/13] Add OpenTelemetry class for validator hub --- guardrails/utils/hub_telemetry_utils.py | 89 +++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 guardrails/utils/hub_telemetry_utils.py diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py new file mode 100644 index 000000000..da898b2dd --- /dev/null +++ b/guardrails/utils/hub_telemetry_utils.py @@ -0,0 +1,89 @@ +# Imports +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + + +class HubTelemetry: + """Singleton class for initializing a tracer for Guardrails Hub""" + + _instance = None + _service_name = None + _endpoint = None + _tracer_name = None + _resource = None + _tracer_provider = None + _processor = None + _tracer = None + _prop = None + _carrier = {} + + def __new__( + cls, + service_name: str = "guardrails-hub", + endpoint: str = "localhost:4317", + tracer_name: str = "gr_hub", + export_locally: bool = True, + ): + if cls._instance is None: + print("Creating HubTelemetry instance...") + cls._instance = super(HubTelemetry, cls).__new__(cls) + print("Initializing HubTelemetry instance...") + cls._instance.initialize_tracer( + service_name, endpoint, tracer_name, export_locally + ) + return cls._instance + + def initialize_tracer( + self, + service_name: str, + endpoint: str, + tracer_name: str, + export_locally: bool, + ): + """Initializes a tracer for Guardrails Hub""" + + self._service_name = service_name + self._endpoint = endpoint + self._tracer_name = tracer_name + + # Create a resource + # Service name is required for most backends + self._resource = Resource(attributes={SERVICE_NAME: self._service_name}) + + # Create a tracer provider and a processor + self._tracer_provider = TracerProvider(resource=self._resource) + + if export_locally: + self._processor = BatchSpanProcessor(ConsoleSpanExporter()) + else: + self._processor = BatchSpanProcessor( + OTLPSpanExporter(endpoint=self._endpoint, insecure=True) + ) + + # Add the processor to the provider + self._tracer_provider.add_span_processor(self._processor) + + # Set the tracer provider and return a tracer + trace.set_tracer_provider(self._tracer_provider) + self._tracer = trace.get_tracer(self._tracer_name) + + self._prop = TraceContextTextMapPropagator() + + def get_tracer(self): + """Returns the tracer""" + + return self._tracer + + def inject_current_context(self) -> None: + """Injects the current context into the carrier""" + self._prop.inject(carrier=self._carrier) + + def extract_current_context(self): + """Extracts the current context from the carrier""" + + context = self._prop.extract(carrier=self._carrier) + return context From 5aaecb9d1503888abd75bad172ac5cb68195b63b Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Wed, 17 Jan 2024 17:29:32 -0500 Subject: [PATCH 02/13] Init global tracer, use it to create nested spans to send usage data --- guardrails/guard.py | 36 +++++++++++++++++++++++++++++++++ guardrails/validator_service.py | 15 ++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/guardrails/guard.py b/guardrails/guard.py index 33b3ffabd..9b4f6d197 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -38,6 +38,8 @@ set_tracer_context, ) from guardrails.validators import Validator +from guardrails.utils.hub_telemetry_utils import HubTelemetry +from guardrails.cli_dir.hub.credentials import Credentials add_destinations(logger.debug) @@ -61,6 +63,10 @@ class Guard(Generic[OT]): _tracer = None _tracer_context = None + _hub_telemetry = None + _hub_tracer = None + _guard_id = None + _user_id = None def __init__( self, @@ -77,6 +83,17 @@ def __init__( self.base_model = base_model self._set_tracer(tracer) + # Initialize Hub Telemetry singleton and get the tracer + self._hub_telemetry = HubTelemetry() + self._hub_tracer = self._hub_telemetry.get_tracer() + + # Get id of guard object (that is unique) + self._guard_id = id(self) # id of guard object; not the class + + # Get unique id of user (from rc file) + self._user_id = Credentials.from_rc_file().id + print(f"Guard init complete with tracer @ location {id(self._hub_tracer)}") + @property def prompt_schema(self) -> Optional[StringSchema]: """Return the input schema.""" @@ -359,6 +376,15 @@ def __call( if prompt_params is None: prompt_params = {} + # TODO: Add tracing: guard usage and user usage for Validator Hub + # Start a span for this guard call + with self._hub_tracer.start_as_current_span("/guard_call") as span: + # Inject the current context + self._hub_telemetry.inject_current_context() + + span.set_attribute("guard_id", self._guard_id) + span.set_attribute("user_id", self._user_id) + set_call_kwargs(kwargs) set_tracer(self._tracer) set_tracer_context(self._tracer_context) @@ -650,6 +676,16 @@ def __parse( final_num_reasks = ( num_reasks if num_reasks is not None else 0 if llm_api is None else None ) + + # TODO: Add tracing: guard usage and user usage for Validator Hub + # Start a span for this guard parse + with self._hub_tracer.start_as_current_span("/guard_parse") as span: + # Inject the current context + self._hub_telemetry.inject_current_context() + + span.set_attribute("guard_id", self._guard_id) + span.set_attribute("user_id", self._user_id) + self.configure(final_num_reasks) if self.num_reasks is None: raise RuntimeError( diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py index ddc146d65..c24c3df8e 100644 --- a/guardrails/validator_service.py +++ b/guardrails/validator_service.py @@ -22,6 +22,7 @@ Validator, ValidatorError, ) +from guardrails.utils.hub_telemetry_utils import HubTelemetry class ValidatorServiceBase: @@ -120,6 +121,20 @@ def run_validator( # If we ever re-use validator instances across multiple properties, # this will have to change. validator_logs.instance_id = to_string(id(validator)) + + # Get HubTelemetry singleton and tracer + hub_telemetry = HubTelemetry() + hub_tracer = hub_telemetry.get_tracer() + print(f"Getting tracer from location: {id(hub_tracer)}") + + with hub_tracer.start_as_current_span( + "/validator_usage", context=hub_telemetry.extract_current_context() + ) as span: + # Inject the current context + hub_telemetry.inject_current_context() + + span.set_attribute("validator_name", validator_class_name) + return validator_logs From 6cfcdf46fb812eedce71dd0a9207238e419f7d70 Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Wed, 17 Jan 2024 17:30:18 -0500 Subject: [PATCH 03/13] Add placeholder cli classes to get creds from local file; will have merge conflicts in future when merging hub-cli --- guardrails/cli_dir/hub/credentials.py | 34 ++++++++++++++++++++++ guardrails/cli_dir/server/serializeable.py | 33 +++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 guardrails/cli_dir/hub/credentials.py create mode 100644 guardrails/cli_dir/server/serializeable.py diff --git a/guardrails/cli_dir/hub/credentials.py b/guardrails/cli_dir/hub/credentials.py new file mode 100644 index 000000000..d85da6ecc --- /dev/null +++ b/guardrails/cli_dir/hub/credentials.py @@ -0,0 +1,34 @@ +import os +import sys +from dataclasses import dataclass +from os.path import expanduser + +from guardrails.cli_dir.server.serializeable import Serializeable + + +@dataclass +class Credentials(Serializeable): + id: str + client_id: str + client_secret: str + no_metrics: bool = False + + @staticmethod + def from_rc_file() -> "Credentials": + try: + home = expanduser("~") + guardrails_rc = os.path.join(home, ".guardrailsrc") + with open(guardrails_rc) as rc_file: + lines = rc_file.readlines() + creds = {} + for line in lines: + key, value = line.split("=", 1) + creds[key.strip()] = value.strip() + rc_file.close() + return Credentials.from_dict(creds) + + except FileNotFoundError as e: + print( + "Guardrails Hub credentials not found!\nSign up to use the Hub here: {insert url}" + ) + sys.exit(1) diff --git a/guardrails/cli_dir/server/serializeable.py b/guardrails/cli_dir/server/serializeable.py new file mode 100644 index 000000000..311085200 --- /dev/null +++ b/guardrails/cli_dir/server/serializeable.py @@ -0,0 +1,33 @@ +import inspect +import json +from dataclasses import InitVar, asdict, dataclass, field, is_dataclass +from json import JSONEncoder +from typing import Any, Dict + + +class SerializeableJSONEncoder(JSONEncoder): + def default(self, o): + if is_dataclass(o): + return asdict(o) + return super().default(o) + + +@dataclass +class Serializeable: + encoder: InitVar[JSONEncoder] = field( + kw_only=True, default=SerializeableJSONEncoder # type: ignore + ) + + @classmethod + def from_dict(cls, data: Dict[str, Any]): + annotations = inspect.get_annotations(cls) + attributes = dict.keys(annotations) + kwargs = {k: data.get(k) for k in data if k in attributes} + return cls(**kwargs) # type: ignore + + @property + def __dict__(self) -> Dict[str, Any]: + return asdict(self) + + def to_json(self): + return json.dumps(self, cls=self.encoder) # type: ignore From 9bfc5f56bd8244b0e14cbcb5e488f0a2d5f2b15a Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Fri, 19 Jan 2024 17:38:22 -0500 Subject: [PATCH 04/13] Update class, endpoint, add new methods for context --- guardrails/utils/hub_telemetry_utils.py | 27 ++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py index da898b2dd..aee6a9511 100644 --- a/guardrails/utils/hub_telemetry_utils.py +++ b/guardrails/utils/hub_telemetry_utils.py @@ -1,9 +1,22 @@ # Imports from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + +# 2 exporters available: HTTP and GRPC, only use one at a time +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, +) # HTTP + +# from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( +# OTLPSpanExporter, +# ) # GRPC + from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, + ConsoleSpanExporter, + SimpleSpanProcessor, +) from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator @@ -24,9 +37,9 @@ class HubTelemetry: def __new__( cls, service_name: str = "guardrails-hub", - endpoint: str = "localhost:4317", + endpoint: str = "http://localhost:4318/v1/traces", # HTTP: 4318, GRPC: 4317 tracer_name: str = "gr_hub", - export_locally: bool = True, + export_locally: bool = False, ): if cls._instance is None: print("Creating HubTelemetry instance...") @@ -58,10 +71,10 @@ def initialize_tracer( self._tracer_provider = TracerProvider(resource=self._resource) if export_locally: - self._processor = BatchSpanProcessor(ConsoleSpanExporter()) + self._processor = SimpleSpanProcessor(ConsoleSpanExporter()) else: - self._processor = BatchSpanProcessor( - OTLPSpanExporter(endpoint=self._endpoint, insecure=True) + self._processor = SimpleSpanProcessor( + OTLPSpanExporter(endpoint=self._endpoint) ) # Add the processor to the provider From 44fcd93dba695c3d5650fc83e015016e50bdf442 Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Thu, 25 Jan 2024 11:07:27 -0500 Subject: [PATCH 05/13] Add new span, update existing spans, one parent, multiple children --- guardrails/guard.py | 10 ++++++++++ guardrails/run.py | 13 +++++++++++++ guardrails/validator_service.py | 7 +++---- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/guardrails/guard.py b/guardrails/guard.py index 9b4f6d197..e0b5fcdc2 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -384,6 +384,11 @@ def __call( span.set_attribute("guard_id", self._guard_id) span.set_attribute("user_id", self._user_id) + span.set_attribute("llm_api", llm_api.__name__ if llm_api else "None") + span.set_attribute("custom_reask_prompt", self.reask_prompt is not None) + span.set_attribute( + "custom_reask_instructions", self.reask_instructions is not None + ) set_call_kwargs(kwargs) set_tracer(self._tracer) @@ -685,6 +690,11 @@ def __parse( span.set_attribute("guard_id", self._guard_id) span.set_attribute("user_id", self._user_id) + span.set_attribute("llm_api", llm_api.__name__ if llm_api else "None") + span.set_attribute("custom_reask_prompt", self.reask_prompt is not None) + span.set_attribute( + "custom_reask_instructions", self.reask_instructions is not None + ) self.configure(final_num_reasks) if self.num_reasks is None: diff --git a/guardrails/run.py b/guardrails/run.py index 5b4325f76..cb3e6e97f 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -26,6 +26,7 @@ reasks_to_dict, ) from guardrails.utils.telemetry_utils import async_trace, trace +from guardrails.utils.hub_telemetry_utils import HubTelemetry from guardrails.validator_base import ValidatorError add_destinations(logger.debug) @@ -195,6 +196,18 @@ def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call prompt_params=prompt_params, include_instructions=include_instructions, ) + + # Log how many times we reasked + # Get HubTelemetry singleton and tracer + hub_telemetry = HubTelemetry() + hub_tracer = hub_telemetry.get_tracer() + print(f"Getting tracer from location: {id(hub_tracer)}") + + with hub_tracer.start_as_current_span( + "/reasks", context=hub_telemetry.extract_current_context() + ) as span: + span.set_attribute("reask_count", index) + except UserFacingException as e: # Because Pydantic v1 doesn't respect property setters call_log._exception = e.original_exception diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py index c24c3df8e..704d930a5 100644 --- a/guardrails/validator_service.py +++ b/guardrails/validator_service.py @@ -130,10 +130,9 @@ def run_validator( with hub_tracer.start_as_current_span( "/validator_usage", context=hub_telemetry.extract_current_context() ) as span: - # Inject the current context - hub_telemetry.inject_current_context() - - span.set_attribute("validator_name", validator_class_name) + span.set_attribute("validator_name", validator.rail_alias) + span.set_attribute("validator_on_fail", validator.on_fail_descriptor) + span.set_attribute("validator_result", result.outcome) return validator_logs From 246784247a9cfc9a7568721f983342a13512ccbd Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Fri, 26 Jan 2024 17:44:51 -0500 Subject: [PATCH 06/13] Update endpoint --- guardrails/utils/hub_telemetry_utils.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py index aee6a9511..2ada32a80 100644 --- a/guardrails/utils/hub_telemetry_utils.py +++ b/guardrails/utils/hub_telemetry_utils.py @@ -1,19 +1,11 @@ # Imports from opentelemetry import trace - -# 2 exporters available: HTTP and GRPC, only use one at a time from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter, -) # HTTP - -# from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( -# OTLPSpanExporter, -# ) # GRPC - +) # HTTP Exporter from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( - BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor, ) @@ -37,7 +29,6 @@ class HubTelemetry: def __new__( cls, service_name: str = "guardrails-hub", - endpoint: str = "http://localhost:4318/v1/traces", # HTTP: 4318, GRPC: 4317 tracer_name: str = "gr_hub", export_locally: bool = False, ): @@ -45,22 +36,20 @@ def __new__( print("Creating HubTelemetry instance...") cls._instance = super(HubTelemetry, cls).__new__(cls) print("Initializing HubTelemetry instance...") - cls._instance.initialize_tracer( - service_name, endpoint, tracer_name, export_locally - ) + cls._instance.initialize_tracer(service_name, tracer_name, export_locally) return cls._instance def initialize_tracer( self, service_name: str, - endpoint: str, tracer_name: str, export_locally: bool, ): """Initializes a tracer for Guardrails Hub""" self._service_name = service_name - self._endpoint = endpoint + # self._endpoint = "http://localhost:4318/v1/traces" # Local Otel Collector (Docker) + self._endpoint = "https://hty0gc1ok3.execute-api.us-east-1.amazonaws.com/v1/traces" # AWS ECS self._tracer_name = tracer_name # Create a resource From 49c83ae95ee81143e8b6274ad09a175ad28c4ef5 Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Fri, 26 Jan 2024 17:49:59 -0500 Subject: [PATCH 07/13] Lint --- guardrails/cli_dir/hub/credentials.py | 4 ++-- guardrails/guard.py | 4 ++-- guardrails/run.py | 2 +- guardrails/utils/hub_telemetry_utils.py | 25 ++++++++++++------------- guardrails/validator_service.py | 2 +- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/guardrails/cli_dir/hub/credentials.py b/guardrails/cli_dir/hub/credentials.py index d85da6ecc..42dd0a17e 100644 --- a/guardrails/cli_dir/hub/credentials.py +++ b/guardrails/cli_dir/hub/credentials.py @@ -27,8 +27,8 @@ def from_rc_file() -> "Credentials": rc_file.close() return Credentials.from_dict(creds) - except FileNotFoundError as e: + except FileNotFoundError: print( - "Guardrails Hub credentials not found!\nSign up to use the Hub here: {insert url}" + "Guardrails Hub credentials not found!\nSign up to use the Hub here: {insert url}" # noqa: E501 ) sys.exit(1) diff --git a/guardrails/guard.py b/guardrails/guard.py index e0b5fcdc2..89e356528 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -24,6 +24,7 @@ from guardrails.classes.generic import Stack from guardrails.classes.history import Call from guardrails.classes.history.call_inputs import CallInputs +from guardrails.cli_dir.hub.credentials import Credentials from guardrails.llm_providers import get_async_llm_ask, get_llm_ask from guardrails.logger import logger, set_scope from guardrails.prompt import Instructions, Prompt @@ -37,9 +38,8 @@ set_tracer, set_tracer_context, ) -from guardrails.validators import Validator from guardrails.utils.hub_telemetry_utils import HubTelemetry -from guardrails.cli_dir.hub.credentials import Credentials +from guardrails.validators import Validator add_destinations(logger.debug) diff --git a/guardrails/run.py b/guardrails/run.py index cb3e6e97f..2298339bd 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -17,6 +17,7 @@ from guardrails.prompt import Instructions, Prompt from guardrails.schema import Schema, StringSchema from guardrails.utils.exception_utils import UserFacingException +from guardrails.utils.hub_telemetry_utils import HubTelemetry from guardrails.utils.llm_response import LLMResponse from guardrails.utils.openai_utils import OPENAI_VERSION from guardrails.utils.reask_utils import ( @@ -26,7 +27,6 @@ reasks_to_dict, ) from guardrails.utils.telemetry_utils import async_trace, trace -from guardrails.utils.hub_telemetry_utils import HubTelemetry from guardrails.validator_base import ValidatorError add_destinations(logger.debug) diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py index 2ada32a80..ef7185d79 100644 --- a/guardrails/utils/hub_telemetry_utils.py +++ b/guardrails/utils/hub_telemetry_utils.py @@ -1,19 +1,16 @@ # Imports from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( # HTTP Exporter OTLPSpanExporter, -) # HTTP Exporter +) from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import ( - ConsoleSpanExporter, - SimpleSpanProcessor, -) +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator class HubTelemetry: - """Singleton class for initializing a tracer for Guardrails Hub""" + """Singleton class for initializing a tracer for Guardrails Hub.""" _instance = None _service_name = None @@ -45,11 +42,13 @@ def initialize_tracer( tracer_name: str, export_locally: bool, ): - """Initializes a tracer for Guardrails Hub""" + """Initializes a tracer for Guardrails Hub.""" self._service_name = service_name - # self._endpoint = "http://localhost:4318/v1/traces" # Local Otel Collector (Docker) - self._endpoint = "https://hty0gc1ok3.execute-api.us-east-1.amazonaws.com/v1/traces" # AWS ECS + # self._endpoint = "http://localhost:4318/v1/traces" + self._endpoint = ( + "https://hty0gc1ok3.execute-api.us-east-1.amazonaws.com/v1/traces" + ) self._tracer_name = tracer_name # Create a resource @@ -76,16 +75,16 @@ def initialize_tracer( self._prop = TraceContextTextMapPropagator() def get_tracer(self): - """Returns the tracer""" + """Returns the tracer.""" return self._tracer def inject_current_context(self) -> None: - """Injects the current context into the carrier""" + """Injects the current context into the carrier.""" self._prop.inject(carrier=self._carrier) def extract_current_context(self): - """Extracts the current context from the carrier""" + """Extracts the current context from the carrier.""" context = self._prop.extract(carrier=self._carrier) return context diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py index 704d930a5..a089281b0 100644 --- a/guardrails/validator_service.py +++ b/guardrails/validator_service.py @@ -9,6 +9,7 @@ from guardrails.datatypes import FieldValidation from guardrails.logger import logger from guardrails.utils.casting_utils import to_string +from guardrails.utils.hub_telemetry_utils import HubTelemetry from guardrails.utils.logs_utils import ValidatorLogs from guardrails.utils.reask_utils import FieldReAsk, ReAsk from guardrails.utils.safe_get import safe_get @@ -22,7 +23,6 @@ Validator, ValidatorError, ) -from guardrails.utils.hub_telemetry_utils import HubTelemetry class ValidatorServiceBase: From ee708e4a4a6e74eb6268fb8483378545dbf79f28 Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Mon, 29 Jan 2024 18:26:53 -0500 Subject: [PATCH 08/13] Update credentials, and add logger --- guardrails/cli_dir/hub/credentials.py | 22 +++++++++++++--------- guardrails/cli_dir/logger.py | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 guardrails/cli_dir/logger.py diff --git a/guardrails/cli_dir/hub/credentials.py b/guardrails/cli_dir/hub/credentials.py index 42dd0a17e..2ba3a5748 100644 --- a/guardrails/cli_dir/hub/credentials.py +++ b/guardrails/cli_dir/hub/credentials.py @@ -1,17 +1,18 @@ import os -import sys from dataclasses import dataclass from os.path import expanduser +from typing import Optional +from guardrails.cli_dir.logger import logger from guardrails.cli_dir.server.serializeable import Serializeable @dataclass class Credentials(Serializeable): - id: str - client_id: str - client_secret: str - no_metrics: bool = False + id: Optional[str] = None + client_id: Optional[str] = None + client_secret: Optional[str] = None + no_metrics: Optional[bool] = False @staticmethod def from_rc_file() -> "Credentials": @@ -27,8 +28,11 @@ def from_rc_file() -> "Credentials": rc_file.close() return Credentials.from_dict(creds) - except FileNotFoundError: - print( - "Guardrails Hub credentials not found!\nSign up to use the Hub here: {insert url}" # noqa: E501 + except FileNotFoundError as e: + logger.error(e) + logger.error( + "Guardrails Hub credentials not found!" + "You will need to sign up to use any authenticated Validators here:" + "{insert url}" ) - sys.exit(1) + return Credentials() diff --git a/guardrails/cli_dir/logger.py b/guardrails/cli_dir/logger.py new file mode 100644 index 000000000..d29662acc --- /dev/null +++ b/guardrails/cli_dir/logger.py @@ -0,0 +1,20 @@ +import logging +import os + +import coloredlogs + +os.environ[ + "COLOREDLOGS_LEVEL_STYLES" +] = "spam=white,faint;success=green,bold;debug=magenta;verbose=blue;notice=cyan,bold;warning=yellow;error=red;critical=background=red" # noqa +LEVELS = { + "SPAM": 5, + "VERBOSE": 15, + "NOTICE": 25, + "SUCCESS": 35, +} +for key in LEVELS: + logging.addLevelName(LEVELS.get(key), key) # type: ignore + + +logger = logging.getLogger("guardrails-cli") +coloredlogs.install(level="DEBUG", logger=logger) From 5e4096be30ceb4df056afbeb6458b214c8e926ad Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Mon, 29 Jan 2024 19:26:10 -0500 Subject: [PATCH 09/13] Add util method that will create spans from anywhere, update existing ways --- guardrails/guard.py | 55 ++++++++++++------------- guardrails/run.py | 19 +++++---- guardrails/utils/hub_telemetry_utils.py | 38 ++++++++++++++--- guardrails/validator_service.py | 24 ++++++----- 4 files changed, 82 insertions(+), 54 deletions(-) diff --git a/guardrails/guard.py b/guardrails/guard.py index 89e356528..98cf4eb07 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -64,7 +64,6 @@ class Guard(Generic[OT]): _tracer = None _tracer_context = None _hub_telemetry = None - _hub_tracer = None _guard_id = None _user_id = None @@ -85,14 +84,12 @@ def __init__( # Initialize Hub Telemetry singleton and get the tracer self._hub_telemetry = HubTelemetry() - self._hub_tracer = self._hub_telemetry.get_tracer() # Get id of guard object (that is unique) self._guard_id = id(self) # id of guard object; not the class # Get unique id of user (from rc file) self._user_id = Credentials.from_rc_file().id - print(f"Guard init complete with tracer @ location {id(self._hub_tracer)}") @property def prompt_schema(self) -> Optional[StringSchema]: @@ -376,19 +373,19 @@ def __call( if prompt_params is None: prompt_params = {} - # TODO: Add tracing: guard usage and user usage for Validator Hub - # Start a span for this guard call - with self._hub_tracer.start_as_current_span("/guard_call") as span: - # Inject the current context - self._hub_telemetry.inject_current_context() - - span.set_attribute("guard_id", self._guard_id) - span.set_attribute("user_id", self._user_id) - span.set_attribute("llm_api", llm_api.__name__ if llm_api else "None") - span.set_attribute("custom_reask_prompt", self.reask_prompt is not None) - span.set_attribute( - "custom_reask_instructions", self.reask_instructions is not None - ) + # Create a new span for this guard call + self._hub_telemetry.create_new_span( + span_name="/guard_call", + attributes=[ + ("guard_id", self._guard_id), + ("user_id", self._user_id), + ("llm_api", llm_api.__name__ if llm_api else "None"), + ("custom_reask_prompt", self.reask_prompt is not None), + ("custom_reask_instructions", self.reask_instructions is not None), + ], + is_parent=True, # It will have children + has_parent=False, # Has no parents + ) set_call_kwargs(kwargs) set_tracer(self._tracer) @@ -682,19 +679,19 @@ def __parse( num_reasks if num_reasks is not None else 0 if llm_api is None else None ) - # TODO: Add tracing: guard usage and user usage for Validator Hub - # Start a span for this guard parse - with self._hub_tracer.start_as_current_span("/guard_parse") as span: - # Inject the current context - self._hub_telemetry.inject_current_context() - - span.set_attribute("guard_id", self._guard_id) - span.set_attribute("user_id", self._user_id) - span.set_attribute("llm_api", llm_api.__name__ if llm_api else "None") - span.set_attribute("custom_reask_prompt", self.reask_prompt is not None) - span.set_attribute( - "custom_reask_instructions", self.reask_instructions is not None - ) + # TODO: Create a new span for this guard parse + self._hub_telemetry.create_new_span( + span_name="/guard_parse", + attributes=[ + ("guard_id", self._guard_id), + ("user_id", self._user_id), + ("llm_api", llm_api.__name__ if llm_api else "None"), + ("custom_reask_prompt", self.reask_prompt is not None), + ("custom_reask_instructions", self.reask_instructions is not None), + ], + is_parent=True, # It will have children + has_parent=False, # Has no parents + ) self.configure(final_num_reasks) if self.num_reasks is None: diff --git a/guardrails/run.py b/guardrails/run.py index 2298339bd..cb2c22687 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -104,6 +104,9 @@ def __init__( self.base_model = base_model self.full_schema_reask = full_schema_reask + # Get the HubTelemetry singleton + self._hub_telemetry = HubTelemetry() + def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call: """Execute the runner by repeatedly calling step until the reask budget is exhausted. @@ -198,15 +201,13 @@ def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call ) # Log how many times we reasked - # Get HubTelemetry singleton and tracer - hub_telemetry = HubTelemetry() - hub_tracer = hub_telemetry.get_tracer() - print(f"Getting tracer from location: {id(hub_tracer)}") - - with hub_tracer.start_as_current_span( - "/reasks", context=hub_telemetry.extract_current_context() - ) as span: - span.set_attribute("reask_count", index) + # Use the HubTelemetry singleton + self._hub_telemetry.create_new_span( + span_name="/reasks", + attributes=[("reask_count", index)], + is_parent=False, # This span has no children + has_parent=True, # This span has a parent + ) except UserFacingException as e: # Because Pydantic v1 doesn't respect property setters diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py index ef7185d79..1bf6ebc0c 100644 --- a/guardrails/utils/hub_telemetry_utils.py +++ b/guardrails/utils/hub_telemetry_utils.py @@ -34,6 +34,8 @@ def __new__( cls._instance = super(HubTelemetry, cls).__new__(cls) print("Initializing HubTelemetry instance...") cls._instance.initialize_tracer(service_name, tracer_name, export_locally) + else: + print("Returning existing HubTelemetry instance...") return cls._instance def initialize_tracer( @@ -74,11 +76,6 @@ def initialize_tracer( self._prop = TraceContextTextMapPropagator() - def get_tracer(self): - """Returns the tracer.""" - - return self._tracer - def inject_current_context(self) -> None: """Injects the current context into the carrier.""" self._prop.inject(carrier=self._carrier) @@ -88,3 +85,34 @@ def extract_current_context(self): context = self._prop.extract(carrier=self._carrier) return context + + def create_new_span( + self, + span_name: str, + attributes: list, + is_parent: bool, # Inject current context if IS a parent span + has_parent: bool, # Extract current context if HAS a parent span + ): + """Creates a new span within the tracer with the given name and attributes. + + If it's a parent span, the current context is injected into the carrier. + If it has a parent span, the current context is extracted from the carrier. + Both the conditions can co-exist e.g. if it's a parent span and also has a parent span. + + Args: + span_name (str): The name of the span. + attributes (list): A list of attributes to set on the span. + is_parent (bool): True if the span is a parent span. + has_parent (bool): True if the span has a parent span. + """ + + with self._tracer.start_as_current_span( + span_name, + context=self.extract_current_context() if has_parent else None, + ) as span: + if is_parent: + # Inject the current context + self.inject_current_context() + + for attribute in attributes: + span.set_attribute(attribute[0], attribute[1]) diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py index a089281b0..6f855ca26 100644 --- a/guardrails/validator_service.py +++ b/guardrails/validator_service.py @@ -122,17 +122,19 @@ def run_validator( # this will have to change. validator_logs.instance_id = to_string(id(validator)) - # Get HubTelemetry singleton and tracer - hub_telemetry = HubTelemetry() - hub_tracer = hub_telemetry.get_tracer() - print(f"Getting tracer from location: {id(hub_tracer)}") - - with hub_tracer.start_as_current_span( - "/validator_usage", context=hub_telemetry.extract_current_context() - ) as span: - span.set_attribute("validator_name", validator.rail_alias) - span.set_attribute("validator_on_fail", validator.on_fail_descriptor) - span.set_attribute("validator_result", result.outcome) + # Get HubTelemetry singleton and create a new span to + # log the validator usage + _hub_telemetry = HubTelemetry() + _hub_telemetry.create_new_span( + span_name="/validator_usage", + attributes=[ + ("validator_name", validator.rail_alias), + ("validator_on_fail", validator.on_fail_descriptor), + ("validator_result", result.outcome), + ], + is_parent=False, # This span will have no children + has_parent=True, # This span has a parent + ) return validator_logs From fc87a7461df598d5f4abd5ca8ed493cfc8f5f87f Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Tue, 13 Feb 2024 14:01:43 -0500 Subject: [PATCH 10/13] Replace print with logging --- guardrails/utils/hub_telemetry_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py index 1bf6ebc0c..cf0c1b58c 100644 --- a/guardrails/utils/hub_telemetry_utils.py +++ b/guardrails/utils/hub_telemetry_utils.py @@ -1,4 +1,6 @@ # Imports +import logging + from opentelemetry import trace from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( # HTTP Exporter OTLPSpanExporter, @@ -30,12 +32,12 @@ def __new__( export_locally: bool = False, ): if cls._instance is None: - print("Creating HubTelemetry instance...") + logging.debug("Creating HubTelemetry instance...") cls._instance = super(HubTelemetry, cls).__new__(cls) - print("Initializing HubTelemetry instance...") + logging.debug("Initializing HubTelemetry instance...") cls._instance.initialize_tracer(service_name, tracer_name, export_locally) else: - print("Returning existing HubTelemetry instance...") + logging.debug("Returning existing HubTelemetry instance...") return cls._instance def initialize_tracer( From af28d4087b04d7b53c67a0a6eb23b24fbf840a04 Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Tue, 13 Feb 2024 14:45:32 -0500 Subject: [PATCH 11/13] Add opt-out --- guardrails/guard.py | 72 ++++++++++++++----------- guardrails/run.py | 21 +++++--- guardrails/utils/hub_telemetry_utils.py | 6 ++- guardrails/validator_service.py | 30 ++++++----- 4 files changed, 76 insertions(+), 53 deletions(-) diff --git a/guardrails/guard.py b/guardrails/guard.py index 98cf4eb07..12219b24d 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -82,14 +82,19 @@ def __init__( self.base_model = base_model self._set_tracer(tracer) - # Initialize Hub Telemetry singleton and get the tracer - self._hub_telemetry = HubTelemetry() + # Get unique id of user from credentials + self._user_id = Credentials.from_rc_file().id + + # Get metrics opt-out from credentials + self._disable_tracer = Credentials.from_rc_file().no_metrics # Get id of guard object (that is unique) self._guard_id = id(self) # id of guard object; not the class - # Get unique id of user (from rc file) - self._user_id = Credentials.from_rc_file().id + # Initialize Hub Telemetry singleton and get the tracer + # if it is not disabled + if not self._disable_tracer: + self._hub_telemetry = HubTelemetry() @property def prompt_schema(self) -> Optional[StringSchema]: @@ -373,19 +378,23 @@ def __call( if prompt_params is None: prompt_params = {} - # Create a new span for this guard call - self._hub_telemetry.create_new_span( - span_name="/guard_call", - attributes=[ - ("guard_id", self._guard_id), - ("user_id", self._user_id), - ("llm_api", llm_api.__name__ if llm_api else "None"), - ("custom_reask_prompt", self.reask_prompt is not None), - ("custom_reask_instructions", self.reask_instructions is not None), - ], - is_parent=True, # It will have children - has_parent=False, # Has no parents - ) + if not self._disable_tracer: + # Create a new span for this guard call + self._hub_telemetry.create_new_span( + span_name="/guard_call", + attributes=[ + ("guard_id", self._guard_id), + ("user_id", self._user_id), + ("llm_api", llm_api.__name__ if llm_api else "None"), + ("custom_reask_prompt", self.reask_prompt is not None), + ( + "custom_reask_instructions", + self.reask_instructions is not None, + ), + ], + is_parent=True, # It will have children + has_parent=False, # Has no parents + ) set_call_kwargs(kwargs) set_tracer(self._tracer) @@ -679,19 +688,22 @@ def __parse( num_reasks if num_reasks is not None else 0 if llm_api is None else None ) - # TODO: Create a new span for this guard parse - self._hub_telemetry.create_new_span( - span_name="/guard_parse", - attributes=[ - ("guard_id", self._guard_id), - ("user_id", self._user_id), - ("llm_api", llm_api.__name__ if llm_api else "None"), - ("custom_reask_prompt", self.reask_prompt is not None), - ("custom_reask_instructions", self.reask_instructions is not None), - ], - is_parent=True, # It will have children - has_parent=False, # Has no parents - ) + if not self._disable_tracer: + self._hub_telemetry.create_new_span( + span_name="/guard_parse", + attributes=[ + ("guard_id", self._guard_id), + ("user_id", self._user_id), + ("llm_api", llm_api.__name__ if llm_api else "None"), + ("custom_reask_prompt", self.reask_prompt is not None), + ( + "custom_reask_instructions", + self.reask_instructions is not None, + ), + ], + is_parent=True, # It will have children + has_parent=False, # Has no parents + ) self.configure(final_num_reasks) if self.num_reasks is None: diff --git a/guardrails/run.py b/guardrails/run.py index cb2c22687..dba3fd3e4 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -6,6 +6,7 @@ from pydantic import BaseModel from guardrails.classes.history import Call, Inputs, Iteration, Outputs +from guardrails.cli_dir.hub.credentials import Credentials from guardrails.datatypes import verify_metadata_requirements from guardrails.llm_providers import ( AsyncPromptCallableBase, @@ -104,8 +105,11 @@ def __init__( self.base_model = base_model self.full_schema_reask = full_schema_reask - # Get the HubTelemetry singleton - self._hub_telemetry = HubTelemetry() + # Get metrics opt-out from credentials + self._disable_tracer = Credentials.from_rc_file().no_metrics + if not self._disable_tracer: + # Get the HubTelemetry singleton + self._hub_telemetry = HubTelemetry() def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call: """Execute the runner by repeatedly calling step until the reask budget @@ -202,12 +206,13 @@ def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call # Log how many times we reasked # Use the HubTelemetry singleton - self._hub_telemetry.create_new_span( - span_name="/reasks", - attributes=[("reask_count", index)], - is_parent=False, # This span has no children - has_parent=True, # This span has a parent - ) + if not self._disable_tracer: + self._hub_telemetry.create_new_span( + span_name="/reasks", + attributes=[("reask_count", index)], + is_parent=False, # This span has no children + has_parent=True, # This span has a parent + ) except UserFacingException as e: # Because Pydantic v1 doesn't respect property setters diff --git a/guardrails/utils/hub_telemetry_utils.py b/guardrails/utils/hub_telemetry_utils.py index cf0c1b58c..e225758c4 100644 --- a/guardrails/utils/hub_telemetry_utils.py +++ b/guardrails/utils/hub_telemetry_utils.py @@ -95,11 +95,13 @@ def create_new_span( is_parent: bool, # Inject current context if IS a parent span has_parent: bool, # Extract current context if HAS a parent span ): - """Creates a new span within the tracer with the given name and attributes. + """Creates a new span within the tracer with the given name and + attributes. If it's a parent span, the current context is injected into the carrier. If it has a parent span, the current context is extracted from the carrier. - Both the conditions can co-exist e.g. if it's a parent span and also has a parent span. + Both the conditions can co-exist e.g. a span can be a parent span which + also has a parent span. Args: span_name (str): The name of the span. diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py index 6f855ca26..5d279e585 100644 --- a/guardrails/validator_service.py +++ b/guardrails/validator_service.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple from guardrails.classes.history import Iteration +from guardrails.cli_dir.hub.credentials import Credentials from guardrails.datatypes import FieldValidation from guardrails.logger import logger from guardrails.utils.casting_utils import to_string @@ -122,19 +123,22 @@ def run_validator( # this will have to change. validator_logs.instance_id = to_string(id(validator)) - # Get HubTelemetry singleton and create a new span to - # log the validator usage - _hub_telemetry = HubTelemetry() - _hub_telemetry.create_new_span( - span_name="/validator_usage", - attributes=[ - ("validator_name", validator.rail_alias), - ("validator_on_fail", validator.on_fail_descriptor), - ("validator_result", result.outcome), - ], - is_parent=False, # This span will have no children - has_parent=True, # This span has a parent - ) + # Get metrics opt-out from credentials + disable_tracer = Credentials.from_rc_file().no_metrics + if not disable_tracer: + # Get HubTelemetry singleton and create a new span to + # log the validator usage + _hub_telemetry = HubTelemetry() + _hub_telemetry.create_new_span( + span_name="/validator_usage", + attributes=[ + ("validator_name", validator.rail_alias), + ("validator_on_fail", validator.on_fail_descriptor), + ("validator_result", result.outcome), + ], + is_parent=False, # This span will have no children + has_parent=True, # This span has a parent + ) return validator_logs From 2054f9156c77b2c5ab2fb45ef3e2c2aefaaec54b Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Tue, 13 Feb 2024 15:10:01 -0500 Subject: [PATCH 12/13] Update str var to bool --- guardrails/guard.py | 23 ++++++++++------------- guardrails/run.py | 5 +++++ guardrails/validator_service.py | 5 +++++ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/guardrails/guard.py b/guardrails/guard.py index 12219b24d..d26e7efa3 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -87,6 +87,10 @@ def __init__( # Get metrics opt-out from credentials self._disable_tracer = Credentials.from_rc_file().no_metrics + if self._disable_tracer.strip().lower() == "true": + self._disable_tracer = True + elif self._disable_tracer.strip().lower() == "false": + self._disable_tracer = False # Get id of guard object (that is unique) self._guard_id = id(self) # id of guard object; not the class @@ -166,9 +170,7 @@ def configure( self.num_reasks = ( num_reasks if num_reasks is not None - else self.num_reasks - if self.num_reasks is not None - else 1 + else self.num_reasks if self.num_reasks is not None else 1 ) def _set_tracer(self, tracer: Tracer = None) -> None: @@ -303,8 +305,7 @@ def __call__( stream: Optional[bool] = False, *args, **kwargs, - ) -> Union[ValidationOutcome[OT], Iterable[str]]: - ... + ) -> Union[ValidationOutcome[OT], Iterable[str]]: ... @overload def __call__( @@ -319,8 +320,7 @@ def __call__( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> Awaitable[ValidationOutcome[OT]]: - ... + ) -> Awaitable[ValidationOutcome[OT]]: ... def __call__( self, @@ -614,8 +614,7 @@ def parse( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> ValidationOutcome[OT]: - ... + ) -> ValidationOutcome[OT]: ... @overload def parse( @@ -628,8 +627,7 @@ def parse( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> Awaitable[ValidationOutcome[OT]]: - ... + ) -> Awaitable[ValidationOutcome[OT]]: ... @overload def parse( @@ -642,8 +640,7 @@ def parse( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> ValidationOutcome[OT]: - ... + ) -> ValidationOutcome[OT]: ... def parse( self, diff --git a/guardrails/run.py b/guardrails/run.py index dba3fd3e4..73f8c7467 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -107,6 +107,11 @@ def __init__( # Get metrics opt-out from credentials self._disable_tracer = Credentials.from_rc_file().no_metrics + if self._disable_tracer.strip().lower() == "true": + self._disable_tracer = True + elif self._disable_tracer.strip().lower() == "false": + self._disable_tracer = False + if not self._disable_tracer: # Get the HubTelemetry singleton self._hub_telemetry = HubTelemetry() diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py index 5d279e585..e6129995e 100644 --- a/guardrails/validator_service.py +++ b/guardrails/validator_service.py @@ -125,6 +125,11 @@ def run_validator( # Get metrics opt-out from credentials disable_tracer = Credentials.from_rc_file().no_metrics + if disable_tracer.strip().lower() == "true": + disable_tracer = True + elif disable_tracer.strip().lower() == "false": + disable_tracer = False + if not disable_tracer: # Get HubTelemetry singleton and create a new span to # log the validator usage From 6500348e6a1ad4c0309a7d3b3ff8aa9ab66a557e Mon Sep 17 00:00:00 2001 From: "karan@guardrailsai.com" Date: Tue, 13 Feb 2024 15:14:56 -0500 Subject: [PATCH 13/13] lint --- guardrails/guard.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/guardrails/guard.py b/guardrails/guard.py index d26e7efa3..af004c3fe 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -170,7 +170,9 @@ def configure( self.num_reasks = ( num_reasks if num_reasks is not None - else self.num_reasks if self.num_reasks is not None else 1 + else self.num_reasks + if self.num_reasks is not None + else 1 ) def _set_tracer(self, tracer: Tracer = None) -> None: @@ -305,7 +307,8 @@ def __call__( stream: Optional[bool] = False, *args, **kwargs, - ) -> Union[ValidationOutcome[OT], Iterable[str]]: ... + ) -> Union[ValidationOutcome[OT], Iterable[str]]: + ... @overload def __call__( @@ -320,7 +323,8 @@ def __call__( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> Awaitable[ValidationOutcome[OT]]: ... + ) -> Awaitable[ValidationOutcome[OT]]: + ... def __call__( self, @@ -614,7 +618,8 @@ def parse( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> ValidationOutcome[OT]: ... + ) -> ValidationOutcome[OT]: + ... @overload def parse( @@ -627,7 +632,8 @@ def parse( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> Awaitable[ValidationOutcome[OT]]: ... + ) -> Awaitable[ValidationOutcome[OT]]: + ... @overload def parse( @@ -640,7 +646,8 @@ def parse( full_schema_reask: Optional[bool] = None, *args, **kwargs, - ) -> ValidationOutcome[OT]: ... + ) -> ValidationOutcome[OT]: + ... def parse( self,