ValidationOutcome(\n",
+ " call_id='4927013424',\n",
+ " raw_llm_output=\"As of my last update in 2023, Jupiter has 95 confirmed moons. The number of known moons can \n",
+ "change as new moons are discovered and confirmed, so it's always a good idea to check the latest information from \n",
+ "reliable sources such as NASA or other astronomical organizations.\",\n",
+ " validated_output=\"As of my last update in 2023, Jupiter has 95 confirmed moons. The number of known moons can \n",
+ "change as new moons are discovered and confirmed, so it's always a good idea to check the latest information from \n",
+ "reliable sources such as NASA or other astronomical organizations.\",\n",
+ " reask=None,\n",
+ " validation_passed=True,\n",
+ " error=None\n",
+ ")\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mValidationOutcome\u001b[0m\u001b[1m(\u001b[0m\n",
+ " \u001b[33mcall_id\u001b[0m=\u001b[32m'4927013424'\u001b[0m,\n",
+ " \u001b[33mraw_llm_output\u001b[0m=\u001b[32m\"As\u001b[0m\u001b[32m of my last update in 2023, Jupiter has 95 confirmed moons. The number of known moons can \u001b[0m\n",
+ "\u001b[32mchange as new moons are discovered and confirmed, so it's always a good idea to check the latest information from \u001b[0m\n",
+ "\u001b[32mreliable sources such as NASA or other astronomical organizations.\"\u001b[0m,\n",
+ " \u001b[33mvalidated_output\u001b[0m=\u001b[32m\"As\u001b[0m\u001b[32m of my last update in 2023, Jupiter has 95 confirmed moons. The number of known moons can \u001b[0m\n",
+ "\u001b[32mchange as new moons are discovered and confirmed, so it's always a good idea to check the latest information from \u001b[0m\n",
+ "\u001b[32mreliable sources such as NASA or other astronomical organizations.\"\u001b[0m,\n",
+ " \u001b[33mreask\u001b[0m=\u001b[3;35mNone\u001b[0m,\n",
+ " \u001b[33mvalidation_passed\u001b[0m=\u001b[3;92mTrue\u001b[0m,\n",
+ " \u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
+ "\u001b[1m)\u001b[0m\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "from rich import print\n",
+ "from guardrails import Guard\n",
+ "from guardrails.hub import RegexMatch\n",
+ "\n",
+ "guard = Guard().use(RegexMatch(\"95\", match_type=\"search\"))\n",
+ "\n",
+ "response = guard(\n",
+ " model=\"gpt-4o\",\n",
+ " instructions=\"You are a helpful assistant.\",\n",
+ " prompt=\"How many moons does jupiter have?\",\n",
+ " messages=[{\n",
+ " \"role\": \"system\",\n",
+ " \"content\": \"You are a helpful assistant.\"\n",
+ " },{\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"How many moons does jupiter have?\"\n",
+ " }]\n",
+ ")\n",
+ "\n",
+ "print(response)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/docs/pydocs/api_reference/actions.py b/docs/pydocs/api_reference/actions.py
new file mode 100644
index 000000000..ee79107a0
--- /dev/null
+++ b/docs/pydocs/api_reference/actions.py
@@ -0,0 +1,60 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from pydoc_markdown.contrib.processors.filter import FilterProcessor
+from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer
+from docs.pydocs.helpers import write_to_file
+
+
+exports = [
+ "guardrails.actions.reask",
+ "ReAsk",
+ "FieldReAsk",
+ "SkeletonReAsk",
+ "NonParseableReAsk",
+ "guardrails.actions.filter",
+ "Filter",
+ "apply_filters",
+ "guardrails.actions.refrain",
+ "Refrain",
+ "apply_refrain",
+]
+export_string = ", ".join([f"'{export}'" for export in exports])
+
+write_to_file(
+ str="# Actions\n\n"
+ + render_loader(
+ PythonLoader(
+ modules=[
+ "guardrails.actions.reask",
+ "guardrails.actions.filter",
+ "guardrails.actions.refrain",
+ ],
+ parser=ParserOptions(print_function=False),
+ ),
+ processor=FilterProcessor(
+ expression=f"name in [{export_string}]", # noqa
+ skip_empty_modules=True,
+ ),
+ renderer=MarkdownRenderer(
+ # Custom
+ data_code_block=True,
+ data_expression_maxlength=250,
+ header_level_by_type={
+ "Class": 2,
+ "Variable": 2,
+ },
+ # Default
+ render_module_header=False,
+ insert_header_anchors=False,
+ descriptive_class_title=False,
+ signature_in_header=False,
+ classdef_code_block=True,
+ classdef_with_decorators=True,
+ signature_code_block=True,
+ signature_with_decorators=True,
+ render_typehint_in_data_header=True,
+ ),
+ ),
+ filename="docs/api_reference_markdown/actions.md",
+)
diff --git a/docs/pydocs/api_reference/errors.py b/docs/pydocs/api_reference/errors.py
new file mode 100644
index 000000000..48d0b1795
--- /dev/null
+++ b/docs/pydocs/api_reference/errors.py
@@ -0,0 +1,24 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from pydoc_markdown.contrib.processors.filter import FilterProcessor
+from docs.pydocs.helpers import write_to_file
+
+
+exports = ["guardrails.errors.__init__", "guardrails.errors", "ValidationError"]
+export_string = ", ".join([f"'{export}'" for export in exports])
+
+write_to_file(
+ str="# Errors\n\n"
+ + render_loader(
+ PythonLoader(
+ modules=["guardrails.errors.__init__"],
+ parser=ParserOptions(print_function=False),
+ ),
+ processor=FilterProcessor(
+ expression=f"name in [{export_string}]", # noqa
+ skip_empty_modules=True,
+ ),
+ ),
+ filename="docs/api_reference_markdown/errors.md",
+)
diff --git a/docs/pydocs/api_reference/formatters.py b/docs/pydocs/api_reference/formatters.py
new file mode 100644
index 000000000..fec945a93
--- /dev/null
+++ b/docs/pydocs/api_reference/formatters.py
@@ -0,0 +1,27 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from docs.pydocs.helpers import write_to_file
+
+
+exports = [
+ "guardrails.formatters.json_formatter",
+ "JsonFormatter",
+ "guardrails.formatters.base_formatter",
+ "BaseFormatter",
+]
+export_string = ", ".join([f"'{export}'" for export in exports])
+
+write_to_file(
+ str="# Formatters\n\n"
+ + render_loader(
+ loader=PythonLoader(
+ modules=[
+ "guardrails.formatters.base_formatter",
+ "guardrails.formatters.json_formatter",
+ ],
+ parser=ParserOptions(print_function=False),
+ ),
+ ),
+ filename="docs/api_reference_markdown/formatters.md",
+)
diff --git a/docs/pydocs/api_reference/generics_and_base_classes.py b/docs/pydocs/api_reference/generics_and_base_classes.py
new file mode 100644
index 000000000..6b454fb59
--- /dev/null
+++ b/docs/pydocs/api_reference/generics_and_base_classes.py
@@ -0,0 +1,27 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from docs.pydocs.helpers import write_to_file
+
+
+exports = [
+ "guardrails.classes.generic.arbitrary_model",
+ "ArbitraryModel",
+ "guardrails.classes.generic.stack",
+ "Stack",
+]
+export_string = ", ".join([f"'{export}'" for export in exports])
+
+write_to_file(
+ str="# Generics And Base Classes\n\n"
+ + render_loader(
+ loader=PythonLoader(
+ modules=[
+ "guardrails.classes.generic.arbitrary_model",
+ "guardrails.classes.generic.stack",
+ ],
+ parser=ParserOptions(print_function=False),
+ ),
+ ),
+ filename="docs/api_reference_markdown/generics_and_base_classes.md",
+)
diff --git a/docs/pydocs/api_reference/guards.py b/docs/pydocs/api_reference/guards.py
new file mode 100644
index 000000000..ad925fb80
--- /dev/null
+++ b/docs/pydocs/api_reference/guards.py
@@ -0,0 +1,84 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from pydoc_markdown.contrib.processors.filter import FilterProcessor
+from docs.pydocs.helpers import write_to_file
+
+
+export_map = {
+ "guardrails/guard.py": [
+ "Guard",
+ "guardrails.guard",
+ "guard",
+ "__init__",
+ "from_rail",
+ "from_rail_string",
+ "from_pydantic",
+ "from_string",
+ "configure",
+ "use",
+ "use_many",
+ "__call__",
+ "parse",
+ "validate",
+ "error_spans_in_output",
+ "add_json_function_calling_tool",
+ "to_dict",
+ "from_dict",
+ "to_runnable",
+ ],
+ "guardrails/async_guard.py": [
+ "AsyncGuard",
+ "guardrails.async_guard",
+ "async_guard",
+ "__init__",
+ "from_rail",
+ "from_rail_string",
+ "from_pydantic",
+ "from_string",
+ "configure",
+ "use",
+ "use_many",
+ "__call__",
+ "parse",
+ "validate",
+ "error_spans_in_output",
+ "add_json_function_calling_tool",
+ "to_dict",
+ "from_dict",
+ "to_runnable",
+ ],
+ "guardrails/classes/validation_outcome.py": [
+ "guardrails.classes.validation_outcome",
+ "ValidationOutcome",
+ "from_guard_history",
+ ],
+}
+
+
+conditionals = []
+for k, v in export_map.items():
+ conditionals.append(
+ f"((name in {v}) if ('{k}' in obj.location.filename) else False)"
+ )
+
+export_string = " or ".join(conditionals)
+
+write_to_file(
+ str="# Guards\n\n"
+ + render_loader(
+ PythonLoader(
+ modules=[
+ "guardrails.guard",
+ "guardrails.async_guard",
+ "guardrails.classes.validation_outcome",
+ ],
+ parser=ParserOptions(print_function=False),
+ ),
+ processor=FilterProcessor(
+ expression=f"({export_string})",
+ skip_empty_modules=True,
+ ),
+ ),
+ filename="docs/api_reference_markdown/guards.md",
+)
diff --git a/docs/pydocs/api_reference/history.py b/docs/pydocs/api_reference/history.py
new file mode 100644
index 000000000..80437f12a
--- /dev/null
+++ b/docs/pydocs/api_reference/history.py
@@ -0,0 +1,22 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from docs.pydocs.helpers import write_to_file
+
+
+write_to_file(
+ str="# History and Logs\n\n"
+ + render_loader(
+ loader=PythonLoader(
+ modules=[
+ "guardrails.classes.history.call",
+ "guardrails.classes.history.iteration",
+ "guardrails.classes.history.inputs",
+ "guardrails.classes.history.outputs",
+ "guardrails.classes.history.call_inputs",
+ ],
+ parser=ParserOptions(print_function=False),
+ ),
+ ),
+ filename="docs/api_reference_markdown/history_and_logs.md",
+)
diff --git a/docs/pydocs/api_reference/llm_interaction.py b/docs/pydocs/api_reference/llm_interaction.py
new file mode 100644
index 000000000..317afb28e
--- /dev/null
+++ b/docs/pydocs/api_reference/llm_interaction.py
@@ -0,0 +1,68 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from pydoc_markdown.contrib.processors.filter import FilterProcessor
+from docs.pydocs.helpers import write_to_file
+
+
+export_map = {
+ "guardrails/prompt/base_prompt.py": [
+ "guardrails.prompt.base_prompt",
+ "BasePrompt",
+ "__init__",
+ "format",
+ "substitute_constants",
+ "get_prompt_variables",
+ "escape",
+ ],
+ "guardrails/prompt/prompt.py": [
+ "guardrails.prompt.prompt",
+ "Prompt",
+ "format",
+ ],
+ "guardrails/prompt/instructions.py": [
+ "guardrails.prompt.instructions",
+ "Instructions",
+ "format",
+ ],
+ "guardrails/llm_providers.py": [
+ "guardrails.llm_providers",
+ "PromptCallableBase",
+ "_invoke_llm",
+ "__call__",
+ ],
+ "guardrails/classes/llm/llm_response.py": [
+ "guardrails.classes.llm.llm_response",
+ "LLMResponse",
+ ],
+}
+
+
+conditionals = []
+for k, v in export_map.items():
+ conditionals.append(
+ f"((name in {v}) if ('{k}' in obj.location.filename) else False)"
+ )
+
+export_string = " or ".join(conditionals)
+
+write_to_file(
+ str="# Helpers for LLM Interactions\n\n"
+ + render_loader(
+ PythonLoader(
+ modules=[
+ "guardrails.prompt.base_prompt",
+ "guardrails.prompt.prompt",
+ "guardrails.prompt.instructions",
+ "guardrails.llm_providers",
+ "guardrails.classes.llm.llm_response",
+ ],
+ parser=ParserOptions(print_function=False),
+ ),
+ processor=FilterProcessor(
+ expression=f"({export_string})", # noqa
+ skip_empty_modules=True,
+ ),
+ ),
+ filename="docs/api_reference_markdown/llm_interaction.md",
+)
diff --git a/docs/pydocs/api_reference/types.py b/docs/pydocs/api_reference/types.py
new file mode 100644
index 000000000..7ddba4872
--- /dev/null
+++ b/docs/pydocs/api_reference/types.py
@@ -0,0 +1,73 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from pydoc_markdown.contrib.processors.filter import FilterProcessor
+from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer
+from docs.pydocs.helpers import write_to_file
+
+
+exports = [
+ "guardrails.types.on_fail",
+ "guardrails.types.rail",
+ "OnFailAction",
+ "RailTypes",
+ "guardrails.types.inputs",
+ "guardrails.types.pydantic",
+ "guardrails.types.validator",
+ "MessageHistory",
+ "ModelOrListOfModels",
+ "ModelOrListOrDict",
+ "ModelOrModelUnion",
+ "PydanticValidatorTuple",
+ "PydanticValidatorSpec",
+ "UseValidatorSpec",
+ "UseManyValidatorTuple",
+ "UseManyValidatorSpec",
+ "ValidatorMap",
+]
+export_string = ", ".join([f"'{export}'" for export in exports])
+
+write_to_file(
+ str="# Types\n\n"
+ + render_loader(
+ PythonLoader(
+ modules=[
+ "guardrails.types.on_fail",
+ "guardrails.types.primitives",
+ "guardrails.types.rail",
+ "guardrails.types.inputs",
+ "guardrails.types.pydantic",
+ "guardrails.types.validator",
+ ],
+ parser=ParserOptions(
+ print_function=False, treat_singleline_comment_blocks_as_docstrings=True
+ ),
+ ),
+ processor=FilterProcessor(
+ expression=f"name in [{export_string}]", # noqa
+ skip_empty_modules=True,
+ ),
+ renderer=MarkdownRenderer(
+ # Custom
+ data_code_block=True,
+ data_expression_maxlength=250,
+ header_level_by_type={
+ "Class": 2,
+ "Method": 2,
+ "Function": 2,
+ "Variable": 2,
+ },
+ # Default
+ render_module_header=False,
+ insert_header_anchors=False,
+ descriptive_class_title=False,
+ signature_in_header=False,
+ classdef_code_block=True,
+ classdef_with_decorators=True,
+ signature_code_block=True,
+ signature_with_decorators=True,
+ render_typehint_in_data_header=True,
+ ),
+ ),
+ filename="docs/api_reference_markdown/types.md",
+)
diff --git a/docs/pydocs/api_reference/validation.py b/docs/pydocs/api_reference/validation.py
new file mode 100644
index 000000000..31c652c6b
--- /dev/null
+++ b/docs/pydocs/api_reference/validation.py
@@ -0,0 +1,71 @@
+from docspec_python import ParserOptions
+from docs.pydocs.pydocs_markdown_impl import render_loader
+from pydoc_markdown.contrib.loaders.python import PythonLoader
+from pydoc_markdown.contrib.processors.filter import FilterProcessor
+from docs.pydocs.helpers import write_to_file
+
+
+# exports = [
+# "docs.pydocs.api_reference.validator",
+# "ValidatorReference",
+# ]
+# export_string = ", ".join([f"'{export}'" for export in exports])
+
+
+export_map = {
+ "guardrails/validator_base.py": [
+ "guardrails.validator_base",
+ "Validator",
+ "__init__",
+ "chunking_function",
+ "validate",
+ "validate_stream",
+ "with_metadata",
+ "to_runnable",
+ "register_validator",
+ ],
+ "guardrails/classes/validation/validation_result.py": [
+ "guardrails.classes.validation.validation_result",
+ "ValidationResult",
+ "PassResult",
+ "FailResult",
+ "ErrorSpan",
+ ],
+ "guardrails/classes/validation/validator_logs.py": [
+ "guardrails.classes.validation.validator_logs",
+ "ValidatorLogs",
+ ],
+ "guardrails/classes/validation/validator_reference.py": [
+ "guardrails.classes.validation.validator_reference",
+ "ValidatorReference",
+ ],
+}
+
+
+conditionals = []
+for k, v in export_map.items():
+ conditionals.append(
+ f"((name in {v}) if ('{k}' in obj.location.filename) else False)"
+ )
+
+export_string = " or ".join(conditionals)
+
+write_to_file(
+ str="# Validation\n\n"
+ + render_loader(
+ PythonLoader(
+ modules=[
+ "guardrails.validator_base",
+ "guardrails.classes.validation.validation_result",
+ "guardrails.classes.validation.validator_logs",
+ "guardrails.classes.validation.validator_reference",
+ ],
+ parser=ParserOptions(print_function=False),
+ ),
+ processor=FilterProcessor(
+ expression=f"({export_string})",
+ skip_empty_modules=False,
+ ),
+ ),
+ filename="docs/api_reference_markdown/validator.md",
+)
diff --git a/docs/pydocs/generate_pydocs.py b/docs/pydocs/generate_pydocs.py
index a0e428ad1..46e38cfbf 100644
--- a/docs/pydocs/generate_pydocs.py
+++ b/docs/pydocs/generate_pydocs.py
@@ -1,98 +1,29 @@
-import os
-# from pydoc_markdown.interfaces import Context
from docspec_python import ParserOptions
+from docs.pydocs.helpers import write_to_file
from docs.pydocs.pydocs_markdown_impl import render_loader
from pydoc_markdown.contrib.loaders.python import PythonLoader
-# from guardrails import Rail, Guard, validators, datatypes
-# from guardrails.classes.validation_outcome import ValidationOutcome
-from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer
-# from guardrails.classes import generic
-# from pydocs_to_md import class_to_string, module_to_string
-from pydoc_markdown.contrib.processors.filter import FilterProcessor
-
-
-def write_to_file(str, filename):
- # if the directory where the filename does not exist, create it
- if not os.path.exists(os.path.dirname(filename)):
- os.makedirs(os.path.dirname(filename))
- with open(filename, "w") as f:
- f.write(str)
- f.close()
-
-
-# write_to_file(
-# # str=class_to_string(Rail, ignore_prefix_list=["load", "_"]),
-# str="# Rail\n\n" + render_loader(
-# PythonLoader(
-# modules=['guardrails.rail'],
-# parser=ParserOptions(
-# print_function=False,
-# ),
-# ),
-# processor = FilterProcessor(
-# expression="not name.startswith('_') and not name.startswith('load') and default()",
-# documented_only=True,
-
-# )
-# ),
-# filename="docs/api_reference_markdown/rail.md",
-# )
-
-
-write_to_file(
- str="# Guard\n\n" + render_loader(
- PythonLoader(
- modules=['guardrails.guard'],
- parser=ParserOptions(
- print_function=False
- ),
- ),
- processor = FilterProcessor(
- expression="name in ['Guard', 'guardrails.guard', 'guard', 'from_rail', 'from_rail_string', 'from_pydantic', 'from_string', 'configure', '__call__', 'parse', 'state']",
- skip_empty_modules=True
- )
- ),
- filename="docs/api_reference_markdown/guard.md",
-)
+from docs.pydocs.api_reference import actions # noqa
+from docs.pydocs.api_reference import errors # noqa
+from docs.pydocs.api_reference import formatters # noqa
+from docs.pydocs.api_reference import generics_and_base_classes # noqa
+from docs.pydocs.api_reference import guards # noqa
+from docs.pydocs.api_reference import history # noqa
+from docs.pydocs.api_reference import llm_interaction # noqa
+from docs.pydocs.api_reference import types # noqa
+from docs.pydocs.api_reference import validation # noqa
write_to_file(
- str="# Validators\n\n" + render_loader(
+ str="# Validators\n\n"
+ + render_loader(
PythonLoader(
- search_path=['validators'],
- parser=ParserOptions(
- print_function=False
- )
+ search_path=["validators"], parser=ParserOptions(print_function=False)
)
),
filename="docs/hub/api_reference_markdown/validators.md",
)
-write_to_file(
- # str=class_to_string(ValidationOutcome, ignore_prefix_list=["load", "_"]),
- str="# Validation Outcome\n\n" + render_loader(
- PythonLoader(
- modules=['guardrails.classes.validation_outcome'],
- parser=ParserOptions(
- print_function=False
- ),
- ),
- processor = FilterProcessor(
- documented_only=True,
- ),
- renderer = MarkdownRenderer(
- render_module_header=False,
- insert_header_anchors=False,
- classdef_code_block=False,
- descriptive_class_title=False,
- classdef_with_decorators=False,
- render_typehint_in_data_header=True,
- data_code_block=True,
- )
- ),
- filename="docs/hub/api_reference_markdown/validation_outcome.md",
-)
# write_to_file(
# str="# Response Structures\n\n" + render_loader(
@@ -102,7 +33,7 @@ def write_to_file(str, filename):
# processor = FilterProcessor(
# expression="""\
# name in \
-# ['guardrails.validator_base', 'ValidationResult', 'PassResult', 'FailResult', 'ValidationError'] \
+# ['guardrails.validator_base', 'ValidationResult', 'PassResult', 'FailResult', 'ValidationError'] \ # noqa
# or obj.parent.name in \
# ['ValidationResult', 'PassResult', 'FailResult', 'ValidationError']\
# """,
@@ -121,7 +52,6 @@ def write_to_file(str, filename):
# )
-
# write_to_file(
# str="# Schema\n\n" + render_loader(PythonLoader(
# modules=['guardrails.schema'],
@@ -158,25 +88,6 @@ def write_to_file(str, filename):
# filename="docs/api_reference_markdown/datatypes.md",
# )
-write_to_file(
- str="# History and Logs\n\n" + render_loader(
- PythonLoader(
- packages=['classes.history'],
- parser=ParserOptions(
- print_function=False
- ),
- ),
- renderer = MarkdownRenderer(
- render_module_header=True,
- insert_header_anchors=False,
- descriptive_class_title=True,
- signature_in_header=True,
- classdef_code_block=False,
- classdef_with_decorators=False,
- ),
- ),
- filename="docs/api_reference_markdown/history_and_logs.md",
-)
# write_to_file(
# str="# Helper Classes\n\n" + render_loader(
diff --git a/docs/pydocs/helpers.py b/docs/pydocs/helpers.py
new file mode 100644
index 000000000..23c460234
--- /dev/null
+++ b/docs/pydocs/helpers.py
@@ -0,0 +1,10 @@
+import os
+
+
+def write_to_file(str, filename):
+ # if the directory where the filename does not exist, create it
+ if not os.path.exists(os.path.dirname(filename)):
+ os.makedirs(os.path.dirname(filename))
+ with open(filename, "w") as f:
+ f.write(str)
+ f.close()
diff --git a/docs/pydocs/pydocs_markdown_impl.py b/docs/pydocs/pydocs_markdown_impl.py
index 16272c75a..67187e7a1 100644
--- a/docs/pydocs/pydocs_markdown_impl.py
+++ b/docs/pydocs/pydocs_markdown_impl.py
@@ -1,21 +1,30 @@
from pydoc_markdown.interfaces import Context
-from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer, MarkdownReferenceResolver
+from pydoc_markdown.contrib.renderers.markdown import (
+ MarkdownRenderer,
+ MarkdownReferenceResolver,
+)
from pydoc_markdown.contrib.processors.filter import FilterProcessor
from pydoc_markdown.contrib.processors.google import GoogleProcessor
+# from pydoc_markdown.contrib.processors.crossref import CrossrefProcessor
-def render_loader(loader, processor = None, renderer = None, context = None):
+
+def render_loader(loader, processor=None, renderer=None, context=None):
if not context:
- context = Context(directory='guardrails')
+ context = Context(directory="guardrails")
if not renderer:
- renderer = MarkdownRenderer(
+ new_renderer = MarkdownRenderer(
render_module_header=False,
insert_header_anchors=False,
descriptive_class_title=False,
- signature_in_header=True,
- classdef_code_block=False,
- classdef_with_decorators=False,
+ signature_in_header=False,
+ classdef_code_block=True,
+ classdef_with_decorators=True,
+ signature_code_block=True,
+ signature_with_decorators=True,
+ render_typehint_in_data_header=True,
)
+ renderer = new_renderer
if not processor:
processor = FilterProcessor(
@@ -24,13 +33,15 @@ def render_loader(loader, processor = None, renderer = None, context = None):
google_processor = GoogleProcessor()
+ # TODO: Add this for api reference
+ # cross_ref_processor = CrossrefProcessor()
+
loader.init(context)
renderer.init(context)
processor.init(context)
-
modules = list(loader.load())
processor.process(modules=modules, resolver=MarkdownReferenceResolver())
google_processor.process(modules=modules, resolver=MarkdownReferenceResolver())
- return renderer.render_to_string(modules)
\ No newline at end of file
+ return renderer.render_to_string(modules)
diff --git a/guardrails/__init__.py b/guardrails/__init__.py
index dd3bf9b1c..742abecaf 100644
--- a/guardrails/__init__.py
+++ b/guardrails/__init__.py
@@ -12,7 +12,7 @@
__all__ = [
"Guard",
"AsyncGuard",
- "PromptCallableBase",
+ "PromptCallableBase", # FIXME: Why is this being exported?
"Validator",
"OnFailAction",
"register_validator",
diff --git a/guardrails/actions/filter.py b/guardrails/actions/filter.py
index 8d25dd9ee..e11bb7d5d 100644
--- a/guardrails/actions/filter.py
+++ b/guardrails/actions/filter.py
@@ -6,6 +6,7 @@ class Filter:
def apply_filters(value: Any) -> Any:
+ """Recursively filter out any values that are instances of Filter."""
if isinstance(value, Filter):
pass
elif isinstance(value, List):
diff --git a/guardrails/actions/reask.py b/guardrails/actions/reask.py
index 3ffa8b0ec..5ed0b8d73 100644
--- a/guardrails/actions/reask.py
+++ b/guardrails/actions/reask.py
@@ -17,6 +17,13 @@
### Classes/Types ###
class ReAsk(IReask):
+ """Base class for ReAsk objects.
+
+ Attributes:
+ incorrect_value (Any): The value that failed validation.
+ fail_results (List[FailResult]): The results of the failed validations.
+ """
+
incorrect_value: Any
fail_results: List[FailResult]
@@ -60,16 +67,36 @@ def from_dict(cls, obj: Dict[str, Any]) -> Optional["ReAsk"]:
class FieldReAsk(ReAsk):
+ """An implementation of ReAsk that is used to reask for a specific field.
+ Inherits from ReAsk.
+
+ Attributes:
+ path (Optional[List[Any]]): a list of keys that
+ designated the path to the field that failed validation.
+ """
+
# FIXME: This shouldn't be optional
# We should be able to assign it on init now
path: Optional[List[Any]] = None
class SkeletonReAsk(ReAsk):
+ """An implementation of ReAsk that is used to reask for structured data
+ when the response does not match the expected schema.
+
+ Inherits from ReAsk.
+ """
+
pass
class NonParseableReAsk(ReAsk):
+ """An implementation of ReAsk that is used to reask for structured data
+ when the response is not parseable as JSON.
+
+ Inherits from ReAsk.
+ """
+
pass
diff --git a/guardrails/actions/refrain.py b/guardrails/actions/refrain.py
index 923b746d3..a4819cc20 100644
--- a/guardrails/actions/refrain.py
+++ b/guardrails/actions/refrain.py
@@ -24,6 +24,10 @@ def check_for_refrain(value: Union[List, Dict]) -> bool:
# Could be a generic instead of Any
def apply_refrain(value: Any, output_type: OutputTypes) -> Any:
+ """Recursively check for any values that are instances of Refrain.
+
+ If found, return an empty value of the appropriate type.
+ """
refrain_value = {}
if output_type == OutputTypes.STRING:
refrain_value = ""
diff --git a/guardrails/async_guard.py b/guardrails/async_guard.py
index fca49a948..911a087d3 100644
--- a/guardrails/async_guard.py
+++ b/guardrails/async_guard.py
@@ -440,7 +440,7 @@ async def __call__(
Args:
llm_api: The LLM API to call
- (e.g. openai.Completion.create or openai.Completion.acreate)
+ (e.g. openai.completions.create or openai.chat.completions.create)
prompt_params: The parameters to pass to the prompt.format() method.
num_reasks: The max times to re-ask the LLM for invalid output.
prompt: The prompt to use for the LLM.
@@ -496,7 +496,7 @@ async def parse(
llm_output: The output being parsed and validated.
metadata: Metadata to pass to the validators.
llm_api: The LLM API to call
- (e.g. openai.Completion.create or openai.Completion.acreate)
+ (e.g. openai.completions.create or openai.Completion.acreate)
num_reasks: The max times to re-ask the LLM for invalid output.
prompt_params: The parameters to pass to the prompt.format() method.
full_schema_reask: When reasking, whether to regenerate the full schema
diff --git a/guardrails/call_tracing/__init__.py b/guardrails/call_tracing/__init__.py
index 62078ab46..cfc312cc5 100644
--- a/guardrails/call_tracing/__init__.py
+++ b/guardrails/call_tracing/__init__.py
@@ -1,10 +1,9 @@
-"""
-For tracing (logging) and reporting the timing of Guard and Validator calls.
+"""For tracing (logging) and reporting the timing of Guard and Validator calls.
sqlite_trace_handler defines most of the actual implementation methods.
-trace_handler provides the singleton that's used for fast global access across threads.
-tracer_mixin defines the interface and can act as a noop.
-trace_entry is just a helpful dataclass.
+trace_handler provides the singleton that's used for fast global access
+across threads. tracer_mixin defines the interface and can act as a
+noop. trace_entry is just a helpful dataclass.
"""
from guardrails.call_tracing.trace_entry import GuardTraceEntry
diff --git a/guardrails/call_tracing/sqlite_trace_handler.py b/guardrails/call_tracing/sqlite_trace_handler.py
index 2831c05d8..1d555d6fd 100644
--- a/guardrails/call_tracing/sqlite_trace_handler.py
+++ b/guardrails/call_tracing/sqlite_trace_handler.py
@@ -1,20 +1,22 @@
-"""
-sqlite_trace_handler.py
-
-This is the metaphorical bread and butter of our tracing implementation, or at least the
-butter. It wraps a SQLite database and configures it to be 'agreeable' in multithreaded
-situations. Normally, when sharing across threads and instances one should consider
-using a larger database solution like Postgres, but in this case we only care about
-_supporting_ writing from multiple places. We don't expect it will be the norm.
-We care about (1) not negatively impacting performance, (2) not crashing when used in
-unusual ways, and (3) not losing data when possible.
-
-The happy path should be reasonably performant. The unhappy path should not crash.
-
-The other part of the multithreaded support comes from the public trace_handler, which
-uses a singleton pattern to only have a single instance of the database per-thread.
-If we _do_ somehow end up shared across threads, the journaling settings and writeahead
-should protect us from odd behavior.
+"""sqlite_trace_handler.py.
+
+This is the metaphorical bread and butter of our tracing implementation,
+or at least the butter. It wraps a SQLite database and configures it to
+be 'agreeable' in multithreaded situations. Normally, when sharing
+across threads and instances one should consider using a larger database
+solution like Postgres, but in this case we only care about _supporting_
+writing from multiple places. We don't expect it will be the norm. We
+care about (1) not negatively impacting performance, (2) not crashing
+when used in unusual ways, and (3) not losing data when possible.
+
+The happy path should be reasonably performant. The unhappy path should
+not crash.
+
+The other part of the multithreaded support comes from the public
+trace_handler, which uses a singleton pattern to only have a single
+instance of the database per-thread. If we _do_ somehow end up shared
+across threads, the journaling settings and writeahead should protect us
+from odd behavior.
"""
import datetime
@@ -194,12 +196,14 @@ def log_validator(self, vlog: ValidatorLogs):
def tail_logs(
self, start_offset_idx: int = 0, follow: bool = False
) -> Iterator[GuardTraceEntry]:
- """Returns an iterator to generate GuardLogEntries.
- @param start_offset_idx : Start printing entries after this IDX. If
- negative, this will instead start printing the LAST start_offset_idx entries.
- @param follow : If follow is True, will re-check the database for new entries
- after the first batch is complete. If False (default), will return when entries
- are exhausted.
+ """Returns an iterator to generate GuardLogEntries. @param
+ start_offset_idx : Start printing entries after this IDX. If.
+
+ negative, this will instead start printing the LAST
+ start_offset_idx entries. @param follow : If follow is True,
+ will re-check the database for new entries after the first batch
+ is complete. If False (default), will return when entries are
+ exhausted.
"""
last_idx = start_offset_idx
cursor = self.db.cursor()
diff --git a/guardrails/call_tracing/trace_entry.py b/guardrails/call_tracing/trace_entry.py
index 259ff3865..6f807a9f1 100644
--- a/guardrails/call_tracing/trace_entry.py
+++ b/guardrails/call_tracing/trace_entry.py
@@ -1,10 +1,10 @@
-"""
-trace_entry.py
+"""trace_entry.py.
-GuardTraceEntry is a dataclass which doesn't explicitly define the schema of our logs,
-but serves as a nice, easy-to-use dataclass for when we want to manipulate things
-programmatically. If performance and filtering is a concern, it's probably worth
-writing the SQL directly instead of filtering these in a for-loop.
+GuardTraceEntry is a dataclass which doesn't explicitly define the
+schema of our logs, but serves as a nice, easy-to-use dataclass for when
+we want to manipulate things programmatically. If performance and
+filtering is a concern, it's probably worth writing the SQL directly
+instead of filtering these in a for-loop.
"""
from dataclasses import dataclass
diff --git a/guardrails/call_tracing/trace_handler.py b/guardrails/call_tracing/trace_handler.py
index cb383c063..1e689f2d0 100644
--- a/guardrails/call_tracing/trace_handler.py
+++ b/guardrails/call_tracing/trace_handler.py
@@ -1,5 +1,4 @@
-"""
-trace_handler.py
+"""trace_handler.py.
A set of tools to track the behavior of guards, specifically with the intent of
collating the pre/post validation text and timing of guard calls. Uses a singleton to
@@ -41,9 +40,12 @@
class TraceHandler(TracerMixin):
- """TraceHandler wraps the internal _SQLiteTraceHandler to make it multi-thread
- safe. Coupled with some write ahead journaling in the _SyncTrace internal, we have
- a faux-multi-write multi-read interface for SQLite."""
+ """TraceHandler wraps the internal _SQLiteTraceHandler to make it multi-
+ thread safe.
+
+ Coupled with some write ahead journaling in the _SyncTrace internal,
+ we have a faux-multi-write multi-read interface for SQLite.
+ """
_instance = None
_lock = threading.Lock()
diff --git a/guardrails/call_tracing/tracer_mixin.py b/guardrails/call_tracing/tracer_mixin.py
index dd56307c5..76b1684f3 100644
--- a/guardrails/call_tracing/tracer_mixin.py
+++ b/guardrails/call_tracing/tracer_mixin.py
@@ -1,9 +1,7 @@
-"""
-tracer_mixin.py
+"""tracer_mixin.py.
-This file defines our preferred tracer interface.
-It has a side effect of acting as a 'noop' when we want to benchmark performance of a
-tracer.
+This file defines our preferred tracer interface. It has a side effect
+of acting as a 'noop' when we want to benchmark performance of a tracer.
"""
import os
diff --git a/guardrails/classes/generic/arbitrary_model.py b/guardrails/classes/generic/arbitrary_model.py
index d26def657..876c978ac 100644
--- a/guardrails/classes/generic/arbitrary_model.py
+++ b/guardrails/classes/generic/arbitrary_model.py
@@ -2,4 +2,6 @@
class ArbitraryModel(BaseModel):
+ """Empty Pydantic model with a config that allows arbitrary types."""
+
model_config = ConfigDict(arbitrary_types_allowed=True)
diff --git a/guardrails/classes/history/__init__.py b/guardrails/classes/history/__init__.py
index 622b0cd18..469fdbfee 100644
--- a/guardrails/classes/history/__init__.py
+++ b/guardrails/classes/history/__init__.py
@@ -4,4 +4,4 @@
from guardrails.classes.history.iteration import Iteration
from guardrails.classes.history.outputs import Outputs
-__all__ = ["CallInputs", "Call", "Inputs", "Iteration", "Outputs"]
+__all__ = ["Call", "Iteration", "Inputs", "Outputs", "CallInputs"]
diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py
index 99252830f..1f2498900 100644
--- a/guardrails/classes/history/call.py
+++ b/guardrails/classes/history/call.py
@@ -29,6 +29,20 @@
# We can't inherit from Iteration because python
# won't let you override a class attribute with a managed attribute
class Call(ICall, ArbitraryModel):
+ """A Call represents a single execution of a Guard. One Call is created
+ each time the user invokes the `Guard.__call__`, `Guard.parse`, or
+ `Guard.validate` method.
+
+ Attributes:
+ iterations (Stack[Iteration]): A stack of iterations
+ for the initial validation round
+ and one for each reask that occurs during a Call.
+ inputs (CallInputs): The inputs as passed in to
+ `Guard.__call__`, `Guard.parse`, or `Guard.validate`
+ exception (Optional[Exception]): The exception that interrupted
+ the Guard execution.
+ """
+
iterations: Stack[Iteration] = Field(
description="A stack of iterations for each"
"step/reask that occurred during this call."
diff --git a/guardrails/classes/history/call_inputs.py b/guardrails/classes/history/call_inputs.py
index bc99a4905..4aa2363f8 100644
--- a/guardrails/classes/history/call_inputs.py
+++ b/guardrails/classes/history/call_inputs.py
@@ -8,6 +8,21 @@
class CallInputs(Inputs, ICallInputs, ArbitraryModel):
+ """CallInputs represent the input data that is passed into the Guard from
+ the user. Inherits from Inputs with the below overrides and additional
+ attributes.
+
+ Attributes:
+ llm_api (Optional[Callable[[Any], Awaitable[Any]]]): The LLM function
+ provided by the user during Guard.__call__ or Guard.parse.
+ prompt (Optional[str]): The prompt string as provided by the user.
+ instructions (Optional[str]): The instructions string as provided by the user.
+ args (List[Any]): Additional arguments for the LLM as provided by the user.
+ Default [].
+ kwargs (Dict[str, Any]): Additional keyword-arguments for
+ the LLM as provided by the user. Default {}.
+ """
+
llm_api: Optional[Callable[[Any], Awaitable[Any]]] = Field(
description="The LLM function provided by the user"
"during Guard.__call__ or Guard.parse.",
diff --git a/guardrails/classes/history/inputs.py b/guardrails/classes/history/inputs.py
index f9ee44423..0e6c2ea8f 100644
--- a/guardrails/classes/history/inputs.py
+++ b/guardrails/classes/history/inputs.py
@@ -10,6 +10,29 @@
class Inputs(IInputs, ArbitraryModel):
+ """Inputs represent the input data that is passed into the validation loop.
+
+ Attributes:
+ llm_api (Optional[PromptCallableBase]): The constructed class
+ for calling the LLM.
+ llm_output (Optional[str]): The string output from an
+ external LLM call provided by the user via Guard.parse.
+ instructions (Optional[Instructions]): The constructed
+ Instructions class for chat model calls.
+ prompt (Optional[Prompt]): The constructed Prompt class.
+ msg_history (Optional[List[Dict]]): The message history
+ provided by the user for chat model calls.
+ prompt_params (Optional[Dict]): The parameters provided
+ by the user that will be formatted into the final LLM prompt.
+ num_reasks (Optional[int]): The total number of reasks allowed;
+ user provided or defaulted.
+ metadata (Optional[Dict[str, Any]]): The metadata provided
+ by the user to be used during validation.
+ full_schema_reask (Optional[bool]): Whether reasks we
+ performed across the entire schema or at the field level.
+ stream (Optional[bool]): Whether or not streaming was used.
+ """
+
llm_api: Optional[PromptCallableBase] = Field(
description="The constructed class for calling the LLM.", default=None
)
diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py
index c440c7e95..45c60baab 100644
--- a/guardrails/classes/history/iteration.py
+++ b/guardrails/classes/history/iteration.py
@@ -19,6 +19,18 @@
class Iteration(IIteration, ArbitraryModel):
+ """An Iteration represents a single iteration of the validation loop
+ including a single call to the LLM if applicable.
+
+ Attributes:
+ id (str): The unique identifier for the iteration.
+ call_id (str): The unique identifier for the Call
+ that this iteration is a part of.
+ index (int): The index of this iteration within the Call.
+ inputs (Inputs): The inputs for the validation loop.
+ outputs (Outputs): The outputs from the validation loop.
+ """
+
# I think these should be containered since their names slightly overlap with
# outputs, but could be convinced otherwise
inputs: Inputs = Field(
diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py
index dda4c7340..a2c21ac1e 100644
--- a/guardrails/classes/history/outputs.py
+++ b/guardrails/classes/history/outputs.py
@@ -20,6 +20,28 @@
class Outputs(IOutputs, ArbitraryModel):
+ """Outputs represent the data that is output from the validation loop.
+
+ Attributes:
+ llm_response_info (Optional[LLMResponse]): Information from the LLM response
+ raw_output (Optional[str]): The exact output from the LLM.
+ parsed_output (Optional[Union[str, List, Dict]]): The output parsed from the LLM
+ response as it was passed into validation.
+ validation_response (Optional[Union[str, ReAsk, List, Dict]]): The response
+ from the validation process.
+ guarded_output (Optional[Union[str, List, Dict]]): Any valid values after
+ undergoing validation.
+ Some values may be "fixed" values that were corrected during validation.
+ This property may be a partial structure if field level reasks occur.
+ reasks (List[ReAsk]): Information from the validation process used to construct
+ a ReAsk to the LLM on validation failure. Default [].
+ validator_logs (List[ValidatorLogs]): The results of each individual
+ validation. Default [].
+ error (Optional[str]): The error message from any exception that raised
+ and interrupted the process.
+ exception (Optional[Exception]): The exception that interrupted the process.
+ """
+
llm_response_info: Optional[LLMResponse] = Field(
description="Information from the LLM response.", default=None
)
@@ -47,7 +69,6 @@ class Outputs(IOutputs, ArbitraryModel):
default_factory=list,
)
# TODO: Rename this;
- # TODO: Add json_path to ValidatorLogs to specify what property it applies to
validator_logs: List[ValidatorLogs] = Field(
description="The results of each individual validation.", default_factory=list
)
diff --git a/guardrails/classes/llm/llm_response.py b/guardrails/classes/llm/llm_response.py
index 6051d2fca..d26e37a35 100644
--- a/guardrails/classes/llm/llm_response.py
+++ b/guardrails/classes/llm/llm_response.py
@@ -13,6 +13,21 @@ def async_to_sync(awaitable):
# TODO: We might be able to delete this
class LLMResponse(ILLMResponse):
+ """Standard information collection from LLM responses to feed the
+ validation loop.
+
+ Attributes:
+ output (str): The output from the LLM.
+ stream_output (Optional[Iterable]): A stream of output from the LLM.
+ Default None.
+ async_stream_output (Optional[AsyncIterable]): An async stream of output
+ from the LLM. Default None.
+ prompt_token_count (Optional[int]): The number of tokens in the prompt.
+ Default None.
+ response_token_count (Optional[int]): The number of tokens in the response.
+ Default None.
+ """
+
# Pydantic Config
model_config = ConfigDict(arbitrary_types_allowed=True)
diff --git a/guardrails/classes/validation/validation_result.py b/guardrails/classes/validation/validation_result.py
index 316f3c415..18e6a0baa 100644
--- a/guardrails/classes/validation/validation_result.py
+++ b/guardrails/classes/validation/validation_result.py
@@ -10,11 +10,19 @@
class ValidationResult(IValidationResult, ArbitraryModel):
+ """ValidationResult is the output type of Validator.validate and the
+ abstract base class for all validation results.
+
+ Attributes:
+ outcome (str): The outcome of the validation. Must be one of "pass" or "fail".
+ metadata (Optional[Dict[str, Any]]): The metadata associated with this
+ validation result.
+ validated_chunk (Optional[Any]): The value argument passed to
+ validator.validate or validator.validate_stream.
+ """
+
outcome: str
metadata: Optional[Dict[str, Any]] = None
-
- # value argument passed to validator.validate
- # or validator.validate_stream
validated_chunk: Optional[Any] = None
@classmethod
@@ -45,6 +53,15 @@ def from_dict(cls, obj: Dict[str, Any]) -> "ValidationResult":
class PassResult(ValidationResult, IPassResult):
+ """PassResult is the output type of Validator.validate when validation
+ succeeds.
+
+ Attributes:
+ outcome (Literal["pass"]): The outcome of the validation. Must be "pass".
+ value_override (Optional[Any]): The value to use as an override
+ if validation passes.
+ """
+
outcome: Literal["pass"] = "pass"
class ValueOverrideSentinel:
@@ -76,21 +93,28 @@ def to_dict(self) -> Dict[str, Any]:
return _dict
-# FIXME: Add this to json schema
-class ErrorSpan(IErrorSpan, ArbitraryModel):
- start: int
- end: int
- # reason validation failed, specific to this chunk
- reason: str
-
-
class FailResult(ValidationResult, IFailResult):
+ """FailResult is the output type of Validator.validate when validation
+ fails.
+
+ Attributes:
+ outcome (Literal["fail"]): The outcome of the validation. Must be "fail".
+ error_message (str): The error message indicating why validation failed.
+ fix_value (Optional[Any]): The auto-fix value that would be applied
+ if the Validator's on_fail method is "fix".
+ error_spans (Optional[List[ErrorSpan]]): Segments that caused
+ validation to fail.
+ """
+
outcome: Literal["fail"] = "fail"
error_message: str
fix_value: Optional[Any] = None
- # segments that caused validation to fail
- error_spans: Optional[List[ErrorSpan]] = None
+ """Segments that caused validation to fail.
+
+ May not exist for non-streamed output.
+ """
+ error_spans: Optional[List["ErrorSpan"]] = None
@classmethod
def from_interface(cls, i_fail_result: IFailResult) -> "FailResult":
@@ -137,3 +161,21 @@ def to_dict(self) -> Dict[str, Any]:
),
}
return _dict
+
+
+class ErrorSpan(IErrorSpan, ArbitraryModel):
+ """ErrorSpan provide additional context for why a validation failed. They
+ specify the start and end index of the segment that caused the failure,
+ which can be useful when validating large chunks of text or validating
+ while streaming with different chunking methods.
+
+ Attributes:
+ start (int): Starting index relative to the validated chunk.
+ end (int): Ending index relative to the validated chunk.
+ reason (str): Reason validation failed for this chunk.
+ """
+
+ start: int
+ end: int
+ # reason validation failed, specific to this chunk
+ reason: str
diff --git a/guardrails/classes/validation/validator_logs.py b/guardrails/classes/validation/validator_logs.py
index 7cf31f193..cb38e22cb 100644
--- a/guardrails/classes/validation/validator_logs.py
+++ b/guardrails/classes/validation/validator_logs.py
@@ -12,7 +12,20 @@
class ValidatorLogs(IValidatorLog, ArbitraryModel):
- """Logs for a single validator."""
+ """Logs for a single validator execution.
+
+ Attributes:
+ validator_name (str): The class name of the validator
+ registered_name (str): The snake_cased id of the validator
+ property_path (str): The JSON path to the property being validated
+ value_before_validation (Any): The value before validation
+ value_after_validation (Optional[Any]): The value after validation;
+ could be different if `value_override`s or `fix`es are applied
+ validation_result (Optional[ValidationResult]): The result of the validation
+ start_time (Optional[datetime]): The time the validation started
+ end_time (Optional[datetime]): The time the validation ended
+ instance_id (Optional[int]): The unique id of this instance of the validator
+ """
validator_name: str
registered_name: str
diff --git a/guardrails/classes/validation/validator_reference.py b/guardrails/classes/validation/validator_reference.py
new file mode 100644
index 000000000..93890d0b0
--- /dev/null
+++ b/guardrails/classes/validation/validator_reference.py
@@ -0,0 +1,19 @@
+from guardrails_api_client import ValidatorReference as IValidatorReference
+
+
+# Docs only
+class ValidatorReference(IValidatorReference):
+ """ValidatorReference is a serialized reference for constructing a
+ Validator.
+
+ Attributes:
+ id (Optional[str]): The unique identifier for this Validator.
+ Often the hub id; e.g. guardrails/regex_match. Default None.
+ on (Optional[str]): A reference to the property this validator should be
+ applied against. Can be a valid JSON path or a meta-property
+ such as `prompt` or `output`. Default None.
+ on_fail (Optional[str]): The OnFailAction to apply during validation.
+ Default None.
+ args (Optional[List[Any]]): Positional arguments. Default None.
+ kwargs (Optional[Dict[str, Any]]): Keyword arguments. Default None.
+ """
diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py
index fe23a1e91..39f26a363 100644
--- a/guardrails/classes/validation_outcome.py
+++ b/guardrails/classes/validation_outcome.py
@@ -16,6 +16,21 @@
class ValidationOutcome(IValidationOutcome, ArbitraryModel, Generic[OT]):
+ """The final output from a Guard execution.
+
+ Attributes:
+ call_id: The id of the Call that produced this ValidationOutcome.
+ raw_llm_output: The raw, unchanged output from the LLM call.
+ validated_output: The validated, and potentially fixed, output from the LLM call
+ after passing through validation.
+ reask: If validation continuously fails and all allocated reasks are used,
+ this field will contain the final reask that would have been sent
+ to the LLM if additional reasks were available.
+ validation_passed: A boolean to indicate whether or not the LLM output
+ passed validation. If this is False, the validated_output may be invalid.
+ error: If the validation failed, this field will contain the error message
+ """
+
raw_llm_output: Optional[str] = Field(
description="The raw, unchanged output from the LLM call.", default=None
)
diff --git a/guardrails/errors/__init__.py b/guardrails/errors/__init__.py
index 64b795688..529467074 100644
--- a/guardrails/errors/__init__.py
+++ b/guardrails/errors/__init__.py
@@ -1,5 +1,11 @@
class ValidationError(Exception):
- """Top level validation error."""
+ """Top level validation error.
+
+ This is thrown from the validation engine when a Validator has
+ on_fail=OnFailActions.EXCEPTION set and validation fails.
+
+ Inherits from Exception.
+ """
class UserFacingException(Exception):
diff --git a/guardrails/formatters/json_formatter.py b/guardrails/formatters/json_formatter.py
index 000423851..791b823f1 100644
--- a/guardrails/formatters/json_formatter.py
+++ b/guardrails/formatters/json_formatter.py
@@ -87,6 +87,9 @@ def _jsonschema_to_jsonformer(
class JsonFormatter(BaseFormatter):
+ """A formatter that uses Jsonformer to ensure the shape of structured data
+ for Hugging Face models."""
+
def __init__(self, schema: dict):
self.output_schema = _jsonschema_to_jsonformer(schema)
diff --git a/guardrails/guard.py b/guardrails/guard.py
index 0b21e1c08..fc8546ec8 100644
--- a/guardrails/guard.py
+++ b/guardrails/guard.py
@@ -31,6 +31,7 @@
from guardrails.api_client import GuardrailsApiClient
from guardrails.classes.output_type import OT
+from guardrails.classes.validation.validation_result import ErrorSpan
from guardrails.classes.validation_outcome import ValidationOutcome
from guardrails.classes.credentials import Credentials
from guardrails.classes.execution import GuardExecutionOptions
@@ -79,7 +80,10 @@
ValidatorMap,
)
-from guardrails.utils.tools_utils import add_json_function_calling_tool
+from guardrails.utils.tools_utils import (
+ # Prevent duplicate declaration in the docs
+ add_json_function_calling_tool as add_json_function_calling_tool_util,
+)
class Guard(IGuard, Generic[OT]):
@@ -118,7 +122,11 @@ def __init__(
validators: Optional[List[ValidatorReference]] = None,
output_schema: Optional[Dict[str, Any]] = None,
):
- """Initialize the Guard with validators and an output schema."""
+ """Initialize the Guard with serialized validator references and an
+ output schema.
+
+ Output schema must be a valid JSON Schema.
+ """
_try_to_load = name is not None
@@ -351,7 +359,8 @@ def from_rail(
name: Optional[str] = None,
description: Optional[str] = None,
):
- """Create a Schema from a `.rail` file.
+ """Create a Guard using a `.rail` file to specify the output schema,
+ prompt, etc.
Args:
rail_file: The path to the `.rail` file.
@@ -399,7 +408,8 @@ def from_rail_string(
name: Optional[str] = None,
description: Optional[str] = None,
):
- """Create a Schema from a `.rail` string.
+ """Create a Guard using a `.rail` string to specify the output schema,
+ prompt, etc..
Args:
rail_string: The `.rail` string.
@@ -452,7 +462,8 @@ def from_pydantic(
description: Optional[str] = None,
output_formatter: Optional[Union[str, BaseFormatter]] = None,
):
- """Create a Guard instance from a Pydantic model.
+ """Create a Guard instance using a Pydantic model to specify the output
+ schema.
Args:
output_class: (Union[Type[BaseModel], List[Type[BaseModel]]]): The pydantic model that describes
@@ -832,7 +843,7 @@ def __call__(
Args:
llm_api: The LLM API to call
- (e.g. openai.Completion.create or openai.Completion.acreate)
+ (e.g. openai.completions.create or openai.Completion.acreate)
prompt_params: The parameters to pass to the prompt.format() method.
num_reasks: The max times to re-ask the LLM for invalid output.
prompt: The prompt to use for the LLM.
@@ -845,7 +856,7 @@ def __call__(
`False` otherwise.
Returns:
- The raw text output from the LLM and the validated output.
+ ValidationOutcome
"""
instructions = instructions or self._exec_opts.instructions
prompt = prompt or self._exec_opts.prompt
@@ -887,15 +898,14 @@ def parse(
llm_output: The output being parsed and validated.
metadata: Metadata to pass to the validators.
llm_api: The LLM API to call
- (e.g. openai.Completion.create or openai.Completion.acreate)
+ (e.g. openai.completions.create or openai.Completion.acreate)
num_reasks: The max times to re-ask the LLM for invalid output.
prompt_params: The parameters to pass to the prompt.format() method.
full_schema_reask: When reasking, whether to regenerate the full schema
or just the incorrect values.
Returns:
- The validated response. This is either a string or a dictionary,
- determined by the object schema defined in the RAILspec.
+ ValidationOutcome
"""
final_num_reasks = (
num_reasks
@@ -929,7 +939,8 @@ def parse(
**kwargs,
)
- def error_spans_in_output(self):
+ def error_spans_in_output(self) -> List[ErrorSpan]:
+ """Get the error spans in the last output."""
try:
call = self.history.last
if call:
@@ -991,8 +1002,6 @@ def use(
- The instructions
- The message history
- *Note*: For on="output", `use` is only available for string output types.
-
Args:
validator: The validator to use. Either the class or an instance.
on: The part of the LLM request to validate. Defaults to "output".
@@ -1017,7 +1026,7 @@ def use_many(
*validators: UseManyValidatorSpec,
on: str = "output",
) -> "Guard":
- """Use a validator to validate results of an LLM request."""
+ """Use multiple validators to validate results of an LLM request."""
# Loop through the validators
for v in validators:
hydrated_validator = get_validator(v)
@@ -1192,6 +1201,7 @@ def _save(self):
self.upsert_guard()
def to_runnable(self) -> Runnable:
+ """Convert a Guard to a LangChain Runnable."""
from guardrails.integrations.langchain.guard_runnable import GuardRunnable
return GuardRunnable(self)
@@ -1213,7 +1223,9 @@ def add_json_function_calling_tool(
self,
tools: list,
) -> List[Dict[str, Any]]:
- tools = add_json_function_calling_tool(
+ """Appends an OpenAI tool that specifies the output structure using
+ JSON Schema for chat models."""
+ tools = add_json_function_calling_tool_util(
tools=tools,
# todo to_dict has a slight bug workaround here
# but should fix in the long run dont have to
diff --git a/guardrails/llm_providers.py b/guardrails/llm_providers.py
index 9525f075d..19c7cee23 100644
--- a/guardrails/llm_providers.py
+++ b/guardrails/llm_providers.py
@@ -178,7 +178,7 @@ def _invoke_llm(
Use Guardrails with OpenAI chat engines by doing
```
raw_llm_response, validated_response, *rest = guard(
- openai.ChatCompletion.create,
+ openai.chat.completions.create,
prompt_params={...},
text=...,
instructions=...,
@@ -754,7 +754,7 @@ async def invoke_llm(
Use Guardrails with OpenAI chat engines by doing
```
raw_llm_response, validated_response, *rest = guard(
- openai.ChatCompletion.create,
+ openai.chat.completions.create,
prompt_params={...},
text=...,
instructions=...,
diff --git a/guardrails/prompt/base_prompt.py b/guardrails/prompt/base_prompt.py
index 1eed29cc1..1ff70559e 100644
--- a/guardrails/prompt/base_prompt.py
+++ b/guardrails/prompt/base_prompt.py
@@ -2,7 +2,7 @@
import re
from string import Template
-from typing import Optional
+from typing import List, Optional
import regex
@@ -21,6 +21,7 @@ def __init__(
*,
xml_output_schema: Optional[str] = None,
):
+ """Initialize and substitute constants in the prompt."""
self._source = source
self.format_instructions_start = self.get_format_instructions_idx(source)
@@ -55,7 +56,7 @@ def variable_names(self):
def format_instructions(self):
return self.source[self.format_instructions_start :]
- def substitute_constants(self, text):
+ def substitute_constants(self, text: str) -> str:
"""Substitute constants in the prompt."""
# Substitute constants by reading the constants file.
# Regex to extract all occurrences of ${gr.}
@@ -70,10 +71,10 @@ def substitute_constants(self, text):
return text
- def get_prompt_variables(self):
+ def get_prompt_variables(self) -> List[str]:
return self.variable_names
- def format(self, **kwargs):
+ def format(self, **kwargs) -> "BasePrompt":
raise NotImplementedError("Subclasses must implement this method.")
def make_vars_optional(self):
diff --git a/guardrails/prompt/instructions.py b/guardrails/prompt/instructions.py
index 110978a04..6aaf5b078 100644
--- a/guardrails/prompt/instructions.py
+++ b/guardrails/prompt/instructions.py
@@ -25,7 +25,7 @@ def __repr__(self) -> str:
def __eq__(self, __value: object) -> bool:
return isinstance(__value, Instructions) and self.source == __value.source
- def format(self, **kwargs):
+ def format(self, **kwargs) -> "Instructions":
"""Format the prompt using the given keyword arguments."""
# Only use the keyword arguments that are present in the prompt.
vars = get_template_variables(self.source)
diff --git a/guardrails/prompt/prompt.py b/guardrails/prompt/prompt.py
index cfd335404..8b656162c 100644
--- a/guardrails/prompt/prompt.py
+++ b/guardrails/prompt/prompt.py
@@ -16,7 +16,7 @@ class Prompt(BasePrompt):
def __eq__(self, __value: object) -> bool:
return isinstance(__value, Prompt) and self.source == __value.source
- def format(self, **kwargs):
+ def format(self, **kwargs) -> "Prompt":
"""Format the prompt using the given keyword arguments."""
# Only use the keyword arguments that are present in the prompt.
vars = get_template_variables(self.source)
diff --git a/guardrails/run/async_stream_runner.py b/guardrails/run/async_stream_runner.py
index cb89f0f9e..a0e8db7bd 100644
--- a/guardrails/run/async_stream_runner.py
+++ b/guardrails/run/async_stream_runner.py
@@ -117,6 +117,7 @@ async def async_step(
llm_response = await self.async_call(
instructions, prompt, msg_history, api, output
)
+ iteration.outputs.llm_response_info = llm_response
stream_output = llm_response.async_stream_output
if not stream_output:
raise ValueError(
@@ -127,6 +128,7 @@ async def async_step(
fragment = ""
parsed_fragment, validated_fragment, valid_op = None, None, None
verified = set()
+ validation_response = ""
if self.output_type == OutputTypes.STRING:
async for chunk in stream_output:
@@ -159,6 +161,7 @@ async def async_step(
"Reasks are not yet supported with streaming. Please "
"remove reasks from schema or disable streaming."
)
+ validation_response += cast(str, validated_fragment)
passed = call_log.status == pass_status
yield ValidationOutcome(
call_id=call_log.id, # type: ignore
@@ -196,6 +199,10 @@ async def async_step(
"remove reasks from schema or disable streaming."
)
+ if self.output_type == OutputTypes.LIST:
+ validation_response = cast(list, validated_fragment)
+ else:
+ validation_response = cast(dict, validated_fragment)
yield ValidationOutcome(
call_id=call_log.id, # type: ignore
raw_llm_output=fragment,
@@ -205,10 +212,8 @@ async def async_step(
iteration.outputs.raw_output = fragment
# FIXME: Handle case where parsing continuously fails/is a reask
- iteration.outputs.parsed_output = parsed_fragment # type: ignore
- iteration.outputs.validation_response = (
- cast(str, validated_fragment) if validated_fragment else None
- )
+ iteration.outputs.parsed_output = parsed_fragment or fragment # type: ignore
+ iteration.outputs.validation_response = validation_response
iteration.outputs.guarded_output = valid_op
def get_chunk_text(self, chunk: Any, api: Union[PromptCallableBase, None]) -> str:
diff --git a/guardrails/run/stream_runner.py b/guardrails/run/stream_runner.py
index a968548e3..e8c87a6c9 100644
--- a/guardrails/run/stream_runner.py
+++ b/guardrails/run/stream_runner.py
@@ -126,6 +126,8 @@ def step(
# Call: run the API that returns a generator wrapped in LLMResponse
llm_response = self.call(instructions, prompt, msg_history, api, output)
+ iteration.outputs.llm_response_info = llm_response
+
# Get the stream (generator) from the LLMResponse
stream = llm_response.stream_output
if stream is None:
@@ -137,6 +139,7 @@ def step(
fragment = ""
parsed_fragment, validated_fragment, valid_op = None, None, None
verified = set()
+ validation_response = ""
# Loop over the stream
# and construct "fragments" of concatenated chunks
# for now, handle string and json schema differently
@@ -184,6 +187,7 @@ def step(
"remove reasks from schema or disable streaming."
)
# 5. Convert validated fragment to a pretty JSON string
+ validation_response += cast(str, validated_text)
passed = call_log.status == pass_status
yield ValidationOutcome(
call_id=call_log.id, # type: ignore
@@ -258,6 +262,10 @@ def step(
"remove reasks from schema or disable streaming."
)
+ if self.output_type == OutputTypes.LIST:
+ validation_response = cast(list, validated_fragment)
+ else:
+ validation_response = cast(dict, validated_fragment)
# 5. Convert validated fragment to a pretty JSON string
yield ValidationOutcome(
call_id=call_log.id, # type: ignore
@@ -270,8 +278,8 @@ def step(
iteration.outputs.raw_output = fragment
# Do we need to care about the type here?
# What happens if parsing continuously fails?
- iteration.outputs.parsed_output = parsed_fragment # type: ignore
- iteration.outputs.validation_response = validated_fragment
+ iteration.outputs.parsed_output = parsed_fragment or fragment # type: ignore
+ iteration.outputs.validation_response = validation_response
iteration.outputs.guarded_output = valid_op
def is_last_chunk(self, chunk: Any, api: Union[PromptCallableBase, None]) -> bool:
diff --git a/guardrails/types/__init__.py b/guardrails/types/__init__.py
index 03b033ce4..632f2781f 100644
--- a/guardrails/types/__init__.py
+++ b/guardrails/types/__init__.py
@@ -17,13 +17,13 @@
)
__all__ = [
- "MessageHistory",
"OnFailAction",
+ "RailTypes",
"PrimitiveTypes",
+ "MessageHistory",
"ModelOrListOfModels",
"ModelOrListOrDict",
"ModelOrModelUnion",
- "RailTypes",
"PydanticValidatorTuple",
"PydanticValidatorSpec",
"UseValidatorSpec",
diff --git a/guardrails/types/inputs.py b/guardrails/types/inputs.py
index deb8d7ab5..07c738d12 100644
--- a/guardrails/types/inputs.py
+++ b/guardrails/types/inputs.py
@@ -2,5 +2,5 @@
from guardrails.prompt.prompt import Prompt
-
+"""List[Dict[str, Union[Prompt, str]]]"""
MessageHistory = List[Dict[str, Union[Prompt, str]]]
diff --git a/guardrails/types/on_fail.py b/guardrails/types/on_fail.py
index 83946aec6..7e9bfa3b9 100644
--- a/guardrails/types/on_fail.py
+++ b/guardrails/types/on_fail.py
@@ -4,6 +4,23 @@
class OnFailAction(str, Enum):
+ """OnFailAction is an Enum that represents the different actions that can
+ be taken when a validation fails.
+
+ Attributes:
+ REASK (Literal["reask"]): On failure, Reask the LLM.
+ FIX (Literal["fix"]): On failure, apply a static fix.
+ FILTER (Literal["filter"]): On failure, filter out the invalid values.
+ REFRAIN (Literal["refrain"]): On failure, refrain from responding;
+ return an empty value.
+ NOOP (Literal["noop"]): On failure, do nothing.
+ EXCEPTION (Literal["exception"]): On failure, raise a ValidationError.
+ FIX_REASK (Literal["fix_reask"]): On failure, apply a static fix,
+ check if the fixed value passed validation, if not then reask the LLM.
+ CUSTOM (Literal["custom"]): On failure, call a custom function with the
+ invalid value and the FailResult's from any validators run on the value.
+ """
+
REASK = "reask"
FIX = "fix"
FILTER = "filter"
diff --git a/guardrails/types/rail.py b/guardrails/types/rail.py
index 912447862..474efad8b 100644
--- a/guardrails/types/rail.py
+++ b/guardrails/types/rail.py
@@ -3,6 +3,25 @@
class RailTypes(str, Enum):
+ """RailTypes is an Enum that represents the builtin tags for RAIL xml.
+
+ Attributes:
+ STRING (Literal["string"]): A string value.
+ INTEGER (Literal["integer"]): An integer value.
+ FLOAT (Literal["float"]): A float value.
+ BOOL (Literal["bool"]): A boolean value.
+ DATE (Literal["date"]): A date value.
+ TIME (Literal["time"]): A time value.
+ DATETIME (Literal["date-time: - A datetime value.
+ PERCENTAGE (Literal["percentage"]): A percentage value represented as a string.
+ Example "20.5%".
+ ENUM (Literal["enum"]): An enum value.
+ LIST (Literal["list"]): A list/array value.
+ OBJECT (Literal["object"]): An object/dictionary value.
+ CHOICE (Literal["choice"]): The options for a discrimated union.
+ CASE (Literal["case"]): A dictionary that contains a discrimated union.
+ """
+
STRING = "string"
INTEGER = "integer"
FLOAT = "float"
diff --git a/guardrails/validators/__init__.py b/guardrails/validators/__init__.py
index 394991271..5054a31f3 100644
--- a/guardrails/validators/__init__.py
+++ b/guardrails/validators/__init__.py
@@ -12,4 +12,5 @@
"ValidationResult",
"PassResult",
"FailResult",
+ "ErrorSpan",
]
diff --git a/package.json b/package.json
index 093c72081..47f21975e 100644
--- a/package.json
+++ b/package.json
@@ -13,7 +13,8 @@
"clear": "docusaurus clear",
"serve": "docusaurus serve",
"write-translations": "docusaurus write-translations",
- "write-heading-ids": "docusaurus write-heading-ids"
+ "write-heading-ids": "docusaurus write-heading-ids",
+ "restart": "rm -rf docs/api_reference_markdown; rm -rf docs-build; npm run start"
},
"dependencies": {
"@docusaurus/core": "^3.1.1",
diff --git a/tests/unit_tests/test_guard_log.py b/tests/unit_tests/test_guard_log.py
index fc332f374..9799c7b36 100644
--- a/tests/unit_tests/test_guard_log.py
+++ b/tests/unit_tests/test_guard_log.py
@@ -23,7 +23,8 @@
def test_multiprocessing_hoisted():
- """Preallocate a shared trace handler and try to log from multiple subprocesses."""
+ """Preallocate a shared trace handler and try to log from multiple
+ subprocesses."""
with Pool(NUM_THREADS) as pool:
pool.map(_hoisted_logger, ["multiproc_hoist" + msg for msg in STOCK_MESSAGES])