Initial Validator Hub Support

.github/workflows/ci.yml

@@ -9,7 +9,6 @@ on:
     branches:
       - main
       - dev
-      - "0.3.0"
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -21,17 +20,17 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Poetry cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry
-          key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}
+      # - name: Poetry cache
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: ~/.cache/pypoetry
+      #     key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}
 
       - name: Install Poetry
         uses: snok/install-poetry@v1
@@ -53,17 +52,17 @@ jobs:
         pydantic-version: ["1.10.9", "2.4.2"]
         openai-version: ["0.28.1", "1.2.4"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Poetry cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry
-          key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ matrix.pydantic-version }}
+      # - name: Poetry cache
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: ~/.cache/pypoetry
+      #     key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ matrix.pydantic-version }}
 
       - name: Install Poetry
         uses: snok/install-poetry@v1
@@ -106,17 +105,17 @@ jobs:
         pydantic-version: ["1.10.9", "2.4.2"]
         openai-version: ["0.28.1", "1.2.4"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Poetry cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry
-          key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ matrix.pydantic-version }}-${{ matrix.openai-version }}
+      # - name: Poetry cache
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: ~/.cache/pypoetry
+      #     key: poetry-cache-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ matrix.pydantic-version }}-${{ matrix.openai-version }}
 
       - name: Install Poetry
         uses: snok/install-poetry@v1
@@ -154,16 +153,16 @@ jobs:
         NLTK_DATA: /tmp/nltk_data
     steps:
         - name: Checkout repository
-          uses: actions/checkout@v2    
+          uses: actions/checkout@v4    
         - name: Set up Python
-          uses: actions/setup-python@v4
+          uses: actions/setup-python@v5
           with: 
             python-version: 3.11.x
-        - name: Poetry cache
-          uses: actions/cache@v3
-          with:
-            path: ~/.cache/pypoetry
-            key: poetry-cache-${{ runner.os }}-3.11.x-${{ env.POETRY_VERSION }}
+        # - name: Poetry cache
+        #   uses: actions/cache@v3
+        #   with:
+        #     path: ~/.cache/pypoetry
+        #     key: poetry-cache-${{ runner.os }}-3.11.x-${{ env.POETRY_VERSION }}
         - name: Install Poetry
           uses: snok/install-poetry@v1
           with:

docs/api_reference/response_structures.md

@@ -6,4 +6,4 @@
             - "ValidationResult"
             - "PassResult"
             - "FailResult"
-            - "ValidatorError"
+            - "ValidationError"

docs/cli.md

@@ -7,4 +7,17 @@ Guardrails can be used from the command line to validate the output of an LLM. C
 
 ```bash
 guardrails validate <path to rail spec> <llm output as string> --out <output path for validated JSON>
-```
+```
+
+## Validator Hub
+In addition to providing a command line interface for validation, the guardrails cli also supports interacting with the Validator Hub.
+
+### Configuration
+In order to access any Validators from the Hub that require authentication, you will need to set up your environment through the `guardrails configure` command.  Before running `guardrails configure`, go to the [Validator Hub]() to generate your access tokens.  [Add more detail on this process].
+
+Once you have your tokens, run `guardrails configure` and enter the client id and client secret you retrieved above.
+
+Also, if you want to opt out of anonymous metrics collection, you can do this now by specifying `--no-metrics=true`.
+
+### Installing Validators
+In order to install a validator from the hub, you can find its identifier on its respective page in the [Validator Hub]().  This identifier is shaped as a namespace and a name: `{namespace}/{validator_name}`.  Once you have this you can go to any project where you have guardrails installed and run `guardrails hub install hub://{namespace}/{validator_name}`.

docs/examples/input_validation.ipynb

@@ -47,7 +47,7 @@
    "source": [
     "When `fix` is specified as the on-fail handler, the prompt will automatically be amended before calling the LLM.\n",
     "\n",
-    "In any other case (for example, `exception`), a `ValidatorError` will be returned in the outcome."
+    "In any other case (for example, `exception`), a `ValidationError` will be returned in the outcome."
    ]
   },
   {
@@ -67,13 +67,13 @@
    ],
    "source": [
     "import openai\n",
-    "from guardrails.errors import ValidatorError\n",
+    "from guardrails.errors import ValidationError\n",
     "\n",
     "try:\n",
     "    guard(\n",
     "        openai.chat.completions.create,\n",
     "    )\n",
-    "except ValidatorError as e:\n",
+    "except ValidationError as e:\n",
     "    print(e)"
    ]
   },
@@ -115,7 +115,7 @@
     "        openai.chat.completions.create,\n",
     "        prompt=\"This is not two words\",\n",
     "    )\n",
-    "except ValidatorError as e:\n",
+    "except ValidationError as e:\n",
     "    print(e)"
    ]
   }
@@ -136,7 +136,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

docs/examples/response_is_on_topic.ipynb

@@ -137,7 +137,7 @@
    "source": [
     "import guardrails as gd\n",
     "from guardrails.validators import OnTopic\n",
-    "from guardrails.errors import ValidatorError\n",
+    "from guardrails.errors import ValidationError\n",
     "\n",
     "# Create the Guard with the OnTopic Validator\n",
     "guard = gd.Guard.from_string(\n",
@@ -159,7 +159,7 @@
     "    guard.parse(\n",
     "        llm_output=text,\n",
     "    )\n",
-    "except ValidatorError as e:\n",
+    "except ValidationError as e:\n",
     "    print(e)\n"
    ]
   },
@@ -205,7 +205,7 @@
     "    guard.parse(\n",
     "        llm_output=text,\n",
     "    )\n",
-    "except ValidatorError as e:\n",
+    "except ValidationError as e:\n",
     "    print(e)"
    ]
   },
@@ -258,7 +258,7 @@
     "    guard.parse(\n",
     "        llm_output=text,\n",
     "    )\n",
-    "except ValidatorError as e:\n",
+    "except ValidationError as e:\n",
     "    print(e)"
    ]
   }
@@ -279,7 +279,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

docs/telemetry.md

@@ -0,0 +1,5 @@
+1. Go to grafana
+2. search for opentelemetry in data sources
+3. Create a token, paste in the resulting file to otel-collector-config.yml
+4. docker-compose up -f telemetry/docker-compose.yml up
+5. run a guard and look for traces

guardrails/classes/__init__.py

@@ -1,4 +1,6 @@
+from guardrails.classes.credentials import Credentials
+from guardrails.classes.input_type import InputType
 from guardrails.classes.output_type import OT
 from guardrails.classes.validation_outcome import ValidationOutcome
 
-__all__ = ["ValidationOutcome", "OT"]
+__all__ = ["ValidationOutcome", "OT", "InputType", "Credentials"]

guardrails/classes/credentials.py

@@ -0,0 +1,31 @@
+import os
+from dataclasses import dataclass
+from os.path import expanduser
+from typing import Optional
+
+from guardrails.classes.generic.serializeable import Serializeable
+
+
+@dataclass
+class Credentials(Serializeable):
+    id: Optional[str] = None
+    client_id: Optional[str] = None
+    client_secret: Optional[str] = None
+    no_metrics: Optional[bool] = False
+
+    @staticmethod
+    def from_rc_file() -> "Credentials":
+        try:
+            home = expanduser("~")
+            guardrails_rc = os.path.join(home, ".guardrailsrc")
+            with open(guardrails_rc) as rc_file:
+                lines = rc_file.readlines()
+                creds = {}
+                for line in lines:
+                    key, value = line.split("=", 1)
+                    creds[key.strip()] = value.strip()
+                rc_file.close()
+                return Credentials.from_dict(creds)
+
+        except FileNotFoundError:
+            return Credentials.from_dict({})  # type: ignore

guardrails/classes/generic/__init__.py

@@ -1,3 +1,4 @@
+from guardrails.classes.generic.serializeable import Serializeable
 from guardrails.classes.generic.stack import Stack
 
-__all__ = ["Stack"]
+__all__ = ["Stack", "Serializeable"]

guardrails/classes/generic/serializeable.py

@@ -0,0 +1,52 @@
+import inspect
+import json
+import sys
+from dataclasses import InitVar, asdict, dataclass, field, is_dataclass
+from json import JSONEncoder
+from typing import Any, Dict
+
+from pydash.strings import snake_case
+
+
+def get_annotations(obj):
+    if sys.version_info.minor >= 10 and hasattr(inspect, "get_annotations"):
+        return inspect.get_annotations(obj)  # type: ignore
+    else:
+        return obj.__annotations__
+
+
+class SerializeableJSONEncoder(JSONEncoder):
+    def default(self, o):
+        if is_dataclass(o):
+            return asdict(o)
+        return super().default(o)
+
+
+encoder_kwargs = {}
+if sys.version_info.minor >= 10:
+    encoder_kwargs["kw_only"] = True
+    encoder_kwargs["default"] = SerializeableJSONEncoder
+
+
+@dataclass
+class Serializeable:
+    encoder: InitVar[JSONEncoder] = field(**encoder_kwargs)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]):
+        annotations = get_annotations(cls)
+        attributes = dict.keys(annotations)
+        snake_case_kwargs = {
+            snake_case(k): data.get(k) for k in data if snake_case(k) in attributes
+        }
+        snake_case_kwargs["encoder"] = snake_case_kwargs.get(
+            "encoder", SerializeableJSONEncoder
+        )
+        return cls(**snake_case_kwargs)  # type: ignore
+
+    @property
+    def __dict__(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    def to_json(self):
+        return json.dumps(self, cls=self.encoder)  # type: ignore

guardrails/classes/generic/stack.py

@@ -49,7 +49,7 @@ def search(self, x: T) -> Optional[int]:
         except ValueError:
             pass
 
-    def at(self, index: int) -> Optional[T]:
+    def at(self, index: int, default: Optional[T] = None) -> Optional[T]:
         """Returns the item located at the index.
 
         If the index does not exist in the stack (Overflow or
@@ -59,7 +59,7 @@ def at(self, index: int) -> Optional[T]:
             value = self[index]
             return value
         except IndexError:
-            pass
+            return default
 
     def copy(self) -> "Stack[T]":
         """Returns a copy of the current Stack."""

guardrails/classes/history/call.py

@@ -19,7 +19,7 @@
     sub_reasks_with_fixed_values,
 )
 from guardrails.utils.safe_get import get_value_from_path
-from guardrails.validator_base import Refrain
+from guardrails.validator_base import Filter, Refrain
 
 
 # We can't inherit from Iteration because python
@@ -315,8 +315,13 @@ def _has_unresolved_failures(self) -> bool:
         output = self.fixed_output
         for failure in self.failed_validations:
             value = get_value_from_path(output, failure.property_path)
-            if value == failure.value_before_validation or isinstance(
-                failure.value_after_validation, Refrain
+            if (
+                # NOTE: this means on_fail="fix" was applied
+                #       to a Validator without a programmatic fix.
+                (value is None and failure.value_before_validation is not None)
+                or value == failure.value_before_validation
+                or isinstance(failure.value_after_validation, Refrain)
+                or isinstance(failure.value_after_validation, Filter)
             ):
                 return True
 

guardrails/classes/input_type.py

@@ -0,0 +1,5 @@
+from typing import TypeVar
+
+from langchain_core.messages import BaseMessage
+
+InputType = TypeVar("InputType", str, BaseMessage)

guardrails/cli/__init__.py

@@ -0,0 +1,12 @@
+import guardrails.cli.configure  # noqa
+import guardrails.cli.validate  # noqa
+from guardrails.cli.guardrails import guardrails as cli
+from guardrails.cli.hub import hub_command
+
+cli.add_typer(
+    hub_command, name="hub", help="Manage validators installed from the Guardrails Hub."
+)
+
+
+if __name__ == "__main__":
+    cli()