Hub cli

.github/workflows/ci.yml

@@ -156,20 +156,20 @@ jobs:
         - name: Checkout repository
           uses: actions/checkout@v2    
         - name: Set up Python
-          uses: actions/setup-python@v2
+          uses: actions/setup-python@v4
           with: 
             python-version: 3.11.x
-        - name: Poetry cache
-          uses: actions/cache@v3
-          with:
-            path: ~/.cache/pypoetry
-            key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }}
         - name: Install Poetry
           uses: snok/install-poetry@v1
           with:
             virtualenvs-create: true
             virtualenvs-in-project: true
             installer-parallel: true
+        - name: Load cached venv
+          uses: actions/cache@v3
+          with:
+            path: .venv
+            key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
         - name: Install dependencies
           run: |
             make full;

docs/cli.md

@@ -7,4 +7,17 @@ Guardrails can be used from the command line to validate the output of an LLM. C
 
 ```bash
 guardrails validate <path to rail spec> <llm output as string> --out <output path for validated JSON>
-```
+```
+
+## Validator Hub
+In addition to providing a command line interface for validation, the guardrails cli also supports interacting with the Validator Hub.
+
+### Configuration
+In order to access any Validators from the Hub that require authentication, you will need to set up your environment through the `guardrails configure` command.  Before running `guardrails configure`, go to the [Validator Hub]() to generate your access tokens.  [Add more detail on this process].
+
+Once you have your tokens, run `guardrails configure` and enter the client id and client secret you retrieved above.
+
+Also, if you want to opt out of anonymous metrics collection, you can do this now by specifying `--no-metrics=true`.
+
+### Installing Validators
+In order to install a validator from the hub, you can find its identifier on its respective page in the [Validator Hub]().  This identifier is shaped as a namespace and a name: `{namespace}/{validator_name}`.  Once you have this you can go to any project where you have guardrails installed and run `guardrails hub install hub://{namespace}/{validator_name}`.

guardrails/classes/generic/stack.py

@@ -49,7 +49,7 @@ def search(self, x: T) -> Optional[int]:
         except ValueError:
             pass
 
-    def at(self, index: int) -> Optional[T]:
+    def at(self, index: int, default: Optional[T] = None) -> Optional[T]:
         """Returns the item located at the index.
 
         If the index does not exist in the stack (Overflow or
@@ -59,7 +59,7 @@ def at(self, index: int) -> Optional[T]:
             value = self[index]
             return value
         except IndexError:
-            pass
+            return default
 
     def copy(self) -> "Stack[T]":
         """Returns a copy of the current Stack."""

guardrails/cli/__init__.py

@@ -0,0 +1,12 @@
+import guardrails.cli.configure  # noqa
+import guardrails.cli.validate  # noqa
+from guardrails.cli.guardrails import guardrails as cli
+from guardrails.cli.hub import hub
+
+cli.add_typer(
+    hub, name="hub", help="Manage validators installed from the Guardrails Hub."
+)
+
+
+if __name__ == "__main__":
+    cli()

guardrails/cli/configure.py

@@ -0,0 +1,79 @@
+import os
+import sys
+import uuid
+from os.path import expanduser
+from typing import Optional
+
+import typer
+
+from guardrails.cli.guardrails import guardrails
+from guardrails.cli.logger import LEVELS, logger
+from guardrails.cli.server.hub_client import AuthenticationError, get_auth
+
+
+def save_configuration_file(
+    client_id: str, client_secret: str, no_metrics: bool
+) -> None:
+    home = expanduser("~")
+    guardrails_rc = os.path.join(home, ".guardrailsrc")
+    with open(guardrails_rc, "w") as rc_file:
+        lines = [
+            f"id={str(uuid.uuid4())}{os.linesep}",
+            f"client_id={client_id}{os.linesep}",
+            f"client_secret={client_secret}{os.linesep}",
+            f"no_metrics={str(no_metrics).lower()}{os.linesep}",
+        ]
+        rc_file.writelines(lines)
+        rc_file.close()
+
+
+@guardrails.command()
+def configure(
+    client_id: Optional[str] = typer.Option(
+        help="Your Guardrails Hub client ID.", default=""
+    ),
+    client_secret: Optional[str] = typer.Option(
+        help="Your Guardrails Hub client secret.", hide_input=True, default=""
+    ),
+    no_metrics: Optional[str] = typer.Option(
+        help="Opt out of anonymous metrics collection.", default=False
+    ),
+):
+    """Set the global configuration for the Guardrails CLI and Hub."""
+    try:
+        if not client_id:
+            client_id = typer.prompt("Client ID")
+        if not client_secret:
+            client_secret = typer.prompt("Client secret", hide_input=True)
+        logger.info("Configuring...")
+        save_configuration_file(client_id, client_secret, no_metrics)  # type: ignore
+
+        logger.info("Validating credentials...")
+        get_auth()
+        success_message = """
+
+    Login successful.
+
+    Get started by installing a validator from the Guardrails Hub!
+
+    guardrails hub install hub://guardrails/lowercase
+
+    Find more validators at https://hub.guardrailsai.com
+    """
+        logger.log(level=LEVELS.get("SUCCESS"), msg=success_message)  # type: ignore
+    except AuthenticationError as auth_error:
+        logger.error(auth_error)
+        logger.error(
+            """
+            Check that your Client ID and Client secret are correct and try again.
+
+            If you don't have your token credentials you can find them here:
+
+            https://hub.guardrailsai.com/tokens
+            """
+        )
+        sys.exit(1)
+    except Exception as e:
+        logger.error("An unexpected error occurred!")
+        logger.error(e)
+        sys.exit(1)

guardrails/cli/guardrails.py

@@ -0,0 +1,3 @@
+import typer
+
+guardrails = typer.Typer()

guardrails/cli/hub/__init__.py

@@ -0,0 +1,4 @@
+import guardrails.cli.hub.install  # noqa
+import guardrails.cli.hub.create_validator  # noqa
+import guardrails.cli.hub.submit  # noqa
+from guardrails.cli.hub.hub import hub  # noqa

guardrails/cli/hub/create_validator.py

@@ -0,0 +1,189 @@
+import os
+from datetime import date
+from pydash import snake_case, pascal_case
+from string import Template
+import typer
+from guardrails.cli.hub.hub import hub
+from guardrails.cli.logger import LEVELS, logger
+
+validator_template = Template("""from typing import Any, Callable, Dict, Optional
+
+from guardrails.validator_base import (
+    FailResult,
+    PassResult,
+    ValidationResult,
+    Validator,
+    register_validator,
+)
+
+
+@register_validator(name="guardrails/${package_name}", data_type="string")
+class ${class_name}(Validator):
+    \"""# Overview
+
+    | Developed by | {Your organization name} |
+    | Date of development | ${dev_date} |
+    | Validator type | Format |
+    | Blog |  |
+    | License | Apache 2 |
+    | Input/Output | Output |
+
+    # Description
+
+    This validator ensures that a generated output is the literal \"pass\".
+
+    # Installation
+
+    ```bash
+    $ guardrails hub install hub://guardrails/${package_name}
+    ```
+
+    # Usage Examples
+
+    ## Validating string output via Python
+
+    In this example, we'll test that a generated word is `pass`.
+
+    ```python
+    # Import Guard and Validator
+    from guardrails.hub import ${class_name}
+    from guardrails import Guard
+
+    # Initialize Validator
+    val = ${class_name}()
+
+    # Setup Guard
+    guard = Guard.from_string(
+        validators=[val, ...],
+    )
+
+    guard.parse(\"pass\")  # Validator passes
+    guard.parse(\"fail\")  # Validator fails
+    ```
+
+    ## Validating JSON output via Python
+
+    In this example, we verify that a processes's status is specified as `pass`.
+
+    ```python
+    # Import Guard and Validator
+    from pydantic import BaseModel
+    from guardrails.hub import ${class_name}
+    from guardrails import Guard
+
+    val = ${class_name}()
+
+    # Create Pydantic BaseModel
+    class Process(BaseModel):
+    		process_name: str
+    		status: str = Field(validators=[val])
+
+    # Create a Guard to check for valid Pydantic output
+    guard = Guard.from_pydantic(output_class=Process)
+
+    # Run LLM output generating JSON through guard
+    guard.parse(\"""
+    {
+    		"process_name": "templating",
+    		"status": "pass"
+    }
+    \""")
+    ```
+
+    # API Reference
+
+    `__init__`
+    - `arg_1`: A placeholder argument to demonstrate how to use init arguments.
+    - `arg_2`: Another placeholder argument to demonstrate how to use init arguments.
+    - `on_fail`: The policy to enact when a validator fails.
+
+    # Dependencies
+
+    ## Production
+    guardrails-ai >= 0.3.2
+
+    ## Development
+    pytest
+    pyright
+    ruff
+    \"""  # noqa
+
+    # If you don't have any init args, you can omit the __init__ method.
+    def __init__(
+        self,
+        arg_1: str,
+        arg_2: str,
+        on_fail: Optional[Callable] = None,
+    ):
+        super().__init__(on_fail=on_fail, arg_1=arg_1, arg_2=arg_2)
+        self._arg_1 = arg_1
+        self._arg_2 = arg_2
+
+    def validate(self, value: Any, metadata: Dict) -> ValidationResult:
+        \"""Validates that {fill in how you validator interacts with the passed value}.\"""
+        # Add your custom validator logic here and return a PassResult or FailResult accordingly.
+        if value != "pass": # FIXME
+            return FailResult(
+                error_message="{A descriptive but concise error message about why validation failed}",
+                fix_value="{The programmtic fix if applicable, otherwise remove this kwarg.}",
+            )
+        return PassResult()
+
+
+# Run tests via `pytest -rP ${filepath}`
+class Test${class_name}:
+    def test_success_case(self):
+        validator = ${class_name}("s")
+        result = validator.validate("pass", {})
+        assert isinstance(result, PassResult) is True
+
+    def test_failure_case(self):
+        validator = ${class_name}("s")
+        result = validator.validate("fail", {})
+        assert isinstance(result, FailResult) is True
+        assert result.error_message == "{A descriptive but concise error message about why validation failed}"
+        assert result.fix_value == "fails"
+""")
+
+
+@hub.command(name='create-validator')
+def create_validator(
+    name: str = typer.Argument(
+        help="The name for your validator."
+    ),
+    filepath: str = typer.Argument(
+        help="The location to write your validator template to",
+        default="./{validator_name}.py"
+    )
+):
+    package_name = snake_case(name)
+    class_name = pascal_case(name)
+    if not filepath or filepath == "./{validator_name}.py":
+        filepath = f"./{package_name}.py"
+
+    template = validator_template.safe_substitute({
+        "package_name": package_name,
+        "class_name": class_name,
+        "filepath": filepath,
+        "dev_date": date.today().strftime("%b %d, %Y")
+    })
+
+    target = os.path.abspath(filepath)
+    with open(target, 'w') as validator_file:
+        validator_file.write(template)
+        validator_file.close()
+
+    success_message = Template(
+        """
+
+    Successfully created validator template at ${filepath}!
+
+    Make any necessary changes then submit for review with the following command:
+
+    guardrails hub submit ${package_name} ${filepath}
+    """
+    ).safe_substitute(
+        {"filepath": filepath, "package_name": package_name}
+    )
+    logger.log(level=LEVELS.get("SUCCESS"), msg=success_message)  # type: ignore
+

guardrails/cli/hub/hub.py

@@ -0,0 +1,3 @@
+import typer
+
+hub = typer.Typer()