From 84341f0e2df6e6f27eec6ea1b4a8d0afc8da04b9 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 8 Feb 2024 09:32:51 -0800 Subject: [PATCH 1/3] start cli upload support --- guardrails/cli/hub/create_validator.py | 79 ++++++++++++++++++++++++++ guardrails/cli/hub/install.py | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 guardrails/cli/hub/create_validator.py diff --git a/guardrails/cli/hub/create_validator.py b/guardrails/cli/hub/create_validator.py new file mode 100644 index 000000000..8445a0c69 --- /dev/null +++ b/guardrails/cli/hub/create_validator.py @@ -0,0 +1,79 @@ +import os +from pydash import snake_case +import typer +from os.path import expanduser +from guardrails.cli.hub.hub import hub +from string import Template + +validator_template = Template("""from typing import Any, Dict + +from guardrails.logger import logger +from guardrails.validator_base import ( + FailResult, + PassResult, + ValidationResult, + Validator, + register_validator, +) + + +@register_validator(name="guardrails/${package_name}", data_type=["string", "list"]) +class ${class_name}(Validator): + \"""Validates that a string or list ends with a given value. + + **Key Properties** + + | Property | Description | + | ----------------------------- | --------------------------------- | + | Name for `format` attribute | `hub://guardrails/${package_name}` | + | Supported data types | `string`, `list` | + | Programmatic fix | Append the given value to the end. | + + Args: + end: The required last element. + \""" + + def __init__(self, end: str, on_fail: str = "fix"): + super().__init__(on_fail=on_fail, end=end) + self._end = end + + def validate(self, value: Any, metadata: Dict) -> ValidationResult: + logger.debug(f"Validating {value} ends with {self._end}...") + + if not value[-1] == self._end: + return FailResult( + error_message=f"{value} must end with {self._end}", + fix_value=value + [self._end], + ) + + return PassResult() + + +class Test${class_name}: + def test_success_case(self): + validator = ${class_name}("s") + result = validator.validate("pass") + assert isintance(result, PassResult) is True + + def test_failure_case(self): + validator = ${class_name}("s") + result = validator.validate("fail") + assert isintance(result, FailResult) is True + assert result.error_message == "fail must end with s" + assert result.fix_value == "fails" +""") + + +@hub.command(name='create-validator') +def create_validator( + name: str = typer.Argument( + help="The name for your validator." + ), + filepath: str = typer.Argument( + help="The location to write your validator template to", + default="./{validator_name}.py" + ) +): + file_name = snake_case(name) + target = os.path.join(os.getcwd(), file_name) + \ No newline at end of file diff --git a/guardrails/cli/hub/install.py b/guardrails/cli/hub/install.py index 0dea65878..9917ac7ca 100644 --- a/guardrails/cli/hub/install.py +++ b/guardrails/cli/hub/install.py @@ -189,7 +189,7 @@ def install_hub_module(module_manifest: ModuleManifest, site_packages: str): @hub.command() def install( package_uri: str = typer.Argument( - help="URI to the package to install. Example: hub://guardrails/regex-match." + help="URI to the package to install. Example: hub://guardrails/regex_match." ), ): """Install a validator from the Hub.""" From f16bb5c11ec33741690c76a390c2ce2113ba6aa4 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Mon, 12 Feb 2024 09:55:11 -0600 Subject: [PATCH 2/3] impl create, start submit --- guardrails/cli/hub/__init__.py | 1 + guardrails/cli/hub/create_validator.py | 104 ++++++++++++++++++------- guardrails/cli/hub/submit.py | 0 3 files changed, 76 insertions(+), 29 deletions(-) create mode 100644 guardrails/cli/hub/submit.py diff --git a/guardrails/cli/hub/__init__.py b/guardrails/cli/hub/__init__.py index ab1f5ba16..101b96d2c 100644 --- a/guardrails/cli/hub/__init__.py +++ b/guardrails/cli/hub/__init__.py @@ -1,2 +1,3 @@ import guardrails.cli.hub.install # noqa +import guardrails.cli.hub.create_validator # noqa from guardrails.cli.hub.hub import hub # noqa diff --git a/guardrails/cli/hub/create_validator.py b/guardrails/cli/hub/create_validator.py index 8445a0c69..e31dca52c 100644 --- a/guardrails/cli/hub/create_validator.py +++ b/guardrails/cli/hub/create_validator.py @@ -1,13 +1,12 @@ import os -from pydash import snake_case +from pydash import snake_case, pascal_case import typer -from os.path import expanduser from guardrails.cli.hub.hub import hub +from guardrails.cli.logger import LEVELS, logger from string import Template -validator_template = Template("""from typing import Any, Dict +validator_template = Template("""from typing import Any, Callable, Dict, Optional -from guardrails.logger import logger from guardrails.validator_base import ( FailResult, PassResult, @@ -16,49 +15,69 @@ register_validator, ) +# List any additional dependencies here. +\""" +dependencies = [ + "guardrails-ai>=0.3.2" +] + +[project.optional-dependencies] +dev = [ + "pytest" +] +\""" + -@register_validator(name="guardrails/${package_name}", data_type=["string", "list"]) +@register_validator(name="guardrails/${package_name}", data_type="string") class ${class_name}(Validator): - \"""Validates that a string or list ends with a given value. + \"""Validates that {fill in how you validator interacts with the passed value}. **Key Properties** - | Property | Description | - | ----------------------------- | --------------------------------- | - | Name for `format` attribute | `hub://guardrails/${package_name}` | - | Supported data types | `string`, `list` | - | Programmatic fix | Append the given value to the end. | + | Property | Description | + | ----------------------------- | --------------------------------- | + | Name for `format` attribute | `guardrails/${package_name}` | + | Supported data types | `string` | + | Programmatic fix | {If you support programmatic fixes, explain it here. Otherwise `None`} | Args: - end: The required last element. - \""" + arg_1 (string): {Description of the argument here} + arg_2 (string): {Description of the argument here} + \""" # noqa - def __init__(self, end: str, on_fail: str = "fix"): - super().__init__(on_fail=on_fail, end=end) - self._end = end + # If you don't have any init args, you can omit the __init__ method. + def __init__( + self, + arg_1: str, + arg_2: str, + on_fail: Optional[Callable] = None, + ): + super().__init__(on_fail=on_fail, arg_1=arg_1, arg_2=arg_2) + self._arg_1 = arg_1 + self._arg_2 = arg_2 def validate(self, value: Any, metadata: Dict) -> ValidationResult: - logger.debug(f"Validating {value} ends with {self._end}...") - - if not value[-1] == self._end: + \"""Validates that {fill in how you validator interacts with the passed value}.\""" + # Add your custom validator logic here and return a PassResult or FailResult accordingly. + if value != "pass": # FIXME return FailResult( - error_message=f"{value} must end with {self._end}", - fix_value=value + [self._end], + error_message="{A descriptive but concise error message about why validation failed}", + fix_value="{The programmtic fix if applicable, otherwise remove this kwarg.}", ) - return PassResult() - + +# Run tests via `pytest -rP ${filepath}` class Test${class_name}: def test_success_case(self): validator = ${class_name}("s") - result = validator.validate("pass") - assert isintance(result, PassResult) is True + result = validator.validate("pass", {}) + assert isinstance(result, PassResult) is True def test_failure_case(self): validator = ${class_name}("s") - result = validator.validate("fail") - assert isintance(result, FailResult) is True + result = validator.validate("fail", {}) + assert isinstance(result, FailResult) is True assert result.error_message == "fail must end with s" assert result.fix_value == "fails" """) @@ -74,6 +93,33 @@ def create_validator( default="./{validator_name}.py" ) ): - file_name = snake_case(name) - target = os.path.join(os.getcwd(), file_name) + package_name = snake_case(name) + class_name = pascal_case(name) + if not filepath or filepath == "./{validator_name}.py": + filepath = f"./{package_name}.py" + + template = validator_template.safe_substitute({ + "package_name": package_name, + "class_name": class_name, + "filepath": filepath + }) + + target = os.path.abspath(filepath) + with open(target, 'w') as validator_file: + validator_file.write(template) + validator_file.close() + + success_message = Template( + """ + + Successfully created validator template at ${filepath}! + + Make any necessary changes then submit for review with the following command: + + guardrails hub submit ${package_name} ${filepath} + """ + ).safe_substitute( + {"filepath": filepath, "package_name": package_name} + ) + logger.log(level=LEVELS.get("SUCCESS"), msg=success_message) # type: ignore \ No newline at end of file diff --git a/guardrails/cli/hub/submit.py b/guardrails/cli/hub/submit.py new file mode 100644 index 000000000..e69de29bb From 2516fa6f27955a3f37dacdc6780f72a5bce0a39b Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 13 Feb 2024 08:29:32 -0600 Subject: [PATCH 3/3] finish create add submit --- guardrails/cli/hub/__init__.py | 1 + guardrails/cli/hub/create_validator.py | 114 +++++++++++++++++++------ guardrails/cli/hub/submit.py | 54 ++++++++++++ guardrails/cli/server/hub_client.py | 32 +++++++ 4 files changed, 176 insertions(+), 25 deletions(-) diff --git a/guardrails/cli/hub/__init__.py b/guardrails/cli/hub/__init__.py index 101b96d2c..e58e5f567 100644 --- a/guardrails/cli/hub/__init__.py +++ b/guardrails/cli/hub/__init__.py @@ -1,3 +1,4 @@ import guardrails.cli.hub.install # noqa import guardrails.cli.hub.create_validator # noqa +import guardrails.cli.hub.submit # noqa from guardrails.cli.hub.hub import hub # noqa diff --git a/guardrails/cli/hub/create_validator.py b/guardrails/cli/hub/create_validator.py index e31dca52c..99f9d270c 100644 --- a/guardrails/cli/hub/create_validator.py +++ b/guardrails/cli/hub/create_validator.py @@ -1,9 +1,10 @@ import os +from datetime import date from pydash import snake_case, pascal_case +from string import Template import typer from guardrails.cli.hub.hub import hub from guardrails.cli.logger import LEVELS, logger -from string import Template validator_template = Template("""from typing import Any, Callable, Dict, Optional @@ -15,34 +16,96 @@ register_validator, ) -# List any additional dependencies here. -\""" -dependencies = [ - "guardrails-ai>=0.3.2" -] - -[project.optional-dependencies] -dev = [ - "pytest" -] -\""" - @register_validator(name="guardrails/${package_name}", data_type="string") class ${class_name}(Validator): - \"""Validates that {fill in how you validator interacts with the passed value}. + \"""# Overview + + | Developed by | {Your organization name} | + | Date of development | ${dev_date} | + | Validator type | Format | + | Blog | | + | License | Apache 2 | + | Input/Output | Output | + + # Description + + This validator ensures that a generated output is the literal \"pass\". + + # Installation + + ```bash + $ guardrails hub install hub://guardrails/${package_name} + ``` + + # Usage Examples + + ## Validating string output via Python + + In this example, we'll test that a generated word is `pass`. + + ```python + # Import Guard and Validator + from guardrails.hub import ${class_name} + from guardrails import Guard + + # Initialize Validator + val = ${class_name}() + + # Setup Guard + guard = Guard.from_string( + validators=[val, ...], + ) + + guard.parse(\"pass\") # Validator passes + guard.parse(\"fail\") # Validator fails + ``` + + ## Validating JSON output via Python + + In this example, we verify that a processes's status is specified as `pass`. + + ```python + # Import Guard and Validator + from pydantic import BaseModel + from guardrails.hub import ${class_name} + from guardrails import Guard + + val = ${class_name}() + + # Create Pydantic BaseModel + class Process(BaseModel): + process_name: str + status: str = Field(validators=[val]) + + # Create a Guard to check for valid Pydantic output + guard = Guard.from_pydantic(output_class=Process) + + # Run LLM output generating JSON through guard + guard.parse(\""" + { + "process_name": "templating", + "status": "pass" + } + \""") + ``` + + # API Reference + + `__init__` + - `arg_1`: A placeholder argument to demonstrate how to use init arguments. + - `arg_2`: Another placeholder argument to demonstrate how to use init arguments. + - `on_fail`: The policy to enact when a validator fails. - **Key Properties** + # Dependencies - | Property | Description | - | ----------------------------- | --------------------------------- | - | Name for `format` attribute | `guardrails/${package_name}` | - | Supported data types | `string` | - | Programmatic fix | {If you support programmatic fixes, explain it here. Otherwise `None`} | + ## Production + guardrails-ai >= 0.3.2 - Args: - arg_1 (string): {Description of the argument here} - arg_2 (string): {Description of the argument here} + ## Development + pytest + pyright + ruff \""" # noqa # If you don't have any init args, you can omit the __init__ method. @@ -78,7 +141,7 @@ def test_failure_case(self): validator = ${class_name}("s") result = validator.validate("fail", {}) assert isinstance(result, FailResult) is True - assert result.error_message == "fail must end with s" + assert result.error_message == "{A descriptive but concise error message about why validation failed}" assert result.fix_value == "fails" """) @@ -101,7 +164,8 @@ def create_validator( template = validator_template.safe_substitute({ "package_name": package_name, "class_name": class_name, - "filepath": filepath + "filepath": filepath, + "dev_date": date.today().strftime("%b %d, %Y") }) target = os.path.abspath(filepath) diff --git a/guardrails/cli/hub/submit.py b/guardrails/cli/hub/submit.py index e69de29bb..1d972c22c 100644 --- a/guardrails/cli/hub/submit.py +++ b/guardrails/cli/hub/submit.py @@ -0,0 +1,54 @@ +import os +import sys +from pydash import snake_case, pascal_case +import typer +from guardrails.cli.hub.hub import hub +from guardrails.cli.logger import LEVELS, logger +from string import Template + +from guardrails.cli.server.hub_client import HttpError, post_validator_submit + + +@hub.command(name='submit') +def submit( + package_name: str = typer.Argument( + help="The package name for your validator." + ), + filepath: str = typer.Argument( + help="The location to your validator file.", + default="./{package_name}.py" + ) +): + try: + if not filepath or filepath == "./{validator_name}.py": + filepath = f"./{package_name}.py" + + target = os.path.abspath(filepath) + with open(target, 'r') as validator_file: + content = validator_file.read() + + post_validator_submit(package_name, content) + + validator_file.close() + + success_message = Template( + """ + + Successfully submitted validator! + + Once your submission is reviewed and published you will be able to install it via: + + guardrails hub install hub://guardrails/${package_name} + """ + ).safe_substitute( + {"package_name": snake_case(package_name)} + ) + logger.log(level=LEVELS.get("SUCCESS"), msg=success_message) # type: ignore + + except HttpError: + logger.error(f"Failed to submit {package_name}!") + sys.exit(1) + except Exception as e: + logger.error("An unexpected error occurred!", e) + sys.exit(1) + \ No newline at end of file diff --git a/guardrails/cli/server/hub_client.py b/guardrails/cli/server/hub_client.py index 50a1cbf52..f1cd5c251 100644 --- a/guardrails/cli/server/hub_client.py +++ b/guardrails/cli/server/hub_client.py @@ -101,3 +101,35 @@ def get_auth(): except Exception as e: logger.error("An unexpected error occurred!", e) raise AuthenticationError("Failed to authenticate!") + + +def post_validator_submit(package_name: str, content: str): + try: + creds = Credentials.from_rc_file() + token = get_auth_token(creds) + submission_url = f"{validator_hub_service}/validator/submit" + + headers = { + "Authorization": f"Bearer {token}", + } + request_body = { + "packageName": package_name, + "content": content + } + req = requests.post(submission_url, data=request_body, headers=headers) + + body = req.json() + if not req.ok: + logger.error(req.status_code) + logger.error(body.get("message")) + http_error = HttpError() + http_error.status = req.status_code + http_error.message = body.get("message") + raise http_error + + return body + except HttpError as http_e: + raise http_e + except Exception as e: + logger.error("An unexpected error occurred!", e) + sys.exit(1) \ No newline at end of file