From 5ab15adb1034999bafdaa25c08633d4eed4472e3 Mon Sep 17 00:00:00 2001 From: Jay Allen Date: Tue, 4 Jun 2024 14:18:21 +0900 Subject: [PATCH 01/11] Integrating recommended changes into telemetry doc --- docs/how_to_guides/telemetry.mdx | 40 ++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/how_to_guides/telemetry.mdx b/docs/how_to_guides/telemetry.mdx index 50b21ce63..100670aac 100644 --- a/docs/how_to_guides/telemetry.mdx +++ b/docs/how_to_guides/telemetry.mdx @@ -2,7 +2,11 @@ import Dashboard from './assets/dashboard.png'; import ViewTraces from './assets/view_traces.mp4'; import GrafanaOtelConfig from './assets/grafana_otel_config.mp4'; -# Capture Metrics for your Guards +# Overview + +In this document, we explain how to set up Guardrails with [OpenTelemetry (OTEL)](https://opentelemetry.io/) using either Grafana or a self-hosted OTEL collector. With this functionality enabled, you can measure latency of Guards, Large Language Models (LLMs), scuccess rates, and other metrics for your Guardrails-protected LLM calls. + +## Metrics you can capture using OTEL This package is instrumented using the OpenTelemetry Python SDK. By viewing the captured traces and derived metrics, we're able to get useful insights into how our Guards, and our LLM apps in general perform. Among other things, we're able to find: @@ -12,12 +16,11 @@ This package is instrumented using the OpenTelemetry Python SDK. By viewing the 4. The rate at which validators Pass and Fail, within a Guard and across Guards 5. Deep dives into singular guard and validator calls -Since we are using OpenTelemetry, traces and metrics can be written to any OpenTelemetry enabled service or OTLP endpoint. This includes all major metrics providers like Grafana, New Relic, Prometheus, and Splunk. +Since we are using OpenTelemetry, traces and metrics can be written to any OpenTelemetry-enabled service or OTLP endpoint. This includes all major metrics providers like Grafana, New Relic, Prometheus, and Splunk. -This guide will show how to set up your python project to log traces to Grafana and an OTEL collector. +This guide will show how to set up your Python project to log traces to Grafana and to a self-hosted OTEL collector. For other OTEL endpoints, consult your metrics provider's documentation on OTEL support. - -## Setup with Grafana +## Configure OTEL for Grafana Grafana Cloud is a free offering by Grafana that's easy to setup and is our preferred location for storing metrics. @@ -41,11 +44,11 @@ OTEL_EXPORTER_OTLP_HEADERS ### Setup a Guard with Telemetry -We first have to install the ```ValidLength``` guardrail from Guardrails Hub. +1. First, install the ```ValidLength``` guardrail from Guardrails Hub. ```guardrails hub install hub://guardrails/valid_length``` -Then, set up your Guard the default tracer provided in the guardrails library. You can still use your desired validators +2. Next, set up your Guard the default tracer provided in the guardrails library. You can still use your desired validators:
main.py
```python @@ -69,7 +72,7 @@ guard( ) ``` -Before running the file, make sure to set the environment variables you got from Grafana +3. Before running the file, make sure to set the environment variables you got from Grafana ```bash export GRAFANA_INSTANCE_ID= @@ -82,7 +85,7 @@ export OTEL_EXPORTER_OTLP_ENDPOINT= export OTEL_EXPORTER_OTLP_HEADERS= ``` -Finally, run the python script +4. Finally, run the python script ```bash @@ -90,26 +93,33 @@ python main.py ``` -### Viewing traces +### View traces + +There are two ways to view traces: using the Explore tab or using the Guardrails Grafana dashboard template. -The simplest way to do this is to go to your grafana stack and click on the "Explore" tab. You should see a list of traces that you can filter by service name, operation name, and more. +#### Use the Explore tab + +The simplest way to do this is to go to your grafana stack and click on the "**Explore** tab. You should see a list of traces that you can filter by service name, operation name, and more. +#### Use the Guardrails Grafana dashboard template + While this is easy to do, it's not the best way to get a big-picture view of how your guards are doing. For that, we should use the Guardrails Grafana dashboard template. -The template can be found here https://grafana.com/grafana/dashboards/20600-standard-guardrails-dash/ -and instructions to use the template are found here https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/ +**[Use the template](https://grafana.com/grafana/dashboards/20600-standard-guardrails-dash/)** + +**[Template instructions](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/)** -## OTEL Collector +## Configure OTEL for a self-hosted OpenTelemetry Collector For advanced use cases (like if you have a metrics provider in a VPC), you can use a self-hosted OpenTelemetry Collector to receive traces and metrics from your Guard. -Standard [open telemetry environment variables](https://opentelemetry-python.readthedocs.io/en/stable/getting-started.html#configure-the-exporter) are used to configure the collector. Use the default_otel_collector_tracer when configuring your guard. +Standard [open telemetry environment variables](https://opentelemetry.io/docs/languages/python/exporters/) are used to configure the collector. Use the `default_otel_collector_tracer` when configuring your guard. ```python from guardrails import Guard, OnFailAction From c76c0c8d6e749970611bc1b8bb034db47e3dad87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:27:50 +0000 Subject: [PATCH 02/11] Bump braces from 3.0.2 to 3.0.3 Bumps [braces](https://github.com/micromatch/braces) from 3.0.2 to 3.0.3. - [Changelog](https://github.com/micromatch/braces/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/braces/compare/3.0.2...3.0.3) --- updated-dependencies: - dependency-name: braces dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 70135e013..c9ab53b3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4554,11 +4554,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -7208,9 +7208,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, From 60448b2ef38e4fa10dadc3a4721b5ec4e3fc7652 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 01:34:10 +0000 Subject: [PATCH 03/11] Bump certifi from 2024.2.2 to 2024.7.4 Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.2.2 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2024.02.02...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index a4cd0ea91..3dd351644 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -573,13 +573,13 @@ files = [ [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] @@ -4091,6 +4091,7 @@ description = "Nvidia JIT LTO Library" optional = true python-versions = ">=3" files = [ + {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83"}, {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57"}, {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1"}, ] From 4ad92dbdc28dac9948b46acb7592c02d2ff836cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:30:05 +0000 Subject: [PATCH 04/11] Bump zipp from 3.18.2 to 3.19.1 Bumps [zipp](https://github.com/jaraco/zipp) from 3.18.2 to 3.19.1. - [Release notes](https://github.com/jaraco/zipp/releases) - [Changelog](https://github.com/jaraco/zipp/blob/main/NEWS.rst) - [Commits](https://github.com/jaraco/zipp/compare/v3.18.2...v3.19.1) --- updated-dependencies: - dependency-name: zipp dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index a4cd0ea91..1c0a0a9d1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -4091,6 +4091,7 @@ description = "Nvidia JIT LTO Library" optional = true python-versions = ">=3" files = [ + {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83"}, {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57"}, {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1"}, ] @@ -8308,18 +8309,18 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.18.2" +version = "3.19.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.18.2-py3-none-any.whl", hash = "sha256:dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e"}, - {file = "zipp-3.18.2.tar.gz", hash = "sha256:6278d9ddbcfb1f1089a88fde84481528b07b0e10474e09dcfe53dad4069fa059"}, + {file = "zipp-3.19.1-py3-none-any.whl", hash = "sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091"}, + {file = "zipp-3.19.1.tar.gz", hash = "sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [extras] anthropic = ["anthropic"] From d8da2c6067f0c7e7493b2acb1722c7d0bbcebf8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jul 2024 22:10:51 +0000 Subject: [PATCH 05/11] Bump urllib3 from 2.2.1 to 2.2.2 Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.2.1 to 2.2.2. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.2.1...2.2.2) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 68495ea02..eb31249b5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -7816,13 +7816,13 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake [[package]] name = "urllib3" -version = "2.2.1" +version = "2.2.2" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, - {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, ] [package.extras] From e75e8efea00929c0a58925f39f1c398e4b3ec9cd Mon Sep 17 00:00:00 2001 From: shiv07tiwari Date: Thu, 11 Jul 2024 13:48:23 +0530 Subject: [PATCH 06/11] fix: replace jwt by pyjwt --- guardrails/cli/server/hub_client.py | 17 +++-- poetry.lock | 32 +++++---- pyproject.toml | 2 +- .../unit_tests/cli/server/test_hub_client.py | 66 +++++++++---------- 4 files changed, 57 insertions(+), 60 deletions(-) diff --git a/guardrails/cli/server/hub_client.py b/guardrails/cli/server/hub_client.py index d29bf2781..2dde206cf 100644 --- a/guardrails/cli/server/hub_client.py +++ b/guardrails/cli/server/hub_client.py @@ -3,8 +3,9 @@ from typing import Any, Dict, Optional import requests -from jwt import JWT -from jwt.exceptions import JWTDecodeError +import jwt +from jwt import ExpiredSignatureError, DecodeError + from guardrails.classes.credentials import Credentials from guardrails.cli.logger import logger @@ -86,13 +87,11 @@ def get_jwt_token(creds: Credentials) -> Optional[str]: # check for jwt expiration if token: try: - JWT().decode(token, do_verify=False) - except JWTDecodeError as e: - # if the error message includes "Expired", then the token is expired - if "Expired" in str(e): - raise ExpiredTokenError(TOKEN_EXPIRED_MESSAGE) - else: - raise InvalidTokenError(TOKEN_INVALID_MESSAGE) + jwt.decode(token, options={"verify_signature": False}) + except ExpiredSignatureError: + raise ExpiredTokenError(TOKEN_EXPIRED_MESSAGE) + except DecodeError: + raise InvalidTokenError(TOKEN_INVALID_MESSAGE) return token diff --git a/poetry.lock b/poetry.lock index eb31249b5..4dc33f925 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2632,19 +2632,6 @@ test-functional = ["pytest", "pytest-randomly", "pytest-xdist"] test-integration = ["ipykernel", "jupyter-server (!=2.11)", "nbconvert", "pytest", "pytest-randomly", "pytest-xdist"] test-ui = ["calysto-bash"] -[[package]] -name = "jwt" -version = "1.3.1" -description = "JSON Web Token library for Python 3." -optional = false -python-versions = ">= 3.6" -files = [ - {file = "jwt-1.3.1-py3-none-any.whl", hash = "sha256:61c9170f92e736b530655e75374681d4fcca9cfa8763ab42be57353b2b203494"}, -] - -[package.dependencies] -cryptography = ">=3.1,<3.4.0 || >3.4.0" - [[package]] name = "keyring" version = "25.2.1" @@ -5062,6 +5049,23 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pymdown-extensions" version = "10.8.1" @@ -8340,4 +8344,4 @@ vectordb = ["faiss-cpu", "numpy"] [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "f2c26b1ad671f41fb695448f5f007d9f29609581db2a23301169a4d0a5ab11f1" +content-hash = "344c86547eec24c2fb96247aad607011d8569dbd8156a599c4b9ce75355a5091" diff --git a/pyproject.toml b/pyproject.toml index 1b97d2e49..653dd5c05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,11 +56,11 @@ langchain-core = ">=0.1,<0.3" coloredlogs = "^15.0.1" requests = "^2.31.0" guardrails-api-client = "^0.2.1" -jwt = "^1.3.1" pip = ">=22" opentelemetry-sdk = "^1.24.0" opentelemetry-exporter-otlp-proto-grpc = "^1.24.0" opentelemetry-exporter-otlp-proto-http = "^1.24.0" +pyjwt = "^2.8.0" [tool.poetry.extras] sql = ["sqlvalidator", "sqlalchemy", "sqlglot"] diff --git a/tests/unit_tests/cli/server/test_hub_client.py b/tests/unit_tests/cli/server/test_hub_client.py index 25062951b..160f6c267 100644 --- a/tests/unit_tests/cli/server/test_hub_client.py +++ b/tests/unit_tests/cli/server/test_hub_client.py @@ -2,12 +2,16 @@ import math import pytest -from jwt import JWT, jwk_from_dict +import jwt +from jwt import ExpiredSignatureError, DecodeError + from guardrails.classes.credentials import Credentials from guardrails.cli.server.hub_client import ( TOKEN_EXPIRED_MESSAGE, TOKEN_INVALID_MESSAGE, + InvalidTokenError, + ExpiredTokenError, get_jwt_token, ) @@ -43,39 +47,29 @@ def test_get_auth(): def test_get_jwt_token(): - expiration = math.floor(datetime.datetime.now().timestamp() + 1000) - - jwk = jwk_from_dict( - { - "alg": "HS256", - "kty": "oct", - "kid": "050bf691-4348-4891-940f-99af8354e82b", - "k": "eCE35cBrbRsO1GhrbxLXnGrVATgUFZDrPyyuOar4crw", - } - ) - - valid_jwt = JWT().encode( - { - "exp": expiration, - }, - jwk, - "HS256", - ) - creds = {"token": valid_jwt} - assert get_jwt_token(Credentials.from_dict(creds)) == valid_jwt - - with pytest.raises(Exception) as e: - expiration = math.floor(datetime.datetime.now().timestamp() - 1000) - expired_jwt = JWT().encode( - { - "exp": expiration, - }, - jwk, - "HS256", - ) - get_jwt_token(Credentials.from_dict({"token": expired_jwt})) - assert str(e.value) == TOKEN_EXPIRED_MESSAGE - - with pytest.raises(Exception) as e: - get_jwt_token(Credentials.from_dict({"token": "invalid_token"})) + # Create a JWT that expires in the future + secret_key = "secret" + expiration = datetime.datetime.now() + datetime.timedelta(seconds=1000) + valid_jwt = jwt.encode({'exp': expiration}, secret_key, algorithm='HS256') + creds = Credentials.from_dict({"token": valid_jwt}) + + # Test valid token + assert get_jwt_token(creds) == valid_jwt + + # Test with an expired JWT + # with pytest.raises(ExpiredTokenError) as e: + # expired = datetime.datetime.now() - datetime.timedelta(seconds=1000) + # expired_jwt = jwt.encode({'exp': expired}, secret_key, algorithm='HS256') + # get_jwt_token(Credentials.from_dict({"token": expired_jwt})) + + expired = datetime.datetime.now() - datetime.timedelta(seconds=1000) + expired_jwt = jwt.encode({'exp': expired}, secret_key, algorithm='HS256') + e = get_jwt_token(Credentials.from_dict({"token": expired_jwt})) + assert str(e) == TOKEN_EXPIRED_MESSAGE + + # Test with an invalid token format + with pytest.assertRaises(InvalidTokenError) as e: + invalid_jwt = "invalid" + get_jwt_token(Credentials.from_dict({"token": invalid_jwt})) + assert str(e.value) == TOKEN_INVALID_MESSAGE From 156ce49d7b152366197ed9d3a4f1cbc25d4f58a7 Mon Sep 17 00:00:00 2001 From: shiv07tiwari Date: Thu, 11 Jul 2024 15:14:28 +0530 Subject: [PATCH 07/11] fix: unit tests --- guardrails/cli/server/hub_client.py | 2 +- .../unit_tests/cli/server/test_hub_client.py | 23 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/guardrails/cli/server/hub_client.py b/guardrails/cli/server/hub_client.py index 2dde206cf..d39d8b41b 100644 --- a/guardrails/cli/server/hub_client.py +++ b/guardrails/cli/server/hub_client.py @@ -87,7 +87,7 @@ def get_jwt_token(creds: Credentials) -> Optional[str]: # check for jwt expiration if token: try: - jwt.decode(token, options={"verify_signature": False}) + jwt.decode(token, options={"verify_signature": False, "verify_exp": True}) except ExpiredSignatureError: raise ExpiredTokenError(TOKEN_EXPIRED_MESSAGE) except DecodeError: diff --git a/tests/unit_tests/cli/server/test_hub_client.py b/tests/unit_tests/cli/server/test_hub_client.py index 160f6c267..7af761fc5 100644 --- a/tests/unit_tests/cli/server/test_hub_client.py +++ b/tests/unit_tests/cli/server/test_hub_client.py @@ -1,9 +1,8 @@ import datetime -import math import pytest import jwt -from jwt import ExpiredSignatureError, DecodeError +from datetime import timezone from guardrails.classes.credentials import Credentials @@ -49,26 +48,24 @@ def test_get_auth(): def test_get_jwt_token(): # Create a JWT that expires in the future secret_key = "secret" - expiration = datetime.datetime.now() + datetime.timedelta(seconds=1000) - valid_jwt = jwt.encode({'exp': expiration}, secret_key, algorithm='HS256') + timedelta = datetime.timedelta(seconds=1000) + expiration = datetime.datetime.now(tz=timezone.utc) + timedelta + valid_jwt = jwt.encode({"exp": expiration}, secret_key, algorithm="HS256") creds = Credentials.from_dict({"token": valid_jwt}) # Test valid token assert get_jwt_token(creds) == valid_jwt # Test with an expired JWT - # with pytest.raises(ExpiredTokenError) as e: - # expired = datetime.datetime.now() - datetime.timedelta(seconds=1000) - # expired_jwt = jwt.encode({'exp': expired}, secret_key, algorithm='HS256') - # get_jwt_token(Credentials.from_dict({"token": expired_jwt})) + with pytest.raises(ExpiredTokenError) as e: + expired = datetime.datetime.now(tz=timezone.utc) - timedelta + expired_jwt = jwt.encode({"exp": expired}, secret_key, algorithm="HS256") + get_jwt_token(Credentials.from_dict({"token": expired_jwt})) - expired = datetime.datetime.now() - datetime.timedelta(seconds=1000) - expired_jwt = jwt.encode({'exp': expired}, secret_key, algorithm='HS256') - e = get_jwt_token(Credentials.from_dict({"token": expired_jwt})) - assert str(e) == TOKEN_EXPIRED_MESSAGE + assert str(e.value) == TOKEN_EXPIRED_MESSAGE # Test with an invalid token format - with pytest.assertRaises(InvalidTokenError) as e: + with pytest.raises(InvalidTokenError) as e: invalid_jwt = "invalid" get_jwt_token(Credentials.from_dict({"token": invalid_jwt})) From ae051a09322b29e268939c2457161c9f0e8902e6 Mon Sep 17 00:00:00 2001 From: Alejandro Date: Thu, 11 Jul 2024 11:27:33 -0700 Subject: [PATCH 08/11] revert change in poetry.lock --- poetry.lock | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index edc9e5e1b..bbc09aa5c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6750,13 +6750,29 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake [[package]] name = "urllib3" -version = "2.2.2" +version = "1.26.19" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "urllib3-1.26.19-py2.py3-none-any.whl", hash = "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3"}, + {file = "urllib3-1.26.19.tar.gz", hash = "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "urllib3" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] From 944fc0ecf5794f959f6f45fd35b3a83798cc7ade Mon Sep 17 00:00:00 2001 From: Alejandro Date: Thu, 11 Jul 2024 11:33:41 -0700 Subject: [PATCH 09/11] poetry lock no update --- poetry.lock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index bbc09aa5c..8e075078e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3861,6 +3861,7 @@ description = "Nvidia JIT LTO Library" optional = true python-versions = ">=3" files = [ + {file = "nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_aarch64.whl", hash = "sha256:004186d5ea6a57758fd6d57052a123c73a4815adf365eb8dd6a85c9eaa7535ff"}, {file = "nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl", hash = "sha256:d9714f27c1d0f0895cd8915c07a87a1d0029a0aa36acaf9156952ec2a8a12189"}, {file = "nvidia_nvjitlink_cu12-12.5.40-py3-none-win_amd64.whl", hash = "sha256:c3401dc8543b52d3a8158007a0c1ab4e9c768fcbd24153a48c86972102197ddd"}, ] @@ -7269,4 +7270,4 @@ vectordb = ["faiss-cpu", "numpy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "72e1b8cf874425425dd8a73b03def13fbcf29f17de881696d3628a54f2dc2e72" +content-hash = "28a7f4b1eecaeaf7e3225a462c45de34b8dd98162bed6a18e71d58de2fdf0427" From db86d50336c4d35141478cff27d316cc4b0f537c Mon Sep 17 00:00:00 2001 From: Alejandro Date: Thu, 11 Jul 2024 11:41:52 -0700 Subject: [PATCH 10/11] updated jwt -> pyjwt in hub token --- guardrails/hub_token/token.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/guardrails/hub_token/token.py b/guardrails/hub_token/token.py index 094c7cfe6..2d63dd3d9 100644 --- a/guardrails/hub_token/token.py +++ b/guardrails/hub_token/token.py @@ -1,6 +1,6 @@ from guardrails.classes.credentials import Credentials -from jwt import JWT -from jwt.exceptions import JWTDecodeError +import jwt +from jwt import ExpiredSignatureError, DecodeError from typing import Optional FIND_NEW_TOKEN = "You can find a new token at https://hub.guardrailsai.com/tokens" @@ -39,11 +39,9 @@ def get_jwt_token(creds: Credentials) -> Optional[str]: # check for jwt expiration if token: try: - JWT().decode(token, do_verify=False) - except JWTDecodeError as e: - # if the error message includes "Expired", then the token is expired - if "Expired" in str(e): - raise ExpiredTokenError(TOKEN_EXPIRED_MESSAGE) - else: - raise InvalidTokenError(TOKEN_INVALID_MESSAGE) + jwt.decode(token, options={"verify_signature": False, "verify_exp": True}) + except ExpiredSignatureError: + raise ExpiredTokenError(TOKEN_EXPIRED_MESSAGE) + except DecodeError: + raise InvalidTokenError(TOKEN_INVALID_MESSAGE) return token From f6002222c4a2e9b8dfadd21bba444d793bf4bd5e Mon Sep 17 00:00:00 2001 From: Alejandro Date: Thu, 11 Jul 2024 12:00:02 -0700 Subject: [PATCH 11/11] updated token urls --- guardrails/cli/server/hub_client.py | 2 +- guardrails/hub_token/token.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/guardrails/cli/server/hub_client.py b/guardrails/cli/server/hub_client.py index 23f312d01..b760242cf 100644 --- a/guardrails/cli/server/hub_client.py +++ b/guardrails/cli/server/hub_client.py @@ -12,7 +12,7 @@ from guardrails.cli.logger import logger from guardrails.cli.server.module_manifest import ModuleManifest -FIND_NEW_TOKEN = "You can find a new token at https://hub.guardrailsai.com/tokens" +FIND_NEW_TOKEN = "You can find a new token at https://hub.guardrailsai.com/keys" TOKEN_EXPIRED_MESSAGE = f"""Your token has expired. Please run `guardrails configure`\ to update your token. diff --git a/guardrails/hub_token/token.py b/guardrails/hub_token/token.py index 2d63dd3d9..f30d1c054 100644 --- a/guardrails/hub_token/token.py +++ b/guardrails/hub_token/token.py @@ -3,7 +3,7 @@ from jwt import ExpiredSignatureError, DecodeError from typing import Optional -FIND_NEW_TOKEN = "You can find a new token at https://hub.guardrailsai.com/tokens" +FIND_NEW_TOKEN = "You can find a new token at https://hub.guardrailsai.com/keys" TOKEN_EXPIRED_MESSAGE = f"""Your token has expired. Please run `guardrails configure`\ to update your token.