From 9053929fafe34f2f228534f9d54d13e3a3799883 Mon Sep 17 00:00:00 2001 From: John <43506685+Coniferish@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:06:11 -0600 Subject: [PATCH 1/6] init decorator and add comments noting human code --- src/unstructured_client/general.py | 4 +++- src/unstructured_client/sdk.py | 4 ++-- src/unstructured_client/utils/_decorators.py | 23 +++++++++++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/unstructured_client/general.py b/src/unstructured_client/general.py index cbd14b1c..ccaa85b7 100644 --- a/src/unstructured_client/general.py +++ b/src/unstructured_client/general.py @@ -4,6 +4,8 @@ from typing import Any, List, Optional from unstructured_client import utils from unstructured_client.models import errors, operations, shared +from unstructured_client.utils._decorators import suggest_defining_url_if_401 # human code + class General: sdk_configuration: SDKConfiguration @@ -12,7 +14,7 @@ def __init__(self, sdk_config: SDKConfiguration) -> None: self.sdk_configuration = sdk_config - + @suggest_defining_url_if_401 # human code def partition(self, request: Optional[shared.PartitionParameters], retries: Optional[utils.RetryConfig] = None) -> operations.PartitionResponse: r"""Pipeline 1""" base_url = utils.template_url(*self.sdk_configuration.get_server_details()) diff --git a/src/unstructured_client/sdk.py b/src/unstructured_client/sdk.py index 9358dbac..ff866c32 100644 --- a/src/unstructured_client/sdk.py +++ b/src/unstructured_client/sdk.py @@ -6,7 +6,7 @@ from typing import Callable, Dict, Union from unstructured_client import utils from unstructured_client.models import shared -from unstructured_client.utils._decorators import clean_server_url +from unstructured_client.utils._decorators import clean_server_url # human code class UnstructuredClient: r"""Unstructured Pipeline API: Partition documents with the Unstructured library""" @@ -14,7 +14,7 @@ class UnstructuredClient: sdk_configuration: SDKConfiguration - @clean_server_url + @clean_server_url # human code def __init__(self, api_key_auth: Union[str, Callable[[], str]], server: str = None, diff --git a/src/unstructured_client/utils/_decorators.py b/src/unstructured_client/utils/_decorators.py index fd891c2a..61fd995e 100644 --- a/src/unstructured_client/utils/_decorators.py +++ b/src/unstructured_client/utils/_decorators.py @@ -1,9 +1,15 @@ from __future__ import annotations import functools -from typing import cast, Callable, Optional +from typing import cast, Callable, TYPE_CHECKING, Optional from typing_extensions import ParamSpec from urllib.parse import urlparse, urlunparse, ParseResult +import warnings + +from unstructured_client.models import errors + +if TYPE_CHECKING: + from unstructured_client.general import General _P = ParamSpec("_P") @@ -44,3 +50,18 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: return func(*args, **kwargs) return wrapper + + +def suggest_defining_url_if_401(func: Callable[_P, None]) -> Callable[_P, None]: + + @functools.wraps(func) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: + try: + return func(*args, **kwargs) + except errors.SDKError: + general_obj: General = args[0] + if not general_obj.sdk_configuration.server_url: + warnings.warn("If intending to use the paid API, please define `server_url` in your request.") + return func(*args, **kwargs) + + return wrapper \ No newline at end of file From 7d5ad3643447840e2d92c67713fa0fc6229ba481 Mon Sep 17 00:00:00 2001 From: John <43506685+Coniferish@users.noreply.github.com> Date: Mon, 5 Feb 2024 22:24:32 -0600 Subject: [PATCH 2/6] add test and refactor --- ...ck_url_protocol.py => test__decorators.py} | 44 ++++++++++++++----- src/unstructured_client/utils/_decorators.py | 8 ++-- 2 files changed, 37 insertions(+), 15 deletions(-) rename _test_unstructured_client/{test_check_url_protocol.py => test__decorators.py} (68%) diff --git a/_test_unstructured_client/test_check_url_protocol.py b/_test_unstructured_client/test__decorators.py similarity index 68% rename from _test_unstructured_client/test_check_url_protocol.py rename to _test_unstructured_client/test__decorators.py index 2bc0b9fd..82939c7b 100644 --- a/_test_unstructured_client/test_check_url_protocol.py +++ b/_test_unstructured_client/test__decorators.py @@ -1,15 +1,11 @@ -import os import pytest from unstructured_client import UnstructuredClient +from unstructured_client.models import shared +from unstructured_client.models.errors import SDKError -def get_api_key(): - api_key = os.getenv("UNS_API_KEY") - if api_key is None: - raise ValueError("""UNS_API_KEY environment variable not set. -Set it in your current shell session with `export UNS_API_KEY=`""") - return api_key +FAKE_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @pytest.mark.parametrize( @@ -25,7 +21,7 @@ def get_api_key(): def test_clean_server_url_on_paid_api_url(server_url: str): client = UnstructuredClient( server_url=server_url, - api_key_auth=get_api_key(), + api_key_auth=FAKE_KEY, ) assert client.general.sdk_configuration.server_url == "https://unstructured-000mock.api.unstructuredapp.io" @@ -42,7 +38,7 @@ def test_clean_server_url_on_paid_api_url(server_url: str): def test_clean_server_url_on_localhost(server_url: str): client = UnstructuredClient( server_url=server_url, - api_key_auth=get_api_key(), + api_key_auth=FAKE_KEY, ) assert client.general.sdk_configuration.server_url == "http://localhost:8000" @@ -50,10 +46,11 @@ def test_clean_server_url_on_localhost(server_url: str): def test_clean_server_url_on_empty_string(): client = UnstructuredClient( server_url="", - api_key_auth=get_api_key(), + api_key_auth=FAKE_KEY, ) assert client.general.sdk_configuration.server_url == "" + @pytest.mark.parametrize( ("server_url"), [ @@ -63,8 +60,33 @@ def test_clean_server_url_on_empty_string(): ) def test_clean_server_url_with_positional_arguments(server_url: str): client = UnstructuredClient( - get_api_key(), + FAKE_KEY, "", server_url, ) assert client.general.sdk_configuration.server_url == "https://unstructured-000mock.api.unstructuredapp.io" + + +def test_suggest_defining_url_if_401(): + with pytest.warns(UserWarning): + + client = UnstructuredClient( + api_key_auth=FAKE_KEY, + ) + + filename = "_sample_docs/layout-parser-paper-fast.pdf" + + with open(filename, "rb") as f: + files=shared.Files( + content=f.read(), + file_name=filename, + ) + + req = shared.PartitionParameters( + files=files, + ) + + try: + client.general.partition(req) + except SDKError as e: + print(e) diff --git a/src/unstructured_client/utils/_decorators.py b/src/unstructured_client/utils/_decorators.py index 61fd995e..a9347b41 100644 --- a/src/unstructured_client/utils/_decorators.py +++ b/src/unstructured_client/utils/_decorators.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse, urlunparse, ParseResult import warnings -from unstructured_client.models import errors +from unstructured_client.models import errors, operations if TYPE_CHECKING: from unstructured_client.general import General @@ -52,14 +52,14 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: return wrapper -def suggest_defining_url_if_401(func: Callable[_P, None]) -> Callable[_P, None]: +def suggest_defining_url_if_401(func: Callable[_P, operations.PartitionResponse]) -> Callable[_P, operations.PartitionResponse]: @functools.wraps(func) - def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> operations.PartitionResponse: try: return func(*args, **kwargs) except errors.SDKError: - general_obj: General = args[0] + general_obj: General = args[0] # type: ignore if not general_obj.sdk_configuration.server_url: warnings.warn("If intending to use the paid API, please define `server_url` in your request.") return func(*args, **kwargs) From 260412f3ec64c714c6b7346610c739a9959c7f11 Mon Sep 17 00:00:00 2001 From: John <43506685+Coniferish@users.noreply.github.com> Date: Tue, 6 Feb 2024 10:31:37 -0600 Subject: [PATCH 3/6] modify decorator so it is only raised on 401 error --- src/unstructured_client/utils/_decorators.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/unstructured_client/utils/_decorators.py b/src/unstructured_client/utils/_decorators.py index a9347b41..3ad58929 100644 --- a/src/unstructured_client/utils/_decorators.py +++ b/src/unstructured_client/utils/_decorators.py @@ -58,10 +58,12 @@ def suggest_defining_url_if_401(func: Callable[_P, operations.PartitionResponse] def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> operations.PartitionResponse: try: return func(*args, **kwargs) - except errors.SDKError: - general_obj: General = args[0] # type: ignore - if not general_obj.sdk_configuration.server_url: - warnings.warn("If intending to use the paid API, please define `server_url` in your request.") + except errors.SDKError as e: + if e.status_code == 401: + general_obj: General = args[0] # type: ignore + if not general_obj.sdk_configuration.server_url: + warnings.warn("If intending to use the paid API, please define `server_url` in your request.") + return func(*args, **kwargs) return wrapper \ No newline at end of file From b4a7c8f2c734b28c7d78168c064f37c664892df2 Mon Sep 17 00:00:00 2001 From: John <43506685+Coniferish@users.noreply.github.com> Date: Tue, 6 Feb 2024 10:35:42 -0600 Subject: [PATCH 4/6] blacken files --- _test_unstructured_client/test__decorators.py | 26 ++++++++++++------- src/unstructured_client/utils/_decorators.py | 18 ++++++++----- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/_test_unstructured_client/test__decorators.py b/_test_unstructured_client/test__decorators.py index 82939c7b..a441b4fd 100644 --- a/_test_unstructured_client/test__decorators.py +++ b/_test_unstructured_client/test__decorators.py @@ -11,29 +11,32 @@ @pytest.mark.parametrize( ("server_url"), [ - ("https://unstructured-000mock.api.unstructuredapp.io"), # correct url + ("https://unstructured-000mock.api.unstructuredapp.io"), # correct url ("unstructured-000mock.api.unstructuredapp.io"), ("http://unstructured-000mock.api.unstructuredapp.io/general/v0/general"), ("https://unstructured-000mock.api.unstructuredapp.io/general/v0/general"), ("unstructured-000mock.api.unstructuredapp.io/general/v0/general"), - ] + ], ) def test_clean_server_url_on_paid_api_url(server_url: str): client = UnstructuredClient( server_url=server_url, api_key_auth=FAKE_KEY, ) - assert client.general.sdk_configuration.server_url == "https://unstructured-000mock.api.unstructuredapp.io" + assert ( + client.general.sdk_configuration.server_url + == "https://unstructured-000mock.api.unstructuredapp.io" + ) @pytest.mark.parametrize( ("server_url"), [ - ("http://localhost:8000"), # correct url + ("http://localhost:8000"), # correct url ("localhost:8000"), ("localhost:8000/general/v0/general"), ("http://localhost:8000/general/v0/general"), - ] + ], ) def test_clean_server_url_on_localhost(server_url: str): client = UnstructuredClient( @@ -56,7 +59,7 @@ def test_clean_server_url_on_empty_string(): [ ("https://unstructured-000mock.api.unstructuredapp.io"), ("unstructured-000mock.api.unstructuredapp.io/general/v0/general"), - ] + ], ) def test_clean_server_url_with_positional_arguments(server_url: str): client = UnstructuredClient( @@ -64,7 +67,10 @@ def test_clean_server_url_with_positional_arguments(server_url: str): "", server_url, ) - assert client.general.sdk_configuration.server_url == "https://unstructured-000mock.api.unstructuredapp.io" + assert ( + client.general.sdk_configuration.server_url + == "https://unstructured-000mock.api.unstructuredapp.io" + ) def test_suggest_defining_url_if_401(): @@ -73,11 +79,11 @@ def test_suggest_defining_url_if_401(): client = UnstructuredClient( api_key_auth=FAKE_KEY, ) - + filename = "_sample_docs/layout-parser-paper-fast.pdf" - + with open(filename, "rb") as f: - files=shared.Files( + files = shared.Files( content=f.read(), file_name=filename, ) diff --git a/src/unstructured_client/utils/_decorators.py b/src/unstructured_client/utils/_decorators.py index 3ad58929..b0cce14c 100644 --- a/src/unstructured_client/utils/_decorators.py +++ b/src/unstructured_client/utils/_decorators.py @@ -45,14 +45,16 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: if url_is_in_kwargs: kwargs["server_url"] = urlunparse(cleaned_url) else: - args = args[:SERVER_URL_ARG_IDX] + (urlunparse(cleaned_url),) + args[SERVER_URL_ARG_IDX+1:] # type: ignore - + args = args[:SERVER_URL_ARG_IDX] + (urlunparse(cleaned_url),) + args[SERVER_URL_ARG_IDX + 1 :] # type: ignore + return func(*args, **kwargs) return wrapper -def suggest_defining_url_if_401(func: Callable[_P, operations.PartitionResponse]) -> Callable[_P, operations.PartitionResponse]: +def suggest_defining_url_if_401( + func: Callable[_P, operations.PartitionResponse] +) -> Callable[_P, operations.PartitionResponse]: @functools.wraps(func) def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> operations.PartitionResponse: @@ -60,10 +62,12 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> operations.PartitionResponse return func(*args, **kwargs) except errors.SDKError as e: if e.status_code == 401: - general_obj: General = args[0] # type: ignore + general_obj: General = args[0] # type: ignore if not general_obj.sdk_configuration.server_url: - warnings.warn("If intending to use the paid API, please define `server_url` in your request.") - + warnings.warn( + "If intending to use the paid API, please define `server_url` in your request." + ) + return func(*args, **kwargs) - return wrapper \ No newline at end of file + return wrapper From eb86cbb5c6af9e3d78609be495ea58e9d7d928a2 Mon Sep 17 00:00:00 2001 From: John <43506685+Coniferish@users.noreply.github.com> Date: Tue, 6 Feb 2024 16:43:57 -0600 Subject: [PATCH 5/6] address comments --- _test_unstructured_client/test__decorators.py | 87 +++++++++++-------- src/unstructured_client/utils/_decorators.py | 13 ++- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/_test_unstructured_client/test__decorators.py b/_test_unstructured_client/test__decorators.py index a441b4fd..20b8a200 100644 --- a/_test_unstructured_client/test__decorators.py +++ b/_test_unstructured_client/test__decorators.py @@ -9,16 +9,18 @@ @pytest.mark.parametrize( - ("server_url"), + "server_url", [ - ("https://unstructured-000mock.api.unstructuredapp.io"), # correct url - ("unstructured-000mock.api.unstructuredapp.io"), - ("http://unstructured-000mock.api.unstructuredapp.io/general/v0/general"), - ("https://unstructured-000mock.api.unstructuredapp.io/general/v0/general"), - ("unstructured-000mock.api.unstructuredapp.io/general/v0/general"), + # -- well-formed url -- + "https://unstructured-000mock.api.unstructuredapp.io", + # -- common malformed urls -- + "unstructured-000mock.api.unstructuredapp.io", + "http://unstructured-000mock.api.unstructuredapp.io/general/v0/general", + "https://unstructured-000mock.api.unstructuredapp.io/general/v0/general", + "unstructured-000mock.api.unstructuredapp.io/general/v0/general", ], ) -def test_clean_server_url_on_paid_api_url(server_url: str): +def test_clean_server_url_fixes_malformed_paid_api_url(server_url: str): client = UnstructuredClient( server_url=server_url, api_key_auth=FAKE_KEY, @@ -30,15 +32,17 @@ def test_clean_server_url_on_paid_api_url(server_url: str): @pytest.mark.parametrize( - ("server_url"), + "server_url", [ - ("http://localhost:8000"), # correct url - ("localhost:8000"), - ("localhost:8000/general/v0/general"), - ("http://localhost:8000/general/v0/general"), + # -- well-formed url -- + "http://localhost:8000", + # -- common malformed urls -- + "localhost:8000", + "localhost:8000/general/v0/general", + "http://localhost:8000/general/v0/general", ], ) -def test_clean_server_url_on_localhost(server_url: str): +def test_clean_server_url_fixes_malformed_localhost_url(server_url: str): client = UnstructuredClient( server_url=server_url, api_key_auth=FAKE_KEY, @@ -46,7 +50,7 @@ def test_clean_server_url_on_localhost(server_url: str): assert client.general.sdk_configuration.server_url == "http://localhost:8000" -def test_clean_server_url_on_empty_string(): +def test_clean_server_url_returns_empty_string_given_empty_string(): client = UnstructuredClient( server_url="", api_key_auth=FAKE_KEY, @@ -54,14 +58,25 @@ def test_clean_server_url_on_empty_string(): assert client.general.sdk_configuration.server_url == "" +def test_clean_server_url_returns_None_given_no_server_url(): + client = UnstructuredClient( + api_key_auth=FAKE_KEY, + ) + assert client.general.sdk_configuration.server_url == None + + @pytest.mark.parametrize( - ("server_url"), + "server_url", [ - ("https://unstructured-000mock.api.unstructuredapp.io"), - ("unstructured-000mock.api.unstructuredapp.io/general/v0/general"), + # -- well-formed url -- + "https://unstructured-000mock.api.unstructuredapp.io", + # -- malformed url -- + "unstructured-000mock.api.unstructuredapp.io/general/v0/general", ], ) -def test_clean_server_url_with_positional_arguments(server_url: str): +def test_clean_server_url_fixes_malformed_urls_with_positional_arguments( + server_url: str, +): client = UnstructuredClient( FAKE_KEY, "", @@ -73,26 +88,26 @@ def test_clean_server_url_with_positional_arguments(server_url: str): ) -def test_suggest_defining_url_if_401(): - with pytest.warns(UserWarning): - - client = UnstructuredClient( - api_key_auth=FAKE_KEY, - ) - - filename = "_sample_docs/layout-parser-paper-fast.pdf" +def test_suggest_defining_url_issues_a_warning_on_a_401(): + client = UnstructuredClient( + api_key_auth=FAKE_KEY, + ) - with open(filename, "rb") as f: - files = shared.Files( - content=f.read(), - file_name=filename, - ) + filename = "_sample_docs/layout-parser-paper-fast.pdf" - req = shared.PartitionParameters( - files=files, + with open(filename, "rb") as f: + files = shared.Files( + content=f.read(), + file_name=filename, ) - try: + req = shared.PartitionParameters( + files=files, + ) + + with pytest.raises(SDKError, match="API error occurred: Status 401"): + with pytest.warns( + UserWarning, + match="If intending to use the paid API, please define `server_url` in your request.", + ): client.general.partition(req) - except SDKError as e: - print(e) diff --git a/src/unstructured_client/utils/_decorators.py b/src/unstructured_client/utils/_decorators.py index b0cce14c..b219c0de 100644 --- a/src/unstructured_client/utils/_decorators.py +++ b/src/unstructured_client/utils/_decorators.py @@ -16,6 +16,13 @@ def clean_server_url(func: Callable[_P, None]) -> Callable[_P, None]: + """A decorator for fixing common types of malformed 'server_url' arguments. + + This decorator addresses the common problem of users omitting or using the wrong url scheme and/or + adding the '/general/v0/general' path to the 'server_url'. + The decorator should be manually applied to the __init__ method of UnstructuredClient after merging + a PR from Speakeasy. + """ @functools.wraps(func) def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: @@ -55,7 +62,11 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: def suggest_defining_url_if_401( func: Callable[_P, operations.PartitionResponse] ) -> Callable[_P, operations.PartitionResponse]: - + """A decorator to suggest defining the 'server_url' parameter if a 401 Unauthorized error is encountered. + + This decorator addresses the common problem of users not passing in the 'server_url' when using their paid api key. + The decorator should be manually applied to General.partition after merging a PR from Speakeasy. + """ @functools.wraps(func) def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> operations.PartitionResponse: try: From a624e46ae87643d882d7e1e5991b5c9a3907b6c0 Mon Sep 17 00:00:00 2001 From: John <43506685+Coniferish@users.noreply.github.com> Date: Wed, 7 Feb 2024 10:14:04 -0600 Subject: [PATCH 6/6] apply additional comments --- _test_unstructured_client/test__decorators.py | 5 +--- src/unstructured_client/utils/_decorators.py | 24 ++++++++++++------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/_test_unstructured_client/test__decorators.py b/_test_unstructured_client/test__decorators.py index 20b8a200..e8781542 100644 --- a/_test_unstructured_client/test__decorators.py +++ b/_test_unstructured_client/test__decorators.py @@ -51,10 +51,7 @@ def test_clean_server_url_fixes_malformed_localhost_url(server_url: str): def test_clean_server_url_returns_empty_string_given_empty_string(): - client = UnstructuredClient( - server_url="", - api_key_auth=FAKE_KEY, - ) + client = UnstructuredClient( server_url="", api_key_auth=FAKE_KEY) assert client.general.sdk_configuration.server_url == "" diff --git a/src/unstructured_client/utils/_decorators.py b/src/unstructured_client/utils/_decorators.py index b219c0de..918725ce 100644 --- a/src/unstructured_client/utils/_decorators.py +++ b/src/unstructured_client/utils/_decorators.py @@ -18,10 +18,9 @@ def clean_server_url(func: Callable[_P, None]) -> Callable[_P, None]: """A decorator for fixing common types of malformed 'server_url' arguments. - This decorator addresses the common problem of users omitting or using the wrong url scheme and/or - adding the '/general/v0/general' path to the 'server_url'. - The decorator should be manually applied to the __init__ method of UnstructuredClient after merging - a PR from Speakeasy. + This decorator addresses the common problem of users omitting or using the wrong url scheme + and/or adding the '/general/v0/general' path to the 'server_url'. The decorator should be + manually applied to the __init__ method of UnstructuredClient after merging a PR from Speakeasy. """ @functools.wraps(func) @@ -52,7 +51,11 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: if url_is_in_kwargs: kwargs["server_url"] = urlunparse(cleaned_url) else: - args = args[:SERVER_URL_ARG_IDX] + (urlunparse(cleaned_url),) + args[SERVER_URL_ARG_IDX + 1 :] # type: ignore + args = ( + args[:SERVER_URL_ARG_IDX] + + (urlunparse(cleaned_url),) + + args[SERVER_URL_ARG_IDX + 1 :] + ) # type: ignore return func(*args, **kwargs) @@ -62,11 +65,14 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: def suggest_defining_url_if_401( func: Callable[_P, operations.PartitionResponse] ) -> Callable[_P, operations.PartitionResponse]: - """A decorator to suggest defining the 'server_url' parameter if a 401 Unauthorized error is encountered. - - This decorator addresses the common problem of users not passing in the 'server_url' when using their paid api key. - The decorator should be manually applied to General.partition after merging a PR from Speakeasy. + """A decorator to suggest defining the 'server_url' parameter if a 401 Unauthorized error is + encountered. + + This decorator addresses the common problem of users not passing in the 'server_url' when + using their paid api key. The decorator should be manually applied to General.partition after + merging a PR from Speakeasy. """ + @functools.wraps(func) def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> operations.PartitionResponse: try: