From 1bb80dee85ed4218c00527a99b04a5859b963b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Thu, 3 Feb 2022 17:14:17 +0100 Subject: [PATCH 1/9] Separate version handling from token handling * Introducing GraphConnectionHandler --- src/hiro_graph_client/__init__.py | 8 +- src/hiro_graph_client/clientlib.py | 227 ++++++++++++++++++----- src/hiro_graph_client/eventswebsocket.py | 2 +- tests/unit/test_client.py | 39 +++- 4 files changed, 220 insertions(+), 56 deletions(-) diff --git a/src/hiro_graph_client/__init__.py b/src/hiro_graph_client/__init__.py index 894afd3..9e05f8d 100644 --- a/src/hiro_graph_client/__init__.py +++ b/src/hiro_graph_client/__init__.py @@ -8,9 +8,9 @@ from hiro_graph_client.authclient import HiroAuth from hiro_graph_client.authzclient import HiroAuthz from hiro_graph_client.client import HiroGraph -from hiro_graph_client.clientlib import AbstractTokenApiHandler, AuthenticationTokenError, FixedTokenError, \ - TokenUnauthorizedError, PasswordAuthTokenApiHandler, FixedTokenApiHandler, EnvironmentTokenApiHandler, \ - SSLConfig +from hiro_graph_client.clientlib import AbstractTokenApiHandler, GraphConnectionHandler, AuthenticationTokenError, \ + FixedTokenError, TokenUnauthorizedError, PasswordAuthTokenApiHandler, FixedTokenApiHandler, \ + EnvironmentTokenApiHandler, SSLConfig from hiro_graph_client.iamclient import HiroIam from hiro_graph_client.kiclient import HiroKi from hiro_graph_client.variablesclient import HiroVariables @@ -19,7 +19,7 @@ this_directory = path.abspath(path.dirname(__file__)) __all__ = [ - 'HiroGraph', 'HiroAuth', 'HiroApp', 'HiroIam', 'HiroKi', 'HiroAuthz', 'HiroVariables', + 'HiroGraph', 'HiroAuth', 'HiroApp', 'HiroIam', 'HiroKi', 'HiroAuthz', 'HiroVariables', 'GraphConnectionHandler', 'AbstractTokenApiHandler', 'PasswordAuthTokenApiHandler', 'FixedTokenApiHandler', 'EnvironmentTokenApiHandler', 'AuthenticationTokenError', 'FixedTokenError', 'TokenUnauthorizedError', '__version__', 'SSLConfig' diff --git a/src/hiro_graph_client/clientlib.py b/src/hiro_graph_client/clientlib.py index 6af84f4..c285fe1 100644 --- a/src/hiro_graph_client/clientlib.py +++ b/src/hiro_graph_client/clientlib.py @@ -101,8 +101,7 @@ class AbstractAPI: _max_tries: int = 2 - _session: requests.Session - """Reference to the requests.Session containing the connection pool""" + _timeout: int = 600 def __init__(self, root_url: str, @@ -110,7 +109,7 @@ def __init__(self, raise_exceptions: bool = True, proxies: dict = None, headers: dict = None, - timeout: int = 600, + timeout: int = None, client_name: str = None, ssl_config: SSLConfig = None, log_communication_on_error: bool = None, @@ -137,14 +136,17 @@ def __init__(self, if not root_url: raise ValueError("'root_url' must not be empty.") + if not session: + raise ValueError("'session' must not be empty.") + self._root_url = root_url self._proxies = proxies self._raise_exceptions = raise_exceptions - self._timeout = timeout + self._timeout = timeout or self._timeout self._log_communication_on_error = log_communication_on_error or False self._session = session - self.ssl_config = ssl_config if ssl_config else SSLConfig() + self.ssl_config = ssl_config or SSLConfig() if not self.ssl_config.verify: AbstractAPI.accept_all_certs = True @@ -527,8 +529,7 @@ def _body_str(body: Union[str, bytes], encoding: str) -> str: ok = self._check_response_ok(res) - if (not ok and self._log_communication_on_error) or logger.isEnabledFor( - logging.DEBUG): + if (not ok and self._log_communication_on_error) or logger.isEnabledFor(logging.DEBUG): log_message = f''' ################ request ################ {res.request.method} {res.request.url} @@ -652,24 +653,29 @@ def _handle_token(self) -> Optional[str]: ################################################################################################################### -# TokenApiHandler classes +# ConnectionHandler class ################################################################################################################### -class AbstractTokenApiHandler(AbstractAPI): +class GraphConnectionHandler(AbstractAPI): """ - Root class for all TokenApiHandler classes. This class also handles resolving the current api endpoints. - Also creates the requests.Session which will be shared among the API Modules using this TokenApiHandler. + Contains information about a Graph Connection. This class also handles resolving the current api endpoints. + Also creates the requests.Session which will be shared among the API Modules using this connection. """ _pool_maxsize = 10 """Default pool_maxsize for requests.adapters.HTTPAdapter.""" + _pool_block = False + + __session: requests.Session + """Stores the connection _session""" + def __init__(self, root_url: str, raise_exceptions: bool = True, proxies: dict = None, headers: dict = None, - timeout: int = 600, + timeout: int = None, client_name: str = None, custom_endpoints: dict = None, ssl_config: SSLConfig = None, @@ -691,8 +697,8 @@ def __init__(self, } This object creates the *requests.Session* and *requests.adapters.HTTPAdapter* for this *root_url*. The - *pool_maxsize* of such a session can be set via the parameter in the constructor. When a TokenApiHandler is - shared between different API objects (like HiroGraph, HiroApp, etc.), this session and its pool are also shared. + *pool_maxsize* of such a _session can be set via the parameter in the constructor. When a TokenApiHandler is + shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also shared. :param root_url: Root url for HIRO, like https://core.arago.co. :param raise_exceptions: Raise exceptions on HTTP status codes that denote an error. Default is True. @@ -700,29 +706,32 @@ def __init__(self, :param headers: Optional custom HTTP headers. Will override the internal headers. Default is None. :param timeout: Optional timeout for requests. Default is 600 (10 min). :param client_name: Optional name for the client. Will also be part of the "User-Agent" header unless *headers* - is given with another value for "User-Agent". Default is "hiro-graph-client". + is given with another value for "User-Agent". Default is "hiro-graph-client". :param custom_endpoints: Optional map of {name:endpoint_path, ...} that overrides or adds to the endpoints taken from /api/version. Example see above. :param ssl_config Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib - will be used. + will be used. :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. + Default is 10. *pool_maxsize* is ignored when *_session* is set. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, - but do not cache them. See requests.adapters.HTTPAdapter. + but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *_session* is set. """ - session = requests.Session() + if not root_url: + raise ValueError("'root_url' must not be empty.") + adapter = requests.adapters.HTTPAdapter( pool_maxsize=pool_maxsize or self._pool_maxsize, pool_connections=1, - pool_block=pool_block or False + pool_block=pool_block or self._pool_block ) - session.mount(root_url, adapter) + self.__session = requests.Session() + self.__session.mount(root_url, adapter) super().__init__(root_url=root_url, - session=session, + session=self.__session, raise_exceptions=raise_exceptions, proxies=proxies, timeout=timeout, @@ -844,6 +853,120 @@ def get_version(self) -> dict: url = self._root_url + '/api/version' return self.get(url) + +################################################################################################################### +# TokenApiHandler classes +################################################################################################################### + +class AbstractTokenApiHandler(AbstractAPI): + """ + Root class for all TokenApiHandler classes. This adds token handling. + """ + + _connection_handler: GraphConnectionHandler + """Reference to an GraphConnectionHandler containing version information and requests.Session""" + + def __init__(self, + root_url: str = None, + raise_exceptions: bool = True, + proxies: dict = None, + headers: dict = None, + timeout: int = None, + client_name: str = None, + custom_endpoints: dict = None, + ssl_config: SSLConfig = None, + log_communication_on_error: bool = None, + max_tries: int = None, + pool_maxsize: int = None, + pool_block: bool = None, + connection_handler: GraphConnectionHandler = None): + """ + Constructor + + Example for custom_endpoints (see params below): + + :: + + { + "graph": "/api/graph/7.2", + "auth": "/api/auth/6.2", + "action-ws": ("/api/action-ws/1.0", "action-1.0.0") + } + + This object creates the *requests.Session* and *requests.adapters.HTTPAdapter* for this *root_url*. The + *pool_maxsize* of such a _session can be set via the parameter in the constructor. When a TokenApiHandler is + shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also shared. + + :param root_url: Root url for HIRO, like https://core.arago.co. + :param raise_exceptions: Raise exceptions on HTTP status codes that denote an error. Default is True. + :param proxies: Proxy configuration for *requests*. Default is None. + :param headers: Optional custom HTTP headers. Will override the internal headers. Default is None. + :param timeout: Optional timeout for requests. Default is 600 (10 min). + :param client_name: Optional name for the client. Will also be part of the "User-Agent" header unless *headers* + is given with another value for "User-Agent". Default is "hiro-graph-client". + :param custom_endpoints: Optional map of {name:endpoint_path, ...} that overrides or adds to the endpoints taken + from /api/version. Example see above. + :param ssl_config Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib + will be used. + :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is + detected. Default is not to do this. + :param max_tries: Max tries for BACKOFF. Default is 2. + :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. + :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, + but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when + *connection_handler* is set and already contains a _session. + :param connection_handler: Use an already existing connection handler. This feature allows for several distinct + TokenApiHandlers to operate on the same connection without querying unnecessary version information for + each or building their own requests.Sessions. Overrides all other parameters. + """ + self._connection_handler = connection_handler or GraphConnectionHandler( + root_url=root_url, + raise_exceptions=raise_exceptions, + proxies=proxies, + headers=headers, + timeout=timeout, + client_name=client_name, + custom_endpoints=custom_endpoints, + ssl_config=ssl_config, + log_communication_on_error=log_communication_on_error, + max_tries=max_tries, + pool_maxsize=pool_maxsize, + pool_block=pool_block + ) + + super().__init__( + root_url=self._connection_handler._root_url, + session=self._connection_handler._session, + raise_exceptions=self._connection_handler._raise_exceptions, + proxies=self._connection_handler._proxies, + timeout=self._connection_handler._timeout, + headers=self._connection_handler._headers, + client_name=self._connection_handler._client_name, + ssl_config=self._connection_handler.ssl_config, + log_communication_on_error=self._connection_handler._log_communication_on_error, + max_tries=self._connection_handler._max_tries + ) + + ############################################################################################################### + # Access connection handler + ############################################################################################################### + + def get_api_endpoint_of(self, api_name: str) -> str: + return self._connection_handler.get_api_endpoint_of(api_name) + + def get_websocket_config(self, api_name: str) -> Tuple[ + str, + str, + Optional[str], + Optional[int], + Optional[dict] + ]: + return self._connection_handler.get_websocket_config(api_name) + + def get_version(self): + return self._connection_handler.get_version() + ############################################################################################################### # Token handling ############################################################################################################### @@ -898,8 +1021,8 @@ class FixedTokenApiHandler(AbstractTokenApiHandler): _token: str def __init__(self, - root_url: str, - token: str, + root_url: str = None, + token: str = None, raise_exceptions: bool = True, proxies: dict = None, headers: dict = None, @@ -910,12 +1033,13 @@ def __init__(self, log_communication_on_error: bool = None, max_tries: int = None, pool_maxsize: int = None, - pool_block: bool = None): + pool_block: bool = None, + connection_handler: GraphConnectionHandler = None): """ Constructor - :param root_url: Root url for HIRO, like https://core.arago.co. :param token: The fixed token to use. + :param root_url: Root url for HIRO, like https://core.arago.co. :param raise_exceptions: Raise exceptions on HTTP status codes that denote an error. Default is True. :param proxies: Proxy configuration for *requests*. Default is None. :param headers: Optional custom HTTP headers. Will override the internal headers. Default is None. @@ -930,9 +1054,13 @@ def __init__(self, detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, - but do not cache them. See requests.adapters.HTTPAdapter. + but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when + *connection_handler* is set and already contains a _session. + :param connection_handler: Use an already existing connection handler. This feature allows for several distinct + TokenApiHandlers to operate on the same connection without querying unnecessary version information for + each or building their own requests.Sessions. Overrides all other connection specific parameters. """ super().__init__( root_url=root_url, @@ -946,7 +1074,8 @@ def __init__(self, log_communication_on_error=log_communication_on_error, max_tries=max_tries, pool_maxsize=pool_maxsize, - pool_block=pool_block + pool_block=pool_block, + connection_handler=connection_handler ) self._token = token @@ -974,7 +1103,7 @@ class EnvironmentTokenApiHandler(AbstractTokenApiHandler): _env_var: str def __init__(self, - root_url: str, + root_url: str = None, env_var: str = 'HIRO_TOKEN', raise_exceptions: bool = True, proxies: dict = None, @@ -986,7 +1115,8 @@ def __init__(self, log_communication_on_error: bool = None, max_tries: int = None, pool_maxsize: int = None, - pool_block: bool = None): + pool_block: bool = None, + connection_handler: GraphConnectionHandler = None): """ Constructor @@ -1006,9 +1136,13 @@ def __init__(self, detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, - but do not cache them. See requests.adapters.HTTPAdapter. + but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when + *connection_handler* is set and already contains a _session. + :param connection_handler: Use an already existing connection handler. This feature allows for several distinct + TokenApiHandlers to operate on the same connection without querying unnecessary version information for + each or building their own requests.Sessions. Overrides all other connection specific parameters. """ super().__init__( root_url=root_url, @@ -1022,7 +1156,8 @@ def __init__(self, log_communication_on_error=log_communication_on_error, max_tries=max_tries, pool_maxsize=pool_maxsize, - pool_block=pool_block + pool_block=pool_block, + connection_handler=connection_handler ) self._env_var = env_var @@ -1170,11 +1305,11 @@ class PasswordAuthTokenApiHandler(AbstractTokenApiHandler): _secure_logging: bool = True def __init__(self, - root_url: str, - username: str, - password: str, - client_id: str, - client_secret: str, + root_url: str = None, + username: str = None, + password: str = None, + client_id: str = None, + client_secret: str = None, secure_logging: bool = True, raise_exceptions: bool = True, proxies: dict = None, @@ -1186,7 +1321,8 @@ def __init__(self, log_communication_on_error: bool = None, max_tries: int = None, pool_maxsize: int = None, - pool_block: bool = None): + pool_block: bool = None, + connection_handler: GraphConnectionHandler = None): """ Constructor @@ -1210,9 +1346,13 @@ def __init__(self, detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, - but do not cache them. See requests.adapters.HTTPAdapter. + but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when + *connection_handler* is set and already contains a _session. + :param connection_handler: Use an already existing connection handler. This feature allows for several distinct + TokenApiHandlers to operate on the same connection without querying unnecessary version information for + each or building their own requests.Sessions. Overrides all other connection specific parameters. """ super().__init__( root_url=root_url, @@ -1226,7 +1366,8 @@ def __init__(self, log_communication_on_error=log_communication_on_error, max_tries=max_tries, pool_maxsize=pool_maxsize, - pool_block=pool_block + pool_block=pool_block, + connection_handler=connection_handler ) self._username = username diff --git a/src/hiro_graph_client/eventswebsocket.py b/src/hiro_graph_client/eventswebsocket.py index ed687b3..337088d 100644 --- a/src/hiro_graph_client/eventswebsocket.py +++ b/src/hiro_graph_client/eventswebsocket.py @@ -356,7 +356,7 @@ def _set_next_token_refresh(self): replace_existing=True) def _token_refresh_thread(self): - logger.debug("Updating token for session") + logger.debug("Updating token for _session") message: dict = { "type": "token", diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 2abac93..07f3534 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1,8 +1,13 @@ -from hiro_graph_client import PasswordAuthTokenApiHandler, HiroGraph, SSLConfig +from hiro_graph_client import PasswordAuthTokenApiHandler, HiroGraph, SSLConfig, GraphConnectionHandler from .testconfig import CONFIG class TestClient: + # connection_handler = GraphConnectionHandler( + # root_url=CONFIG.get('URL'), + # ssl_config=SSLConfig(verify=False) + # ) + hiro_api_handler = PasswordAuthTokenApiHandler( root_url=CONFIG.get('URL'), username=CONFIG.get('USERNAME'), @@ -13,14 +18,32 @@ class TestClient: ssl_config=SSLConfig(verify=False) ) + hiro_api_handler2 = PasswordAuthTokenApiHandler( + root_url=CONFIG.get('URL'), + username=CONFIG.get('USERNAME'), + password=CONFIG.get('PASSWORD'), + client_id=CONFIG.get('CLIENT_ID'), + client_secret=CONFIG.get('CLIENT_SECRET'), + secure_logging=False, + ssl_config=SSLConfig(verify=False) + ) + def test_simple_query(self): hiro_client: HiroGraph = HiroGraph(api_handler=self.hiro_api_handler) - attributes = { - "/start_date": None, - "/end_date": None, - "/status": None, - "/inActiveTimespan": None - } + hiro_client.get_node(node_id="ckqjkt42s0fgf0883pf0cb0hx_ckqjl014l0hvr0883hxcvmcwq", meta=True) + + hiro_client: HiroGraph = HiroGraph(api_handler=self.hiro_api_handler2) + + hiro_client.get_node(node_id="ckqjkt42s0fgf0883pf0cb0hx_ckqjl014l0hvr0883hxcvmcwq", meta=True) - hiro_client.update_node(node_id="ckqjkt42s0fgf0883pf0cb0hx_ckshomsv84i9x0783cla0g2bv", data=attributes) + # def query(_id: str): + # hiro_client.get_node(node_id=_id) + # + # t1 = threading.Thread(target=query, args=("ckqjkt42s0fgf0883pf0cb0hx_ckqjl014l0hvr0883hxcvmcwq",)) + # t2 = threading.Thread(target=query, args=("ckqjkt42s0fgf0883pf0cb0hx_ckshomsv84i9x0783cla0g2bv",)) + # t3 = threading.Thread(target=query, args=("ckqjkt42s0fgf0883pf0cb0hx_ckshomsv84i9x0783cla0g2bv",)) + # + # t1.start() + # t2.start() + # t3.start() From dcd768075f8d3ffea8555307d4fb419ff28e6a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 01:03:24 +0100 Subject: [PATCH 2/9] Some optimizations --- src/hiro_graph_client/clientlib.py | 120 +++++++++++++---------------- tests/unit/test_client.py | 14 ++-- 2 files changed, 58 insertions(+), 76 deletions(-) diff --git a/src/hiro_graph_client/clientlib.py b/src/hiro_graph_client/clientlib.py index c285fe1..ee61397 100644 --- a/src/hiro_graph_client/clientlib.py +++ b/src/hiro_graph_client/clientlib.py @@ -104,8 +104,8 @@ class AbstractAPI: _timeout: int = 600 def __init__(self, - root_url: str, - session: requests.Session, + root_url: str = None, + session: requests.Session = None, raise_exceptions: bool = True, proxies: dict = None, headers: dict = None, @@ -113,7 +113,8 @@ def __init__(self, client_name: str = None, ssl_config: SSLConfig = None, log_communication_on_error: bool = None, - max_tries: int = None): + max_tries: int = None, + abstract_api=None): """ Constructor @@ -131,41 +132,44 @@ def __init__(self, :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. + :param abstract_api: Set all parameters by copying them from another instance. Overrides all other parameters. """ + self._root_url = getattr(abstract_api, '_root_url', root_url) + self._session = getattr(abstract_api, '_session', session) - if not root_url: + if not self._root_url: raise ValueError("'root_url' must not be empty.") - if not session: + if not self._session: raise ValueError("'session' must not be empty.") - self._root_url = root_url - self._proxies = proxies - self._raise_exceptions = raise_exceptions - self._timeout = timeout or self._timeout - self._log_communication_on_error = log_communication_on_error or False - self._session = session + self._proxies = getattr(abstract_api, '_proxies', proxies) + self._raise_exceptions = getattr(abstract_api, '_raise_exceptions', raise_exceptions) + self._timeout = getattr(abstract_api, '_timeout', timeout or self._timeout) + self._log_communication_on_error = getattr(abstract_api, '_log_communication_on_error', + log_communication_on_error or False) - self.ssl_config = ssl_config or SSLConfig() + self.ssl_config = getattr(abstract_api, 'ssl_config', ssl_config or SSLConfig()) if not self.ssl_config.verify: AbstractAPI.accept_all_certs = True requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) - if client_name: - self._client_name = client_name + self._client_name = getattr(abstract_api, '_client_name', client_name or self._client_name) - self._headers = { - 'Content-Type': 'application/json', - 'Accept': 'text/plain, application/json', - 'User-Agent': f"{self._client_name} {__version__}" - } + if abstract_api: + self._headers = getattr(abstract_api, '_headers', None) + else: + self._headers = { + 'Content-Type': 'application/json', + 'Accept': 'text/plain, application/json', + 'User-Agent': f"{self._client_name} {__version__}" + } - if headers: - self._headers.update({self._capitalize_header(k): v for k, v in headers.items()}) + if headers: + self._headers.update({self._capitalize_header(k): v for k, v in headers.items()}) - if max_tries is not None: - self._max_tries = max_tries + self._max_tries = getattr(abstract_api, '_max_tries', max_tries) def _get_max_tries(self): return self._max_tries @@ -667,11 +671,8 @@ class GraphConnectionHandler(AbstractAPI): _pool_block = False - __session: requests.Session - """Stores the connection _session""" - def __init__(self, - root_url: str, + root_url: str = None, raise_exceptions: bool = True, proxies: dict = None, headers: dict = None, @@ -698,7 +699,8 @@ def __init__(self, This object creates the *requests.Session* and *requests.adapters.HTTPAdapter* for this *root_url*. The *pool_maxsize* of such a _session can be set via the parameter in the constructor. When a TokenApiHandler is - shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also shared. + shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also + shared. :param root_url: Root url for HIRO, like https://core.arago.co. :param raise_exceptions: Raise exceptions on HTTP status codes that denote an error. Default is True. @@ -727,11 +729,11 @@ def __init__(self, pool_connections=1, pool_block=pool_block or self._pool_block ) - self.__session = requests.Session() - self.__session.mount(root_url, adapter) + session = requests.Session() + session.mount(root_url, adapter) super().__init__(root_url=root_url, - session=self.__session, + session=session, raise_exceptions=raise_exceptions, proxies=proxies, timeout=timeout, @@ -766,10 +768,7 @@ def get_api_endpoint_of(self, api_name: str) -> str: if endpoint: return self._remove_slash(self._root_url + endpoint) - if not self._version_info: - self._version_info = self.get_version() - - api_entry: dict = self._version_info.get(api_name) + api_entry: dict = self.get_version().get(api_name) if not api_entry: raise ValueError("No API named '{}' found.".format(api_name)) @@ -830,10 +829,7 @@ def _construct_result(_endpoint: str, _protocol: str) -> Tuple[ if endpoint: return _construct_result(endpoint, protocol) - if not self._version_info: - self._version_info = self.get_version() - - api_entry: dict = self._version_info.get(api_name) + api_entry: dict = self.get_version().get(api_name) if not api_entry: raise ValueError("No WS API named '{}' found.".format(api_name)) @@ -850,8 +846,11 @@ def get_version(self) -> dict: :return: The result payload """ - url = self._root_url + '/api/version' - return self.get(url) + if not self._version_info: + url = self._root_url + '/api/version' + self._version_info = self.get(url) + + return self._version_info ################################################################################################################### @@ -895,7 +894,8 @@ def __init__(self, This object creates the *requests.Session* and *requests.adapters.HTTPAdapter* for this *root_url*. The *pool_maxsize* of such a _session can be set via the parameter in the constructor. When a TokenApiHandler is - shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also shared. + shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also + shared. :param root_url: Root url for HIRO, like https://core.arago.co. :param raise_exceptions: Raise exceptions on HTTP status codes that denote an error. Default is True. @@ -912,7 +912,8 @@ def __init__(self, detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a + _session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. @@ -935,18 +936,7 @@ def __init__(self, pool_block=pool_block ) - super().__init__( - root_url=self._connection_handler._root_url, - session=self._connection_handler._session, - raise_exceptions=self._connection_handler._raise_exceptions, - proxies=self._connection_handler._proxies, - timeout=self._connection_handler._timeout, - headers=self._connection_handler._headers, - client_name=self._connection_handler._client_name, - ssl_config=self._connection_handler.ssl_config, - log_communication_on_error=self._connection_handler._log_communication_on_error, - max_tries=self._connection_handler._max_tries - ) + super().__init__(abstract_api=self._connection_handler) ############################################################################################################### # Access connection handler @@ -1054,7 +1044,8 @@ def __init__(self, detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a + _session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. @@ -1136,7 +1127,8 @@ def __init__(self, detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a + _session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. @@ -1346,7 +1338,8 @@ def __init__(self, detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a _session. + Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a + _session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. @@ -1535,16 +1528,7 @@ def __init__(self, if not api_handler or not api_name: raise ValueError("Cannot authenticate against HIRO without *api_handler* and *api_name*.") - super().__init__(root_url=api_handler._root_url, - session=api_handler._session, - raise_exceptions=api_handler._raise_exceptions, - proxies=api_handler._proxies, - headers=api_handler._headers, - timeout=api_handler._timeout, - client_name=api_handler._client_name, - ssl_config=api_handler.ssl_config, - log_communication_on_error=api_handler._log_communication_on_error, - max_tries=api_handler._get_max_tries()) + super().__init__(abstract_api=api_handler) self._api_handler = api_handler self._api_name = api_name diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 07f3534..ca7a1ee 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -3,29 +3,27 @@ class TestClient: - # connection_handler = GraphConnectionHandler( - # root_url=CONFIG.get('URL'), - # ssl_config=SSLConfig(verify=False) - # ) + connection_handler = GraphConnectionHandler( + root_url=CONFIG.get('URL'), + ssl_config=SSLConfig(verify=False) + ) hiro_api_handler = PasswordAuthTokenApiHandler( - root_url=CONFIG.get('URL'), username=CONFIG.get('USERNAME'), password=CONFIG.get('PASSWORD'), client_id=CONFIG.get('CLIENT_ID'), client_secret=CONFIG.get('CLIENT_SECRET'), secure_logging=False, - ssl_config=SSLConfig(verify=False) + connection_handler=connection_handler ) hiro_api_handler2 = PasswordAuthTokenApiHandler( - root_url=CONFIG.get('URL'), username=CONFIG.get('USERNAME'), password=CONFIG.get('PASSWORD'), client_id=CONFIG.get('CLIENT_ID'), client_secret=CONFIG.get('CLIENT_SECRET'), secure_logging=False, - ssl_config=SSLConfig(verify=False) + connection_handler=connection_handler ) def test_simple_query(self): From c2605bb67380ff2a793a87859de765e5a1b5e217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 07:52:38 +0100 Subject: [PATCH 3/9] Refactoring * Make TokenApiHandler derive from GraphConnectionHandler instead of using an internal reference. --- src/hiro_graph_client/clientlib.py | 105 ++++++++++++++--------------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/src/hiro_graph_client/clientlib.py b/src/hiro_graph_client/clientlib.py index ee61397..ca074bb 100644 --- a/src/hiro_graph_client/clientlib.py +++ b/src/hiro_graph_client/clientlib.py @@ -132,7 +132,8 @@ def __init__(self, :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. - :param abstract_api: Set all parameters by copying them from another instance. Overrides all other parameters. + :param abstract_api: Set all parameters by copying them from the instance given by this parameter. Overrides + all other parameters. """ self._root_url = getattr(abstract_api, '_root_url', root_url) self._session = getattr(abstract_api, '_session', session) @@ -671,6 +672,8 @@ class GraphConnectionHandler(AbstractAPI): _pool_block = False + _version_info: dict = None + def __init__(self, root_url: str = None, raise_exceptions: bool = True, @@ -683,7 +686,8 @@ def __init__(self, log_communication_on_error: bool = None, max_tries: int = None, pool_maxsize: int = None, - pool_block: bool = None): + pool_block: bool = None, + connection_handler=None): """ Constructor @@ -720,7 +724,11 @@ def __init__(self, Default is 10. *pool_maxsize* is ignored when *_session* is set. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *_session* is set. + :param connection_handler: Copy parameters from this already existing connection handler. Overrides all other + parameters. """ + root_url = getattr(connection_handler, '_root_url', root_url) + if not root_url: raise ValueError("'root_url' must not be empty.") @@ -732,19 +740,24 @@ def __init__(self, session = requests.Session() session.mount(root_url, adapter) - super().__init__(root_url=root_url, - session=session, - raise_exceptions=raise_exceptions, - proxies=proxies, - timeout=timeout, - headers=headers, - client_name=client_name, - ssl_config=ssl_config, - log_communication_on_error=log_communication_on_error, - max_tries=max_tries) + super().__init__( + root_url=root_url, + session=session, + raise_exceptions=raise_exceptions, + proxies=proxies, + timeout=timeout, + headers=headers, + client_name=client_name, + ssl_config=ssl_config, + log_communication_on_error=log_communication_on_error, + max_tries=max_tries, + abstract_api=connection_handler + ) + + self.custom_endpoints = getattr(connection_handler, '_custom_endpoints', custom_endpoints) + self._version_info = getattr(connection_handler, '_version_info', None) - self._version_info = None - self.custom_endpoints = custom_endpoints + self.get_version() @staticmethod def _remove_slash(endpoint: str) -> str: @@ -840,13 +853,14 @@ def _construct_result(_endpoint: str, _protocol: str) -> Tuple[ # REST API operations ############################################################################################################### - def get_version(self) -> dict: + def get_version(self, force_update: bool = False) -> dict: """ HIRO REST query API: `GET self._endpoint + '/api/version'` + :param force_update: Force updating the internal cache with version_info via API request. :return: The result payload """ - if not self._version_info: + if not self._version_info or force_update: url = self._root_url + '/api/version' self._version_info = self.get(url) @@ -857,14 +871,11 @@ def get_version(self) -> dict: # TokenApiHandler classes ################################################################################################################### -class AbstractTokenApiHandler(AbstractAPI): +class AbstractTokenApiHandler(GraphConnectionHandler): """ Root class for all TokenApiHandler classes. This adds token handling. """ - _connection_handler: GraphConnectionHandler - """Reference to an GraphConnectionHandler containing version information and requests.Session""" - def __init__(self, root_url: str = None, raise_exceptions: bool = True, @@ -917,11 +928,12 @@ def __init__(self, :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. - :param connection_handler: Use an already existing connection handler. This feature allows for several distinct - TokenApiHandlers to operate on the same connection without querying unnecessary version information for - each or building their own requests.Sessions. Overrides all other parameters. + :param connection_handler: Copy all parameters from this already existing connection handler. This feature + allows for several distinct TokenApiHandlers to operate on the same connection without querying + unnecessary version information for each or building their own requests.Sessions. Overrides all other + parameters. """ - self._connection_handler = connection_handler or GraphConnectionHandler( + super().__init__( root_url=root_url, raise_exceptions=raise_exceptions, proxies=proxies, @@ -933,30 +945,10 @@ def __init__(self, log_communication_on_error=log_communication_on_error, max_tries=max_tries, pool_maxsize=pool_maxsize, - pool_block=pool_block + pool_block=pool_block, + connection_handler=connection_handler ) - super().__init__(abstract_api=self._connection_handler) - - ############################################################################################################### - # Access connection handler - ############################################################################################################### - - def get_api_endpoint_of(self, api_name: str) -> str: - return self._connection_handler.get_api_endpoint_of(api_name) - - def get_websocket_config(self, api_name: str) -> Tuple[ - str, - str, - Optional[str], - Optional[int], - Optional[dict] - ]: - return self._connection_handler.get_websocket_config(api_name) - - def get_version(self): - return self._connection_handler.get_version() - ############################################################################################################### # Token handling ############################################################################################################### @@ -1049,9 +1041,10 @@ def __init__(self, :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. - :param connection_handler: Use an already existing connection handler. This feature allows for several distinct - TokenApiHandlers to operate on the same connection without querying unnecessary version information for - each or building their own requests.Sessions. Overrides all other connection specific parameters. + :param connection_handler: Copy all parameters from this already existing connection handler. This feature + allows for several distinct TokenApiHandlers to operate on the same connection without querying + unnecessary version information for each or building their own requests.Sessions. Overrides all other + parameters. """ super().__init__( root_url=root_url, @@ -1132,9 +1125,10 @@ def __init__(self, :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. - :param connection_handler: Use an already existing connection handler. This feature allows for several distinct - TokenApiHandlers to operate on the same connection without querying unnecessary version information for - each or building their own requests.Sessions. Overrides all other connection specific parameters. + :param connection_handler: Copy all parameters from this already existing connection handler. This feature + allows for several distinct TokenApiHandlers to operate on the same connection without querying + unnecessary version information for each or building their own requests.Sessions. Overrides all other + parameters. """ super().__init__( root_url=root_url, @@ -1343,9 +1337,10 @@ def __init__(self, :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *connection_handler* is set and already contains a _session. - :param connection_handler: Use an already existing connection handler. This feature allows for several distinct - TokenApiHandlers to operate on the same connection without querying unnecessary version information for - each or building their own requests.Sessions. Overrides all other connection specific parameters. + :param connection_handler: Copy all parameters from this already existing connection handler. This feature + allows for several distinct TokenApiHandlers to operate on the same connection without querying + unnecessary version information for each or building their own requests.Sessions. Overrides all other + parameters. """ super().__init__( root_url=root_url, From 4acaa3b09344a9e0cb2a156e982221fd82da8aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 08:18:23 +0100 Subject: [PATCH 4/9] Better parameter passing in constructors --- src/hiro_graph_client/clientlib.py | 162 ++++------------------------- 1 file changed, 23 insertions(+), 139 deletions(-) diff --git a/src/hiro_graph_client/clientlib.py b/src/hiro_graph_client/clientlib.py index ca074bb..dc292c1 100644 --- a/src/hiro_graph_client/clientlib.py +++ b/src/hiro_graph_client/clientlib.py @@ -674,20 +674,17 @@ class GraphConnectionHandler(AbstractAPI): _version_info: dict = None + _lock: threading.RLock + """Reentrant mutex for thread safety""" + def __init__(self, root_url: str = None, - raise_exceptions: bool = True, - proxies: dict = None, - headers: dict = None, - timeout: int = None, - client_name: str = None, custom_endpoints: dict = None, - ssl_config: SSLConfig = None, - log_communication_on_error: bool = None, - max_tries: int = None, pool_maxsize: int = None, pool_block: bool = None, - connection_handler=None): + connection_handler=None, + *args, + **kwargs): """ Constructor @@ -727,6 +724,8 @@ def __init__(self, :param connection_handler: Copy parameters from this already existing connection handler. Overrides all other parameters. """ + self._lock = threading.RLock() + root_url = getattr(connection_handler, '_root_url', root_url) if not root_url: @@ -743,15 +742,8 @@ def __init__(self, super().__init__( root_url=root_url, session=session, - raise_exceptions=raise_exceptions, - proxies=proxies, - timeout=timeout, - headers=headers, - client_name=client_name, - ssl_config=ssl_config, - log_communication_on_error=log_communication_on_error, - max_tries=max_tries, - abstract_api=connection_handler + abstract_api=connection_handler, + **kwargs ) self.custom_endpoints = getattr(connection_handler, '_custom_endpoints', custom_endpoints) @@ -860,11 +852,12 @@ def get_version(self, force_update: bool = False) -> dict: :param force_update: Force updating the internal cache with version_info via API request. :return: The result payload """ - if not self._version_info or force_update: - url = self._root_url + '/api/version' - self._version_info = self.get(url) + with self._lock: + if not self._version_info or force_update: + url = self._root_url + '/api/version' + self._version_info = self.get(url) - return self._version_info + return self._version_info ################################################################################################################### @@ -876,20 +869,7 @@ class AbstractTokenApiHandler(GraphConnectionHandler): Root class for all TokenApiHandler classes. This adds token handling. """ - def __init__(self, - root_url: str = None, - raise_exceptions: bool = True, - proxies: dict = None, - headers: dict = None, - timeout: int = None, - client_name: str = None, - custom_endpoints: dict = None, - ssl_config: SSLConfig = None, - log_communication_on_error: bool = None, - max_tries: int = None, - pool_maxsize: int = None, - pool_block: bool = None, - connection_handler: GraphConnectionHandler = None): + def __init__(self, *args, **kwargs): """ Constructor @@ -933,21 +913,7 @@ def __init__(self, unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. """ - super().__init__( - root_url=root_url, - raise_exceptions=raise_exceptions, - proxies=proxies, - headers=headers, - timeout=timeout, - client_name=client_name, - custom_endpoints=custom_endpoints, - ssl_config=ssl_config, - log_communication_on_error=log_communication_on_error, - max_tries=max_tries, - pool_maxsize=pool_maxsize, - pool_block=pool_block, - connection_handler=connection_handler - ) + super().__init__(*args, **kwargs) ############################################################################################################### # Token handling @@ -1002,21 +968,7 @@ class FixedTokenApiHandler(AbstractTokenApiHandler): _token: str - def __init__(self, - root_url: str = None, - token: str = None, - raise_exceptions: bool = True, - proxies: dict = None, - headers: dict = None, - timeout: int = 600, - client_name: str = None, - custom_endpoints: dict = None, - ssl_config: SSLConfig = None, - log_communication_on_error: bool = None, - max_tries: int = None, - pool_maxsize: int = None, - pool_block: bool = None, - connection_handler: GraphConnectionHandler = None): + def __init__(self, token: str = None, *args, **kwargs): """ Constructor @@ -1046,21 +998,7 @@ def __init__(self, unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. """ - super().__init__( - root_url=root_url, - raise_exceptions=raise_exceptions, - proxies=proxies, - timeout=timeout, - headers=headers, - client_name=client_name, - custom_endpoints=custom_endpoints, - ssl_config=ssl_config, - log_communication_on_error=log_communication_on_error, - max_tries=max_tries, - pool_maxsize=pool_maxsize, - pool_block=pool_block, - connection_handler=connection_handler - ) + super().__init__(*args, **kwargs) self._token = token @@ -1086,21 +1024,7 @@ class EnvironmentTokenApiHandler(AbstractTokenApiHandler): _env_var: str - def __init__(self, - root_url: str = None, - env_var: str = 'HIRO_TOKEN', - raise_exceptions: bool = True, - proxies: dict = None, - headers: dict = None, - timeout: int = 600, - client_name: str = None, - custom_endpoints: dict = None, - ssl_config: SSLConfig = None, - log_communication_on_error: bool = None, - max_tries: int = None, - pool_maxsize: int = None, - pool_block: bool = None, - connection_handler: GraphConnectionHandler = None): + def __init__(self, env_var: str = 'HIRO_TOKEN', *args, **kwargs): """ Constructor @@ -1130,21 +1054,7 @@ def __init__(self, unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. """ - super().__init__( - root_url=root_url, - raise_exceptions=raise_exceptions, - proxies=proxies, - headers=headers, - timeout=timeout, - client_name=client_name, - custom_endpoints=custom_endpoints, - ssl_config=ssl_config, - log_communication_on_error=log_communication_on_error, - max_tries=max_tries, - pool_maxsize=pool_maxsize, - pool_block=pool_block, - connection_handler=connection_handler - ) + super().__init__(*args, **kwargs) self._env_var = env_var @@ -1291,24 +1201,12 @@ class PasswordAuthTokenApiHandler(AbstractTokenApiHandler): _secure_logging: bool = True def __init__(self, - root_url: str = None, username: str = None, password: str = None, client_id: str = None, client_secret: str = None, secure_logging: bool = True, - raise_exceptions: bool = True, - proxies: dict = None, - headers: dict = None, - timeout: int = 600, - client_name: str = None, - custom_endpoints: dict = None, - ssl_config: SSLConfig = None, - log_communication_on_error: bool = None, - max_tries: int = None, - pool_maxsize: int = None, - pool_block: bool = None, - connection_handler: GraphConnectionHandler = None): + *args, **kwargs): """ Constructor @@ -1342,21 +1240,7 @@ def __init__(self, unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. """ - super().__init__( - root_url=root_url, - raise_exceptions=raise_exceptions, - proxies=proxies, - headers=headers, - timeout=timeout, - client_name=client_name, - custom_endpoints=custom_endpoints, - ssl_config=ssl_config, - log_communication_on_error=log_communication_on_error, - max_tries=max_tries, - pool_maxsize=pool_maxsize, - pool_block=pool_block, - connection_handler=connection_handler - ) + super().__init__(*args, **kwargs) self._username = username self._password = password From 3b13d6de073a7fead2480d56222b493c3e3b041e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 08:32:32 +0100 Subject: [PATCH 5/9] Version 5.2.0 --- src/hiro_graph_client/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hiro_graph_client/VERSION b/src/hiro_graph_client/VERSION index acf69b4..7cbea07 100644 --- a/src/hiro_graph_client/VERSION +++ b/src/hiro_graph_client/VERSION @@ -1 +1 @@ -5.1.0 \ No newline at end of file +5.2.0 \ No newline at end of file From 2fe179cb189320b198f4225ac368f4caad1aee19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 08:41:47 +0100 Subject: [PATCH 6/9] Also copy session --- src/hiro_graph_client/clientlib.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/hiro_graph_client/clientlib.py b/src/hiro_graph_client/clientlib.py index dc292c1..1a0014f 100644 --- a/src/hiro_graph_client/clientlib.py +++ b/src/hiro_graph_client/clientlib.py @@ -93,6 +93,10 @@ class AbstractAPI: tool methods for handling headers, url query parts and response error checking. """ + _root_url: str = None + + _session: requests.Session = None + accept_all_certs: bool = False ssl_config: SSLConfig @@ -727,17 +731,19 @@ def __init__(self, self._lock = threading.RLock() root_url = getattr(connection_handler, '_root_url', root_url) + session = getattr(connection_handler, '_session', None) if not root_url: raise ValueError("'root_url' must not be empty.") - adapter = requests.adapters.HTTPAdapter( - pool_maxsize=pool_maxsize or self._pool_maxsize, - pool_connections=1, - pool_block=pool_block or self._pool_block - ) - session = requests.Session() - session.mount(root_url, adapter) + if not session: + adapter = requests.adapters.HTTPAdapter( + pool_maxsize=pool_maxsize or self._pool_maxsize, + pool_connections=1, + pool_block=pool_block or self._pool_block + ) + session = requests.Session() + session.mount(root_url, adapter) super().__init__( root_url=root_url, From ae982d43e7c315cd8d7317b5ddc9e7f0ab46104e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 08:47:18 +0100 Subject: [PATCH 7/9] Add missing `*args` --- src/hiro_graph_client/clientlib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hiro_graph_client/clientlib.py b/src/hiro_graph_client/clientlib.py index 1a0014f..fe89ed7 100644 --- a/src/hiro_graph_client/clientlib.py +++ b/src/hiro_graph_client/clientlib.py @@ -749,6 +749,7 @@ def __init__(self, root_url=root_url, session=session, abstract_api=connection_handler, + *args, **kwargs ) From e5d1fbfc984c8c52616626d8c489007dd7d7ddf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 09:06:14 +0100 Subject: [PATCH 8/9] Documentation --- src/README.md | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/README.md b/src/README.md index 69e2cb6..490e878 100644 --- a/src/README.md +++ b/src/README.md @@ -142,7 +142,7 @@ query_result = hiro_client.query('ogit\\/_type:"ogit/MARS/Machine"') print(query_result) ``` -## Handler sharing +## Token Handler sharing When you need to access multiple APIs of HIRO, share the TokenApiHandler between the API objects to avoid unnecessary requests for token- and version-information against HIRO. The TokenApiHandler will share a `requests.Session`, token- @@ -168,6 +168,48 @@ hiro_app_client: HiroApp = HiroApp( ) ``` +## Connection sharing + +You can also let TokenApiHandlers share a common connection instead of letting each of them create their own. This might +prove useful in a multithreading environment where tokens have to be set externally or change often (i.e. one token per +user per thread). This also ensures, that version-requests happen only once when the connection is initialized. + +```python +from hiro_graph_client import HiroGraph, HiroApp, FixedTokenApiHandler, GraphConnectionHandler + +connection_handler = GraphConnectionHandler( + root_url="https://core.arago.co", + client_name="Your Graph Client 0.0.1" # optional parameter +) + +# Work with token of user 1 + +user1_client: HiroGraph = HiroGraph( + api_handler=FixedTokenApiHandler( + connection_handler=connection_handler, + token='token user 1' + ) +) + +# Work with token of user 2 (Shared Token Handler) + +user2_api_handler = FixedTokenApiHandler( + connection_handler=connection_handler, + token='token user 2' +) + +user2_client: HiroGraph = HiroGraph( + api_handler=user2_api_handler +) + +hiro_app_client: HiroApp = HiroApp( + api_handler=user2_api_handler +) + +``` + +Everything written in [Token Handler Sharing](#token-handler-sharing) still applies. + ## SSL Configuration SSL parameters are configured using the class `SSLConfig`. This class translates the parameters given to the required From 6dd8fb0eaba935d90f49c8ebb7fd2673f97e74f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20H=C3=BCbner?= Date: Fri, 4 Feb 2022 12:37:02 +0100 Subject: [PATCH 9/9] Documentation fixes --- src/README.md | 15 ++++++--- src/hiro_graph_client/clientlib.py | 52 ++++++++++++++++++------------ 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/README.md b/src/README.md index 490e878..621e808 100644 --- a/src/README.md +++ b/src/README.md @@ -170,16 +170,23 @@ hiro_app_client: HiroApp = HiroApp( ## Connection sharing -You can also let TokenApiHandlers share a common connection instead of letting each of them create their own. This might -prove useful in a multithreading environment where tokens have to be set externally or change often (i.e. one token per -user per thread). This also ensures, that version-requests happen only once when the connection is initialized. +You can also let TokenApiHandlers share a common connection session instead of letting each of them create their own. +This might prove useful in a multithreading environment where tokens have to be set externally or change often (i.e. +one token per user per thread). This also ensures, that version-requests happen only once when the connection is +initialized. + +Use the parameters `pool_maxsize` and `pool_block` to further tune the connection parameters for parallel access to +the backend. See [requests Session Objects](https://docs.python-requests.org/en/latest/user/advanced/#session-objects) +and Python documentation of `requests.adapters.HTTPAdapter` for more information. ```python from hiro_graph_client import HiroGraph, HiroApp, FixedTokenApiHandler, GraphConnectionHandler connection_handler = GraphConnectionHandler( root_url="https://core.arago.co", - client_name="Your Graph Client 0.0.1" # optional parameter + pool_maxsize=200, # Optional: Max pool of cached connections for this connection session + pool_block=True, # Optional: Do not allow more parallel connections than pool_maxsize + client_name="Your Graph Client 0.0.1" # Optional: Will be used in the header 'User-Agent' ) # Work with token of user 1 diff --git a/src/hiro_graph_client/clientlib.py b/src/hiro_graph_client/clientlib.py index fe89ed7..36d3df2 100644 --- a/src/hiro_graph_client/clientlib.py +++ b/src/hiro_graph_client/clientlib.py @@ -426,7 +426,7 @@ def _delete() -> Any: def _get_proxies(self) -> dict: """ - Create a copy of proxies if they exists or return None + Create a copy of proxies if they exist or return None :return: copy of self._proxies or None """ @@ -612,7 +612,7 @@ def _check_content_type(res: requests.Response, expected_media_type: str) -> Non :param res: The response object :param expected_media_type: The expected Media-Type. :raise WrongContentTypeError: When the Media-Type of the Content-Type is not *expected_media_type* or - the header is is missing completely. + the header is missing completely. """ content_type = res.headers.get('Content-Type') if not content_type: @@ -703,8 +703,8 @@ def __init__(self, } This object creates the *requests.Session* and *requests.adapters.HTTPAdapter* for this *root_url*. The - *pool_maxsize* of such a _session can be set via the parameter in the constructor. When a TokenApiHandler is - shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also + *pool_maxsize* of such a session can be set via the parameter in the constructor. When a TokenApiHandler is + shared between different API objects (like HiroGraph, HiroApp, etc.), this session and its pool are also shared. :param root_url: Root url for HIRO, like https://core.arago.co. @@ -716,17 +716,19 @@ def __init__(self, is given with another value for "User-Agent". Default is "hiro-graph-client". :param custom_endpoints: Optional map of {name:endpoint_path, ...} that overrides or adds to the endpoints taken from /api/version. Example see above. - :param ssl_config Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib - will be used. + :param ssl_config: Optional configuration for SSL connections. If this is omitted, the defaults of `requests` + lib will be used. :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. - Default is 10. *pool_maxsize* is ignored when *_session* is set. + Default is 10. *pool_maxsize* is ignored when *session* is set. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, - but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *_session* is set. + but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when *session* is set. :param connection_handler: Copy parameters from this already existing connection handler. Overrides all other parameters. + :param args: Unnamed parameters for parent class. See there. + :param kwargs: Named parameters for parent class. See there. """ self._lock = threading.RLock() @@ -891,8 +893,8 @@ def __init__(self, *args, **kwargs): } This object creates the *requests.Session* and *requests.adapters.HTTPAdapter* for this *root_url*. The - *pool_maxsize* of such a _session can be set via the parameter in the constructor. When a TokenApiHandler is - shared between different API objects (like HiroGraph, HiroApp, etc.), this _session and its pool are also + *pool_maxsize* of such a session can be set via the parameter in the constructor. When a TokenApiHandler is + shared between different API objects (like HiroGraph, HiroApp, etc.), this session and its pool are also shared. :param root_url: Root url for HIRO, like https://core.arago.co. @@ -904,21 +906,23 @@ def __init__(self, *args, **kwargs): is given with another value for "User-Agent". Default is "hiro-graph-client". :param custom_endpoints: Optional map of {name:endpoint_path, ...} that overrides or adds to the endpoints taken from /api/version. Example see above. - :param ssl_config Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib + :param ssl_config: Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib will be used. :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a - _session. + session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when - *connection_handler* is set and already contains a _session. + *connection_handler* is set and already contains a session. :param connection_handler: Copy all parameters from this already existing connection handler. This feature allows for several distinct TokenApiHandlers to operate on the same connection without querying unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. + :param args: Unnamed parameters for parent class. See there. + :param kwargs: Named parameters for parent class. See there. """ super().__init__(*args, **kwargs) @@ -989,21 +993,23 @@ def __init__(self, token: str = None, *args, **kwargs): is given with another value for "User-Agent". Default is "hiro-graph-client". :param custom_endpoints: Optional map of [name:endpoint_path] that overrides or adds to the endpoints taken from /api/version. - :param ssl_config Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib + :param ssl_config: Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib will be used. :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a - _session. + session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when - *connection_handler* is set and already contains a _session. + *connection_handler* is set and already contains a session. :param connection_handler: Copy all parameters from this already existing connection handler. This feature allows for several distinct TokenApiHandlers to operate on the same connection without querying unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. + :param args: Unnamed parameters for parent class. See there. + :param kwargs: Named parameters for parent class. See there. """ super().__init__(*args, **kwargs) @@ -1045,21 +1051,23 @@ def __init__(self, env_var: str = 'HIRO_TOKEN', *args, **kwargs): is given with another value for "User-Agent". Default is "hiro-graph-client". :param custom_endpoints: Optional map of [name:endpoint_path] that overrides or adds to the endpoints taken from /api/version. - :param ssl_config Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib + :param ssl_config: Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib will be used. :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. :param max_tries: Max tries for BACKOFF. Default is 2. :param pool_maxsize: Size of a connection pool for a single connection. See requests.adapters.HTTPAdapter. Default is 10. *pool_maxsize* is ignored when *connection_handler* is set and already contains a - _session. + session. :param pool_block: Block any connections that exceed the pool_maxsize. Default is False: Allow more connections, but do not cache them. See requests.adapters.HTTPAdapter. *pool_block* is ignored when - *connection_handler* is set and already contains a _session. + *connection_handler* is set and already contains a session. :param connection_handler: Copy all parameters from this already existing connection handler. This feature allows for several distinct TokenApiHandlers to operate on the same connection without querying unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. + :param args: Unnamed parameters for parent class. See there. + :param kwargs: Named parameters for parent class. See there. """ super().__init__(*args, **kwargs) @@ -1189,7 +1197,7 @@ class PasswordAuthTokenApiHandler(AbstractTokenApiHandler): API Tokens will be fetched using this class. It does not handle any automatic token fetching, refresh or token expiry. This has to be checked and triggered by the *caller*. - The methods of this class are thread-safe so it can be shared between several HIRO objects. + The methods of this class are thread-safe, so it can be shared between several HIRO objects. It is built this way to avoid endless calling loops when resolving tokens. """ @@ -1231,7 +1239,7 @@ def __init__(self, is given with another value for "User-Agent". Default is "hiro-graph-client". :param custom_endpoints: Optional map of [name:endpoint_path] that overrides or adds to the endpoints taken from /api/version. - :param ssl_config Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib + :param ssl_config: Optional configuration for SSL connections. If this is omitted, the defaults of `requests` lib will be used. :param log_communication_on_error: Log socket communication when an error (status_code of HTTP Response) is detected. Default is not to do this. @@ -1246,6 +1254,8 @@ def __init__(self, allows for several distinct TokenApiHandlers to operate on the same connection without querying unnecessary version information for each or building their own requests.Sessions. Overrides all other parameters. + :param args: Unnamed parameters for parent class. See there. + :param kwargs: Named parameters for parent class. See there. """ super().__init__(*args, **kwargs)