From bfbcf6368b0a0846d8d2c739964d94d0fbe2cdde Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Wed, 14 Apr 2021 10:26:33 +0200 Subject: [PATCH 1/4] Log errors from invoking API endpoints. With a HTTP proxy configured, calls to the default "http://0.0.0.0/api/call/update" would silently fail. --- src/pyff/api.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pyff/api.py b/src/pyff/api.py index 45b85b73..0fda2c4d 100644 --- a/src/pyff/api.py +++ b/src/pyff/api.py @@ -116,8 +116,13 @@ def _fmt(data, accepter): raise exc.exception_response(406) -def call(entry): - requests.post('{}/api/call/{}'.format(config.base_url, entry)) +def call(entry: str) -> None: + url = f'{config.base_url}/api/call/{entry}' + log.debug(f'Calling API endpoint at {url}') + resp = requests.post(url) + if resp.status_code >= 300: + log.error(f'POST request to API endpoint at {url} failed: {resp.status_code} {resp.reason}') + return None def request_handler(request): From 80e55a9476f7abe42a733f3e314afb7e0dec43b1 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Wed, 14 Apr 2021 10:27:08 +0200 Subject: [PATCH 2/4] spelling --- src/pyff/builtins.py | 14 +++++++------- src/pyff/resource.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index 9479e1e9..c0ef1edc 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -596,7 +596,7 @@ def load(req, *opts): - max_workers <5> : Number of parallel threads to use for loading MD files - timeout <120> : Socket timeout when downloading files - validate : When true downloaded metadata files are validated (schema validation) - - fail_on_error : Control whether an error during download, parsing or (optional)validatation of a MD file + - fail_on_error : Control whether an error during download, parsing or (optional)validation of a MD file does not abort processing of the pipeline. When true a failure aborts and causes pyff to exit with a non zero exit code. Otherwise errors are logged but ignored. - filter_invalid : Controls validation behaviour. When true Entities that fail validation are filtered @@ -713,7 +713,7 @@ def select(req, *opts): This would select all SPs Select statements are not cumulative - a select followed by another select in the plumbing resets the - working douments to the result of the second select. + working documents to the result of the second select. Most statements except local and remote depend on having a select somewhere in your plumbing and will stop the plumbing if the current working document is empty. For instance, running @@ -799,7 +799,7 @@ def _match(q, elt): raise PipeException("empty select - stop") if req.plumbing.id != name: - log.debug("storing synthentic collection {}".format(name)) + log.debug("storing synthetic collection {}".format(name)) req.store.update(ot, name) return ot @@ -886,7 +886,7 @@ def first(req, *opts): :return: returns the first entity descriptor if the working document only contains one Sometimes (eg when running an MDX pipeline) it is usually expected that if a single EntityDescriptor is being returned - then the outer EntitiesDescriptor is stripped. This method does exactly that: + then the outer EntitiesDescriptor is stripped. This method does exactly that. """ if req.t is None: @@ -918,7 +918,7 @@ def _discojson(req, *opts): cache & converted to data: URIs :param req: The request - :param opts: Options (unusued) + :param opts: Options (unused) :return: returns a JSON array """ @@ -1453,10 +1453,10 @@ def finalize(req, *opts): :return: returns the working document with @Name, @cacheDuration and @validUntil set Set Name, ID, cacheDuration and validUntil on the toplevel EntitiesDescriptor element of the working document. - Unlessexplicit provided the @Name is set from the request URI if the pipeline is executed in the pyFF server. The + Unless explicitly provided the @Name is set from the request URI if the pipeline is executed in the pyFF server. The @ID is set to a string representing the current date/time and will be prefixed with the string provided, which defaults to '_'. The @cacheDuration element must be a valid xsd duration (eg PT5H for 5 hrs) and @validUntil can - be either an absolute ISO 8601 time string or (more comonly) a relative time on the form + be either an absolute ISO 8601 time string or (more commonly) a relative time in the form .. code-block:: none diff --git a/src/pyff/resource.py b/src/pyff/resource.py index 6425aa46..4d0175ff 100644 --- a/src/pyff/resource.py +++ b/src/pyff/resource.py @@ -1,6 +1,6 @@ """ -An abstraction layer for metadata fetchers. Supports both syncronous and asyncronous fetchers with cache. +An abstraction layer for metadata fetchers. Supports both synchronous and asynchronous fetchers with cache. """ From 043f802ec49ba09d1b59b01b6f74a10185c4c600 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Wed, 14 Apr 2021 10:27:52 +0200 Subject: [PATCH 3/4] fix errors reported by 'make typecheck' --- src/pyff/locks.py | 4 ++-- src/pyff/utils.py | 18 ++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/pyff/locks.py b/src/pyff/locks.py index 129cec0a..fe692023 100644 --- a/src/pyff/locks.py +++ b/src/pyff/locks.py @@ -106,7 +106,7 @@ def acquireRead(self, blocking=True, timeout=None): finally: self.__condition.release() - @property + @property # type: ignore @contextmanager def readlock(self): """Yields a read lock""" @@ -116,7 +116,7 @@ def readlock(self): finally: self.release() - @property + @property # type: ignore @contextmanager def writelock(self): """Yields a write lock""" diff --git a/src/pyff/utils.py b/src/pyff/utils.py index e58f0c18..b2a0cd41 100644 --- a/src/pyff/utils.py +++ b/src/pyff/utils.py @@ -50,15 +50,6 @@ from .exceptions import * from .logs import get_log -try: - from redis import StrictRedis -except ImportError as ex: - StrictRedis = None - -try: - from PIL import Image -except ImportError as ex: - Image = None etree.set_default_parser(etree.XMLParser(resolve_entities=False)) @@ -244,7 +235,9 @@ def redis(): if not hasattr(thread_data, 'redis'): thread_data.redis = None - if StrictRedis is None: + try: + from redis import StrictRedis + except ImportError: raise ValueError("redis_py missing from dependencies") if thread_data.redis is None: @@ -748,6 +741,11 @@ def img_to_data(data, content_type): if len(data) > config.icon_maxsize: return None + try: + from PIL import Image + except ImportError: + Image = None + if Image is not None: try: im = Image.open(io.BytesIO(data)) From 3757d7dfb6ecceed466fb049a6437418424bb525 Mon Sep 17 00:00:00 2001 From: Fredrik Thulin Date: Wed, 14 Apr 2021 10:28:13 +0200 Subject: [PATCH 4/4] add a tiny bit of type checking --- src/pyff/utils.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/pyff/utils.py b/src/pyff/utils.py index b2a0cd41..b3de2b36 100644 --- a/src/pyff/utils.py +++ b/src/pyff/utils.py @@ -24,13 +24,12 @@ from itertools import chain from threading import local from time import gmtime, strftime +from typing import AnyStr, Optional, Union import iso8601 import pkg_resources import requests -import six import xmlsec -import yaml from _collections_abc import Mapping, MutableMapping from apscheduler.executors.pool import ThreadPoolExecutor from apscheduler.jobstores.memory import MemoryJobStore @@ -43,7 +42,7 @@ from requests.structures import CaseInsensitiveDict from requests_cache import CachedSession from requests_file import FileAdapter -from six.moves.urllib_parse import quote_plus, urlparse +from six.moves.urllib_parse import urlparse from . import __version__ from .constants import NS, config @@ -720,13 +719,13 @@ def url_get(url): return r -def safe_b64e(data): - if not isinstance(data, six.binary_type): +def safe_b64e(data: Union[str, bytes]) -> str: + if not isinstance(data, bytes): data = data.encode("utf-8") return base64.b64encode(data).decode('ascii') -def safe_b64d(s): +def safe_b64d(s: str) -> bytes: return base64.b64decode(s) @@ -734,7 +733,7 @@ def safe_b64d(s): # data:;base64, -def img_to_data(data, content_type): +def img_to_data(data: bytes, content_type: str) -> Optional[str]: """Convert a file (specified by a path) into a data URI.""" mime_type, options = cgi.parse_header(content_type) data64 = None