From 65c86346a7983b63f8660743288ed8bafd3fdee4 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 11:42:03 -0400 Subject: [PATCH 01/13] add static analysis tooling configs --- .flake8 | 19 +++--------------- .pre-commit-config.yaml | 43 +++++++++++++++++++++++++++++++++++++++++ doc8.ini | 4 ++++ mypy.ini | 12 ++++++++++++ requirements-dev.txt | 23 +++++++++++++++++----- setup.py | 43 ++++++++++++++++------------------------- 6 files changed, 97 insertions(+), 47 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 doc8.ini create mode 100644 mypy.ini diff --git a/.flake8 b/.flake8 index d559d456..90b81357 100644 --- a/.flake8 +++ b/.flake8 @@ -1,17 +1,4 @@ [flake8] -max-line-length = 125 - -## IGNORES - -# E126: yapf conflicts with "continuation line over-indented for hanging indent" - -# E127: flake8 reporting incorrect continuation line indent errors -# on multi-line and multi-level indents - -# W503: flake8 reports this as incorrect, and scripts/format_code -# changes code to it, so let format_code win. - -ignore = E126,E127,W503 - -exclude = - docs/build +max-line-length = 88 +extend-ignore = E203, W503, E731, E722 +per-file-ignores = __init__.py:F401 \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..73b344ad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +# Configuration file for pre-commit (https://pre-commit.com/). +# Please run `pre-commit run --all-files` when adding or changing entries. + +repos: + - repo: local + hooks: + - id: black + name: black + entry: black + language: system + stages: [commit] + types: [python] + + - id: codespell + name: codespell + entry: codespell + args: [--ignore-words=.codespellignore] + language: system + stages: [commit] + types_or: [jupyter, markdown, python, shell] + + - id: doc8 + name: doc8 + entry: doc8 + language: system + files: \.rst$ + require_serial: true + + - id: flake8 + name: flake8 + entry: flake8 + language: system + stages: [commit] + types: [python] + + - id: mypy + name: mypy + entry: mypy + args: [--no-incremental] + language: system + stages: [commit] + types: [python] + require_serial: true \ No newline at end of file diff --git a/doc8.ini b/doc8.ini new file mode 100644 index 00000000..9bfdc2f9 --- /dev/null +++ b/doc8.ini @@ -0,0 +1,4 @@ +[doc8] + +ignore-path=docs/_build,docs/tutorials +max-line-length=88 \ No newline at end of file diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..476febc1 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,12 @@ +[mypy] +show_error_codes = True +strict = True + +[mypy-jinja2.*] +ignore_missing_imports = True + +[mypy-jsonschema.*] +ignore_missing_imports = True + +[mypy-setuptools.*] +ignore_missing_imports = True \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index db76b26d..90192ec8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,3 @@ -codespell==1.17.1 -coverage==5.2.* -flake8==3.8.* pytest~=6.2.3 pytest-benchmark~=3.4.1 pytest-cov~=2.11.1 @@ -9,5 +6,21 @@ pytest-console-scripts~=1.1.0 recommonmark~=0.7.1 requests-mock~=1.9.3 Sphinx~=3.5.1 -toml~=0.10.2 -yapf==0.30.* + +mypy~=0.960 +flake8~=4.0.1 +black~=22.3.0 +codespell~=2.1.0 + +jsonschema~=4.5.1 +coverage~=6.3.2 +doc8~=0.11.1 + +types-python-dateutil~=2.8.15 +types-orjson~=3.6.2 +types-requests~=2.27.29 + +pre-commit==2.19.0 + +# optional dependencies +orjson==3.6.8 \ No newline at end of file diff --git a/setup.py b/setup.py index ae85e01f..8eec9983 100644 --- a/setup.py +++ b/setup.py @@ -3,29 +3,30 @@ from setuptools import setup, find_packages from glob import glob -__version__ = load_source('pystac_client.version', 'pystac_client/version.py').__version__ +__version__ = load_source( + "pystac_client.version", "pystac_client/version.py" +).__version__ -from os.path import ( - basename, - splitext -) +from os.path import basename, splitext here = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(here, 'README.md')) as readme_file: +with open(os.path.join(here, "README.md")) as readme_file: readme = readme_file.read() setup( - name='pystac-client', + name="pystac-client", version=__version__, - description=("Python library for working with Spatiotemporal Asset Catalog (STAC)."), + description=( + "Python library for working with Spatiotemporal Asset Catalog (STAC)." + ), long_description=readme, long_description_content_type="text/markdown", author="Jon Duckworth, Matthew Hanson", - author_email='duckontheweb@gmail.com, matt.a.hanson@gmail.com', - url='https://github.com/stac-utils/pystac-client.git', + author_email="duckontheweb@gmail.com, matt.a.hanson@gmail.com", + url="https://github.com/stac-utils/pystac-client.git", packages=find_packages(exclude=("tests",)), - py_modules=[splitext(basename(path))[0] for path in glob('pystac_client/*.py')], + py_modules=[splitext(basename(path))[0] for path in glob("pystac_client/*.py")], include_package_data=False, python_requires=">=3.7", install_requires=[ @@ -33,18 +34,10 @@ "pystac>=1.4.0", "python-dateutil>=2.7.0", ], - extras_require={ - "validation": ["jsonschema==3.2.0"] - }, + extras_require={"validation": ["jsonschema==3.2.0"]}, license="Apache Software License 2.0", zip_safe=False, - keywords=[ - 'pystac', - 'imagery', - 'raster', - 'catalog', - 'STAC' - ], + keywords=["pystac", "imagery", "raster", "catalog", "STAC"], classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 3", @@ -59,10 +52,8 @@ "License :: OSI Approved :: MIT License", "Topic :: Scientific/Engineering :: GIS", "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules" + "Topic :: Software Development :: Libraries :: Python Modules", ], - test_suite='tests', - entry_points={ - 'console_scripts': ['stac-client=pystac_client.cli:cli'] - } + test_suite="tests", + entry_points={"console_scripts": ["stac-client=pystac_client.cli:cli"]}, ) From 86c09d54b5b7a4d251b8acebc3e3089981e375eb Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 12:47:29 -0400 Subject: [PATCH 02/13] fixup for static analysis compliance --- .flake8 | 2 +- .isort.cfg | 2 + .pre-commit-config.yaml | 60 ++-- doc8.ini | 3 +- docs/api.rst | 23 +- docs/conf.py | 48 +-- docs/contributing.rst | 6 +- docs/design/design_decisions.rst | 2 +- docs/quickstart.rst | 2 +- docs/tutorials.rst | 4 +- docs/usage.rst | 10 +- pystac_client/__init__.py | 4 +- pystac_client/cli.py | 224 +++++++------ pystac_client/client.py | 55 +-- pystac_client/collection_client.py | 11 +- pystac_client/conformance.py | 24 +- pystac_client/item_search.py | 263 +++++++++------ pystac_client/stac_api_io.py | 112 +++---- pystac_client/version.py | 2 +- requirements-dev.txt | 4 - scripts/format | 7 +- scripts/lint | 28 ++ scripts/test | 18 +- setup.py | 5 +- tests/helpers.py | 4 +- tests/test_cli.py | 9 +- tests/test_client.py | 216 +++++++----- tests/test_collection_client.py | 12 +- tests/test_item_search.py | 519 ++++++++++++++++------------- tests/test_stac_api_io.py | 36 +- 30 files changed, 969 insertions(+), 746 deletions(-) create mode 100644 .isort.cfg create mode 100644 scripts/lint diff --git a/.flake8 b/.flake8 index 90b81357..02826407 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] max-line-length = 88 -extend-ignore = E203, W503, E731, E722 +extend-ignore = E203, W503, E731, E722, E501 per-file-ignores = __init__.py:F401 \ No newline at end of file diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..6860bdb0 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile = black \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 73b344ad..1c02490d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,42 +2,38 @@ # Please run `pre-commit run --all-files` when adding or changing entries. repos: - - repo: local + - repo: https://github.com/psf/black + rev: 22.3.0 hooks: - id: black - name: black - entry: black - language: system - stages: [commit] - types: [python] - + - repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: - id: codespell - name: codespell - entry: codespell args: [--ignore-words=.codespellignore] - language: system - stages: [commit] types_or: [jupyter, markdown, python, shell] - + - repo: https://github.com/PyCQA/doc8 + rev: 0.11.1 + hooks: - id: doc8 - name: doc8 - entry: doc8 - language: system - files: \.rst$ - require_serial: true - + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: - id: flake8 - name: flake8 - entry: flake8 - language: system - stages: [commit] - types: [python] - - - id: mypy - name: mypy - entry: mypy - args: [--no-incremental] - language: system - stages: [commit] - types: [python] - require_serial: true \ No newline at end of file + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort (python) +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: v0.960 +# hooks: +# - id: mypy +# # TODO lint test and scripts too +## files: "^pystac_client/.*\\.py$" +# args: +# - --ignore-missing-imports +# additional_dependencies: +# - pystac +# - types-requests +# - types-python-dateutil \ No newline at end of file diff --git a/doc8.ini b/doc8.ini index 9bfdc2f9..196d2054 100644 --- a/doc8.ini +++ b/doc8.ini @@ -1,4 +1,5 @@ [doc8] ignore-path=docs/_build,docs/tutorials -max-line-length=88 \ No newline at end of file +max-line-length=88 +ignore=D001 \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst index 7674875a..bf020f31 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,16 +1,16 @@ API Reference ============= -This section is autogenerated from in-line code documentation. It is mostly useful as a reference for the various -classes, methods, and other objects in the library, but is not intended to function as a starting point for working -with ``pystac_client``. +This section is autogenerated from in-line code documentation. It is mostly useful as a +reference for the various classes, methods, and other objects in the library, but is +not intended to function as a starting point for working with ``pystac_client``. Client ------ Client is the base PySTAC-Client that inherits from :class:`Catalog `. -In addition to the PySTAC functionality, Client allows opening of API URLs, understanding of -conformance, and support for searching and paging through results. +In addition to the PySTAC functionality, Client allows opening of API URLs, +understanding of conformance, and support for searching and paging through results. .. autoclass:: pystac_client.Client :members: @@ -20,9 +20,10 @@ conformance, and support for searching and paging through results. Collection Client ----------------- -Client is the a PySTAC-Client that inherits from :class:`Collection `. -In addition to the PySTAC functionality, CollectionClient allows opening of API URLs, and iterating -through items at a search endpoint, if supported. +Client is the a PySTAC-Client that inherits from +:class:`Collection `. In addition to the PySTAC functionality, +CollectionClient allows opening of API URLs, and iterating through items at a search +endpoint, if supported. .. autoclass:: pystac_client.CollectionClient :members: @@ -40,10 +41,10 @@ The `ItemSearch` class represents a search of a STAC API. STAC API IO ----------- -The StacApiIO class inherits from the :class:`Collection ` class and allows -for reading over http, such as with REST APIs. +The StacApiIO class inherits from the :class:`Collection ` +class and allows for reading over http, such as with REST APIs. -.. autoclass:: pystac_client.stac_api_io.StacApiIO +.. autoclass:: pystac_client.stac_api_io.StacApiIO :members: :undoc-members: :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index cd347013..5a165848 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,15 +26,15 @@ # -- Project information ----------------------------------------------------- -project = 'pystac-client' -copyright = '2021, Jon Duckworth' -author = 'Matthew Hanson, Jon Duckworth' -github_user = 'stac-utils' -github_repo = 'pystac-client' -package_description = 'A Python client for the STAC and STAC-API specs' +project = "pystac-client" +copyright = "2021, Jon Duckworth" +author = "Matthew Hanson, Jon Duckworth" +github_user = "stac-utils" +github_repo = "pystac-client" +package_description = "A Python client for the STAC and STAC-API specs" # The full version, including alpha/beta/rc tags -version = re.fullmatch(r'^(\d+\.\d+\.\d).*$', __version__).group(1) +version = re.fullmatch(r"^(\d+\.\d+\.\d).*$", __version__).group(1) release = __version__ @@ -44,8 +44,14 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', - 'sphinx.ext.extlinks', 'sphinxcontrib.fulltoc', 'nbsphinx', 'myst_parser' + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.extlinks", + "sphinxcontrib.fulltoc", + "nbsphinx", + "myst_parser", ] extlinks = { @@ -59,13 +65,13 @@ nbsphinx_allow_errors = True # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. source_suffix = [".rst", "*.md", "*.ipynb"] -exclude_patterns = ['build/*'] +exclude_patterns = ["build/*"] # -- Options for HTML output ------------------------------------------------- @@ -73,15 +79,15 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" html_theme_options = { # 'sidebar_collapse': False, - 'fixed_sidebar': True, - 'github_button': True, - 'github_user': github_user, - 'github_repo': github_repo, - 'description': package_description + "fixed_sidebar": True, + "github_button": True, + "github_user": github_user, + "github_repo": github_repo, + "description": package_description, } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -91,10 +97,10 @@ # -- Options for intersphinx extension --------------------------------------- intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None), - 'requests': ('https://requests.readthedocs.io/en/master', None), - 'pystac': ('https://pystac.readthedocs.io/en/latest', None), - 'dateutil': ('https://dateutil.readthedocs.io/en/stable/', None), + "python": ("https://docs.python.org/3", None), + "requests": ("https://requests.readthedocs.io/en/master", None), + "pystac": ("https://pystac.readthedocs.io/en/latest", None), + "dateutil": ("https://dateutil.readthedocs.io/en/stable/", None), } # -- Options for autodoc extension ------------------------------------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index dd14718c..fc4a3baf 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,8 +7,8 @@ way is to coordinate with the core developers via an issue or pull request conve Development installation ^^^^^^^^^^^^^^^^^^^^^^^^ -Fork PySTAC-Client into your GitHub account. Then, clone the repo and install it locally with -pip as follows: +Fork PySTAC-Client into your GitHub account. Then, clone the repo and install +it locally with pip as follows: .. code-block:: bash @@ -91,4 +91,4 @@ few steps: PR. For more information on changelogs and how to write a good entry, see `keep a changelog -`_. \ No newline at end of file +`_. diff --git a/docs/design/design_decisions.rst b/docs/design/design_decisions.rst index d3807ec1..5f464dbe 100644 --- a/docs/design/design_decisions.rst +++ b/docs/design/design_decisions.rst @@ -9,4 +9,4 @@ library. In general, this library makes an attempt to follow the design patterns :glob: :maxdepth: 1 - * \ No newline at end of file + * diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 98c5fe29..8daf209b 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -41,7 +41,7 @@ another process such as $ stac-client search ${STAC_API_URL} -c sentinel-s2-l2a-cogs --bbox -72.5 40.5 -72 41 --datetime 2020-01-01/2020-01-31 | stacterm cal --label platform .. figure:: images/stacterm-cal.png - :alt: + :alt: If the ``--save`` switch is provided instead, the results will not be output to stdout, but instead will be saved to the specified file. diff --git a/docs/tutorials.rst b/docs/tutorials.rst index a952c304..3be6a487 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -24,5 +24,5 @@ CQL Filtering - :tutorial:`GitHub version ` -This tutorial gives an introduction to using CQL-JSON filtering in searches to -search by arbitrary STAC Item properties. \ No newline at end of file +This tutorial gives an introduction to using CQL-JSON filtering in searches to +search by arbitrary STAC Item properties. diff --git a/docs/usage.rst b/docs/usage.rst index 3bbd8e93..a4ce4990 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -5,7 +5,7 @@ PySTAC-Client (pystac-client) builds upon `PySTAC `__ section. See the :mod:`Paging ` docs for details on how to customize this behavior. Query Extension ------------- +--------------- If the Catalog supports the `Query extension `__, diff --git a/pystac_client/__init__.py b/pystac_client/__init__.py index cc5ef3a0..f137af73 100644 --- a/pystac_client/__init__.py +++ b/pystac_client/__init__.py @@ -1,7 +1,7 @@ # flake8: noqa -from pystac_client.version import __version__ -from pystac_client.item_search import ItemSearch from pystac_client.client import Client from pystac_client.collection_client import CollectionClient from pystac_client.conformance import ConformanceClasses +from pystac_client.item_search import ItemSearch +from pystac_client.version import __version__ diff --git a/pystac_client/cli.py b/pystac_client/cli.py index f39c4961..8babb672 100644 --- a/pystac_client/cli.py +++ b/pystac_client/cli.py @@ -10,19 +10,19 @@ logger = logging.getLogger(__name__) -def search(client, method='GET', matched=False, save=None, **kwargs): - """ Main function for performing a search """ +def search(client, method="GET", matched=False, save=None, **kwargs): + """Main function for performing a search""" try: search = client.search(method=method, **kwargs) if matched: matched = search.matched() - print('%s items matched' % matched) + print("%s items matched" % matched) else: feature_collection = search.get_all_items_as_dict() if save: - with open(save, 'w') as f: + with open(save, "w") as f: f.write(json.dumps(feature_collection)) else: print(json.dumps(feature_collection)) @@ -33,12 +33,12 @@ def search(client, method='GET', matched=False, save=None, **kwargs): def collections(client, save=None, **kwargs): - """ Fetch collections from collections endpoint """ + """Fetch collections from collections endpoint""" try: collections = client.get_all_collections(**kwargs) collections_dicts = [c.to_dict() for c in collections] if save: - with open(save, 'w') as f: + with open(save, "w") as f: f.write(json.dumps(collections_dicts)) else: print(json.dumps(collections_dicts)) @@ -48,111 +48,143 @@ def collections(client, save=None, **kwargs): def parse_args(args): - desc = 'STAC Client' + desc = "STAC Client" dhf = argparse.ArgumentDefaultsHelpFormatter parser0 = argparse.ArgumentParser(description=desc) - parser0.add_argument('--version', - help='Print version and exit', - action='version', - version=__version__) + parser0.add_argument( + "--version", + help="Print version and exit", + action="version", + version=__version__, + ) parent = argparse.ArgumentParser(add_help=False) - parent.add_argument('url', help='Root Catalog URL') - parent.add_argument('--logging', default='INFO', help='DEBUG, INFO, WARN, ERROR, CRITICAL') - parent.add_argument('--headers', - nargs='*', - help='Additional request headers (KEY=VALUE pairs)', - default=None) - parent.add_argument('--ignore-conformance', - dest='ignore_conformance', - default=False, - action='store_true') - - subparsers = parser0.add_subparsers(dest='command') + parent.add_argument("url", help="Root Catalog URL") + parent.add_argument( + "--logging", default="INFO", help="DEBUG, INFO, WARN, ERROR, CRITICAL" + ) + parent.add_argument( + "--headers", + nargs="*", + help="Additional request headers (KEY=VALUE pairs)", + default=None, + ) + parent.add_argument( + "--ignore-conformance", + dest="ignore_conformance", + default=False, + action="store_true", + ) + + subparsers = parser0.add_subparsers(dest="command") # collections command - parser = subparsers.add_parser('collections', - help='Get all collections in this Catalog', - parents=[parent], - formatter_class=dhf) - output_group = parser.add_argument_group('output options') - output_group.add_argument('--save', help='Filename to save collections to', default=None) + parser = subparsers.add_parser( + "collections", + help="Get all collections in this Catalog", + parents=[parent], + formatter_class=dhf, + ) + output_group = parser.add_argument_group("output options") + output_group.add_argument( + "--save", help="Filename to save collections to", default=None + ) # search command - parser = subparsers.add_parser('search', - help='Perform new search of items', - parents=[parent], - formatter_class=dhf) - - search_group = parser.add_argument_group('search options') - search_group.add_argument('-c', '--collections', help='One or more collection IDs', nargs='*') - search_group.add_argument('--ids', - help='One or more Item IDs (ignores other parameters)', - nargs='*') - search_group.add_argument('--bbox', - help='Bounding box (min lon, min lat, max lon, max lat)', - nargs=4) - search_group.add_argument('--intersects', help='GeoJSON Feature or geometry (file or string)') - search_group.add_argument('--datetime', - help='Single date/time or begin and end date/time ' - '(e.g., 2017-01-01/2017-02-15)') - search_group.add_argument('-q', - '--query', - nargs='*', - help='Query properties of form ' - 'KEY=VALUE (<, >, <=, >=, = supported)') + parser = subparsers.add_parser( + "search", + help="Perform new search of items", + parents=[parent], + formatter_class=dhf, + ) + + search_group = parser.add_argument_group("search options") search_group.add_argument( - '--filter', help='Filter on queryables using language specified in filter-lang parameter') - search_group.add_argument('--filter-lang', - help='Filter language used within the filter parameter', - default="cql-json") - search_group.add_argument('--sortby', help='Sort by fields', nargs='*') - search_group.add_argument('--fields', help='Control what fields get returned', nargs='*') - search_group.add_argument('--limit', help='Page size limit', type=int, default=100) - search_group.add_argument('--max-items', - dest='max_items', - help='Max items to retrieve from search', - type=int) - search_group.add_argument('--method', help='GET or POST', default='POST') - - output_group = parser.add_argument_group('output options') - output_group.add_argument('--matched', - help='Print number matched', - default=False, - action='store_true') - output_group.add_argument('--save', help='Filename to save Item collection to', default=None) - - parsed_args = {k: v for k, v in vars(parser0.parse_args(args)).items() if v is not None} + "-c", "--collections", help="One or more collection IDs", nargs="*" + ) + search_group.add_argument( + "--ids", help="One or more Item IDs (ignores other parameters)", nargs="*" + ) + search_group.add_argument( + "--bbox", help="Bounding box (min lon, min lat, max lon, max lat)", nargs=4 + ) + search_group.add_argument( + "--intersects", help="GeoJSON Feature or geometry (file or string)" + ) + search_group.add_argument( + "--datetime", + help="Single date/time or begin and end date/time " + "(e.g., 2017-01-01/2017-02-15)", + ) + search_group.add_argument( + "-q", + "--query", + nargs="*", + help="Query properties of form " "KEY=VALUE (<, >, <=, >=, = supported)", + ) + search_group.add_argument( + "--filter", + help="Filter on queryables using language specified in filter-lang parameter", + ) + search_group.add_argument( + "--filter-lang", + help="Filter language used within the filter parameter", + default="cql-json", + ) + search_group.add_argument("--sortby", help="Sort by fields", nargs="*") + search_group.add_argument( + "--fields", help="Control what fields get returned", nargs="*" + ) + search_group.add_argument("--limit", help="Page size limit", type=int, default=100) + search_group.add_argument( + "--max-items", + dest="max_items", + help="Max items to retrieve from search", + type=int, + ) + search_group.add_argument("--method", help="GET or POST", default="POST") + + output_group = parser.add_argument_group("output options") + output_group.add_argument( + "--matched", help="Print number matched", default=False, action="store_true" + ) + output_group.add_argument( + "--save", help="Filename to save Item collection to", default=None + ) + + parsed_args = { + k: v for k, v in vars(parser0.parse_args(args)).items() if v is not None + } if "command" not in parsed_args: parser0.print_usage() return [] # if intersects is JSON file, read it in - if 'intersects' in parsed_args: - if os.path.exists(parsed_args['intersects']): - with open(parsed_args['intersects']) as f: + if "intersects" in parsed_args: + if os.path.exists(parsed_args["intersects"]): + with open(parsed_args["intersects"]) as f: data = json.loads(f.read()) - if data['type'] == 'Feature': - parsed_args['intersects'] = data['geometry'] - elif data['type'] == 'FeatureCollection': - parsed_args['intersects'] = data['features'][0]['geometry'] + if data["type"] == "Feature": + parsed_args["intersects"] = data["geometry"] + elif data["type"] == "FeatureCollection": + parsed_args["intersects"] = data["features"][0]["geometry"] else: - parsed_args['intersects'] = data + parsed_args["intersects"] = data # if headers provided, parse it - if 'headers' in parsed_args: + if "headers" in parsed_args: new_headers = {} - for head in parsed_args['headers']: - parts = head.split('=') + for head in parsed_args["headers"]: + parts = head.split("=") if len(parts) == 2: new_headers[parts[0]] = parts[1] else: logger.warning(f"Unable to parse header {head}") - parsed_args['headers'] = new_headers + parsed_args["headers"] = new_headers - if 'filter' in parsed_args: + if "filter" in parsed_args: if "json" in parsed_args["filter_lang"]: - parsed_args['filter'] = json.loads(parsed_args['filter']) + parsed_args["filter"] = json.loads(parsed_args["filter"]) return parsed_args @@ -162,26 +194,28 @@ def cli(): if not args: return None - loglevel = args.pop('logging') - if args.get('save', False) or args.get('matched', False): + loglevel = args.pop("logging") + if args.get("save", False) or args.get("matched", False): logging.basicConfig(level=loglevel) # quiet loggers logging.getLogger("urllib3").propagate = False - cmd = args.pop('command') + cmd = args.pop("command") try: - url = args.pop('url') - headers = args.pop('headers', {}) - ignore_conformance = args.pop('ignore_conformance') - client = Client.open(url, headers=headers, ignore_conformance=ignore_conformance) + url = args.pop("url") + headers = args.pop("headers", {}) + ignore_conformance = args.pop("ignore_conformance") + client = Client.open( + url, headers=headers, ignore_conformance=ignore_conformance + ) except Exception as e: print(e) return 1 - if cmd == 'search': + if cmd == "search": return search(client, **args) - elif cmd == 'collections': + elif cmd == "collections": return collections(client, **args) diff --git a/pystac_client/client.py b/pystac_client/client.py index 7ad321de..58e7d206 100644 --- a/pystac_client/client.py +++ b/pystac_client/client.py @@ -1,10 +1,10 @@ from functools import lru_cache -from typing import Any, Iterable, Dict, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional import pystac import pystac.validation -from pystac_client.collection_client import CollectionClient +from pystac_client.collection_client import CollectionClient from pystac_client.conformance import ConformanceClasses from pystac_client.exceptions import APIError from pystac_client.item_search import ItemSearch @@ -24,8 +24,9 @@ class Client(pystac.Catalog): APIs that have a ``"conformsTo"`` indicate that it supports additional functionality on top of a normal STAC Catalog, such as searching items (e.g., /search endpoint). """ + def __repr__(self): - return ''.format(self.id) + return "".format(self.id) @classmethod def open( @@ -52,8 +53,9 @@ def open( search_link = cat.get_search_link() # if there is a search link, but no conformsTo advertised, ignore conformance entirely # NOTE: this behavior to be deprecated as implementations become conformant - if ignore_conformance or ('conformsTo' not in cat.extra_fields.keys() - and len(search_link) > 0): + if ignore_conformance or ( + "conformsTo" not in cat.extra_fields.keys() and len(search_link) > 0 + ): cat._stac_io.set_conformance(None) return cat @@ -75,7 +77,7 @@ def from_file( cat = super().from_file(href, stac_io) - cat._stac_io._conformance = cat.extra_fields.get('conformsTo', []) + cat._stac_io._conformance = cat.extra_fields.get("conformsTo", []) return cat @@ -91,7 +93,9 @@ def get_collection(self, collection_id: str) -> CollectionClient: """ if self._stac_io.conforms_to(ConformanceClasses.COLLECTIONS): url = f"{self.get_self_href()}/collections/{collection_id}" - collection = CollectionClient.from_dict(self._stac_io.read_json(url), root=self) + collection = CollectionClient.from_dict( + self._stac_io.read_json(url), root=self + ) return collection else: for col in self.get_collections(): @@ -99,7 +103,7 @@ def get_collection(self, collection_id: str) -> CollectionClient: return col def get_collections(self) -> Iterable[CollectionClient]: - """ Get Collections in this Catalog + """Get Collections in this Catalog Gets the collections from the /collections endpoint if supported, otherwise fall back to Catalog behavior of following child links @@ -108,11 +112,11 @@ def get_collections(self) -> Iterable[CollectionClient]: Iterable[CollectionClient]: Iterator through Collections in Catalog/API """ if self._stac_io.conforms_to(ConformanceClasses.COLLECTIONS): - url = self.get_self_href() + '/collections' + url = self.get_self_href() + "/collections" for page in self._stac_io.get_pages(url): - if 'collections' not in page: + if "collections" not in page: raise APIError("Invalid response from /collections") - for col in page['collections']: + for col in page["collections"]: collection = CollectionClient.from_dict(col, root=self) yield collection else: @@ -171,14 +175,19 @@ def search(self, **kwargs: Any) -> ItemSearch: a ``"rel"`` type of ``"search"``. """ if not self._stac_io.conforms_to(ConformanceClasses.ITEM_SEARCH): - raise NotImplementedError("This catalog does not support search because it " - f"does not conform to \"{ConformanceClasses.ITEM_SEARCH}\"") + raise NotImplementedError( + "This catalog does not support search because it " + f'does not conform to "{ConformanceClasses.ITEM_SEARCH}"' + ) search_link = self.get_search_link() if search_link is None: raise NotImplementedError( - 'No link with "rel" type of "search" could be found in this catalog') + 'No link with "rel" type of "search" could be found in this catalog' + ) - return ItemSearch(search_link.target, stac_io=self._stac_io, client=self, **kwargs) + return ItemSearch( + search_link.target, stac_io=self._stac_io, client=self, **kwargs + ) def get_search_link(self) -> Optional[pystac.Link]: """Returns this client's search link. @@ -188,7 +197,15 @@ def get_search_link(self) -> Optional[pystac.Link]: Returns: Optional[pystac.Link]: The search link, or None if there is not one found. """ - return next((link for link in self.links - if link.rel == "search" and (link.media_type == pystac.MediaType.GEOJSON - or link.media_type == pystac.MediaType.JSON)), - None) + return next( + ( + link + for link in self.links + if link.rel == "search" + and ( + link.media_type == pystac.MediaType.GEOJSON + or link.media_type == pystac.MediaType.JSON + ) + ), + None, + ) diff --git a/pystac_client/collection_client.py b/pystac_client/collection_client.py index 991176a5..5aaeae09 100644 --- a/pystac_client/collection_client.py +++ b/pystac_client/collection_client.py @@ -1,6 +1,7 @@ -from typing import (Iterable, TYPE_CHECKING) +from typing import TYPE_CHECKING, Iterable import pystac + from pystac_client.item_search import ItemSearch if TYPE_CHECKING: @@ -9,7 +10,7 @@ class CollectionClient(pystac.Collection): def __repr__(self): - return ''.format(self.id) + return "".format(self.id) def get_items(self) -> Iterable["Item_Type"]: """Return all items in this Collection. @@ -21,9 +22,11 @@ def get_items(self) -> Iterable["Item_Type"]: Iterable[Item]: Generator of items whose parent is this catalog. """ - link = self.get_single_link('items') + link = self.get_single_link("items") if link is not None: - search = ItemSearch(link.href, method='GET', stac_io=self.get_root()._stac_io) + search = ItemSearch( + link.href, method="GET", stac_io=self.get_root()._stac_io + ) yield from search.get_items() else: yield from super().get_items() diff --git a/pystac_client/conformance.py b/pystac_client/conformance.py index 040cbada..3a668eca 100644 --- a/pystac_client/conformance.py +++ b/pystac_client/conformance.py @@ -1,23 +1,23 @@ -from enum import Enum import re +from enum import Enum class ConformanceClasses(Enum): - """Enumeration class for Conformance Classes - - """ + """Enumeration class for Conformance Classes""" stac_prefix = re.escape("https://api.stacspec.org/v1.0.") # defined conformance classes regexes - CORE = fr"{stac_prefix}(.*){re.escape('/core')}" - ITEM_SEARCH = fr"{stac_prefix}(.*){re.escape('/item-search')}" - CONTEXT = fr"{stac_prefix}(.*){re.escape('/item-search#context')}" - FIELDS = fr"{stac_prefix}(.*){re.escape('/item-search#fields')}" - SORT = fr"{stac_prefix}(.*){re.escape('/item-search#sort')}" - QUERY = fr"{stac_prefix}(.*){re.escape('/item-search#query')}" - FILTER = fr"{stac_prefix}(.*){re.escape('/item-search#filter')}" - COLLECTIONS = re.escape("http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30") + CORE = rf"{stac_prefix}(.*){re.escape('/core')}" + ITEM_SEARCH = rf"{stac_prefix}(.*){re.escape('/item-search')}" + CONTEXT = rf"{stac_prefix}(.*){re.escape('/item-search#context')}" + FIELDS = rf"{stac_prefix}(.*){re.escape('/item-search#fields')}" + SORT = rf"{stac_prefix}(.*){re.escape('/item-search#sort')}" + QUERY = rf"{stac_prefix}(.*){re.escape('/item-search#query')}" + FILTER = rf"{stac_prefix}(.*){re.escape('/item-search#filter')}" + COLLECTIONS = re.escape( + "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30" + ) CONFORMANCE_URIS = {c.name: c.value for c in ConformanceClasses} diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 16fbda0d..8f201c5a 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -1,26 +1,29 @@ -from functools import lru_cache -from dateutil.tz import tzutc -from dateutil.relativedelta import relativedelta import json import re +import warnings from collections.abc import Iterable, Mapping from copy import deepcopy -from datetime import timezone, datetime as datetime_ -from typing import Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Union -import warnings +from datetime import datetime as datetime_ +from datetime import timezone +from functools import lru_cache +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union +from dateutil.relativedelta import relativedelta +from dateutil.tz import tzutc from pystac import Collection, Item, ItemCollection from pystac.stac_io import StacIO -from pystac_client.stac_api_io import StacApiIO from pystac_client.conformance import ConformanceClasses +from pystac_client.stac_api_io import StacApiIO if TYPE_CHECKING: from pystac_client.client import Client -DATETIME_REGEX = re.compile(r"(?P\d{4})(\-(?P\d{2})(\-(?P\d{2})" - r"(?P(T|t)\d{2}:\d{2}:\d{2}(\.\d+)?" - r"(?PZ|([-+])(\d{2}):(\d{2}))?)?)?)?") +DATETIME_REGEX = re.compile( + r"(?P\d{4})(\-(?P\d{2})(\-(?P\d{2})" + r"(?P(T|t)\d{2}:\d{2}:\d{2}(\.\d+)?" + r"(?PZ|([-+])(\d{2}):(\d{2}))?)?)?)?" +) # todo: add runtime_checkable when we drop 3.7 support # class GeoInterface(Protocol): @@ -29,8 +32,12 @@ DatetimeOrTimestamp = Optional[Union[datetime_, str]] Datetime = Union[Tuple[str], Tuple[str, str]] -DatetimeLike = Union[DatetimeOrTimestamp, Tuple[DatetimeOrTimestamp, DatetimeOrTimestamp], - List[DatetimeOrTimestamp], Iterator[DatetimeOrTimestamp]] +DatetimeLike = Union[ + DatetimeOrTimestamp, + Tuple[DatetimeOrTimestamp, DatetimeOrTimestamp], + List[DatetimeOrTimestamp], + Iterator[DatetimeOrTimestamp], +] BBox = Tuple[float, ...] BBoxLike = Union[BBox, List[float], Iterator[float], str] @@ -57,12 +64,12 @@ Fields = List[str] FieldsLike = Union[Fields, str] -OP_MAP = {'>=': 'gte', '<=': 'lte', '=': 'eq', '>': 'gt', '<': 'lt'} +OP_MAP = {">=": "gte", "<=": "lte", "=": "eq", ">": "gt", "<": "lt"} # from https://gist.github.com/angstwad/bf22d1822c38a92ec0a9#gistcomment-2622319 def dict_merge(dct: Dict, merge_dct: Dict, add_keys: bool = True) -> Dict: - """ Recursive dict merge. + """Recursive dict merge. Inspired by :meth:``dict.update()``, instead of updating only top-level keys, dict_merge recurses down into dicts nested @@ -84,7 +91,7 @@ def dict_merge(dct: Dict, merge_dct: Dict, add_keys: bool = True) -> Dict: merge_dct = {k: merge_dct[k] for k in set(dct).intersection(set(merge_dct))} for k, v in merge_dct.items(): - if (k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], Mapping)): + if k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], Mapping): dct[k] = dict_merge(dct[k], merge_dct[k], add_keys=add_keys) else: dct[k] = merge_dct[k] @@ -141,40 +148,48 @@ class ItemSearch: intersects: A string or dictionary representing a GeoJSON geometry, or an object that implements a ``__geo_interface__`` property as supported by several libraries including Shapely, ArcPy, PySAL, and geojson. Results filtered to only those intersecting the geometry. - ids: List of Item ids to return. All other filter parameters that further restrict the number of search results + ids: List of Item ids to return. All other filter parameters that further + restrict the number of search results (except ``limit``) are ignored. collections: List of one or more Collection IDs or :class:`pystac.Collection` instances. Only Items in one of the provided Collections will be searched query: List or JSON of query parameters as per the STAC API `query` extension filter: JSON of query parameters as per the STAC API `filter` extension - filter_lang: Language variant used in the filter body. If `filter` is a dictionary or not provided, defaults + filter_lang: Language variant used in the filter body. If `filter` is a + dictionary or not provided, defaults to 'cql2-json'. If `filter` is a string, defaults to `cql2-text`. sortby: A single field or list of fields to sort the response by - fields: A list of fields to include in the response. Note this may result in invalid STAC objects, as they - may not have required fields. Use `get_all_items_as_dict` to avoid object unmarshalling errors. - max_items: The maximum number of items to get, even if there are more matched items. + fields: A list of fields to include in the response. Note this may + result in invalid STAC objects, as they may not have required fields. + Use `get_all_items_as_dict` to avoid object unmarshalling errors. + max_items: The maximum number of items to get, even if there are more + matched items. method: The http method, 'GET' or 'POST' - stac_io: An instance of StacIO for retrieving results. Normally comes from the Client that returns this ItemSearch - client: An instance of a root Client used to set the root on resulting Items + stac_io: An instance of StacIO for retrieving results. Normally comes + from the Client that returns this ItemSearch client: An instance of a + root Client used to set the root on resulting Items """ - def __init__(self, - url: str, - *, - limit: Optional[int] = 100, - bbox: Optional[BBoxLike] = None, - datetime: Optional[DatetimeLike] = None, - intersects: Optional[IntersectsLike] = None, - ids: Optional[IDsLike] = None, - collections: Optional[CollectionsLike] = None, - query: Optional[QueryLike] = None, - filter: Optional[FilterLike] = None, - filter_lang: Optional[FilterLangLike] = None, - sortby: Optional[SortbyLike] = None, - fields: Optional[FieldsLike] = None, - max_items: Optional[int] = None, - method: Optional[str] = 'POST', - stac_io: Optional[StacIO] = None, - client: Optional["Client"] = None): + + def __init__( + self, + url: str, + *, + limit: Optional[int] = 100, + bbox: Optional[BBoxLike] = None, + datetime: Optional[DatetimeLike] = None, + intersects: Optional[IntersectsLike] = None, + ids: Optional[IDsLike] = None, + collections: Optional[CollectionsLike] = None, + query: Optional[QueryLike] = None, + filter: Optional[FilterLike] = None, + filter_lang: Optional[FilterLangLike] = None, + sortby: Optional[SortbyLike] = None, + fields: Optional[FieldsLike] = None, + max_items: Optional[int] = None, + method: Optional[str] = "POST", + stac_io: Optional[StacIO] = None, + client: Optional["Client"] = None, + ): self.url = url self.client = client @@ -194,49 +209,49 @@ def __init__(self, self.method = method params = { - 'limit': limit, - 'bbox': self._format_bbox(bbox), - 'datetime': self._format_datetime(datetime), - 'ids': self._format_ids(ids), - 'collections': self._format_collections(collections), - 'intersects': self._format_intersects(intersects), - 'query': self._format_query(query), - 'filter': self._format_filter(filter), - 'filter-lang': self._format_filter_lang(filter, filter_lang), - 'sortby': self._format_sortby(sortby), - 'fields': self._format_fields(fields) + "limit": limit, + "bbox": self._format_bbox(bbox), + "datetime": self._format_datetime(datetime), + "ids": self._format_ids(ids), + "collections": self._format_collections(collections), + "intersects": self._format_intersects(intersects), + "query": self._format_query(query), + "filter": self._format_filter(filter), + "filter-lang": self._format_filter_lang(filter, filter_lang), + "sortby": self._format_sortby(sortby), + "fields": self._format_fields(fields), } self._parameters = {k: v for k, v in params.items() if v is not None} - def get_parameters(self): - if self.method == 'POST': + def get_parameters(self) -> Dict[str, Any]: + if self.method == "POST": return self._parameters - elif self.method == 'GET': + elif self.method == "GET": params = deepcopy(self._parameters) - if 'bbox' in params: - params['bbox'] = ','.join(map(str, params['bbox'])) - if 'ids' in params: - params['ids'] = ','.join(params['ids']) - if 'collections' in params: - params['collections'] = ','.join(params['collections']) - if 'intersects' in params: - params['intersects'] = json.dumps(params['intersects']) - if 'sortby' in params: - params['sortby'] = self.sortby_json_to_str(params['sortby']) + if "bbox" in params: + params["bbox"] = ",".join(map(str, params["bbox"])) + if "ids" in params: + params["ids"] = ",".join(params["ids"]) + if "collections" in params: + params["collections"] = ",".join(params["collections"]) + if "intersects" in params: + params["intersects"] = json.dumps(params["intersects"]) + if "sortby" in params: + params["sortby"] = self.sortby_json_to_str(params["sortby"]) return params else: raise Exception(f"Unsupported method {self.method}") @staticmethod - def _format_query(value: List[QueryLike]) -> Optional[dict]: + def _format_query(value: List[QueryLike]) -> Optional[Dict[str, Any]]: if value is None: return None if isinstance(value, list): query = {} for q in value: - for op in ['>=', '<=', '=', '>', '<']: + for op in [">=", "<=", "=", ">", "<"]: parts = q.split(op) if len(parts) == 2: param = parts[0] @@ -251,7 +266,9 @@ def _format_query(value: List[QueryLike]) -> Optional[dict]: return query @staticmethod - def _format_filter_lang(_filter: FilterLike, value: FilterLangLike) -> Optional[str]: + def _format_filter_lang( + _filter: FilterLike, value: FilterLangLike + ) -> Optional[str]: if _filter is None: return None @@ -259,10 +276,10 @@ def _format_filter_lang(_filter: FilterLike, value: FilterLangLike) -> Optional[ return value if isinstance(_filter, str): - return 'cql2-text' + return "cql2-text" if isinstance(_filter, dict): - return 'cql2-json' + return "cql2-json" return None @@ -279,7 +296,7 @@ def _format_bbox(value: Optional[BBoxLike]) -> Optional[BBox]: return None if isinstance(value, str): - bbox = tuple(map(float, value.split(','))) + bbox = tuple(map(float, value.split(","))) else: bbox = tuple(map(float, value)) @@ -295,11 +312,14 @@ def _to_utc_isoformat(dt): def _to_isoformat_range(component: DatetimeOrTimestamp): """Converts a single DatetimeOrTimestamp into one or two Datetimes. - This is required to expand a single value like "2017" out to the whole year. This function returns two values. - The first value is always a valid Datetime. The second value can be None or a Datetime. If it is None, this - means that the first value was an exactly specified value (e.g. a `datetime.datetime`). If the second value is - a Datetime, then it will be the end of the range at the resolution of the component, e.g. if the component - were "2017" the second value would be the last second of the last day of 2017. + This is required to expand a single value like "2017" out to the whole + year. This function returns two values. The first value is always a + valid Datetime. The second value can be None or a Datetime. If it is + None, this means that the first value was an exactly specified value + (e.g. a `datetime.datetime`). If the second value is a Datetime, then + it will be the end of the range at the resolution of the component, + e.g. if the component were "2017" the second value would be the last + second of the last day of 2017. """ if component is None: return "..", None @@ -321,16 +341,20 @@ def _to_isoformat_range(component: DatetimeOrTimestamp): optional_day = match.group("day") if optional_day is not None: - start = datetime_(year, - int(optional_month), - int(optional_day), - 0, - 0, - 0, - tzinfo=tzutc()) + start = datetime_( + year, + int(optional_month), + int(optional_day), + 0, + 0, + 0, + tzinfo=tzutc(), + ) end = start + relativedelta(days=1, seconds=-1) elif optional_month is not None: - start = datetime_(year, int(optional_month), 1, 0, 0, 0, tzinfo=tzutc()) + start = datetime_( + year, int(optional_month), 1, 0, 0, 0, tzinfo=tzutc() + ) end = start + relativedelta(months=1, seconds=-1) else: start = datetime_(year, 1, 1, 0, 0, 0, tzinfo=tzutc()) @@ -362,7 +386,9 @@ def _to_isoformat_range(component: DatetimeOrTimestamp): return f"{start}/{end or backup_end}" else: raise Exception( - f"too many datetime components (max=2, actual={len(components)}): {value}") + "too many datetime components " + f"(max=2, actual={len(components)}): {value}" + ) @staticmethod def _format_collections(value: Optional[CollectionsLike]) -> Optional[Collections]: @@ -377,9 +403,9 @@ def _format(c): if value is None: return None if isinstance(value, str): - return tuple(map(_format, value.split(','))) + return tuple(map(_format, value.split(","))) if isinstance(value, Collection): - return _format(value), + return (_format(value),) return _format(value) @@ -389,7 +415,7 @@ def _format_ids(value: Optional[IDsLike]) -> Optional[IDs]: return None if isinstance(value, str): - return tuple(value.split(',')) + return tuple(value.split(",")) return tuple(value) @@ -400,7 +426,7 @@ def _format_sortby(self, value: Optional[SortbyLike]) -> Optional[Sortby]: self._stac_io.assert_conforms_to(ConformanceClasses.SORT) if isinstance(value, str): - return [self.sortby_part_to_json(part) for part in value.split(',')] + return [self.sortby_part_to_json(part) for part in value.split(",")] if isinstance(value, list): if value and isinstance(value[0], str): @@ -408,7 +434,9 @@ def _format_sortby(self, value: Optional[SortbyLike]) -> Optional[Sortby]: elif value and isinstance(value[0], dict): return value - raise Exception("sortby must be of type None, str, List[str], or List[Dict[str, str]") + raise Exception( + "sortby must be of type None, str, List[str], or List[Dict[str, str]" + ) @staticmethod def sortby_part_to_json(part: str) -> Dict[str, str]: @@ -422,7 +450,11 @@ def sortby_part_to_json(part: str) -> Dict[str, str]: @staticmethod def sortby_json_to_str(sortby: Sortby) -> str: return ",".join( - [f"{'+' if sort['direction'] == 'asc' else '-'}{sort['field']}" for sort in sortby]) + [ + f"{'+' if sort['direction'] == 'asc' else '-'}{sort['field']}" + for sort in sortby + ] + ) def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]: if value is None: @@ -431,7 +463,7 @@ def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]: self._stac_io.assert_conforms_to(ConformanceClasses.FIELDS) if isinstance(value, str): - return tuple(value.split(',')) + return tuple(value.split(",")) return tuple(value) @@ -443,47 +475,53 @@ def _format_intersects(value: Optional[IntersectsLike]) -> Optional[Intersects]: return deepcopy(value) if isinstance(value, str): return json.loads(value) - if hasattr(value, '__geo_interface__'): - return deepcopy(getattr(value, '__geo_interface__')) + if hasattr(value, "__geo_interface__"): + return deepcopy(getattr(value, "__geo_interface__")) raise Exception( - "intersects must be of type None, str, dict, or an object that implements __geo_interface__" + "intersects must be of type None, str, dict, or an object that " + "implements __geo_interface__" ) @lru_cache(1) def matched(self) -> int: """Return number matched for search - Returns the value from the `numberMatched` or `context.matched` field. Not all APIs - will support counts in which case a warning will be issued + Returns the value from the `numberMatched` or `context.matched` field. + Not all APIs will support counts in which case a warning will be issued Returns: - int: Total count of matched items. If counts are not supported `None` is returned. + int: Total count of matched items. If counts are not supported `None` + is returned. """ params = {**self.get_parameters(), "limit": 1} resp = self._stac_io.read_json(self.url, method=self.method, parameters=params) found = None - if 'context' in resp: - found = resp['context']['matched'] - elif 'numberMatched' in resp: - found = resp['numberMatched'] + if "context" in resp: + found = resp["context"]["matched"] + elif "numberMatched" in resp: + found = resp["numberMatched"] if found is None: warnings.warn("numberMatched or context.matched not in response") return found def get_item_collections(self) -> Iterator[ItemCollection]: - """Iterator that yields ItemCollection objects. Each ItemCollection is a page of results - from the search. + """Iterator that yields ItemCollection objects. Each ItemCollection is + a page of results from the search. Yields: Iterable[Item] : pystac_client.ItemCollection """ - for page in self._stac_io.get_pages(self.url, self.method, self.get_parameters()): + for page in self._stac_io.get_pages( + self.url, self.method, self.get_parameters() + ): yield ItemCollection.from_dict(page, preserve_dict=False, root=self.client) def get_items(self) -> Iterator[Item]: - """Iterator that yields :class:`pystac.Item` instances for each item matching the given search parameters. Calls + """Iterator that yields :class:`pystac.Item` instances for each item matching + the given search parameters. Calls :meth:`ItemSearch.item_collections()` internally and yields from - :attr:`ItemCollection.features ` for each page of results. + :attr:`ItemCollection.features ` for + each page of results. Return: Iterable[Item] : Iterate through resulting Items @@ -505,8 +543,10 @@ def get_all_items_as_dict(self) -> Dict: Dict : A GeoJSON FeatureCollection """ features = [] - for page in self._stac_io.get_pages(self.url, self.method, self.get_parameters()): - for feature in page['features']: + for page in self._stac_io.get_pages( + self.url, self.method, self.get_parameters() + ): + for feature in page["features"]: features.append(feature) if self._max_items and len(features) >= self._max_items: return {"type": "FeatureCollection", "features": features} @@ -514,10 +554,13 @@ def get_all_items_as_dict(self) -> Dict: @lru_cache(1) def get_all_items(self) -> ItemCollection: - """Convenience method that builds an :class:`ItemCollection` from all items matching the given search parameters. + """Convenience method that builds an :class:`ItemCollection` from all items + matching the given search parameters. Return: item_collection : ItemCollection """ feature_collection = self.get_all_items_as_dict() - return ItemCollection.from_dict(feature_collection, preserve_dict=False, root=self.client) + return ItemCollection.from_dict( + feature_collection, preserve_dict=False, root=self.client + ) diff --git a/pystac_client/stac_api_io.py b/pystac_client/stac_api_io.py index 1c3ff2bb..0e49bea1 100644 --- a/pystac_client/stac_api_io.py +++ b/pystac_client/stac_api_io.py @@ -1,36 +1,29 @@ -from copy import deepcopy import json import logging -from typing import ( - Any, - Dict, - Iterator, - List, - Optional, - TYPE_CHECKING, - Union, -) -from urllib.parse import urlparse import re -from requests import Request, Session +from copy import deepcopy +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union +from urllib.parse import urlparse import pystac from pystac.link import Link from pystac.serialization import ( - merge_common_properties, - identify_stac_object_type, identify_stac_object, + identify_stac_object_type, + merge_common_properties, migrate_to_latest, ) from pystac.stac_io import DefaultStacIO +from requests import Request, Session import pystac_client + +from .conformance import CONFORMANCE_URIS, ConformanceClasses from .exceptions import APIError -from .conformance import ConformanceClasses, CONFORMANCE_URIS if TYPE_CHECKING: - from pystac.stac_object import STACObject as STACObject_Type from pystac.catalog import Catalog as Catalog_Type + from pystac.stac_object import STACObject as STACObject_Type logger = logging.getLogger(__name__) @@ -60,11 +53,13 @@ def __init__( self._conformance = conformance - def read_text(self, - source: Union[str, Link], - *args: Any, - parameters: Optional[dict] = {}, - **kwargs: Any) -> str: + def read_text( + self, + source: Union[str, Link], + *args: Any, + parameters: Optional[dict] = {}, + **kwargs: Any, + ) -> str: """Read text from the given URI. Overwrites the default method for reading text from a URL or file to allow :class:`urllib.request.Request` @@ -81,29 +76,33 @@ def read_text(self, return href_contents elif isinstance(source, Link): link = source.to_dict() - href = link['href'] + href = link["href"] # get headers and body from Link and add to request from simple stac resolver - merge = bool(link.get('merge', False)) + merge = bool(link.get("merge", False)) # If the link object includes a "method" property, use that. If not fall back to 'GET'. - method = link.get('method', 'GET') + method = link.get("method", "GET") # If the link object includes a "headers" property, use that and respect the "merge" property. - headers = link.get('headers', None) + headers = link.get("headers", None) # If "POST" use the body object that and respect the "merge" property. - link_body = link.get('body', {}) - if method == 'POST': + link_body = link.get("body", {}) + if method == "POST": parameters = {**parameters, **link_body} if merge else link_body else: # parameters are already in the link href parameters = {} - return self.request(href, *args, method=method, headers=headers, parameters=parameters) + return self.request( + href, *args, method=method, headers=headers, parameters=parameters + ) - def request(self, - href: str, - method: Optional[str] = 'GET', - headers: Optional[dict] = {}, - parameters: Optional[dict] = {}) -> str: + def request( + self, + href: str, + method: Optional[str] = "GET", + headers: Optional[dict] = {}, + parameters: Optional[dict] = {}, + ) -> str: """Makes a request to an http endpoint Args: @@ -118,17 +117,17 @@ def request(self, Return: str: The decoded response from the endpoint """ - if method == 'POST': + if method == "POST": request = Request(method=method, url=href, headers=headers, json=parameters) else: params = deepcopy(parameters) - if 'intersects' in params: - params['intersects'] = json.dumps(params['intersects']) + if "intersects" in params: + params["intersects"] = json.dumps(params["intersects"]) request = Request(method=method, url=href, headers=headers, params=params) try: prepped = self.session.prepare_request(request) msg = f"{prepped.method} {prepped.url} Headers: {prepped.headers}" - if method == 'POST': + if method == "POST": msg += f" Payload: {json.dumps(request.json)}" logger.debug(msg) resp = self.session.send(prepped) @@ -170,37 +169,33 @@ def stac_object_from_dict( collection_cache = root._resolved_objects.as_collection_cache() # Merge common properties in case this is an older STAC object. - merge_common_properties(d, json_href=href, collection_cache=collection_cache) + merge_common_properties( + d, json_href=href, collection_cache=collection_cache + ) info = identify_stac_object(d) d = migrate_to_latest(d, info) if info.object_type == pystac.STACObjectType.CATALOG: - result = pystac_client.Client.from_dict(d, - href=href, - root=root, - migrate=False, - preserve_dict=preserve_dict) + result = pystac_client.Client.from_dict( + d, href=href, root=root, migrate=False, preserve_dict=preserve_dict + ) result._stac_io = self return result if info.object_type == pystac.STACObjectType.COLLECTION: - return pystac_client.CollectionClient.from_dict(d, - href=href, - root=root, - migrate=False, - preserve_dict=preserve_dict) + return pystac_client.CollectionClient.from_dict( + d, href=href, root=root, migrate=False, preserve_dict=preserve_dict + ) if info.object_type == pystac.STACObjectType.ITEM: - return pystac.Item.from_dict(d, - href=href, - root=root, - migrate=False, - preserve_dict=preserve_dict) + return pystac.Item.from_dict( + d, href=href, root=root, migrate=False, preserve_dict=preserve_dict + ) raise ValueError(f"Unknown STAC object type {info.object_type}") - def get_pages(self, url, method='GET', parameters={}) -> Iterator[Dict]: + def get_pages(self, url, method="GET", parameters={}) -> Iterator[Dict]: """Iterator that yields dictionaries for each page at a STAC paging endpoint, e.g., /collections, /search Return: @@ -209,15 +204,18 @@ def get_pages(self, url, method='GET', parameters={}) -> Iterator[Dict]: page = self.read_json(url, method=method, parameters=parameters) yield page - next_link = next((link for link in page.get('links', []) if link['rel'] == 'next'), None) + next_link = next( + (link for link in page.get("links", []) if link["rel"] == "next"), None + ) while next_link: link = Link.from_dict(next_link) page = self.read_json(link, parameters=parameters) yield page # get the next link and make the next request - next_link = next((link for link in page.get('links', []) if link['rel'] == 'next'), - None) + next_link = next( + (link for link in page.get("links", []) if link["rel"] == "next"), None + ) def assert_conforms_to(self, conformance_class: ConformanceClasses) -> None: """Raises a :exc:`NotImplementedError` if the API does not publish the given conformance class. This method diff --git a/pystac_client/version.py b/pystac_client/version.py index bfeb9e74..334b8995 100644 --- a/pystac_client/version.py +++ b/pystac_client/version.py @@ -1 +1 @@ -__version__ = '0.3.4' +__version__ = "0.3.4" diff --git a/requirements-dev.txt b/requirements-dev.txt index 90192ec8..c9a3d1f7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,10 +16,6 @@ jsonschema~=4.5.1 coverage~=6.3.2 doc8~=0.11.1 -types-python-dateutil~=2.8.15 -types-orjson~=3.6.2 -types-requests~=2.27.29 - pre-commit==2.19.0 # optional dependencies diff --git a/scripts/format b/scripts/format index 8ad178f9..26a03c60 100755 --- a/scripts/format +++ b/scripts/format @@ -9,7 +9,7 @@ fi function usage() { echo -n \ "Usage: $(basename "$0") -Format code with yapf +Format code with black " } @@ -17,6 +17,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then if [ "${1:-}" = "--help" ]; then usage else - yapf -ipr pystac_client tests + pre-commit run black --all-files + pre-commit run isort --all-files fi -fi +fi \ No newline at end of file diff --git a/scripts/lint b/scripts/lint new file mode 100644 index 00000000..e1b762d3 --- /dev/null +++ b/scripts/lint @@ -0,0 +1,28 @@ + + +#!/bin/bash + +set -e + +if [[ -n "${CI}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Execute project linters. +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + if [ "${1:-}" = "--help" ]; then + usage + else + pre-commit run codespell --all-files + pre-commit run doc8 --all-files + pre-commit run flake8 --all-files + pre-commit run isort --all-files + pre-commit run mypy --all-files + fi +fi \ No newline at end of file diff --git a/scripts/test b/scripts/test index a87fbb90..c089dada 100755 --- a/scripts/test +++ b/scripts/test @@ -17,22 +17,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then if [ "${1:-}" = "--help" ]; then usage else - # Lint - flake8 pystac_client tests - - # Code formatting - yapf -dpr pystac_client tests - - # Code spelling - codespell -I .codespellignore -f \ - pystac_client/*.py pystac_client/**/*.py \ - tests/*.py tests/**/*.py \ - docs/*.rst docs/**/*.rst \ - docs/*.ipynb docs/**/*.ipynb \ - scripts/* \ - *.py \ - *.md - + ./scripts/lint + ./scripts/format # Test suite with coverage enabled pytest -s --block-network --cov pystac_client --cov-report term-missing coverage xml diff --git a/setup.py b/setup.py index 8eec9983..74bd3a76 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ import os -from imp import load_source -from setuptools import setup, find_packages from glob import glob +from imp import load_source + +from setuptools import find_packages, setup __version__ = load_source( "pystac_client.version", "pystac_client/version.py" diff --git a/tests/helpers.py b/tests/helpers.py index 70c8b8bc..ac250511 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,7 +1,7 @@ import json from pathlib import Path -TEST_DATA = Path(__file__).parent / 'data' +TEST_DATA = Path(__file__).parent / "data" STAC_URLS = { "PLANETARY-COMPUTER": "https://planetarycomputer.microsoft.com/api/stac/v1", @@ -10,7 +10,7 @@ } -def read_data_file(file_name: str, mode='r', parse_json=False): +def read_data_file(file_name: str, mode="r", parse_json=False): file_path = TEST_DATA / file_name with file_path.open(mode=mode) as src: if parse_json: diff --git a/tests/test_cli.py b/tests/test_cli.py index 5d5d652b..9e867810 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,8 +8,13 @@ class TestCLI: @pytest.mark.vcr def test_item_search(self, script_runner: ScriptRunner): args = [ - "stac-client", "search", STAC_URLS['PLANETARY-COMPUTER'], "-c", "naip", "--max-items", - "20" + "stac-client", + "search", + STAC_URLS["PLANETARY-COMPUTER"], + "-c", + "naip", + "--max-items", + "20", ] result = script_runner.run(*args, print_result=False) assert result.success diff --git a/tests/test_client.py b/tests/test_client.py index 79d88c95..90d22a12 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2,11 +2,11 @@ import os.path from datetime import datetime from tempfile import TemporaryDirectory -from urllib.parse import urlsplit, parse_qs +from urllib.parse import parse_qs, urlsplit -from dateutil.tz import tzutc import pystac import pytest +from dateutil.tz import tzutc from pystac import MediaType from pystac_client import Client @@ -18,33 +18,35 @@ class TestAPI: @pytest.mark.vcr def test_instance(self): - api = Client.open(STAC_URLS['PLANETARY-COMPUTER']) + api = Client.open(STAC_URLS["PLANETARY-COMPUTER"]) # An API instance is also a Catalog instance assert isinstance(api, pystac.Catalog) - assert str(api) == '' + assert str(api) == "" @pytest.mark.vcr def test_links(self): - api = Client.open(STAC_URLS['PLANETARY-COMPUTER']) + api = Client.open(STAC_URLS["PLANETARY-COMPUTER"]) # Should be able to get collections via links as with a typical PySTAC Catalog - collection_links = api.get_links('child') + collection_links = api.get_links("child") assert len(collection_links) > 0 collections = list(api.get_collections()) assert len(collection_links) == len(collections) - first_collection = api.get_single_link('child').resolve_stac_object(root=api).target + first_collection = ( + api.get_single_link("child").resolve_stac_object(root=api).target + ) assert isinstance(first_collection, pystac.Collection) def test_spec_conformance(self): """Testing conformance against a ConformanceClass should allow APIs using legacy URIs to pass.""" - client = Client.from_file(str(TEST_DATA / 'planetary-computer-root.json')) + client = Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) # Set conformsTo URIs to conform with STAC API - Core using official URI - client._stac_io._conformance = ['https://api.stacspec.org/v1.0.0-beta.1/core'] + client._stac_io._conformance = ["https://api.stacspec.org/v1.0.0-beta.1/core"] assert client._stac_io.conforms_to(ConformanceClasses.CORE) @@ -52,7 +54,7 @@ def test_spec_conformance(self): def test_no_conformance(self): """Should raise a NotImplementedError if no conformance info can be found. Luckily, the test API doesn't publish a "conformance" link so we can just remove the "conformsTo" attribute to test this.""" - client = Client.from_file(str(TEST_DATA / 'planetary-computer-root.json')) + client = Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) client._stac_io._conformance = [] with pytest.raises(NotImplementedError): @@ -64,7 +66,7 @@ def test_no_conformance(self): @pytest.mark.vcr def test_no_stac_core_conformance(self): """Should raise a NotImplementedError if the API does not conform to the STAC API - Core spec.""" - client = Client.from_file(str(TEST_DATA / 'planetary-computer-root.json')) + client = Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) client._stac_io._conformance = client._stac_io._conformance[1:] with pytest.raises(NotImplementedError): @@ -74,9 +76,9 @@ def test_no_stac_core_conformance(self): @pytest.mark.vcr def test_from_file(self): - api = Client.from_file(STAC_URLS['PLANETARY-COMPUTER']) + api = Client.from_file(STAC_URLS["PLANETARY-COMPUTER"]) - assert api.title == 'Microsoft Planetary Computer STAC API' + assert api.title == "Microsoft Planetary Computer STAC API" def test_invalid_url(self): with pytest.raises(TypeError): @@ -85,23 +87,25 @@ def test_invalid_url(self): def test_get_collections_with_conformance(self, requests_mock): """Checks that the "data" endpoint is used if the API published the collections conformance class.""" pc_root_text = read_data_file("planetary-computer-root.json") - pc_collection_dict = read_data_file("planetary-computer-aster-l1t-collection.json", - parse_json=True) + pc_collection_dict = read_data_file( + "planetary-computer-aster-l1t-collection.json", parse_json=True + ) # Mock the root catalog - requests_mock.get(STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text) + requests_mock.get( + STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text + ) api = Client.open(STAC_URLS["PLANETARY-COMPUTER"]) assert api._stac_io.conforms_to(ConformanceClasses.COLLECTIONS) # Get & mock the collections (rel type "data") link collections_link = api.get_single_link("data") - requests_mock.get(collections_link.href, - status_code=200, - json={ - "collections": [pc_collection_dict], - "links": [] - }) + requests_mock.get( + collections_link.href, + status_code=200, + json={"collections": [pc_collection_dict], "links": []}, + ) _ = next(api.get_collections()) history = requests_mock.request_history @@ -110,14 +114,20 @@ def test_get_collections_with_conformance(self, requests_mock): def test_custom_request_parameters(self, requests_mock): pc_root_text = read_data_file("planetary-computer-root.json") - pc_collection_dict = read_data_file("planetary-computer-collection.json", parse_json=True) + pc_collection_dict = read_data_file( + "planetary-computer-collection.json", parse_json=True + ) - requests_mock.get(STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text) + requests_mock.get( + STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text + ) init_qp_name = "my-param" init_qp_value = "some-value" - api = Client.open(STAC_URLS['PLANETARY-COMPUTER'], parameters={init_qp_name: init_qp_value}) + api = Client.open( + STAC_URLS["PLANETARY-COMPUTER"], parameters={init_qp_name: init_qp_value} + ) # Ensure that the the Client will use the /collections endpoint and not fall back # to traversing child links. @@ -127,12 +137,11 @@ def test_custom_request_parameters(self, requests_mock): collections_link = api.get_single_link("data") # Mock the request - requests_mock.get(collections_link.href, - status_code=200, - json={ - "collections": [pc_collection_dict], - "links": [] - }) + requests_mock.get( + collections_link.href, + status_code=200, + json={"collections": [pc_collection_dict], "links": []}, + ) # Make the collections request _ = next(api.get_collections()) @@ -148,45 +157,53 @@ def test_custom_request_parameters(self, requests_mock): assert len(actual_qp[init_qp_name]) == 1 assert actual_qp[init_qp_name][0] == init_qp_value - def test_custom_query_params_get_collections_propagation(self, requests_mock) -> None: + def test_custom_query_params_get_collections_propagation( + self, requests_mock + ) -> None: """Checks that query params passed to the init method are added to requests for CollectionClients fetched from the /collections endpoint.""" pc_root_text = read_data_file("planetary-computer-root.json") - pc_collection_dict = read_data_file("planetary-computer-collection.json", parse_json=True) + pc_collection_dict = read_data_file( + "planetary-computer-collection.json", parse_json=True + ) - requests_mock.get(STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text) + requests_mock.get( + STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text + ) init_qp_name = "my-param" init_qp_value = "some-value" - client = Client.open(STAC_URLS['PLANETARY-COMPUTER'], - parameters={init_qp_name: init_qp_value}) + client = Client.open( + STAC_URLS["PLANETARY-COMPUTER"], parameters={init_qp_name: init_qp_value} + ) # Get the /collections endpoint collections_link = client.get_single_link("data") # Mock the request - requests_mock.get(collections_link.href, - status_code=200, - json={ - "collections": [pc_collection_dict], - "links": [] - }) + requests_mock.get( + collections_link.href, + status_code=200, + json={"collections": [pc_collection_dict], "links": []}, + ) # Make the collections request collection = next(client.get_collections()) # Mock the items endpoint - items_link = collection.get_single_link('items') + items_link = collection.get_single_link("items") assert items_link is not None - requests_mock.get(items_link.href, - status_code=200, - json={ - "type": "FeatureCollection", - "stac_version": "1.0.0", - "features": [], - "links": [] - }) + requests_mock.get( + items_link.href, + status_code=200, + json={ + "type": "FeatureCollection", + "stac_version": "1.0.0", + "features": [], + "links": [], + }, + ) # Make the items request _ = list(collection.get_items()) @@ -202,20 +219,27 @@ def test_custom_query_params_get_collections_propagation(self, requests_mock) -> assert len(actual_qp[init_qp_name]) == 1 assert actual_qp[init_qp_name][0] == init_qp_value - def test_custom_query_params_get_collection_propagation(self, requests_mock) -> None: + def test_custom_query_params_get_collection_propagation( + self, requests_mock + ) -> None: """Checks that query params passed to the init method are added to requests for CollectionClients fetched from the /collections endpoint.""" pc_root_text = read_data_file("planetary-computer-root.json") - pc_collection_dict = read_data_file("planetary-computer-collection.json", parse_json=True) + pc_collection_dict = read_data_file( + "planetary-computer-collection.json", parse_json=True + ) pc_collection_id = pc_collection_dict["id"] - requests_mock.get(STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text) + requests_mock.get( + STAC_URLS["PLANETARY-COMPUTER"], status_code=200, text=pc_root_text + ) init_qp_name = "my-param" init_qp_value = "some-value" - client = Client.open(STAC_URLS['PLANETARY-COMPUTER'], - parameters={init_qp_name: init_qp_value}) + client = Client.open( + STAC_URLS["PLANETARY-COMPUTER"], parameters={init_qp_name: init_qp_value} + ) # Get the /collections endpoint collections_link = client.get_single_link("data") @@ -228,16 +252,18 @@ def test_custom_query_params_get_collection_propagation(self, requests_mock) -> collection = client.get_collection(pc_collection_id) # Mock the items endpoint - items_link = collection.get_single_link('items') + items_link = collection.get_single_link("items") assert items_link is not None - requests_mock.get(items_link.href, - status_code=200, - json={ - "type": "FeatureCollection", - "stac_version": "1.0.0", - "features": [], - "links": [] - }) + requests_mock.get( + items_link.href, + status_code=200, + json={ + "type": "FeatureCollection", + "stac_version": "1.0.0", + "features": [], + "links": [], + }, + ) # Make the items request _ = list(collection.get_items()) @@ -256,23 +282,31 @@ def test_custom_query_params_get_collection_propagation(self, requests_mock) -> def test_get_collections_without_conformance(self, requests_mock): """Checks that the "data" endpoint is used if the API published the collections conformance class.""" pc_root_dict = read_data_file("planetary-computer-root.json", parse_json=True) - pc_collection_dict = read_data_file("planetary-computer-aster-l1t-collection.json", - parse_json=True) + pc_collection_dict = read_data_file( + "planetary-computer-aster-l1t-collection.json", parse_json=True + ) # Remove the collections conformance class pc_root_dict["conformsTo"].remove( - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30") + "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30" + ) # Remove all child links except for the collection that we are mocking - pc_collection_href = next(link["href"] for link in pc_collection_dict["links"] - if link["rel"] == "self") + pc_collection_href = next( + link["href"] + for link in pc_collection_dict["links"] + if link["rel"] == "self" + ) pc_root_dict["links"] = [ - link for link in pc_root_dict["links"] + link + for link in pc_root_dict["links"] if link["rel"] != "child" or link["href"] == pc_collection_href ] # Mock the root catalog - requests_mock.get(STAC_URLS["PLANETARY-COMPUTER"], status_code=200, json=pc_root_dict) + requests_mock.get( + STAC_URLS["PLANETARY-COMPUTER"], status_code=200, json=pc_root_dict + ) api = Client.open(STAC_URLS["PLANETARY-COMPUTER"]) assert not api._stac_io.conforms_to(ConformanceClasses.COLLECTIONS) @@ -288,9 +322,9 @@ def test_get_collections_without_conformance(self, requests_mock): class TestAPISearch: - @pytest.fixture(scope='function') + @pytest.fixture(scope="function") def api(self): - return Client.from_file(str(TEST_DATA / 'planetary-computer-root.json')) + return Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) def test_search_conformance_error(self, api): """Should raise a NotImplementedError if the API doesn't conform to the Item Search spec. Message should @@ -299,19 +333,19 @@ def test_search_conformance_error(self, api): api._stac_io._conformance = [api._stac_io._conformance[0]] with pytest.raises(NotImplementedError) as excinfo: - api.search(limit=10, max_items=10, collections='mr-peebles') + api.search(limit=10, max_items=10, collections="mr-peebles") assert str(ConformanceClasses.ITEM_SEARCH) in str(excinfo.value) def test_no_search_link(self, api): # Remove the search link - api.remove_links('search') + api.remove_links("search") with pytest.raises(NotImplementedError) as excinfo: - api.search(limit=10, max_items=10, collections='naip') + api.search(limit=10, max_items=10, collections="naip") assert 'No link with "rel" type of "search"' in str(excinfo.value) - def test_no_conforms_to(self): - with open(str(TEST_DATA / 'planetary-computer-root.json')) as f: + def test_no_conforms_to(self) -> None: + with open(str(TEST_DATA / "planetary-computer-root.json")) as f: data = json.load(f) del data["conformsTo"] with TemporaryDirectory() as temporary_directory: @@ -321,21 +355,23 @@ def test_no_conforms_to(self): api = Client.from_file(path) with pytest.raises(NotImplementedError) as excinfo: - api.search(limit=10, max_items=10, collections='naip') - assert 'does not support search' in str(excinfo.value) + api.search(limit=10, max_items=10, collections="naip") + assert "does not support search" in str(excinfo.value) def test_search(self, api): - results = api.search(bbox=[-73.21, 43.99, -73.12, 44.05], - collections='naip', - limit=10, - max_items=20, - datetime=[datetime(2020, 1, 1, 0, 0, 0, tzinfo=tzutc()), None]) + results = api.search( + bbox=[-73.21, 43.99, -73.12, 44.05], + collections="naip", + limit=10, + max_items=20, + datetime=[datetime(2020, 1, 1, 0, 0, 0, tzinfo=tzutc()), None], + ) assert results._parameters == { - 'bbox': (-73.21, 43.99, -73.12, 44.05), - 'collections': ('naip', ), - 'limit': 10, - 'datetime': '2020-01-01T00:00:00Z/..' + "bbox": (-73.21, 43.99, -73.12, 44.05), + "collections": ("naip",), + "limit": 10, + "datetime": "2020-01-01T00:00:00Z/..", } def test_json_search_link(self, api: Client) -> None: diff --git a/tests/test_collection_client.py b/tests/test_collection_client.py index 2d2af0f2..a410cd6b 100644 --- a/tests/test_collection_client.py +++ b/tests/test_collection_client.py @@ -9,16 +9,16 @@ class TestCollectionClient: @pytest.mark.vcr def test_instance(self): - client = Client.open(STAC_URLS['PLANETARY-COMPUTER']) - collection = client.get_collection('aster-l1t') + client = Client.open(STAC_URLS["PLANETARY-COMPUTER"]) + collection = client.get_collection("aster-l1t") assert isinstance(collection, CollectionClient) - assert str(collection) == '' + assert str(collection) == "" @pytest.mark.vcr def test_get_items(self): - client = Client.open(STAC_URLS['PLANETARY-COMPUTER']) - collection = client.get_collection('aster-l1t') + client = Client.open(STAC_URLS["PLANETARY-COMPUTER"]) + collection = client.get_collection("aster-l1t") for item in collection.get_items(): - assert (item.collection_id == collection.id) + assert item.collection_id == collection.id return diff --git a/tests/test_item_search.py b/tests/test_item_search.py index 49f66aba..a571cb8b 100644 --- a/tests/test_item_search.py +++ b/tests/test_item_search.py @@ -5,6 +5,7 @@ import pytest import requests from dateutil.tz import gettz, tzutc + from pystac_client import Client from pystac_client.item_search import ItemSearch @@ -12,10 +13,16 @@ SEARCH_URL = f"{STAC_URLS['PLANETARY-COMPUTER']}/search" INTERSECTS_EXAMPLE = { - 'type': - 'Polygon', - 'coordinates': [[[-73.21, 43.99], [-73.21, 44.05], [-73.12, 44.05], [-73.12, 43.99], - [-73.21, 43.99]]] + "type": "Polygon", + "coordinates": [ + [ + [-73.21, 43.99], + [-73.21, 44.05], + [-73.12, 44.05], + [-73.12, 43.99], + [-73.21, 43.99], + ] + ], } ITEM_EXAMPLE = {"collections": "io-lulc", "ids": "60U-2020"} @@ -23,17 +30,18 @@ @pytest.mark.skip(reason="Performance testing skipped in normal test run") class TestItemPerformance: - @pytest.fixture(scope='function') - def single_href(self): + @pytest.fixture(scope="function") + def single_href(self) -> None: item_href = "https://planetarycomputer.microsoft.com/api/stac/v1/collections/{collections}/items/{ids}".format( - collections=ITEM_EXAMPLE['collections'], ids=ITEM_EXAMPLE['ids']) + collections=ITEM_EXAMPLE["collections"], ids=ITEM_EXAMPLE["ids"] + ) return item_href def test_requests(self, benchmark, single_href): response = benchmark(requests.get, single_href) assert response.status_code == 200 - assert response.json()['id'] == ITEM_EXAMPLE["ids"] + assert response.json()["id"] == ITEM_EXAMPLE["ids"] def test_single_item(self, benchmark, single_href): item = benchmark(pystac.Item.from_file, single_href) @@ -50,116 +58,154 @@ def test_single_item_search(self, benchmark, single_href): class TestItemSearchParams: - @pytest.fixture(scope='function') - def sample_client(self): - api_content = read_data_file('planetary-computer-root.json', parse_json=True) + @pytest.fixture(scope="function") + def sample_client(self) -> None: + api_content = read_data_file("planetary-computer-root.json", parse_json=True) return Client.from_dict(api_content) - def test_tuple_bbox(self): + def test_tuple_bbox(self) -> None: # Tuple input search = ItemSearch(url=SEARCH_URL, bbox=(-104.5, 44.0, -104.0, 45.0)) - assert search.get_parameters()['bbox'] == (-104.5, 44.0, -104.0, 45.0) + assert search.get_parameters()["bbox"] == (-104.5, 44.0, -104.0, 45.0) - def test_list_bbox(self): + def test_list_bbox(self) -> None: # List input search = ItemSearch(url=SEARCH_URL, bbox=[-104.5, 44.0, -104.0, 45.0]) - assert search.get_parameters()['bbox'] == (-104.5, 44.0, -104.0, 45.0) + assert search.get_parameters()["bbox"] == (-104.5, 44.0, -104.0, 45.0) - def test_string_bbox(self): + def test_string_bbox(self) -> None: # String Input - search = ItemSearch(url=SEARCH_URL, bbox='-104.5,44.0,-104.0,45.0') - assert search.get_parameters()['bbox'] == (-104.5, 44.0, -104.0, 45.0) + search = ItemSearch(url=SEARCH_URL, bbox="-104.5,44.0,-104.0,45.0") + assert search.get_parameters()["bbox"] == (-104.5, 44.0, -104.0, 45.0) - def test_generator_bbox(self): + def test_generator_bbox(self) -> None: # Generator Input def bboxer(): yield from [-104.5, 44.0, -104.0, 45.0] search = ItemSearch(url=SEARCH_URL, bbox=bboxer()) - assert search.get_parameters()['bbox'] == (-104.5, 44.0, -104.0, 45.0) + assert search.get_parameters()["bbox"] == (-104.5, 44.0, -104.0, 45.0) - def test_single_string_datetime(self): + def test_single_string_datetime(self) -> None: # Single timestamp input - search = ItemSearch(url=SEARCH_URL, datetime='2020-02-01T00:00:00Z') - assert search.get_parameters()['datetime'] == '2020-02-01T00:00:00Z' + search = ItemSearch(url=SEARCH_URL, datetime="2020-02-01T00:00:00Z") + assert search.get_parameters()["datetime"] == "2020-02-01T00:00:00Z" - def test_range_string_datetime(self): + def test_range_string_datetime(self) -> None: # Timestamp range input - search = ItemSearch(url=SEARCH_URL, datetime='2020-02-01T00:00:00Z/2020-02-02T00:00:00Z') - assert search.get_parameters()['datetime'] == '2020-02-01T00:00:00Z/2020-02-02T00:00:00Z' + search = ItemSearch( + url=SEARCH_URL, datetime="2020-02-01T00:00:00Z/2020-02-02T00:00:00Z" + ) + assert ( + search.get_parameters()["datetime"] + == "2020-02-01T00:00:00Z/2020-02-02T00:00:00Z" + ) - def test_list_of_strings_datetime(self): + def test_list_of_strings_datetime(self) -> None: # Timestamp list input - search = ItemSearch(url=SEARCH_URL, - datetime=['2020-02-01T00:00:00Z', '2020-02-02T00:00:00Z']) - assert search.get_parameters()['datetime'] == '2020-02-01T00:00:00Z/2020-02-02T00:00:00Z' + search = ItemSearch( + url=SEARCH_URL, datetime=["2020-02-01T00:00:00Z", "2020-02-02T00:00:00Z"] + ) + assert ( + search.get_parameters()["datetime"] + == "2020-02-01T00:00:00Z/2020-02-02T00:00:00Z" + ) - def test_open_range_string_datetime(self): + def test_open_range_string_datetime(self) -> None: # Open timestamp range input - search = ItemSearch(url=SEARCH_URL, datetime='2020-02-01T00:00:00Z/..') - assert search.get_parameters()['datetime'] == '2020-02-01T00:00:00Z/..' + search = ItemSearch(url=SEARCH_URL, datetime="2020-02-01T00:00:00Z/..") + assert search.get_parameters()["datetime"] == "2020-02-01T00:00:00Z/.." - def test_single_datetime_object(self): + def test_single_datetime_object(self) -> None: start = datetime(2020, 2, 1, 0, 0, 0, tzinfo=tzutc()) # Single datetime input search = ItemSearch(url=SEARCH_URL, datetime=start) - assert search.get_parameters()['datetime'] == '2020-02-01T00:00:00Z' + assert search.get_parameters()["datetime"] == "2020-02-01T00:00:00Z" - def test_list_of_datetimes(self): + def test_list_of_datetimes(self) -> None: start = datetime(2020, 2, 1, 0, 0, 0, tzinfo=tzutc()) end = datetime(2020, 2, 2, 0, 0, 0, tzinfo=tzutc()) # Datetime range input search = ItemSearch(url=SEARCH_URL, datetime=[start, end]) - assert search.get_parameters()['datetime'] == '2020-02-01T00:00:00Z/2020-02-02T00:00:00Z' + assert ( + search.get_parameters()["datetime"] + == "2020-02-01T00:00:00Z/2020-02-02T00:00:00Z" + ) - def test_open_list_of_datetimes(self): + def test_open_list_of_datetimes(self) -> None: start = datetime(2020, 2, 1, 0, 0, 0, tzinfo=tzutc()) # Open datetime range input search = ItemSearch(url=SEARCH_URL, datetime=(start, None)) - assert search.get_parameters()['datetime'] == '2020-02-01T00:00:00Z/..' + assert search.get_parameters()["datetime"] == "2020-02-01T00:00:00Z/.." - def test_localized_datetime_converted_to_utc(self): + def test_localized_datetime_converted_to_utc(self) -> None: # Localized datetime input (should be converted to UTC) - start_localized = datetime(2020, 2, 1, 0, 0, 0, tzinfo=gettz('US/Eastern')) + start_localized = datetime(2020, 2, 1, 0, 0, 0, tzinfo=gettz("US/Eastern")) search = ItemSearch(url=SEARCH_URL, datetime=start_localized) - assert search.get_parameters()['datetime'] == '2020-02-01T05:00:00Z' + assert search.get_parameters()["datetime"] == "2020-02-01T05:00:00Z" - def test_single_year(self): - search = ItemSearch(url=SEARCH_URL, datetime='2020') - assert search.get_parameters()['datetime'] == "2020-01-01T00:00:00Z/2020-12-31T23:59:59Z" + def test_single_year(self) -> None: + search = ItemSearch(url=SEARCH_URL, datetime="2020") + assert ( + search.get_parameters()["datetime"] + == "2020-01-01T00:00:00Z/2020-12-31T23:59:59Z" + ) - def test_range_of_years(self): - search = ItemSearch(url=SEARCH_URL, datetime='2019/2020') - assert search.get_parameters()['datetime'] == "2019-01-01T00:00:00Z/2020-12-31T23:59:59Z" + def test_range_of_years(self) -> None: + search = ItemSearch(url=SEARCH_URL, datetime="2019/2020") + assert ( + search.get_parameters()["datetime"] + == "2019-01-01T00:00:00Z/2020-12-31T23:59:59Z" + ) - def test_single_month(self): - search = ItemSearch(url=SEARCH_URL, datetime='2020-06') - assert search.get_parameters()['datetime'] == "2020-06-01T00:00:00Z/2020-06-30T23:59:59Z" + def test_single_month(self) -> None: + search = ItemSearch(url=SEARCH_URL, datetime="2020-06") + assert ( + search.get_parameters()["datetime"] + == "2020-06-01T00:00:00Z/2020-06-30T23:59:59Z" + ) - def test_range_of_months(self): - search = ItemSearch(url=SEARCH_URL, datetime='2020-04/2020-06') - assert search.get_parameters()['datetime'] == "2020-04-01T00:00:00Z/2020-06-30T23:59:59Z" + def test_range_of_months(self) -> None: + search = ItemSearch(url=SEARCH_URL, datetime="2020-04/2020-06") + assert ( + search.get_parameters()["datetime"] + == "2020-04-01T00:00:00Z/2020-06-30T23:59:59Z" + ) - def test_single_date(self): - search = ItemSearch(url=SEARCH_URL, datetime='2020-06-10') - assert search.get_parameters()['datetime'] == "2020-06-10T00:00:00Z/2020-06-10T23:59:59Z" + def test_single_date(self) -> None: + search = ItemSearch(url=SEARCH_URL, datetime="2020-06-10") + assert ( + search.get_parameters()["datetime"] + == "2020-06-10T00:00:00Z/2020-06-10T23:59:59Z" + ) - def test_range_of_dates(self): - search = ItemSearch(url=SEARCH_URL, datetime='2020-06-10/2020-06-20') - assert search.get_parameters()['datetime'] == "2020-06-10T00:00:00Z/2020-06-20T23:59:59Z" + def test_range_of_dates(self) -> None: + search = ItemSearch(url=SEARCH_URL, datetime="2020-06-10/2020-06-20") + assert ( + search.get_parameters()["datetime"] + == "2020-06-10T00:00:00Z/2020-06-20T23:59:59Z" + ) - def test_mixed_simple_date_strings(self): + def test_mixed_simple_date_strings(self) -> None: search = ItemSearch(url=SEARCH_URL, datetime="2019/2020-06-10") - assert search.get_parameters()['datetime'] == "2019-01-01T00:00:00Z/2020-06-10T23:59:59Z" + assert ( + search.get_parameters()["datetime"] + == "2019-01-01T00:00:00Z/2020-06-10T23:59:59Z" + ) - def test_time(self): - search = ItemSearch(url=SEARCH_URL, datetime="2019-01-01T00:00:00Z/2019-01-01T00:12:00") - assert search.get_parameters()['datetime'] == "2019-01-01T00:00:00Z/2019-01-01T00:12:00Z" + def test_time(self) -> None: + search = ItemSearch( + url=SEARCH_URL, datetime="2019-01-01T00:00:00Z/2019-01-01T00:12:00" + ) + assert ( + search.get_parameters()["datetime"] + == "2019-01-01T00:00:00Z/2019-01-01T00:12:00Z" + ) - def test_many_datetimes(self): + def test_many_datetimes(self) -> None: datetimes = [ "1985-04-12T23:20:50.52Z", "1996-12-19T16:39:57-08:00", @@ -187,7 +233,7 @@ def test_many_datetimes(self): for date_time in datetimes: ItemSearch(url=SEARCH_URL, datetime=date_time) - def test_three_datetimes(self): + def test_three_datetimes(self) -> None: start = datetime(2020, 2, 1, 0, 0, 0, tzinfo=tzutc()) middle = datetime(2020, 2, 2, 0, 0, 0, tzinfo=tzutc()) end = datetime(2020, 2, 3, 0, 0, 0, tzinfo=tzutc()) @@ -195,172 +241,175 @@ def test_three_datetimes(self): with pytest.raises(Exception): ItemSearch(url=SEARCH_URL, datetime=[start, middle, end]) - def test_single_collection_string(self): + def test_single_collection_string(self) -> None: # Single ID string - search = ItemSearch(url=SEARCH_URL, collections='naip') - assert search.get_parameters()['collections'] == ('naip', ) + search = ItemSearch(url=SEARCH_URL, collections="naip") + assert search.get_parameters()["collections"] == ("naip",) - def test_multiple_collection_string(self): + def test_multiple_collection_string(self) -> None: # Comma-separated ID string - search = ItemSearch(url=SEARCH_URL, collections='naip,landsat8_l1tp') - assert search.get_parameters()['collections'] == ('naip', 'landsat8_l1tp') + search = ItemSearch(url=SEARCH_URL, collections="naip,landsat8_l1tp") + assert search.get_parameters()["collections"] == ("naip", "landsat8_l1tp") - def test_list_of_collection_strings(self): + def test_list_of_collection_strings(self) -> None: # List of ID strings - search = ItemSearch(url=SEARCH_URL, collections=['naip', 'landsat8_l1tp']) - assert search.get_parameters()['collections'] == ('naip', 'landsat8_l1tp') + search = ItemSearch(url=SEARCH_URL, collections=["naip", "landsat8_l1tp"]) + assert search.get_parameters()["collections"] == ("naip", "landsat8_l1tp") - def test_generator_of_collection_strings(self): + def test_generator_of_collection_strings(self) -> None: # Generator of ID strings def collectioner(): - yield from ['naip', 'landsat8_l1tp'] + yield from ["naip", "landsat8_l1tp"] search = ItemSearch(url=SEARCH_URL, collections=collectioner()) - assert search.get_parameters()['collections'] == ('naip', 'landsat8_l1tp') + assert search.get_parameters()["collections"] == ("naip", "landsat8_l1tp") - def test_single_id_string(self): + def test_single_id_string(self) -> None: # Single ID - search = ItemSearch(url=SEARCH_URL, ids='m_3510836_se_12_060_20180508_20190331') - assert search.get_parameters()['ids'] == ('m_3510836_se_12_060_20180508_20190331', ) + search = ItemSearch(url=SEARCH_URL, ids="m_3510836_se_12_060_20180508_20190331") + assert search.get_parameters()["ids"] == ( + "m_3510836_se_12_060_20180508_20190331", + ) - def test_multiple_id_string(self): + def test_multiple_id_string(self) -> None: # Comma-separated ID string search = ItemSearch( url=SEARCH_URL, - ids='m_3510836_se_12_060_20180508_20190331,m_3510840_se_12_060_20180504_20190331') - assert search.get_parameters()['ids'] == ('m_3510836_se_12_060_20180508_20190331', - 'm_3510840_se_12_060_20180504_20190331') + ids="m_3510836_se_12_060_20180508_20190331,m_3510840_se_12_060_20180504_20190331", + ) + assert search.get_parameters()["ids"] == ( + "m_3510836_se_12_060_20180508_20190331", + "m_3510840_se_12_060_20180504_20190331", + ) - def test_list_of_id_strings(self): + def test_list_of_id_strings(self) -> None: # List of IDs search = ItemSearch( url=SEARCH_URL, - ids=['m_3510836_se_12_060_20180508_20190331', 'm_3510840_se_12_060_20180504_20190331']) - assert search.get_parameters()['ids'] == ('m_3510836_se_12_060_20180508_20190331', - 'm_3510840_se_12_060_20180504_20190331') + ids=[ + "m_3510836_se_12_060_20180508_20190331", + "m_3510840_se_12_060_20180504_20190331", + ], + ) + assert search.get_parameters()["ids"] == ( + "m_3510836_se_12_060_20180508_20190331", + "m_3510840_se_12_060_20180504_20190331", + ) - def test_generator_of_id_string(self): + def test_generator_of_id_string(self) -> None: # Generator of IDs def ids(): yield from [ - 'm_3510836_se_12_060_20180508_20190331', 'm_3510840_se_12_060_20180504_20190331' + "m_3510836_se_12_060_20180508_20190331", + "m_3510840_se_12_060_20180504_20190331", ] search = ItemSearch(url=SEARCH_URL, ids=ids()) - assert search.get_parameters()['ids'] == ('m_3510836_se_12_060_20180508_20190331', - 'm_3510840_se_12_060_20180504_20190331') + assert search.get_parameters()["ids"] == ( + "m_3510836_se_12_060_20180508_20190331", + "m_3510840_se_12_060_20180504_20190331", + ) - def test_intersects_dict(self): + def test_intersects_dict(self) -> None: # Dict input search = ItemSearch(url=SEARCH_URL, intersects=INTERSECTS_EXAMPLE) - assert search.get_parameters()['intersects'] == INTERSECTS_EXAMPLE + assert search.get_parameters()["intersects"] == INTERSECTS_EXAMPLE - def test_intersects_json_string(self): + def test_intersects_json_string(self) -> None: # JSON string input search = ItemSearch(url=SEARCH_URL, intersects=json.dumps(INTERSECTS_EXAMPLE)) - assert search.get_parameters()['intersects'] == INTERSECTS_EXAMPLE + assert search.get_parameters()["intersects"] == INTERSECTS_EXAMPLE - def test_intersects_non_geo_interface_object(self): + def test_intersects_non_geo_interface_object(self) -> None: with pytest.raises(Exception): ItemSearch(url=SEARCH_URL, intersects=object()) - def test_filter_lang_default_for_dict(self): + def test_filter_lang_default_for_dict(self) -> None: search = ItemSearch(url=SEARCH_URL, filter={}) - assert search.get_parameters()['filter-lang'] == 'cql2-json' + assert search.get_parameters()["filter-lang"] == "cql2-json" - def test_filter_lang_default_for_str(self): + def test_filter_lang_default_for_str(self) -> None: search = ItemSearch(url=SEARCH_URL, filter="") - assert search.get_parameters()['filter-lang'] == 'cql2-text' + assert search.get_parameters()["filter-lang"] == "cql2-text" - def test_filter_lang_cql2_text(self): + def test_filter_lang_cql2_text(self) -> None: # Use specified filter_lang search = ItemSearch(url=SEARCH_URL, filter_lang="cql2-text", filter={}) - assert search.get_parameters()['filter-lang'] == 'cql2-text' + assert search.get_parameters()["filter-lang"] == "cql2-text" - def test_filter_lang_cql2_json(self): + def test_filter_lang_cql2_json(self) -> None: # Use specified filter_lang search = ItemSearch(url=SEARCH_URL, filter_lang="cql2-json", filter="") - assert search.get_parameters()['filter-lang'] == 'cql2-json' + assert search.get_parameters()["filter-lang"] == "cql2-json" - def test_filter_lang_without_filter(self): + def test_filter_lang_without_filter(self) -> None: # No filter provided search = ItemSearch(url=SEARCH_URL) - assert 'filter-lang' not in search.get_parameters() + assert "filter-lang" not in search.get_parameters() - def test_sortby(self): + def test_sortby(self) -> None: search = ItemSearch(url=SEARCH_URL, sortby="properties.datetime") - assert search.get_parameters()['sortby'] == [{ - 'direction': 'asc', - 'field': 'properties.datetime' - }] + assert search.get_parameters()["sortby"] == [ + {"direction": "asc", "field": "properties.datetime"} + ] search = ItemSearch(url=SEARCH_URL, sortby="+properties.datetime") - assert search.get_parameters()['sortby'] == [{ - 'direction': 'asc', - 'field': 'properties.datetime' - }] + assert search.get_parameters()["sortby"] == [ + {"direction": "asc", "field": "properties.datetime"} + ] search = ItemSearch(url=SEARCH_URL, sortby="-properties.datetime") - assert search.get_parameters()['sortby'] == [{ - 'direction': 'desc', - 'field': 'properties.datetime' - }] - - search = ItemSearch(url=SEARCH_URL, sortby="-properties.datetime,+id,collection") - assert search.get_parameters()['sortby'] == [{ - 'direction': 'desc', - 'field': 'properties.datetime' - }, { - 'direction': 'asc', - 'field': 'id' - }, { - 'direction': 'asc', - 'field': 'collection' - }] - - search = ItemSearch(url=SEARCH_URL, - sortby=[{ - 'direction': 'desc', - 'field': 'properties.datetime' - }, { - 'direction': 'asc', - 'field': 'id' - }, { - 'direction': 'asc', - 'field': 'collection' - }]) - assert search.get_parameters()['sortby'] == [{ - 'direction': 'desc', - 'field': 'properties.datetime' - }, { - 'direction': 'asc', - 'field': 'id' - }, { - 'direction': 'asc', - 'field': 'collection' - }] - - search = ItemSearch(url=SEARCH_URL, sortby=['-properties.datetime', 'id', 'collection']) - assert search.get_parameters()['sortby'] == [{ - 'direction': 'desc', - 'field': 'properties.datetime' - }, { - 'direction': 'asc', - 'field': 'id' - }, { - 'direction': 'asc', - 'field': 'collection' - }] - - search = ItemSearch(url=SEARCH_URL, - method="GET", - sortby=['-properties.datetime', 'id', 'collection']) - assert search.get_parameters()['sortby'] == '-properties.datetime,+id,+collection' - - search = ItemSearch(url=SEARCH_URL, - method="GET", - sortby='-properties.datetime,id,collection') - assert search.get_parameters()['sortby'] == '-properties.datetime,+id,+collection' + assert search.get_parameters()["sortby"] == [ + {"direction": "desc", "field": "properties.datetime"} + ] + + search = ItemSearch( + url=SEARCH_URL, sortby="-properties.datetime,+id,collection" + ) + assert search.get_parameters()["sortby"] == [ + {"direction": "desc", "field": "properties.datetime"}, + {"direction": "asc", "field": "id"}, + {"direction": "asc", "field": "collection"}, + ] + + search = ItemSearch( + url=SEARCH_URL, + sortby=[ + {"direction": "desc", "field": "properties.datetime"}, + {"direction": "asc", "field": "id"}, + {"direction": "asc", "field": "collection"}, + ], + ) + assert search.get_parameters()["sortby"] == [ + {"direction": "desc", "field": "properties.datetime"}, + {"direction": "asc", "field": "id"}, + {"direction": "asc", "field": "collection"}, + ] + + search = ItemSearch( + url=SEARCH_URL, sortby=["-properties.datetime", "id", "collection"] + ) + assert search.get_parameters()["sortby"] == [ + {"direction": "desc", "field": "properties.datetime"}, + {"direction": "asc", "field": "id"}, + {"direction": "asc", "field": "collection"}, + ] + + search = ItemSearch( + url=SEARCH_URL, + method="GET", + sortby=["-properties.datetime", "id", "collection"], + ) + assert ( + search.get_parameters()["sortby"] == "-properties.datetime,+id,+collection" + ) + + search = ItemSearch( + url=SEARCH_URL, method="GET", sortby="-properties.datetime,id,collection" + ) + assert ( + search.get_parameters()["sortby"] == "-properties.datetime,+id,+collection" + ) with pytest.raises(Exception): ItemSearch(url=SEARCH_URL, sortby=1) @@ -370,29 +419,29 @@ def test_sortby(self): class TestItemSearch: - @pytest.fixture(scope='function') - def astraea_api(self): - api_content = read_data_file('astraea_api.json', parse_json=True) + @pytest.fixture(scope="function") + def astraea_api(self) -> None: + api_content = read_data_file("astraea_api.json", parse_json=True) return Client.from_dict(api_content) - def test_method(self): + def test_method(self) -> None: # Default method should be POST... search = ItemSearch(url=SEARCH_URL) - assert search.method == 'POST' + assert search.method == "POST" # "method" argument should take precedence over presence of "intersects" - search = ItemSearch(url=SEARCH_URL, method='GET', intersects=INTERSECTS_EXAMPLE) - assert search.method == 'GET' + search = ItemSearch(url=SEARCH_URL, method="GET", intersects=INTERSECTS_EXAMPLE) + assert search.method == "GET" - def test_method_params(self): + def test_method_params(self) -> None: params_in = { - 'bbox': (-72, 41, -71, 42), - 'ids': ( - 'idone', - 'idtwo', + "bbox": (-72, 41, -71, 42), + "ids": ( + "idone", + "idtwo", ), - 'collections': ('collectionone', ), - 'intersects': INTERSECTS_EXAMPLE + "collections": ("collectionone",), + "intersects": INTERSECTS_EXAMPLE, } # For POST this is pass through search = ItemSearch(url=SEARCH_URL, **params_in) @@ -400,16 +449,16 @@ def test_method_params(self): assert params == search.get_parameters() # For GET requests, parameters are in query string and must be serialized - search = ItemSearch(url=SEARCH_URL, method='GET', **params_in) + search = ItemSearch(url=SEARCH_URL, method="GET", **params_in) params = search.get_parameters() assert all(key in params for key in params_in) assert all(isinstance(params[key], str) for key in params_in) @pytest.mark.vcr - def test_results(self): + def test_results(self) -> None: search = ItemSearch( url=SEARCH_URL, - collections='naip', + collections="naip", max_items=20, limit=10, ) @@ -418,10 +467,10 @@ def test_results(self): assert all(isinstance(item, pystac.Item) for item in results) @pytest.mark.vcr - def test_ids_results(self): + def test_ids_results(self) -> None: ids = [ - 'S2B_MSIL2A_20210610T115639_N0212_R066_T33XXG_20210613T185024.SAFE', - 'fl_m_2608004_nw_17_060_20191215_20200113' + "S2B_MSIL2A_20210610T115639_N0212_R066_T33XXG_20210613T185024.SAFE", + "fl_m_2608004_nw_17_060_20191215_20200113", ] search = ItemSearch( url=SEARCH_URL, @@ -433,9 +482,9 @@ def test_ids_results(self): assert all(item.id in ids for item in results) @pytest.mark.vcr - def test_datetime_results(self): + def test_datetime_results(self) -> None: # Datetime range string - datetime_ = '2019-01-01T00:00:01Z/2019-01-01T00:00:10Z' + datetime_ = "2019-01-01T00:00:01Z/2019-01-01T00:00:10Z" search = ItemSearch(url=SEARCH_URL, datetime=datetime_) results = list(search.get_items()) assert len(results) == 33 @@ -444,39 +493,51 @@ def test_datetime_results(self): max_datetime = datetime(2019, 1, 1, 0, 0, 10, tzinfo=tzutc()) search = ItemSearch(url=SEARCH_URL, datetime=(min_datetime, max_datetime)) results = search.get_items() - assert all(min_datetime <= item.datetime <= (max_datetime + timedelta(seconds=1)) - for item in results) + assert all( + min_datetime <= item.datetime <= (max_datetime + timedelta(seconds=1)) + for item in results + ) @pytest.mark.vcr - def test_intersects_results(self): + def test_intersects_results(self) -> None: # GeoJSON-like dict intersects_dict = { - 'type': - 'Polygon', - 'coordinates': [[[-73.21, 43.99], [-73.21, 44.05], [-73.12, 44.05], [-73.12, 43.99], - [-73.21, 43.99]]] + "type": "Polygon", + "coordinates": [ + [ + [-73.21, 43.99], + [-73.21, 44.05], + [-73.12, 44.05], + [-73.12, 43.99], + [-73.21, 43.99], + ] + ], } - search = ItemSearch(url=SEARCH_URL, intersects=intersects_dict, collections='naip') + search = ItemSearch( + url=SEARCH_URL, intersects=intersects_dict, collections="naip" + ) results = list(search.get_items()) assert len(results) == 30 # Geo-interface object class MockGeoObject: @property - def __geo_interface__(self): + def __geo_interface__(self) -> None: return intersects_dict intersects_obj = MockGeoObject() - search = ItemSearch(url=SEARCH_URL, intersects=intersects_obj, collections='naip') + search = ItemSearch( + url=SEARCH_URL, intersects=intersects_obj, collections="naip" + ) results = search.get_items() assert all(isinstance(item, pystac.Item) for item in results) @pytest.mark.vcr - def test_result_paging(self): + def test_result_paging(self) -> None: search = ItemSearch( url=SEARCH_URL, bbox=(-73.21, 43.99, -73.12, 44.05), - collections='naip', + collections="naip", limit=10, max_items=20, ) @@ -488,11 +549,11 @@ def test_result_paging(self): assert pages[1].items != pages[2].items @pytest.mark.vcr - def test_get_all_items(self): + def test_get_all_items(self) -> None: search = ItemSearch( url=SEARCH_URL, bbox=(-73.21, 43.99, -73.12, 44.05), - collections='naip', + collections="naip", limit=10, max_items=20, ) @@ -502,21 +563,23 @@ def test_get_all_items(self): class TestItemSearchQuery: @pytest.mark.vcr - def test_query_shortcut_syntax(self): - search = ItemSearch(url=SEARCH_URL, - bbox=(-73.21, 43.99, -73.12, 44.05), - query=["gsd=10"], - max_items=1) + def test_query_shortcut_syntax(self) -> None: + search = ItemSearch( + url=SEARCH_URL, + bbox=(-73.21, 43.99, -73.12, 44.05), + query=["gsd=10"], + max_items=1, + ) items1 = list(search.get_items()) - search = ItemSearch(url=SEARCH_URL, - bbox=(-73.21, 43.99, -73.12, 44.05), - query={"gsd": { - "eq": 10 - }}, - max_items=1) + search = ItemSearch( + url=SEARCH_URL, + bbox=(-73.21, 43.99, -73.12, 44.05), + query={"gsd": {"eq": 10}}, + max_items=1, + ) items2 = list(search.get_items()) - assert (len(items1) == 1) - assert (len(items2) == 1) - assert (items1[0].id == items2[0].id) + assert len(items1) == 1 + assert len(items2) == 1 + assert items1[0].id == items2[0].id diff --git a/tests/test_stac_api_io.py b/tests/test_stac_api_io.py index ad9d73ef..c2988676 100644 --- a/tests/test_stac_api_io.py +++ b/tests/test_stac_api_io.py @@ -1,29 +1,31 @@ +from pathlib import Path from urllib.parse import parse_qs, urlsplit import pytest -from pystac_client.conformance import ConformanceClasses +from pystac_client.conformance import ConformanceClasses from pystac_client.exceptions import APIError from pystac_client.stac_api_io import StacApiIO + from .helpers import STAC_URLS class TestSTAC_IOOverride: @pytest.mark.vcr - def test_request_input(self): + def test_request_input(self) -> None: stac_api_io = StacApiIO() - response = stac_api_io.read_text(STAC_URLS['PLANETARY-COMPUTER']) + response = stac_api_io.read_text(STAC_URLS["PLANETARY-COMPUTER"]) assert isinstance(response, str) @pytest.mark.vcr - def test_str_input(self): + def test_str_input(self) -> None: stac_api_io = StacApiIO() - response = stac_api_io.read_text(STAC_URLS['PLANETARY-COMPUTER']) + response = stac_api_io.read_text(STAC_URLS["PLANETARY-COMPUTER"]) assert isinstance(response, str) @pytest.mark.vcr - def test_http_error(self): + def test_http_error(self) -> None: stac_api_io = StacApiIO() # Attempt to access an authenticated endpoint with pytest.raises(APIError) as excinfo: @@ -31,33 +33,37 @@ def test_http_error(self): assert isinstance(excinfo.value, APIError) - def test_local_file(self, tmp_path): + def test_local_file(self, tmp_path: Path) -> None: stac_api_io = StacApiIO() - test_file = tmp_path / 'test.txt' - with test_file.open('w') as dst: - dst.write('Hi there!') + test_file = tmp_path / "test.txt" + with test_file.open("w") as dst: + dst.write("Hi there!") response = stac_api_io.read_text(str(test_file)) - assert response == 'Hi there!' + assert response == "Hi there!" - def test_assert_conforms_to(self): + def test_assert_conforms_to(self) -> None: nonconformant = StacApiIO(conformance=[]) with pytest.raises(NotImplementedError): nonconformant.assert_conforms_to(ConformanceClasses.CORE) - conformant_io = StacApiIO(conformance=["https://api.stacspec.org/v1.0.0-beta.1/core"]) + conformant_io = StacApiIO( + conformance=["https://api.stacspec.org/v1.0.0-beta.1/core"] + ) # Check that this does not raise an exception conformant_io.assert_conforms_to(ConformanceClasses.CORE) - def test_conforms_to(self): + def test_conforms_to(self) -> None: nonconformant = StacApiIO(conformance=[]) assert not nonconformant.conforms_to(ConformanceClasses.CORE) - conformant_io = StacApiIO(conformance=["https://api.stacspec.org/v1.0.0-beta.1/core"]) + conformant_io = StacApiIO( + conformance=["https://api.stacspec.org/v1.0.0-beta.1/core"] + ) # Check that this does not raise an exception assert conformant_io.conforms_to(ConformanceClasses.CORE) From 8452f6662c30f3de55e060ce127addce54440427 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 13:37:09 -0400 Subject: [PATCH 03/13] add some type annotations, disable mypy in pre-commit and ci --- .pre-commit-config.yaml | 3 +-- pystac_client/cli.py | 36 ++++++++++++++++++++---------- pystac_client/collection_client.py | 8 +++---- pystac_client/item_search.py | 2 +- pystac_client/stac_api_io.py | 4 ++-- scripts/lint | 4 +--- 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c02490d..e525fc59 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,8 +29,7 @@ repos: # rev: v0.960 # hooks: # - id: mypy -# # TODO lint test and scripts too -## files: "^pystac_client/.*\\.py$" +# files: "^pystac_client/.*\\.py$" # args: # - --ignore-missing-imports # additional_dependencies: diff --git a/pystac_client/cli.py b/pystac_client/cli.py index 8babb672..0ee4831e 100644 --- a/pystac_client/cli.py +++ b/pystac_client/cli.py @@ -3,6 +3,7 @@ import logging import os import sys +from typing import Any, Dict, List, Optional from .client import Client from .version import __version__ @@ -10,44 +11,50 @@ logger = logging.getLogger(__name__) -def search(client, method="GET", matched=False, save=None, **kwargs): +def search( + client: Client, + method: str = "GET", + matched: bool = False, + save: Optional[str] = None, + **kwargs: Dict[str, Any], +) -> int: """Main function for performing a search""" try: - search = client.search(method=method, **kwargs) + result = client.search(method=method, **kwargs) if matched: - matched = search.matched() - print("%s items matched" % matched) + print(f"{result.matched()} items matched") else: - feature_collection = search.get_all_items_as_dict() + feature_collection = result.get_all_items_as_dict() if save: with open(save, "w") as f: f.write(json.dumps(feature_collection)) else: print(json.dumps(feature_collection)) + return 0 except Exception as e: print(e) return 1 -def collections(client, save=None, **kwargs): +def collections(client: Client, save: Optional[str] = None) -> int: """Fetch collections from collections endpoint""" try: - collections = client.get_all_collections(**kwargs) - collections_dicts = [c.to_dict() for c in collections] + collections_dicts = [c.to_dict() for c in client.get_all_collections()] if save: with open(save, "w") as f: f.write(json.dumps(collections_dicts)) else: print(json.dumps(collections_dicts)) + return 0 except Exception as e: print(e) return 1 -def parse_args(args): +def parse_args(args: List[str]) -> Dict[str, Any]: desc = "STAC Client" dhf = argparse.ArgumentDefaultsHelpFormatter parser0 = argparse.ArgumentParser(description=desc) @@ -157,7 +164,7 @@ def parse_args(args): } if "command" not in parsed_args: parser0.print_usage() - return [] + return {} # if intersects is JSON file, read it in if "intersects" in parsed_args: @@ -189,10 +196,10 @@ def parse_args(args): return parsed_args -def cli(): +def cli() -> int: args = parse_args(sys.argv[1:]) if not args: - return None + return 1 loglevel = args.pop("logging") if args.get("save", False) or args.get("matched", False): @@ -217,6 +224,11 @@ def cli(): return search(client, **args) elif cmd == "collections": return collections(client, **args) + else: + print( + f"Command '{cmd}' is not a valid command, must be 'search' or 'collections'" + ) + return 1 if __name__ == "__main__": diff --git a/pystac_client/collection_client.py b/pystac_client/collection_client.py index 5aaeae09..b20906d2 100644 --- a/pystac_client/collection_client.py +++ b/pystac_client/collection_client.py @@ -9,7 +9,7 @@ class CollectionClient(pystac.Collection): - def __repr__(self): + def __repr__(self) -> str: return "".format(self.id) def get_items(self) -> Iterable["Item_Type"]: @@ -23,10 +23,8 @@ def get_items(self) -> Iterable["Item_Type"]: """ link = self.get_single_link("items") - if link is not None: - search = ItemSearch( - link.href, method="GET", stac_io=self.get_root()._stac_io - ) + if link is not None and (root := self.get_root()): + search = ItemSearch(link.href, method="GET", stac_io=root._stac_io) yield from search.get_items() else: yield from super().get_items() diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 8f201c5a..662754ae 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -392,7 +392,7 @@ def _to_isoformat_range(component: DatetimeOrTimestamp): @staticmethod def _format_collections(value: Optional[CollectionsLike]) -> Optional[Collections]: - def _format(c): + def _format(c: Any) -> Any: if isinstance(c, str): return c if isinstance(c, Iterable): diff --git a/pystac_client/stac_api_io.py b/pystac_client/stac_api_io.py index 0e49bea1..136dd166 100644 --- a/pystac_client/stac_api_io.py +++ b/pystac_client/stac_api_io.py @@ -31,9 +31,9 @@ class StacApiIO(DefaultStacIO): def __init__( self, - headers: Optional[Dict] = None, + headers: Optional[Dict[str, str]] = None, conformance: Optional[List[str]] = None, - parameters: Optional[Dict] = None, + parameters: Optional[Dict[str, Any]] = None, ): """Initialize class for API IO diff --git a/scripts/lint b/scripts/lint index e1b762d3..f72f9468 100644 --- a/scripts/lint +++ b/scripts/lint @@ -1,5 +1,3 @@ - - #!/bin/bash set -e @@ -23,6 +21,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then pre-commit run doc8 --all-files pre-commit run flake8 --all-files pre-commit run isort --all-files - pre-commit run mypy --all-files + # pre-commit run mypy --all-files fi fi \ No newline at end of file From 91840c7a003054f6aa4f9546c80a819883675eb6 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 13:41:53 -0400 Subject: [PATCH 04/13] add pre-commit to readme --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 46444dfb..7a9ce2fe 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,20 @@ $ pip install -e . $ pip install -r requirements-dev.txt ``` -To run just the tests +[pre-commit](https://pre-commit.com/) is used to ensure a standard set of formatting and +linting is run before every commit. These hooks should be installed with: + +```shell +$ pre-commit install +``` + +These can then be run independent of a commit with: + +```shell +$ pre-commit run --all-files +``` + +To run just the tests: ```shell $ pytest -v -s --block-network --cov pystac_client --cov-report term-missing From dbadbf1f052bf64f4b34fad5293be4c28061a91f Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 20:24:59 -0400 Subject: [PATCH 05/13] eofs --- .flake8 | 2 +- .isort.cfg | 2 +- .pre-commit-config.yaml | 2 +- doc8.ini | 2 +- mypy.ini | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.flake8 b/.flake8 index 02826407..b4b49632 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] max-line-length = 88 extend-ignore = E203, W503, E731, E722, E501 -per-file-ignores = __init__.py:F401 \ No newline at end of file +per-file-ignores = __init__.py:F401 diff --git a/.isort.cfg b/.isort.cfg index 6860bdb0..f238bf7e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -profile = black \ No newline at end of file +profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e525fc59..3331a455 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,4 +35,4 @@ repos: # additional_dependencies: # - pystac # - types-requests -# - types-python-dateutil \ No newline at end of file +# - types-python-dateutil diff --git a/doc8.ini b/doc8.ini index 196d2054..5e39e145 100644 --- a/doc8.ini +++ b/doc8.ini @@ -2,4 +2,4 @@ ignore-path=docs/_build,docs/tutorials max-line-length=88 -ignore=D001 \ No newline at end of file +ignore=D001 diff --git a/mypy.ini b/mypy.ini index 476febc1..5fcd1391 100644 --- a/mypy.ini +++ b/mypy.ini @@ -9,4 +9,4 @@ ignore_missing_imports = True ignore_missing_imports = True [mypy-setuptools.*] -ignore_missing_imports = True \ No newline at end of file +ignore_missing_imports = True From 5765f5aae28970c22eb9fcafad6b4fd332df2a8b Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 20:27:54 -0400 Subject: [PATCH 06/13] get rid of walrus operator --- pystac_client/collection_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pystac_client/collection_client.py b/pystac_client/collection_client.py index b20906d2..848655bb 100644 --- a/pystac_client/collection_client.py +++ b/pystac_client/collection_client.py @@ -23,7 +23,8 @@ def get_items(self) -> Iterable["Item_Type"]: """ link = self.get_single_link("items") - if link is not None and (root := self.get_root()): + root = self.get_root() + if link is not None and root is not None: search = ItemSearch(link.href, method="GET", stac_io=root._stac_io) yield from search.get_items() else: From cfd9a61f1fbbede472231747171704e65630c26a Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 27 May 2022 09:08:16 -0400 Subject: [PATCH 07/13] fix test with no arguments, should return 1 and failure --- tests/test_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9e867810..699f3366 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,4 +22,5 @@ def test_item_search(self, script_runner: ScriptRunner): def test_no_arguments(self, script_runner: ScriptRunner): args = ["stac-client"] result = script_runner.run(*args, print_result=False) - assert result.success + assert not result.success + assert result.returncode == 1 From f51222916eb699c027df416457c16f32469abcac Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 27 May 2022 09:36:15 -0400 Subject: [PATCH 08/13] fix long lines in all docs --- .pre-commit-config.yaml | 4 + doc8.ini | 5 - docs/design/design_decisions.rst | 5 +- docs/index.rst | 20 ++-- docs/usage.rst | 158 +++++++++++++++++++------------ setup.cfg | 4 + 6 files changed, 117 insertions(+), 79 deletions(-) delete mode 100644 doc8.ini diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3331a455..57a51262 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,10 @@ repos: rev: 0.11.1 hooks: - id: doc8 +# - repo: https://github.com/myint/rstcheck +# rev: v6.0.0rc2 +# hooks: +# - id: rstcheck - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: diff --git a/doc8.ini b/doc8.ini deleted file mode 100644 index 5e39e145..00000000 --- a/doc8.ini +++ /dev/null @@ -1,5 +0,0 @@ -[doc8] - -ignore-path=docs/_build,docs/tutorials -max-line-length=88 -ignore=D001 diff --git a/docs/design/design_decisions.rst b/docs/design/design_decisions.rst index 5f464dbe..bc879afb 100644 --- a/docs/design/design_decisions.rst +++ b/docs/design/design_decisions.rst @@ -2,8 +2,9 @@ Design Decisions ================ `Architectural Design Records (ADRs) -`__ for major design decisions related to the -library. In general, this library makes an attempt to follow the design patterns laid out in the PySTAC library. +`__ for major +design decisions related to the library. In general, this library makes an attempt to +follow the design patterns laid out in the PySTAC library. .. toctree:: :glob: diff --git a/docs/index.rst b/docs/index.rst index ef674800..38ae8a6d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,11 +6,12 @@ PySTAC Client Documentation =========================== -The STAC Python Client (``pystac_client``) is a Python package for working with STAC Catalogs and APIs -that conform to the `STAC `__ and +The STAC Python Client (``pystac_client``) is a Python package for working with STAC +Catalogs and APIs that conform to the +`STAC `__ and `STAC API `__ specs in a seamless way. -PySTAC Client builds upon PySTAC through higher-level functionality and ability to leverage -STAC API search endpoints. +PySTAC Client builds upon PySTAC through higher-level functionality and ability to +leverage STAC API search endpoints. STAC Versions ============= @@ -32,15 +33,16 @@ Installation ``pystac_client`` requires `Python >=3.7 `__. -This will install the dependencies :doc:`PySTAC `, :doc:`python-dateutil `, -and :doc:`requests `. +This will install the dependencies :doc:`PySTAC `, +:doc:`python-dateutil `, and :doc:`requests `. Acknowledgements ---------------- -This package builds upon the great work of the PySTAC library for working with STAC objects in Python. -It also uses concepts from the `sat-search `__ library -for working with STAC API search endpoints. +This package builds upon the great work of the PySTAC library for working with +STAC objects in Python. It also uses concepts from the +`sat-search `__ library for working with STAC +API search endpoints. Table of Contents ----------------- diff --git a/docs/usage.rst b/docs/usage.rst index a4ce4990..9850f118 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,17 +1,21 @@ Usage ##### -PySTAC-Client (pystac-client) builds upon `PySTAC `_ library to add support -for STAC APIs in addition to static STACs. PySTAC-Client can be used with static or dynamic (i.e., API) -catalogs. Currently, pystac-client does not offer much in the way of additional functionality if using with -static catalogs, as the additional features are for support STAC API endpoints such as `search`. However, -in the future it is expected that pystac-client will offer additional convenience functions that may be -useful for static and dynamic catalogs alike. - -The most basic implementation of a STAC API is an endpoint that returns a valid STAC Catalog, but also contains -a ``"conformsTo"`` attribute that is a list of conformance URIs for the standards that the API supports. - -This section is organized by the classes that are used, which mirror parent classes from PySTAC: +PySTAC-Client (pystac-client) builds upon +`PySTAC `_ library to add support +for STAC APIs in addition to static STACs. PySTAC-Client can be used with static or +dynamic (i.e., API) catalogs. Currently, pystac-client does not offer much in the way of +additional functionality if using with static catalogs, as the additional features are +for support STAC API endpoints such as `search`. However, in the future it is expected +that pystac-client will offer additional convenience functions that may be useful for +static and dynamic catalogs alike. + +The most basic implementation of a STAC API is an endpoint that returns a valid STAC +Catalog, but also contains a ``"conformsTo"`` attribute that is a list of conformance +URIs for the standards that the API supports. + +This section is organized by the classes that are used, which mirror parent classes +from PySTAC: +------------------+------------+ | pystac-client | pystac | @@ -21,24 +25,28 @@ This section is organized by the classes that are used, which mirror parent clas | CollectionClient | Collection | +------------------+------------+ -The classes offer all of the same functions for accessing and traversing Catalogs as in PySTAC. The documentation -for pystac-client only includes new functions, it does not duplicate documentation for inherited functions. +The classes offer all of the same functions for accessing and traversing Catalogs as +in PySTAC. The documentation for pystac-client only includes new functions, it does +not duplicate documentation for inherited functions. Client ++++++ -The :class:`pystac_client.Client` class is the main interface for working with services that conform to the STAC API spec. -This class inherits from the :class:`pystac.Catalog` class and in addition to the methods and attributes implemented by -a Catalog, it also includes convenience methods and attributes for: +The :class:`pystac_client.Client` class is the main interface for working with services +that conform to the STAC API spec. This class inherits from the :class:`pystac.Catalog` +class and in addition to the methods and attributes implemented by a Catalog, it also +includes convenience methods and attributes for: * Checking conformance to various specs * Querying a search endpoint (if the API conforms to the STAC API - Item Search spec) -The preferred way to interact with any STAC Catalog or API is to create an :class:`pystac_client.Client` instance -with the ``pystac_client.Client.open`` method on a root Catalog. This calls the :meth:`pystac.STACObject.from_file` -except properly configures conformance and IO for reading from remote servers. +The preferred way to interact with any STAC Catalog or API is to create an +:class:`pystac_client.Client` instance with the ``pystac_client.Client.open`` method +on a root Catalog. This calls the :meth:`pystac.STACObject.from_file` except +properly configures conformance and IO for reading from remote servers. -The following code creates an instance by making a call to the Microsoft Planetary Computer root catalog. +The following code creates an instance by making a call to the Microsoft Planetary +Computer root catalog. .. code-block:: python @@ -47,27 +55,33 @@ The following code creates an instance by making a call to the Microsoft Planeta >>> api.title 'microsoft-pc' -Some functions, such as ``Client.search`` will throw an error if the provided Catalog/API does -not support the required Conformance Class. In other cases, such as ``Client.get_collections``, API endpoints will be -used if the API conforms, otherwise it will fall back to default behavior provided by :class:`pystac.Catalog`. +Some functions, such as ``Client.search`` will throw an error if the provided +Catalog/API does not support the required Conformance Class. In other cases, +such as ``Client.get_collections``, API endpoints will be used if the API +conforms, otherwise it will fall back to default behavior provided by +:class:`pystac.Catalog`. -Users may optionally provide an ``ignore_conformance`` argument when opening, in which case pystac-client will not check -for conformance and will assume this is a fully featured API. This can cause unusual errors to be thrown if the API +Users may optionally provide an ``ignore_conformance`` argument when opening, +in which case pystac-client will not check for conformance and will assume +this is a fully featured API. This can cause unusual errors to be thrown if the API does not in fact conform to the expected behavior. -In addition to the methods and attributes inherited from :class:`pystac.Catalog`, this class offers more efficient -methods (if used with an API) for getting collections and items, as well as a search capability, utilizing the +In addition to the methods and attributes inherited from :class:`pystac.Catalog`, +this class offers more efficient methods (if used with an API) for getting collections +and items, as well as a search capability, utilizing the :class:`pystac_client.ItemSearch` class. API Conformance --------------- -This library is intended to work with any STAC static catalog or STAC API. A static catalog will be usable more or less -the same as with PySTAC, except that pystac-client supports providing custom headers to API endpoints. (e.g., authenticating +This library is intended to work with any STAC static catalog or STAC API. A static +catalog will be usable more or less the same as with PySTAC, except that pystac-client +supports providing custom headers to API endpoints. (e.g., authenticating to an API with a token). -A STAC API is a STAC Catalog that is required to advertise its capabilities in a `conformsTo` field and implements -the `STAC API - Core` spec along with other optional specifications: +A STAC API is a STAC Catalog that is required to advertise its capabilities in a +`conformsTo` field and implements the `STAC API - Core` spec along with other +optional specifications: * `STAC API - Core `__ * `STAC API - Item Search `__ @@ -79,9 +93,10 @@ the `STAC API - Core` spec along with other optional specifications: * `STAC API - Features `__ (based on `OGC API - Features `__) -The :meth:`pystac_client.Client.conforms_to` method is used to check conformance against conformance classes (specs). -To check an API for support for a given spec, pass the `conforms_to` function the :class:`ConformanceClasses` attribute -as a parameter. +The :meth:`pystac_client.Client.conforms_to` method is used to check conformance +against conformance classes (specs). To check an API for support for a given spec, +pass the `conforms_to` function the :class:`ConformanceClasses` attribute as a +parameter. .. code-block:: python @@ -92,9 +107,11 @@ as a parameter. CollectionClient ++++++++++++++++ -STAC APIs may provide a curated list of catalogs and collections via their ``"links"`` attribute. Links with a ``"rel"`` -type of ``"child"`` represent catalogs or collections provided by the API. Since :class:`~pystac_client.Client` instances are -also :class:`pystac.Catalog` instances, we can use the methods defined on that class to get collections: +STAC APIs may provide a curated list of catalogs and collections via their ``"links"`` +attribute. Links with a ``"rel"`` type of ``"child"`` represent catalogs or collections +provided by the API. Since :class:`~pystac_client.Client` instances are also +:class:`pystac.Catalog` instances, we can use the methods defined on that class to +get collections: .. code-block:: python @@ -107,24 +124,27 @@ also :class:`pystac.Catalog` instances, we can use the methods defined on that c >>> first_collection.title 'Landsat 8 C1 T1' -CollectionClient overrides the :meth:`pystac.Collection.get_items` method. PySTAC will get items by -iterating through all children until it gets to an `item` link. If the `CollectionClient` instance -contains an `items` link, this will instead iterate through items using the API endpoint instead: -`/collections//items`. If no such link is present it will fall back to the -PySTAC Collection behavior. +CollectionClient overrides the :meth:`pystac.Collection.get_items` method. PySTAC will +get items by iterating through all children until it gets to an `item` link. If the +`CollectionClient` instance contains an `items` link, this will instead iterate through +items using the API endpoint instead: `/collections//items`. If no such +link is present it will fall back to the PySTAC Collection behavior. ItemSearch ++++++++++ -STAC API services may optionally implement a ``/search`` endpoint as describe in the `STAC API - Item Search spec -`__. This endpoint allows clients to query -STAC Items across the entire service using a variety of filter parameters. See the `Query Parameter Table -`__ from that spec for -details on the meaning of each parameter. +STAC API services may optionally implement a ``/search`` endpoint as describe in the +`STAC API - Item Search spec +`__. This +endpoint allows clients to query STAC Items across the entire service using a variety +of filter parameters. See the `Query Parameter Table +`__ +from that spec for details on the meaning of each parameter. -The :meth:`pystac_client.Client.search` method provides an interface for making requests to a service's -"search" endpoint. This method returns a :class:`pystac_client.ItemSearch` instance. +The :meth:`pystac_client.Client.search` method provides an interface for making +requests to a service's "search" endpoint. This method returns a +:class:`pystac_client.ItemSearch` instance. .. code-block:: python @@ -136,22 +156,29 @@ The :meth:`pystac_client.Client.search` method provides an interface for making ... max_items=5 ... ) -Instances of :class:`~pystac_client.ItemSearch` have 2 methods for iterating over results: +Instances of :class:`~pystac_client.ItemSearch` have 2 methods for iterating +over results: -* :meth:`ItemSearch.get_item_collections `: iterates over *pages* of results, +* :meth:`ItemSearch.get_item_collections `: + iterates over *pages* of results, yielding an :class:`~pystac.ItemCollection` for each page of results. -* :meth:`ItemSearch.get_items `: iterate over individual results, yielding a - :class:`pystac.Item` instance for all items that match the search criteria. +* :meth:`ItemSearch.get_items `: iterate over + individual results, yielding a :class:`pystac.Item` instance for all items + that match the search criteria. In addition three additional convenience methods are provided: -* :meth:`ItemSearch.matched `: returns the number of hits (items) for this search. +* :meth:`ItemSearch.matched `: returns the number + of hits (items) for this search. Not all APIs support returning a total count, in which case a warning will be issued. -* :meth:`ItemSearch.get_all_items `: Rather than return an iterator, this function will +* :meth:`ItemSearch.get_all_items `: Rather + than return an iterator, this function will fetch all items and return them as a single :class:`~pystac.ItemCollection`. -* :meth:`ItemSearch.get_all_items_as_dict `: Like `get_all_items` this fetches all items - but returns them as a GeoJSON FeatureCollection dictionary rather than a PySTAC object. This can be more efficient if - only a dictionary of the results is needed. +* :meth:`ItemSearch.get_all_items_as_dict + ` : Like `get_all_items` this fetches + all items but returns them as a GeoJSON FeatureCollection dictionary rather than a + PySTAC object. This can be more efficient if only a dictionary of the results is + needed. .. code-block:: python @@ -163,11 +190,16 @@ In addition three additional convenience methods are provided: MYD11A1.A2019002.h12v04.006.2019003174703 MYD11A1.A2019001.h12v04.006.2019002165238 -The :meth:`~pystac_client.ItemSearch.get_items` and related methods handle retrieval of successive pages of results -by finding links with a ``"rel"`` type of ``"next"`` and parsing them to construct the next request. The default -implementation of this ``"next"`` link parsing assumes that the link follows the spec for an extended STAC link as -described in the `STAC API - Item Search: Paging `__ -section. See the :mod:`Paging ` docs for details on how to customize this behavior. +The :meth:`~pystac_client.ItemSearch.get_items` and related methods handle retrieval of +successive pages of results +by finding links with a ``"rel"`` type of ``"next"`` and parsing them to construct the +next request. The default +implementation of this ``"next"`` link parsing assumes that the link follows the spec for +an extended STAC link as +described in the +`STAC API - Item Search: Paging `__ +section. See the :mod:`Paging ` docs for details on how to +customize this behavior. Query Extension --------------- diff --git a/setup.cfg b/setup.cfg index 0c9e0fc1..c47a1fa3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,6 @@ [metadata] license_file = LICENSE + +[doc8] +ignore-path=docs/_build,docs/tutorials +max-line-length=88 From 2546126794f853ea94dda6719c7ba52d0bc88d06 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 27 May 2022 10:48:13 -0400 Subject: [PATCH 09/13] fix line lengths --- .flake8 | 4 -- .isort.cfg | 2 - .pre-commit-config.yaml | 4 -- .style.yapf | 2 - mypy.ini | 12 ---- pystac_client/client.py | 54 ++++++++++------- pystac_client/collection_client.py | 5 +- pystac_client/item_search.py | 95 +++++++++++++++++++----------- pystac_client/stac_api_io.py | 44 +++++++++----- pytest.ini | 3 - setup.cfg | 25 ++++++++ tests/test_client.py | 29 +++++---- 12 files changed, 170 insertions(+), 109 deletions(-) delete mode 100644 .flake8 delete mode 100644 .isort.cfg delete mode 100644 .style.yapf delete mode 100644 mypy.ini delete mode 100644 pytest.ini diff --git a/.flake8 b/.flake8 deleted file mode 100644 index b4b49632..00000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 88 -extend-ignore = E203, W503, E731, E722, E501 -per-file-ignores = __init__.py:F401 diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index f238bf7e..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57a51262..3331a455 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,10 +16,6 @@ repos: rev: 0.11.1 hooks: - id: doc8 -# - repo: https://github.com/myint/rstcheck -# rev: v6.0.0rc2 -# hooks: -# - id: rstcheck - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index a5af2734..00000000 --- a/.style.yapf +++ /dev/null @@ -1,2 +0,0 @@ -[style] -column_limit = 100 \ No newline at end of file diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 5fcd1391..00000000 --- a/mypy.ini +++ /dev/null @@ -1,12 +0,0 @@ -[mypy] -show_error_codes = True -strict = True - -[mypy-jinja2.*] -ignore_missing_imports = True - -[mypy-jsonschema.*] -ignore_missing_imports = True - -[mypy-setuptools.*] -ignore_missing_imports = True diff --git a/pystac_client/client.py b/pystac_client/client.py index 4e1f0a97..3578b4ec 100644 --- a/pystac_client/client.py +++ b/pystac_client/client.py @@ -17,11 +17,15 @@ class Client(pystac.Catalog): """A Client for interacting with the root of a STAC Catalog or API - Instances of the ``Client`` class inherit from :class:`pystac.Catalog` and provide a convenient way of interacting - with STAC Catalogs OR STAC APIs that conform to the `STAC API spec `_. + Instances of the ``Client`` class inherit from :class:`pystac.Catalog` + and provide a convenient way of interacting + with STAC Catalogs OR STAC APIs that conform to the `STAC API spec + `_. In addition to being a valid - `STAC Catalog `_ - APIs that have a ``"conformsTo"`` indicate that it supports additional functionality on top of a normal STAC Catalog, + `STAC Catalog + `_ + APIs that have a ``"conformsTo"`` indicate that it supports additional + functionality on top of a normal STAC Catalog, such as searching items (e.g., /search endpoint). """ @@ -40,10 +44,14 @@ def open( This function will read the root catalog of a STAC Catalog or API Args: - url : The URL of a STAC Catalog. If not specified, this will use the `STAC_URL` environment variable. - headers : A dictionary of additional headers to use in all requests made to any part of this Catalog/API. - ignore_conformance : Ignore any advertised Conformance Classes in this Catalog/API. This means that - functions will skip checking conformance, and may throw an unknown error if that feature is + url : The URL of a STAC Catalog. If not specified, this will use the + `STAC_URL` environment variable. + headers : A dictionary of additional headers to use in all requests + made to any part of this Catalog/API. + ignore_conformance : Ignore any advertised Conformance Classes in this + Catalog/API. This means that + functions will skip checking conformance, and may throw an unknown + error if that feature is not supported, rather than a :class:`NotImplementedError`. Return: @@ -51,7 +59,8 @@ def open( """ cat = cls.from_file(url, headers=headers, parameters=parameters) search_link = cat.get_search_link() - # if there is a search link, but no conformsTo advertised, ignore conformance entirely + # if there is a search link, but no conformsTo advertised, ignore + # conformance entirely # NOTE: this behavior to be deprecated as implementations become conformant if ignore_conformance or ( "conformsTo" not in cat.extra_fields.keys() @@ -108,8 +117,8 @@ def get_collection(self, collection_id: str) -> CollectionClient: def get_collections(self) -> Iterable[CollectionClient]: """Get Collections in this Catalog - Gets the collections from the /collections endpoint if supported, otherwise fall - back to Catalog behavior of following child links + Gets the collections from the /collections endpoint if supported, + otherwise fall back to Catalog behavior of following child links Return: Iterable[CollectionClient]: Iterator through Collections in Catalog/API @@ -154,27 +163,32 @@ def get_all_items(self) -> Iterable["Item_Type"]: def search(self, **kwargs: Any) -> ItemSearch: """Query the ``/search`` endpoint using the given parameters. - This method returns an :class:`~pystac_client.ItemSearch` instance, see that class's documentation - for details on how to get the number of matches and iterate over results. All keyword arguments are passed - directly to the :class:`~pystac_client.ItemSearch` instance. + This method returns an :class:`~pystac_client.ItemSearch` instance, see that + class's documentation for details on how to get the number of matches and + iterate over results. All keyword arguments are passed directly to the + :class:`~pystac_client.ItemSearch` instance. .. warning:: This method is only implemented if the API conforms to the - `STAC API - Item Search `__ spec - *and* contains a link with a ``"rel"`` type of ``"search"`` in its root catalog. - If the API does not meet either of these criteria, this method will raise a :exc:`NotImplementedError`. + `STAC API - Item Search + `__ + spec *and* contains a link with a ``"rel"`` type of ``"search"`` in its + root catalog. If the API does not meet either of these criteria, this + method will raise a :exc:`NotImplementedError`. Args: - **kwargs : Any parameter to the :class:`~pystac_client.ItemSearch` class, other than `url`, `conformance`, - and `stac_io` which are set from this Client instance + **kwargs : Any parameter to the :class:`~pystac_client.ItemSearch` class, + other than `url`, `conformance`, and `stac_io` which are set from this + Client instance Returns: search : An ItemSearch instance that can be used to iterate through Items. Raises: NotImplementedError: If the API does not conform to the `Item Search spec - `__ or does not have a link with + `__ + or does not have a link with a ``"rel"`` type of ``"search"``. """ if not self._stac_io.conforms_to(ConformanceClasses.ITEM_SEARCH): diff --git a/pystac_client/collection_client.py b/pystac_client/collection_client.py index 848655bb..25eaeaf0 100644 --- a/pystac_client/collection_client.py +++ b/pystac_client/collection_client.py @@ -15,8 +15,9 @@ def __repr__(self) -> str: def get_items(self) -> Iterable["Item_Type"]: """Return all items in this Collection. - If the Collection contains a link of with a `rel` value of `items`, that link will be - used to iterate through items. Otherwise, the default PySTAC behavior is assumed. + If the Collection contains a link of with a `rel` value of `items`, + that link will be used to iterate through items. Otherwise, the default + PySTAC behavior is assumed. Return: Iterable[Item]: Generator of items whose parent is this catalog. diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 662754ae..1c01085c 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -75,8 +75,9 @@ def dict_merge(dct: Dict, merge_dct: Dict, add_keys: bool = True) -> Dict: updating only top-level keys, dict_merge recurses down into dicts nested to an arbitrary depth, updating keys. The ``merge_dct`` is merged into ``dct``. This version will return a copy of the dictionary and leave the original - arguments untouched. The optional argument ``add_keys``, determines whether keys which are - present in ``merge_dict`` but not ``dct`` should be included in the new dict. + arguments untouched. The optional argument ``add_keys``, determines whether keys + which are present in ``merge_dict`` but not ``dct`` should be included in the new + dict. Args: dct (dict) onto which the merge is executed @@ -101,57 +102,85 @@ def dict_merge(dct: Dict, merge_dct: Dict, add_keys: bool = True) -> Dict: class ItemSearch: """Represents a deferred query to a STAC search endpoint as described in the - `STAC API - Item Search spec `__. + `STAC API - Item Search spec + `__. - No request is sent to the API until a function is called to fetch or iterate through the resulting STAC Items, - either the :meth:`ItemSearch.item_collections` or :meth:`ItemSearch.items` method is called and iterated over. + No request is sent to the API until a function is called to fetch or iterate + through the resulting STAC Items, + either the :meth:`ItemSearch.item_collections` or :meth:`ItemSearch.items` + method is called and iterated over. - All "Parameters", with the exception of ``max_items``, ``method``, and ``url`` correspond to query parameters + All "Parameters", with the exception of ``max_items``, ``method``, and + ``url`` correspond to query parameters described in the `STAC API - Item Search: Query Parameters Table - `__ docs. Please refer + `__ + docs. Please refer to those docs for details on how these parameters filter search results. Args: url : The URL to the item-search endpoint - method : The HTTP method to use when making a request to the service. This must be either ``"GET"``, ``"POST"``, or - ``None``. If ``None``, this will default to ``"POST"`` if the ``intersects`` argument is present and ``"GET"`` - if not. If a ``"POST"`` request receives a ``405`` status for the response, it will automatically retry with a + method : The HTTP method to use when making a request to the service. + This must be either ``"GET"``, ``"POST"``, or + ``None``. If ``None``, this will default to ``"POST"`` if the + ``intersects`` argument is present and ``"GET"`` + if not. If a ``"POST"`` request receives a ``405`` status for + the response, it will automatically retry with a ``"GET"`` request for all subsequent requests. - max_items : The maximum number of items to return from the search. *Note that this is not a STAC API - Item Search - parameter and is instead used by the client to limit the total number of returned items*. - limit : The maximum number of items to return *per page*. Defaults to ``None``, which falls back to the limit set + max_items : The maximum number of items to return from the search. *Note + that this is not a STAC API - Item Search + parameter and is instead used by the client to limit the total number + of returned items*. + limit : The maximum number of items to return *per page*. Defaults to + ``None``, which falls back to the limit set by the service. - bbox: May be a list, tuple, or iterator representing a bounding box of 2D or 3D coordinates. Results will be filtered + bbox: May be a list, tuple, or iterator representing a bounding box of 2D + or 3D coordinates. Results will be filtered to only those intersecting the bounding box. - datetime: Either a single datetime or datetime range used to filter results. You may express a single datetime - using a :class:`datetime.datetime` instance, a `RFC 3339-compliant `__ - timestamp, or a simple date string (see below). Instances of :class:`datetime.datetime` may be either - timezone aware or unaware. Timezone aware instances will be converted to a UTC timestamp before being passed - to the endpoint. Timezone unaware instances are assumed to represent UTC timestamps. You may represent a - datetime range using a ``"/"`` separated string as described in the spec, or a list, tuple, or iterator - of 2 timestamps or datetime instances. For open-ended ranges, use either ``".."`` (``'2020-01-01:00:00:00Z/..'``, - ``['2020-01-01:00:00:00Z', '..']``) or a value of ``None`` (``['2020-01-01:00:00:00Z', None]``). - - If using a simple date string, the datetime can be specified in ``YYYY-mm-dd`` format, optionally truncating - to ``YYYY-mm`` or just ``YYYY``. Simple date strings will be expanded to include the entire time period, for + datetime: Either a single datetime or datetime range used to filter results. + You may express a single datetime using a :class:`datetime.datetime` + instance, a `RFC 3339-compliant `__ + timestamp, or a simple date string (see below). Instances of + :class:`datetime.datetime` may be either + timezone aware or unaware. Timezone aware instances will be converted to + a UTC timestamp before being passed + to the endpoint. Timezone unaware instances are assumed to represent UTC + timestamps. You may represent a + datetime range using a ``"/"`` separated string as described in the spec, + or a list, tuple, or iterator + of 2 timestamps or datetime instances. For open-ended ranges, use either + ``".."`` (``'2020-01-01:00:00:00Z/..'``, + ``['2020-01-01:00:00:00Z', '..']``) or a value of ``None`` + (``['2020-01-01:00:00:00Z', None]``). + + If using a simple date string, the datetime can be specified in + ``YYYY-mm-dd`` format, optionally truncating + to ``YYYY-mm`` or just ``YYYY``. Simple date strings will be expanded to + include the entire time period, for example: - ``2017`` expands to ``2017-01-01T00:00:00Z/2017-12-31T23:59:59Z`` - ``2017-06`` expands to ``2017-06-01T00:00:00Z/2017-06-30T23:59:59Z`` - ``2017-06-10`` expands to ``2017-06-10T00:00:00Z/2017-06-10T23:59:59Z`` - If used in a range, the end of the range expands to the end of that day/month/year, for example: - - - ``2017/2018`` expands to ``2017-01-01T00:00:00Z/2018-12-31T23:59:59Z`` - - ``2017-06/2017-07`` expands to ``2017-06-01T00:00:00Z/2017-07-31T23:59:59Z`` - - ``2017-06-10/2017-06-11`` expands to ``2017-06-10T00:00:00Z/2017-06-11T23:59:59Z`` - intersects: A string or dictionary representing a GeoJSON geometry, or an object that implements a - ``__geo_interface__`` property as supported by several libraries including Shapely, ArcPy, PySAL, and + If used in a range, the end of the range expands to the end of that + day/month/year, for example: + + - ``2017/2018`` expands to + ``2017-01-01T00:00:00Z/2018-12-31T23:59:59Z`` + - ``2017-06/2017-07`` expands to + ``2017-06-01T00:00:00Z/2017-07-31T23:59:59Z`` + - ``2017-06-10/2017-06-11`` expands to + ``2017-06-10T00:00:00Z/2017-06-11T23:59:59Z`` + intersects: A string or dictionary representing a GeoJSON geometry, or + an object that implements a + ``__geo_interface__`` property as supported by several libraries + including Shapely, ArcPy, PySAL, and geojson. Results filtered to only those intersecting the geometry. ids: List of Item ids to return. All other filter parameters that further restrict the number of search results (except ``limit``) are ignored. - collections: List of one or more Collection IDs or :class:`pystac.Collection` instances. Only Items in one + collections: List of one or more Collection IDs or :class:`pystac.Collection` + instances. Only Items in one of the provided Collections will be searched query: List or JSON of query parameters as per the STAC API `query` extension filter: JSON of query parameters as per the STAC API `filter` extension diff --git a/pystac_client/stac_api_io.py b/pystac_client/stac_api_io.py index 136dd166..50fac4dd 100644 --- a/pystac_client/stac_api_io.py +++ b/pystac_client/stac_api_io.py @@ -41,7 +41,8 @@ def __init__( headers : Optional dictionary of headers to include in all requests conformance : Optional list of `Conformance Classes `__. - parameters: Optional dictionary of query string parameters to include in all requests. + parameters: Optional dictionary of query string parameters to + include in all requests. Return: StacApiIO : StacApiIO instance @@ -62,8 +63,9 @@ def read_text( ) -> str: """Read text from the given URI. - Overwrites the default method for reading text from a URL or file to allow :class:`urllib.request.Request` - instances as input. This method also raises any :exc:`urllib.error.HTTPError` exceptions rather than catching + Overwrites the default method for reading text from a URL or file to allow + :class:`urllib.request.Request` instances as input. This method also raises + any :exc:`urllib.error.HTTPError` exceptions rather than catching them to allow us to handle different response status codes as needed. """ if isinstance(source, str): @@ -77,12 +79,15 @@ def read_text( elif isinstance(source, Link): link = source.to_dict() href = link["href"] - # get headers and body from Link and add to request from simple stac resolver + # get headers and body from Link and add to request from simple STAC + # resolver merge = bool(link.get("merge", False)) - # If the link object includes a "method" property, use that. If not fall back to 'GET'. + # If the link object includes a "method" property, use that. If not + # fall back to 'GET'. method = link.get("method", "GET") - # If the link object includes a "headers" property, use that and respect the "merge" property. + # If the link object includes a "headers" property, use that and + # respect the "merge" property. headers = link.get("headers", None) # If "POST" use the body object that and respect the "merge" property. @@ -107,9 +112,12 @@ def request( Args: href (str): The request URL - method (Optional[str], optional): The http method to use, 'GET' or 'POST'. Defaults to 'GET'. - headers (Optional[dict], optional): Additional headers to include in request. Defaults to {}. - parameters (Optional[dict], optional): parameters to send with request. Defaults to {}. + method (Optional[str], optional): The http method to use, 'GET' or 'POST'. + Defaults to 'GET'. + headers (Optional[dict], optional): Additional headers to include in + request. Defaults to {}. + parameters (Optional[dict], optional): parameters to send with request. + Defaults to {}. Raises: APIError: raised if the server returns an error response @@ -196,7 +204,8 @@ def stac_object_from_dict( raise ValueError(f"Unknown STAC object type {info.object_type}") def get_pages(self, url, method="GET", parameters={}) -> Iterator[Dict]: - """Iterator that yields dictionaries for each page at a STAC paging endpoint, e.g., /collections, /search + """Iterator that yields dictionaries for each page at a STAC paging + endpoint, e.g., /collections, /search Return: Dict : JSON content from a single page @@ -218,20 +227,23 @@ def get_pages(self, url, method="GET", parameters={}) -> Iterator[Dict]: ) def assert_conforms_to(self, conformance_class: ConformanceClasses) -> None: - """Raises a :exc:`NotImplementedError` if the API does not publish the given conformance class. This method - only checks against the ``"conformsTo"`` property from the API landing page and does not make any additional + """Raises a :exc:`NotImplementedError` if the API does not publish the given + conformance class. This method only checks against the ``"conformsTo"`` + property from the API landing page and does not make any additional calls to a ``/conformance`` endpoint even if the API provides such an endpoint. Args: - conformance_class: The ``ConformanceClasses`` key to check conformance against. + conformance_class: The ``ConformanceClasses`` key to check conformance + against. """ if not self.conforms_to(conformance_class): raise NotImplementedError(f"{conformance_class} not supported") def conforms_to(self, conformance_class: ConformanceClasses) -> bool: - """Whether the API conforms to the given standard. This method only checks against the ``"conformsTo"`` - property from the API landing page and does not make any additional calls to a ``/conformance`` endpoint - even if the API provides such an endpoint. + """Whether the API conforms to the given standard. This method only checks + against the ``"conformsTo"`` property from the API landing page and does not + make any additional calls to a ``/conformance`` endpoint even if the API + provides such an endpoint. Args: key : The ``ConformanceClasses`` key to check conformance against. diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 302a4708..00000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -markers = - vcr: records network activity \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index c47a1fa3..33094480 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,31 @@ [metadata] license_file = LICENSE +[isort] +profile=black + [doc8] ignore-path=docs/_build,docs/tutorials max-line-length=88 + +[flake8] +max-line-length = 88 +extend-ignore = E203, W503, E731, E722 +per-file-ignores = __init__.py:F401,test_item_search.py:E501 + +[pytest] +markers = + vcr: records network activity + +[mypy] +show_error_codes = True +strict = True + +[mypy-jinja2.*] +ignore_missing_imports = True + +[mypy-jsonschema.*] +ignore_missing_imports = True + +[mypy-setuptools.*] +ignore_missing_imports = True diff --git a/tests/test_client.py b/tests/test_client.py index 90d22a12..b06b0f6c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -42,7 +42,8 @@ def test_links(self): assert isinstance(first_collection, pystac.Collection) def test_spec_conformance(self): - """Testing conformance against a ConformanceClass should allow APIs using legacy URIs to pass.""" + """Testing conformance against a ConformanceClass should allow APIs using legacy + URIs to pass.""" client = Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) # Set conformsTo URIs to conform with STAC API - Core using official URI @@ -52,8 +53,9 @@ def test_spec_conformance(self): @pytest.mark.vcr def test_no_conformance(self): - """Should raise a NotImplementedError if no conformance info can be found. Luckily, the test API doesn't publish - a "conformance" link so we can just remove the "conformsTo" attribute to test this.""" + """Should raise a NotImplementedError if no conformance info can be found. + Luckily, the test API doesn't publish a "conformance" link so we can just + remove the "conformsTo" attribute to test this.""" client = Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) client._stac_io._conformance = [] @@ -65,7 +67,8 @@ def test_no_conformance(self): @pytest.mark.vcr def test_no_stac_core_conformance(self): - """Should raise a NotImplementedError if the API does not conform to the STAC API - Core spec.""" + """Should raise a NotImplementedError if the API does not conform to the + STAC API - Core spec.""" client = Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) client._stac_io._conformance = client._stac_io._conformance[1:] @@ -85,7 +88,8 @@ def test_invalid_url(self): Client.open() def test_get_collections_with_conformance(self, requests_mock): - """Checks that the "data" endpoint is used if the API published the collections conformance class.""" + """Checks that the "data" endpoint is used if the API published the + STAC API Collections conformance class.""" pc_root_text = read_data_file("planetary-computer-root.json") pc_collection_dict = read_data_file( "planetary-computer-aster-l1t-collection.json", parse_json=True @@ -129,7 +133,7 @@ def test_custom_request_parameters(self, requests_mock): STAC_URLS["PLANETARY-COMPUTER"], parameters={init_qp_name: init_qp_value} ) - # Ensure that the the Client will use the /collections endpoint and not fall back + # Ensure that the Client will use the /collections endpoint and not fall back # to traversing child links. assert api._stac_io.conforms_to(ConformanceClasses.COLLECTIONS) @@ -160,7 +164,8 @@ def test_custom_request_parameters(self, requests_mock): def test_custom_query_params_get_collections_propagation( self, requests_mock ) -> None: - """Checks that query params passed to the init method are added to requests for CollectionClients fetched from + """Checks that query params passed to the init method are added to requests for + CollectionClients fetched from the /collections endpoint.""" pc_root_text = read_data_file("planetary-computer-root.json") pc_collection_dict = read_data_file( @@ -222,8 +227,8 @@ def test_custom_query_params_get_collections_propagation( def test_custom_query_params_get_collection_propagation( self, requests_mock ) -> None: - """Checks that query params passed to the init method are added to requests for CollectionClients fetched from - the /collections endpoint.""" + """Checks that query params passed to the init method are added to + requests for CollectionClients fetched from the /collections endpoint.""" pc_root_text = read_data_file("planetary-computer-root.json") pc_collection_dict = read_data_file( "planetary-computer-collection.json", parse_json=True @@ -280,7 +285,8 @@ def test_custom_query_params_get_collection_propagation( assert actual_qp[init_qp_name][0] == init_qp_value def test_get_collections_without_conformance(self, requests_mock): - """Checks that the "data" endpoint is used if the API published the collections conformance class.""" + """Checks that the "data" endpoint is used if the API published + the Collections conformance class.""" pc_root_dict = read_data_file("planetary-computer-root.json", parse_json=True) pc_collection_dict = read_data_file( "planetary-computer-aster-l1t-collection.json", parse_json=True @@ -327,7 +333,8 @@ def api(self): return Client.from_file(str(TEST_DATA / "planetary-computer-root.json")) def test_search_conformance_error(self, api): - """Should raise a NotImplementedError if the API doesn't conform to the Item Search spec. Message should + """Should raise a NotImplementedError if the API doesn't conform + to the Item Search spec. Message should include information about the spec that was not conformed to.""" # Set the conformance to only STAC API - Core api._stac_io._conformance = [api._stac_io._conformance[0]] From 3b045bc2dec33d00ebaa25f790e010c52b873c84 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 27 May 2022 10:50:22 -0400 Subject: [PATCH 10/13] add dependabot config --- .github/dependabot.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..9f9e6247 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + - package-ecosystem: pip + directory: "/.github/workflows" + schedule: + interval: weekly + - package-ecosystem: pip + directory: "." + schedule: + interval: daily From c49cd029496f6cdb8ab101d8549dda808bde2bd6 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 27 May 2022 10:57:51 -0400 Subject: [PATCH 11/13] update setup.cfg for pytest --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 33094480..7de93485 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ max-line-length = 88 extend-ignore = E203, W503, E731, E722 per-file-ignores = __init__.py:F401,test_item_search.py:E501 -[pytest] +[tool:pytest] markers = vcr: records network activity From b9726af4f0b17aa3ed021486729d7989c048c216 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 27 May 2022 13:23:40 -0400 Subject: [PATCH 12/13] add sort documentation, general wordsmithing --- docs/usage.rst | 58 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 9850f118..dae75619 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -3,12 +3,12 @@ Usage PySTAC-Client (pystac-client) builds upon `PySTAC `_ library to add support -for STAC APIs in addition to static STACs. PySTAC-Client can be used with static or -dynamic (i.e., API) catalogs. Currently, pystac-client does not offer much in the way of -additional functionality if using with static catalogs, as the additional features are -for support STAC API endpoints such as `search`. However, in the future it is expected -that pystac-client will offer additional convenience functions that may be useful for -static and dynamic catalogs alike. +for STAC APIs in addition to static STAC catalogs. PySTAC-Client can be used with static +or dynamic (i.e., API) catalogs. Currently, pystac-client does not offer much in the way +of additional functionality if using with static catalogs, as the additional features +are for support STAC API endpoints such as `search`. However, in the future it is +expected that pystac-client will offer additional convenience functions that may be +useful for static and dynamic catalogs alike. The most basic implementation of a STAC API is an endpoint that returns a valid STAC Catalog, but also contains a ``"conformsTo"`` attribute that is a list of conformance @@ -148,8 +148,8 @@ requests to a service's "search" endpoint. This method returns a .. code-block:: python - >>> from pystac_client import API - >>> api = API.from_file('https://planetarycomputer.microsoft.com/api/stac/v1') + >>> from pystac_client import Client + >>> api = Client.from_file('https://planetarycomputer.microsoft.com/api/stac/v1') >>> results = api.search( ... bbox=[-73.21, 43.99, -73.12, 44.05], ... datetime=['2019-01-01T00:00:00Z', '2019-01-02T00:00:00Z'], @@ -160,16 +160,16 @@ Instances of :class:`~pystac_client.ItemSearch` have 2 methods for iterating over results: * :meth:`ItemSearch.get_item_collections `: - iterates over *pages* of results, + an iterable over *pages* of results, yielding an :class:`~pystac.ItemCollection` for each page of results. -* :meth:`ItemSearch.get_items `: iterate over - individual results, yielding a :class:`pystac.Item` instance for all items - that match the search criteria. +* :meth:`ItemSearch.get_items `: an iterable over + individual Item objects, yielding a :class:`pystac.Item` instance for Item + that matches the search criteria. In addition three additional convenience methods are provided: * :meth:`ItemSearch.matched `: returns the number - of hits (items) for this search. + of hits (items) for this search if the API supports the STAC API Context Extension. Not all APIs support returning a total count, in which case a warning will be issued. * :meth:`ItemSearch.get_all_items `: Rather than return an iterator, this function will @@ -230,3 +230,35 @@ The query filter will also accept complete JSON as per the specification. Any number of properties can be included, and each can be included more than once to use additional operators. + +Sort Extension +--------------- + +If the Catalog supports the `Sort +extension `__, +the search request can specify the order in which the results should be sorted with +the ``sortby`` parameter. The ``sortby`` parameter can either be a string +(e.g., ``"-properties.datetime,+id,collection"``), a list of strings +(e.g., ``["-properties.datetime", "+id", "+collection"]``), or a dictionary representing +the POST JSON format of sortby. In the string and list formats, a ``-`` prefix means a +descending sort and a ``+`` prefix or no prefix means an ascending sort. + +.. code-block:: python + + >>> from pystac_client import Client + >>> results = Client.from_file('https://planetarycomputer.microsoft.com/api/stac/v1').search( + ... sortby="properties.datetime" + ... ) + >>> results = Client.from_file('https://planetarycomputer.microsoft.com/api/stac/v1').search( + ... sortby="-properties.datetime,+id,+collection" + ... ) + >>> results = Client.from_file('https://planetarycomputer.microsoft.com/api/stac/v1').search( + ... sortby=["-properties.datetime", "+id" , "+collection" ] + ... ) + >>> results = Client.from_file('https://planetarycomputer.microsoft.com/api/stac/v1').search( + ... sortby=[ + {"direction": "desc", "field": "properties.datetime"}, + {"direction": "asc", "field": "id"}, + {"direction": "asc", "field": "collection"}, + ] + ... ) From 29be6a044d459fea526c4902e08c40a3121d6975 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 31 May 2022 12:43:39 -0400 Subject: [PATCH 13/13] fix problem with merge --- docs/usage.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index b0eac5ff..644a5922 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -3,21 +3,12 @@ Usage PySTAC-Client (pystac-client) builds upon `PySTAC `_ library to add support -<<<<<<< HEAD for STAC APIs in addition to static STAC catalogs. PySTAC-Client can be used with static or dynamic (i.e., API) catalogs. Currently, pystac-client does not offer much in the way of additional functionality if using with static catalogs, as the additional features are for support STAC API endpoints such as `search`. However, in the future it is expected that pystac-client will offer additional convenience functions that may be useful for static and dynamic catalogs alike. -======= -for STAC APIs in addition to static STACs. PySTAC-Client can be used with static or -dynamic (i.e., API) catalogs. Currently, pystac-client does not offer much in the way of -additional functionality if using with static catalogs, as the additional features are -for support STAC API endpoints such as `search`. However, in the future it is expected -that pystac-client will offer additional convenience functions that may be useful for -static and dynamic catalogs alike. ->>>>>>> main The most basic implementation of a STAC API is an endpoint that returns a valid STAC Catalog, but also contains a ``"conformsTo"`` attribute that is a list of conformance