From c5b76e92ae0e5008b0f2f33c6f378e77e54dac45 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 20:18:17 -0400 Subject: [PATCH 1/5] fix search fields parameter typing and behavior --- CHANGELOG.md | 13 +++++++----- pystac_client/item_search.py | 38 ++++++++++++++++++++++++++++++++---- tests/test_item_search.py | 25 ++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbfc101b..65deba12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [v0.3.5] - 2022-05-26 - -### Fixed - -- Search against earth-search v0 failed with message "object of type 'Link' has no len()" [#179](https://github.com/stac-utils/pystac-client/pull/179) +## [Unreleased] - TBD ### Added @@ -21,6 +17,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Search sortby parameter now has correct typing and correctly handles both GET and POST JSON parameter formats. [#175](https://github.com/stac-utils/pystac-client/pull/175) +- Search fields parameter now has correct typing and correctly handles both GET and POST JSON parameter formats. + +## [v0.3.5] - 2022-05-26 + +### Fixed + +- Search against earth-search v0 failed with message "object of type 'Link' has no len()" [#179](https://github.com/stac-utils/pystac-client/pull/179) ## [v0.3.4] - 2022-05-18 diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 16fbda0d..9cdfa101 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -8,6 +8,7 @@ from datetime import timezone, datetime as datetime_ from typing import Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Union import warnings +from itertools import chain from pystac import Collection, Item, ItemCollection from pystac.stac_io import StacIO @@ -54,8 +55,8 @@ Sortby = List[Dict[str, str]] SortbyLike = Union[Sortby, str, List[str]] -Fields = List[str] -FieldsLike = Union[Fields, str] +Fields = Dict[str, List[str]] +FieldsLike = Union[Fields, str, List[str]] OP_MAP = {'>=': 'gte', '<=': 'lte', '=': 'eq', '>': 'gt', '<': 'lt'} @@ -157,6 +158,7 @@ class ItemSearch: stac_io: An instance of StacIO for retrieving results. Normally comes from the Client that returns this ItemSearch client: An instance of a root Client used to set the root on resulting Items """ + def __init__(self, url: str, *, @@ -224,6 +226,8 @@ def get_parameters(self): params['intersects'] = json.dumps(params['intersects']) if 'sortby' in params: params['sortby'] = self.sortby_json_to_str(params['sortby']) + if 'fields' in params: + params['fields'] = self.fields_json_to_str(params['fields']) return params else: raise Exception(f"Unsupported method {self.method}") @@ -430,10 +434,36 @@ def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]: self._stac_io.assert_conforms_to(ConformanceClasses.FIELDS) + Fields = Dict[str, List[str]] + FieldsLike = Union[Fields, str, List[str]] + if isinstance(value, str): - return tuple(value.split(',')) + return self.fields_to_json(value.split(',')) + if isinstance(value, list): + return self.fields_to_json(value) + if isinstance(value, dict): + return value - return tuple(value) + raise Exception("sortby must be of type None, str, List[str], or List[Dict[str, str]") + + @staticmethod + def fields_to_json(fields: List[str]) -> Fields: + includes: List[str] = [] + excludes: List[str] = [] + for field in fields: + if field.startswith("-"): + excludes.append(field[1:]) + elif field.startswith("+"): + includes.append(field[1:]) + else: + includes.append(field) + return {"includes": includes, "excludes": excludes} + + @staticmethod + def fields_json_to_str(fields: Fields) -> str: + includes = [f"+{x}" for x in fields.get('includes', [])] + excludes = [f"-{x}" for x in fields.get('excludes', [])] + return ",".join(chain(includes, excludes)) @staticmethod def _format_intersects(value: Optional[IntersectsLike]) -> Optional[Intersects]: diff --git a/tests/test_item_search.py b/tests/test_item_search.py index 49f66aba..54af59ff 100644 --- a/tests/test_item_search.py +++ b/tests/test_item_search.py @@ -368,6 +368,31 @@ def test_sortby(self): with pytest.raises(Exception): ItemSearch(url=SEARCH_URL, sortby=[1]) + def test_fields(self): + + with pytest.raises(Exception): + ItemSearch(url=SEARCH_URL, fields=1) + + with pytest.raises(Exception): + ItemSearch(url=SEARCH_URL, fields=[1]) + + search = ItemSearch(url=SEARCH_URL, fields="id,collection,+foo,-bar") + assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection', 'foo']} + + search = ItemSearch(url=SEARCH_URL, fields=["id","collection", "+foo", "-bar"]) + assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection', 'foo']} + + search = ItemSearch(url=SEARCH_URL, fields={'excludes': ["bar"], 'includes': ['id', 'collection']}) + assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection']} + + search = ItemSearch(url=SEARCH_URL, method="GET", fields="id,collection,+foo,-bar") + assert search.get_parameters()['fields'] == "+id,+collection,+foo,-bar" + + search = ItemSearch(url=SEARCH_URL, method="GET", fields=["id","collection", "+foo", "-bar"]) + assert search.get_parameters()['fields'] == "+id,+collection,+foo,-bar" + + search = ItemSearch(url=SEARCH_URL, method="GET", fields={'excludes': ["bar"], 'includes': ['id', 'collection']}) + assert search.get_parameters()['fields'] == "+id,+collection,-bar" class TestItemSearch: @pytest.fixture(scope='function') From 146c1649e609e1069d4d0ee6538ca363a60c64ac Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 26 May 2022 20:19:47 -0400 Subject: [PATCH 2/5] linting --- pystac_client/item_search.py | 3 --- tests/test_item_search.py | 5 +++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 9cdfa101..1e0721f9 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -434,9 +434,6 @@ def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]: self._stac_io.assert_conforms_to(ConformanceClasses.FIELDS) - Fields = Dict[str, List[str]] - FieldsLike = Union[Fields, str, List[str]] - if isinstance(value, str): return self.fields_to_json(value.split(',')) if isinstance(value, list): diff --git a/tests/test_item_search.py b/tests/test_item_search.py index 54af59ff..329ab664 100644 --- a/tests/test_item_search.py +++ b/tests/test_item_search.py @@ -379,7 +379,7 @@ def test_fields(self): search = ItemSearch(url=SEARCH_URL, fields="id,collection,+foo,-bar") assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection', 'foo']} - search = ItemSearch(url=SEARCH_URL, fields=["id","collection", "+foo", "-bar"]) + search = ItemSearch(url=SEARCH_URL, fields=["id", "collection", "+foo", "-bar"]) assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection', 'foo']} search = ItemSearch(url=SEARCH_URL, fields={'excludes': ["bar"], 'includes': ['id', 'collection']}) @@ -388,12 +388,13 @@ def test_fields(self): search = ItemSearch(url=SEARCH_URL, method="GET", fields="id,collection,+foo,-bar") assert search.get_parameters()['fields'] == "+id,+collection,+foo,-bar" - search = ItemSearch(url=SEARCH_URL, method="GET", fields=["id","collection", "+foo", "-bar"]) + search = ItemSearch(url=SEARCH_URL, method="GET", fields=["id", "collection", "+foo", "-bar"]) assert search.get_parameters()['fields'] == "+id,+collection,+foo,-bar" search = ItemSearch(url=SEARCH_URL, method="GET", fields={'excludes': ["bar"], 'includes': ['id', 'collection']}) assert search.get_parameters()['fields'] == "+id,+collection,-bar" + class TestItemSearch: @pytest.fixture(scope='function') def astraea_api(self): From 79a8e5f2b165a8949c0abdc8e4c534c682b1b4eb Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Fri, 27 May 2022 09:00:05 -0400 Subject: [PATCH 3/5] format --- pystac_client/item_search.py | 1 - tests/test_item_search.py | 32 ++++++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index 1e0721f9..4bad4fa3 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -158,7 +158,6 @@ class ItemSearch: stac_io: An instance of StacIO for retrieving results. Normally comes from the Client that returns this ItemSearch client: An instance of a root Client used to set the root on resulting Items """ - def __init__(self, url: str, *, diff --git a/tests/test_item_search.py b/tests/test_item_search.py index 329ab664..c905cc32 100644 --- a/tests/test_item_search.py +++ b/tests/test_item_search.py @@ -377,21 +377,41 @@ def test_fields(self): ItemSearch(url=SEARCH_URL, fields=[1]) search = ItemSearch(url=SEARCH_URL, fields="id,collection,+foo,-bar") - assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection', 'foo']} + assert search.get_parameters()['fields'] == { + 'excludes': ["bar"], + 'includes': ['id', 'collection', 'foo'] + } search = ItemSearch(url=SEARCH_URL, fields=["id", "collection", "+foo", "-bar"]) - assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection', 'foo']} + assert search.get_parameters()['fields'] == { + 'excludes': ["bar"], + 'includes': ['id', 'collection', 'foo'] + } - search = ItemSearch(url=SEARCH_URL, fields={'excludes': ["bar"], 'includes': ['id', 'collection']}) - assert search.get_parameters()['fields'] == {'excludes': ["bar"], 'includes': ['id', 'collection']} + search = ItemSearch(url=SEARCH_URL, + fields={ + 'excludes': ["bar"], + 'includes': ['id', 'collection'] + }) + assert search.get_parameters()['fields'] == { + 'excludes': ["bar"], + 'includes': ['id', 'collection'] + } search = ItemSearch(url=SEARCH_URL, method="GET", fields="id,collection,+foo,-bar") assert search.get_parameters()['fields'] == "+id,+collection,+foo,-bar" - search = ItemSearch(url=SEARCH_URL, method="GET", fields=["id", "collection", "+foo", "-bar"]) + search = ItemSearch(url=SEARCH_URL, + method="GET", + fields=["id", "collection", "+foo", "-bar"]) assert search.get_parameters()['fields'] == "+id,+collection,+foo,-bar" - search = ItemSearch(url=SEARCH_URL, method="GET", fields={'excludes': ["bar"], 'includes': ['id', 'collection']}) + search = ItemSearch(url=SEARCH_URL, + method="GET", + fields={ + 'excludes': ["bar"], + 'includes': ['id', 'collection'] + }) assert search.get_parameters()['fields'] == "+id,+collection,-bar" From 146b2384a43e789a94019c65324779b54b6d2988 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 31 May 2022 12:25:50 -0400 Subject: [PATCH 4/5] update changelong --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65deba12..d93e295f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Search sortby parameter now has correct typing and correctly handles both GET and POST JSON parameter formats. [#175](https://github.com/stac-utils/pystac-client/pull/175) -- Search fields parameter now has correct typing and correctly handles both GET and POST JSON parameter formats. +- Search fields parameter now has correct typing and correctly handles both GET and POST JSON parameter formats. [#184](https://github.com/stac-utils/pystac-client/pull/184) ## [v0.3.5] - 2022-05-26 From 814fcc4386f0b7724f1c6b3df2b85bff7aa1aa62 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 31 May 2022 12:34:42 -0400 Subject: [PATCH 5/5] rename methods --- pystac_client/item_search.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pystac_client/item_search.py b/pystac_client/item_search.py index f00b91ba..286bd79f 100644 --- a/pystac_client/item_search.py +++ b/pystac_client/item_search.py @@ -268,9 +268,9 @@ def get_parameters(self) -> Dict[str, Any]: if "intersects" in params: params["intersects"] = json.dumps(params["intersects"]) if "sortby" in params: - params["sortby"] = self.sortby_json_to_str(params["sortby"]) + params["sortby"] = self.sortby_dict_to_str(params["sortby"]) if "fields" in params: - params["fields"] = self.fields_json_to_str(params["fields"]) + params["fields"] = self.fields_dict_to_str(params["fields"]) return params else: raise Exception(f"Unsupported method {self.method}") @@ -458,11 +458,11 @@ def _format_sortby(self, value: Optional[SortbyLike]) -> Optional[Sortby]: self._stac_io.assert_conforms_to(ConformanceClasses.SORT) if isinstance(value, str): - return [self.sortby_part_to_json(part) for part in value.split(",")] + return [self.sortby_part_to_dict(part) for part in value.split(",")] if isinstance(value, list): if value and isinstance(value[0], str): - return [self.sortby_part_to_json(v) for v in value] + return [self.sortby_part_to_dict(v) for v in value] elif value and isinstance(value[0], dict): return value @@ -471,7 +471,7 @@ def _format_sortby(self, value: Optional[SortbyLike]) -> Optional[Sortby]: ) @staticmethod - def sortby_part_to_json(part: str) -> Dict[str, str]: + def sortby_part_to_dict(part: str) -> Dict[str, str]: if part.startswith("-"): return {"field": part[1:], "direction": "desc"} elif part.startswith("+"): @@ -480,7 +480,7 @@ def sortby_part_to_json(part: str) -> Dict[str, str]: return {"field": part, "direction": "asc"} @staticmethod - def sortby_json_to_str(sortby: Sortby) -> str: + def sortby_dict_to_str(sortby: Sortby) -> str: return ",".join( [ f"{'+' if sort['direction'] == 'asc' else '-'}{sort['field']}" @@ -495,9 +495,9 @@ def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]: self._stac_io.assert_conforms_to(ConformanceClasses.FIELDS) if isinstance(value, str): - return self.fields_to_json(value.split(",")) + return self.fields_to_dict(value.split(",")) if isinstance(value, list): - return self.fields_to_json(value) + return self.fields_to_dict(value) if isinstance(value, dict): return value @@ -506,7 +506,7 @@ def _format_fields(self, value: Optional[FieldsLike]) -> Optional[Fields]: ) @staticmethod - def fields_to_json(fields: List[str]) -> Fields: + def fields_to_dict(fields: List[str]) -> Fields: includes: List[str] = [] excludes: List[str] = [] for field in fields: @@ -519,7 +519,7 @@ def fields_to_json(fields: List[str]) -> Fields: return {"includes": includes, "excludes": excludes} @staticmethod - def fields_json_to_str(fields: Fields) -> str: + def fields_dict_to_str(fields: Fields) -> str: includes = [f"+{x}" for x in fields.get("includes", [])] excludes = [f"-{x}" for x in fields.get("excludes", [])] return ",".join(chain(includes, excludes))