From dab127398b4aab492f83b926cabb2a9f399db4ad Mon Sep 17 00:00:00 2001 From: David Huard Date: Mon, 14 Jun 2021 16:05:53 -0400 Subject: [PATCH 1/9] add spatial filters in fes2.py, add gml:Point object, fix gml namespace link. --- owslib/fes2.py | 52 +++++++++++++++++++++++++++++++++++ owslib/gml.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ owslib/namespaces.py | 4 +-- 3 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 owslib/gml.py diff --git a/owslib/fes2.py b/owslib/fes2.py index f7141cc9e..b733dfc94 100644 --- a/owslib/fes2.py +++ b/owslib/fes2.py @@ -18,6 +18,7 @@ from owslib.etree import etree from owslib import util from owslib.namespaces import Namespaces +from abc import ABCMeta, abstractmethod # default variables @@ -391,6 +392,57 @@ def toXML(self): return tmp +class TopologicalOpType(OgcExpression, metaclass=ABCMeta): + """Abstract base class for topological operators.""" + @property + @abstractmethod + def operation(self): + """This is a mechanism to ensure this class is subclassed by an actual operation.""" + pass + + def __init__(self, propertyname, geometry): + self.propertyname = propertyname + self.geometry = geometry + + def toXML(self): + p = etree.Element(util.nspath_eval(f"fes:Filter", namespaces)) + + node = etree.Element(util.nspath_eval(f"fes:{self.operation}", namespaces)) + etree.SubElement(node, util.nspath_eval("fes:ValueReference", namespaces)).text = self.propertyname + node.append(self.geometry.toXML()) + + p.append(node) + return p + + +class Intersects(TopologicalOpType): + operation = "Intersects" + + +class Contains(TopologicalOpType): + operation = "Contains" + + +class Disjoint(TopologicalOpType): + operation = "Disjoint" + + +class Within(TopologicalOpType): + operation = "Within" + + +class Touches(TopologicalOpType): + operation = "Touches" + + +class Overlaps(TopologicalOpType): + operation = "Overlaps" + + +class Equals(TopologicalOpType): + operation = "Equals" + + # BINARY class BinaryLogicOpType(OgcExpression): """ Binary Operators: And / Or """ diff --git a/owslib/gml.py b/owslib/gml.py new file mode 100644 index 000000000..8b664ed72 --- /dev/null +++ b/owslib/gml.py @@ -0,0 +1,65 @@ +from dataclasses import dataclass +from typing import Sequence +from owslib.etree import etree +from owslib import util +from owslib.namespaces import Namespaces + +# default variables +def get_namespaces(): + n = Namespaces() + ns = n.get_namespaces(["gml", "ogc", "xsd"]) + ns[None] = n.get_namespace("ogc") + return ns + + +namespaces = get_namespaces() + +prefix = lambda x: util.nspath_eval(x, namespaces) + + +@dataclass +class AbstractGMLType(): + id: str + + +@dataclass +class AbstractGeometryType(AbstractGMLType): + srsName: str = None + srsDimension: int = None + # axisLabels: str = None + # uomLabels: str = None + + description: str = None + descriptionReference: str = None + identifier: str = None + name: str = None + + +@dataclass +class _PointBase: + pos: Sequence[float] + + +@dataclass +class Point(AbstractGeometryType, _PointBase): + """GML Point object.""" + + def toXML(self): + """Return `lxml.etree.Element` object.""" + node = etree.Element(prefix("gml:Point")) + for key in ["id", "srsName"]: + if getattr(self, key, None) is not None: + node.set(prefix(f"gml:{key}"), getattr(self, key)) + + for key in ["description", "descriptionReference", "identifier", "name"]: + content = getattr(self, key) + if content is not None: + etree.SubElement(node, prefix(f"gml:{key}")).text = content + + coords = etree.SubElement(node, prefix("gml:pos")) + coords.text = " ".join([str(c) for c in self.pos]) + for key in ["srsDimension"]: + if getattr(self, key, None) is not None: + node.set(prefix(f"gml:{key}"), getattr(self, key)) + + return node diff --git a/owslib/namespaces.py b/owslib/namespaces.py index 98299856e..c2c2d6cea 100644 --- a/owslib/namespaces.py +++ b/owslib/namespaces.py @@ -19,8 +19,8 @@ class Namespaces(object): 'gm03': 'http://www.interlis.ch/INTERLIS2.3', 'gmd': 'http://www.isotc211.org/2005/gmd', 'gmi': 'http://www.isotc211.org/2005/gmi', - 'gml': 'http://www.opengis.net/gml', - 'gml311': 'http://www.opengis.net/gml', + 'gml': 'http://www.opengis.net/gml/3.2', + 'gml311': 'http://www.opengis.net/gml', # Dead link 'gml32': 'http://www.opengis.net/gml/3.2', 'gmx': 'http://www.isotc211.org/2005/gmx', 'gts': 'http://www.isotc211.org/2005/gts', From 6a462b277aaf5f02632def4c326834cbb8115a9a Mon Sep 17 00:00:00 2001 From: David Huard Date: Mon, 14 Jun 2021 16:31:53 -0400 Subject: [PATCH 2/9] use gml32 explicitly. bypass filter construction from string --- owslib/feature/postrequest.py | 8 ++++++-- owslib/gml.py | 12 ++++++------ owslib/namespaces.py | 4 ++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/owslib/feature/postrequest.py b/owslib/feature/postrequest.py index 83c6b3dd2..c9ca945c5 100644 --- a/owslib/feature/postrequest.py +++ b/owslib/feature/postrequest.py @@ -173,8 +173,12 @@ def set_filter(self, filter): Cannot be used with set_bbox() or set_featureid(). """ - f = etree.fromstring(filter) - sub_elem = f.find(util.nspath("Filter", FES_NAMESPACE)) + if isinstance(filter, str): + f = etree.fromstring(filter) + sub_elem = f.find(util.nspath("Filter", FES_NAMESPACE)) + else: + sub_elem = filter + self._query.append(sub_elem) def set_maxfeatures(self, maxfeatures): diff --git a/owslib/gml.py b/owslib/gml.py index 8b664ed72..26c688eef 100644 --- a/owslib/gml.py +++ b/owslib/gml.py @@ -7,7 +7,7 @@ # default variables def get_namespaces(): n = Namespaces() - ns = n.get_namespaces(["gml", "ogc", "xsd"]) + ns = n.get_namespaces(["gml32", "ogc", "xsd"]) ns[None] = n.get_namespace("ogc") return ns @@ -46,20 +46,20 @@ class Point(AbstractGeometryType, _PointBase): def toXML(self): """Return `lxml.etree.Element` object.""" - node = etree.Element(prefix("gml:Point")) + node = etree.Element(prefix("gml32:Point")) for key in ["id", "srsName"]: if getattr(self, key, None) is not None: - node.set(prefix(f"gml:{key}"), getattr(self, key)) + node.set(prefix(f"gml32:{key}"), getattr(self, key)) for key in ["description", "descriptionReference", "identifier", "name"]: content = getattr(self, key) if content is not None: - etree.SubElement(node, prefix(f"gml:{key}")).text = content + etree.SubElement(node, prefix(f"gml32:{key}")).text = content - coords = etree.SubElement(node, prefix("gml:pos")) + coords = etree.SubElement(node, prefix("gml32:pos")) coords.text = " ".join([str(c) for c in self.pos]) for key in ["srsDimension"]: if getattr(self, key, None) is not None: - node.set(prefix(f"gml:{key}"), getattr(self, key)) + node.set(prefix(f"gml32:{key}"), getattr(self, key)) return node diff --git a/owslib/namespaces.py b/owslib/namespaces.py index c2c2d6cea..98299856e 100644 --- a/owslib/namespaces.py +++ b/owslib/namespaces.py @@ -19,8 +19,8 @@ class Namespaces(object): 'gm03': 'http://www.interlis.ch/INTERLIS2.3', 'gmd': 'http://www.isotc211.org/2005/gmd', 'gmi': 'http://www.isotc211.org/2005/gmi', - 'gml': 'http://www.opengis.net/gml/3.2', - 'gml311': 'http://www.opengis.net/gml', # Dead link + 'gml': 'http://www.opengis.net/gml', + 'gml311': 'http://www.opengis.net/gml', 'gml32': 'http://www.opengis.net/gml/3.2', 'gmx': 'http://www.isotc211.org/2005/gmx', 'gts': 'http://www.isotc211.org/2005/gts', From 56a47ac24409f3313c2406ae1571a17818b5eec0 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 17 Jun 2021 10:25:07 -0400 Subject: [PATCH 3/9] install dataclasses if python < 3.7 --- owslib/gml.py | 3 ++- requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/owslib/gml.py b/owslib/gml.py index 26c688eef..13571b88b 100644 --- a/owslib/gml.py +++ b/owslib/gml.py @@ -4,7 +4,7 @@ from owslib import util from owslib.namespaces import Namespaces -# default variables + def get_namespaces(): n = Namespaces() ns = n.get_namespaces(["gml32", "ogc", "xsd"]) @@ -37,6 +37,7 @@ class AbstractGeometryType(AbstractGMLType): @dataclass class _PointBase: + """This is used to avoid issues arising from non-optional attributes defined after optional attributes.""" pos: Sequence[float] diff --git a/requirements.txt b/requirements.txt index fcca5839b..e508ba74f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ pytz requests>=1.0 pyproj >=2 pyyaml +dataclasses; python_version < '3.7' From 645316eb4070652d0e54051e2357e43cfb40fb34 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 17 Jun 2021 16:04:31 -0400 Subject: [PATCH 4/9] add Filter class to wrap topological operations, allowing combinations with And & Or --- owslib/fes2.py | 15 +++++++++++---- tests/test_fes2.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 tests/test_fes2.py diff --git a/owslib/fes2.py b/owslib/fes2.py index b733dfc94..6508b5ad4 100644 --- a/owslib/fes2.py +++ b/owslib/fes2.py @@ -392,6 +392,16 @@ def toXML(self): return tmp +class Filter(OgcExpression): + def __init__(self, filter): + self.filter = filter + + def toXML(self): + node = etree.Element(util.nspath_eval(f"fes:Filter", namespaces)) + node.append(self.filter.toXML()) + return node + + class TopologicalOpType(OgcExpression, metaclass=ABCMeta): """Abstract base class for topological operators.""" @property @@ -405,14 +415,11 @@ def __init__(self, propertyname, geometry): self.geometry = geometry def toXML(self): - p = etree.Element(util.nspath_eval(f"fes:Filter", namespaces)) - node = etree.Element(util.nspath_eval(f"fes:{self.operation}", namespaces)) etree.SubElement(node, util.nspath_eval("fes:ValueReference", namespaces)).text = self.propertyname node.append(self.geometry.toXML()) - p.append(node) - return p + return node class Intersects(TopologicalOpType): diff --git a/tests/test_fes2.py b/tests/test_fes2.py new file mode 100644 index 000000000..2e6732da2 --- /dev/null +++ b/tests/test_fes2.py @@ -0,0 +1,15 @@ +from owslib import fes2 +from owslib.gml import Point +from lxml.etree import tostring + + +def test_filter(): + point = Point(id="qc", srsName="http://www.opengis.net/gml/srs/epsg.xml#4326", pos=[-71, 46]) + f = fes2.Filter( + fes2.And([fes2.Intersects(propertyname="the_geom", geometry=point), + fes2.PropertyIsLike("name", "value")] + ) + ) + + xml = f.toXML() + assert tostring(xml) == b'the_geom-71 46namevalue' From ea095dc66353887efa6fdfd06107e4b381ce31dd Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 17 Jun 2021 16:20:53 -0400 Subject: [PATCH 5/9] use owslib.etree --- tests/test_fes2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_fes2.py b/tests/test_fes2.py index 2e6732da2..e8adbc851 100644 --- a/tests/test_fes2.py +++ b/tests/test_fes2.py @@ -1,6 +1,6 @@ from owslib import fes2 from owslib.gml import Point -from lxml.etree import tostring +from owslib.etree import etree def test_filter(): @@ -12,4 +12,5 @@ def test_filter(): ) xml = f.toXML() - assert tostring(xml) == b'the_geom-71 46namevalue' + assert etree.tostring(xml) == b'the_geom-71 46namevalue' From 7c8bdb9b82ccb09944f10670fbf6b72dc30f19de Mon Sep 17 00:00:00 2001 From: David Huard Date: Wed, 30 Jun 2021 10:32:45 -0400 Subject: [PATCH 6/9] pep8 --- owslib/iso.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/owslib/iso.py b/owslib/iso.py index 7b68ecd71..f29e6312c 100644 --- a/owslib/iso.py +++ b/owslib/iso.py @@ -548,7 +548,8 @@ def __init__(self, md=None, identtype=None): self.status = _testCodeListValue(md.find(util.nspath_eval('gmd:status/gmd:MD_ProgressCode', namespaces))) self.graphicoverview = [] - for val in md.findall(util.nspath_eval('gmd:graphicOverview/gmd:MD_BrowseGraphic/gmd:fileName/gco:CharacterString', namespaces)): + for val in md.findall(util.nspath_eval( + 'gmd:graphicOverview/gmd:MD_BrowseGraphic/gmd:fileName/gco:CharacterString', namespaces)): if val is not None: val2 = util.testXMLValue(val) if val2 is not None: From 82fdc6399457970768c8ce21600bb4015c519367 Mon Sep 17 00:00:00 2001 From: David Huard Date: Wed, 30 Jun 2021 12:00:32 -0400 Subject: [PATCH 7/9] fixed test. added live filtering test --- tests/test_fes2.py | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/tests/test_fes2.py b/tests/test_fes2.py index e8adbc851..32634631e 100644 --- a/tests/test_fes2.py +++ b/tests/test_fes2.py @@ -1,9 +1,21 @@ +import pytest +from owslib.wfs import WebFeatureService from owslib import fes2 from owslib.gml import Point -from owslib.etree import etree +from owslib.namespaces import Namespaces +from owslib import util +import geojson +n = Namespaces() +FES_NAMESPACE = n.get_namespace("fes") +GML32_NAMESPACE = n.get_namespace("gml32") -def test_filter(): + +SERVICE_URL = "http://soggy.zoology.ubc.ca:8080/geoserver/wfs" + + +def test_raw_filter(): + """Just inspect the filter object (not embedded in a getfeature request).""" point = Point(id="qc", srsName="http://www.opengis.net/gml/srs/epsg.xml#4326", pos=[-71, 46]) f = fes2.Filter( fes2.And([fes2.Intersects(propertyname="the_geom", geometry=point), @@ -12,5 +24,23 @@ def test_filter(): ) xml = f.toXML() - assert etree.tostring(xml) == b'the_geom-71 46namevalue' + + # Fairly basic test + xml.find(util.nspath("Filter", FES_NAMESPACE)) + xml.find(util.nspath("And", FES_NAMESPACE)) + xml.find(util.nspath("Intersects", FES_NAMESPACE)) + xml.find(util.nspath("Point", GML32_NAMESPACE)) + + +@pytest.mark.online +def test_filter(): + """A request without filtering will yield 600 entries. With filtering we expect only one. + + Note that this type of topological filtering only works (so far) with WFS 2.0.0 and POST requests. + """ + wfs = WebFeatureService(SERVICE_URL, version="2.0.0") + layer = "bcmca:commercialfish_crab" + point = Point(id="random", srsName="http://www.opengis.net/gml/srs/epsg.xml#4326", pos=[-129.8, 55.44]) + f = fes2.Filter(fes2.Contains(propertyname="geom", geometry=point)) + r = wfs.getfeature(layer, outputFormat="application/json", method="POST", filter=f.toXML()) + assert geojson.load(r)["totalFeatures"] == 1 From b50a97d2e8c330972e7c2df0489c12c588fbab74 Mon Sep 17 00:00:00 2001 From: David Huard Date: Wed, 30 Jun 2021 13:40:16 -0400 Subject: [PATCH 8/9] replace geojson by json --- tests/test_fes2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_fes2.py b/tests/test_fes2.py index 32634631e..6bb12be58 100644 --- a/tests/test_fes2.py +++ b/tests/test_fes2.py @@ -4,7 +4,7 @@ from owslib.gml import Point from owslib.namespaces import Namespaces from owslib import util -import geojson +import json n = Namespaces() FES_NAMESPACE = n.get_namespace("fes") @@ -43,4 +43,4 @@ def test_filter(): point = Point(id="random", srsName="http://www.opengis.net/gml/srs/epsg.xml#4326", pos=[-129.8, 55.44]) f = fes2.Filter(fes2.Contains(propertyname="geom", geometry=point)) r = wfs.getfeature(layer, outputFormat="application/json", method="POST", filter=f.toXML()) - assert geojson.load(r)["totalFeatures"] == 1 + assert json.load(r)["totalFeatures"] == 1 From 943f40fe0617801ddf844b9f1f0cc25185e2d229 Mon Sep 17 00:00:00 2001 From: David Huard Date: Wed, 30 Jun 2021 14:15:10 -0400 Subject: [PATCH 9/9] pep8 --- owslib/fes2.py | 2 +- owslib/gml.py | 5 ++++- tests/test_wfs_generic.py | 9 +-------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/owslib/fes2.py b/owslib/fes2.py index 6508b5ad4..795104fd4 100644 --- a/owslib/fes2.py +++ b/owslib/fes2.py @@ -397,7 +397,7 @@ def __init__(self, filter): self.filter = filter def toXML(self): - node = etree.Element(util.nspath_eval(f"fes:Filter", namespaces)) + node = etree.Element(util.nspath_eval("fes:Filter", namespaces)) node.append(self.filter.toXML()) return node diff --git a/owslib/gml.py b/owslib/gml.py index 13571b88b..3bea9eb82 100644 --- a/owslib/gml.py +++ b/owslib/gml.py @@ -14,7 +14,10 @@ def get_namespaces(): namespaces = get_namespaces() -prefix = lambda x: util.nspath_eval(x, namespaces) + +def prefix(x): + """Shorthand to insert namespaces.""" + return util.nspath_eval(x, namespaces) @dataclass diff --git a/tests/test_wfs_generic.py b/tests/test_wfs_generic.py index 57af94c61..343c6d059 100644 --- a/tests/test_wfs_generic.py +++ b/tests/test_wfs_generic.py @@ -2,7 +2,7 @@ from owslib.util import ServiceException from urllib.parse import urlparse from tests.utils import resource_file, sorted_url_query, service_ok - +import json import pytest SERVICE_URL = 'https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288' @@ -54,7 +54,6 @@ def test_getfeature(): @pytest.mark.skipif(not service_ok(SERVICE_URL), reason="WFS service is unreachable") def test_outputformat_wfs_100(): - import json wfs = WebFeatureService('https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288', version='1.0.0') feature = wfs.getfeature( @@ -66,7 +65,6 @@ def test_outputformat_wfs_100(): @pytest.mark.skipif(not service_ok(SERVICE_URL), reason="WFS service is unreachable") def test_outputformat_wfs_110(): - import json wfs = WebFeatureService('https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288', version='1.1.0') feature = wfs.getfeature( @@ -78,7 +76,6 @@ def test_outputformat_wfs_110(): @pytest.mark.skipif(not service_ok(SERVICE_URL), reason="WFS service is unreachable") def test_outputformat_wfs_200(): - import json wfs = WebFeatureService('https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288', version='2.0.0') feature = wfs.getfeature( @@ -90,7 +87,6 @@ def test_outputformat_wfs_200(): @pytest.mark.skipif(not service_ok(SERVICE_URL), reason="WFS service is unreachable") def test_srsname_wfs_100(): - import json wfs = WebFeatureService('https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288', version='1.0.0') # ServiceException: Unable to support srsName: EPSG:99999999 @@ -99,7 +95,6 @@ def test_srsname_wfs_100(): typename=['sb:Project_Area'], maxfeatures=1, propertyname=None, outputFormat='application/json', srsname="EPSG:99999999") - import json wfs = WebFeatureService('https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288', version='1.0.0') feature = wfs.getfeature( @@ -112,7 +107,6 @@ def test_srsname_wfs_100(): @pytest.mark.skipif(not service_ok(SERVICE_URL), reason="WFS service is unreachable") def test_srsname_wfs_110(): - import json wfs = WebFeatureService( 'https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288', version='1.1.0') @@ -122,7 +116,6 @@ def test_srsname_wfs_110(): typename=['sb:Project_Area'], maxfeatures=1, propertyname=None, outputFormat='application/json', srsname="EPSG:99999999") - import json wfs = WebFeatureService( 'https://www.sciencebase.gov/catalogMaps/mapping/ows/53398e51e4b0db25ad10d288', version='1.0.0')