upgrade OWSLib to 0.20.0 and fix WFS3 (OAPIF) Probes (#333)

* upgrade OWSLib to 0.20.0 and fix OAPIF Probes

* upgrade OWSLib to 0.20.0 and fix OAPIF Probes - fix test for pygeoapi

* #333 comment and strings changes OAFeat

GeoHealthCheck/config_main.py

@@ -154,7 +154,7 @@
         'probe_class': 'GeoHealthCheck.plugins.probe.sta.StaCaps'
     },
     'OGC:WFS3': {
-        'probe_class': 'GeoHealthCheck.plugins.probe.wfs3.WFS3Drilldown'
+        'probe_class': 'GeoHealthCheck.plugins.probe.wfs3.WFS3Caps'
     },
     'ESRI:FS': {
         'probe_class': 'GeoHealthCheck.plugins.probe.esrifs.ESRIFSDrilldown'

GeoHealthCheck/healthcheck.py

@@ -238,7 +238,7 @@ def sniff_test_resource(config, resource_type, url):
             if resource_type == 'OGC:STA':
                 title = 'OGC STA'
             elif resource_type == 'OGC:WFS3':
-                title = 'OGC WFS3'
+                title = 'OGC WFS3 (OAPIF)'
             elif resource_type == 'ESRI:FS':
                 title = 'ESRI ArcGIS FS'
             else:
@@ -279,8 +279,8 @@ def geonode_get_ows(base_url):
     base_name = 'GeoNode {}: {{}}'.format(url.hostname)
     status_code = r.getcode()
     if status_code != 200:
-        msg = "Errorous response from GeoNode at {}: {}".format(base_url,
-                                                                r.text)
+        msg = "Error response from GeoNode at {}: {}".format(
+                                                    base_url, r.text)
         raise ValueError(msg)
 
     try:

GeoHealthCheck/plugins/probe/tms.py

@@ -110,7 +110,8 @@ def get_metadata(self, resource, version='1.0.0'):
         :param version:
         :return: Metadata object
         """
-        return TileMapService(resource.url, version=version)
+        return TileMapService(resource.url, version=version,
+                              headers=self.get_request_headers())
 
     # Overridden: expand param-ranges from WMS metadata
     def expand_params(self, resource):

GeoHealthCheck/plugins/probe/wfs.py

@@ -127,7 +127,9 @@ def get_metadata(self, resource, version='1.1.0'):
         :param version:
         :return: Metadata object
         """
-        return WebFeatureService(resource.url, version=version)
+        return WebFeatureService(resource.url,
+                                 version=version,
+                                 headers=self.get_request_headers())
 
     # Overridden: expand param-ranges from WFS metadata
     def expand_params(self, resource):

GeoHealthCheck/plugins/probe/wfs3.py

@@ -1,25 +1,64 @@
-import requests
-from owslib.wfs import WebFeatureService
+from owslib.ogcapi.features import Features
 from openapi_spec_validator import openapi_v3_spec_validator
 
 from GeoHealthCheck.probe import Probe
 from GeoHealthCheck.result import Result, push_result
 
 
+class WFS3Caps(Probe):
+    """Probe for OGC WFS3 API (OAFeat) main endpoint url"""
+
+    NAME = 'OGC WFS3 (OAFeat) API Capabilities'
+    DESCRIPTION = 'Perform OGC WFS3 (OAFeat) API Capabilities ' \
+                  'Operation and check validity'
+    RESOURCE_TYPE = 'OGC:WFS3'
+
+    REQUEST_METHOD = 'GET'
+    REQUEST_HEADERS = {'Accept': 'application/json'}
+
+    # e.g. https://demo.pygeoapi.io/master
+    REQUEST_TEMPLATE = ''
+
+    def __init__(self):
+        Probe.__init__(self)
+
+    CHECKS_AVAIL = {
+        'GeoHealthCheck.plugins.check.checks.HttpStatusNoError': {
+            'default': True
+        },
+        'GeoHealthCheck.plugins.check.checks.JsonParse': {
+            'default': True
+        },
+        'GeoHealthCheck.plugins.check.checks.ContainsStrings': {
+            'set_params': {
+                'strings': {
+                    'name': 'Contains required strings',
+                    'value': ['/conformance', '/collections',
+                              'service', 'links']
+                }
+            },
+            'default': True
+        },
+    }
+    """Check for OGC WFS3 API (OGC API Features) service endpoint
+    availability"""
+
+
 class WFS3Drilldown(Probe):
     """
-    Probe for WFS3 endpoint "drilldown": starting
+    Probe for WFS3 (OpenAPI Features or OAFeat) endpoint "drilldown": starting
     with top endpoint: get Collections and do
-    GetItems on them etc. Using OWSLib.WebFeatureService.
+    GetItems on them etc. Using OWSLib owslib.ogcapi package.
 
-    TODO: needs finalization.
+    TODO: needs renaming: WFS3 is now OAFeat.
     """
 
-    NAME = 'WFS3 Drilldown'
-    DESCRIPTION = 'Traverses a OGC WFS3 (REST) API endpoint by drilling down'
+    NAME = 'WFS3 (OAFeat) Drilldown'
+    DESCRIPTION = 'Traverses a OGC WFS3 (OAFeat) API endpoint by drilling down'
     RESOURCE_TYPE = 'OGC:WFS3'
 
     REQUEST_METHOD = 'GET'
+    REQUEST_HEADERS = {'Accept': 'application/json'}
 
     PARAM_DEFS = {
         'drilldown_level': {
@@ -43,14 +82,15 @@ def perform_request(self):
         See https://github.com/geopython/OWSLib/blob/
         master/tests/doctests/wfs3_GeoServerCapabilities.txt
         """
-        wfs3 = None
+        oa_feat = None
         collections = None
 
         # 1.1 Test Landing Page
         result = Result(True, 'Test Landing Page')
         result.start()
         try:
-            wfs3 = WebFeatureService(self._resource.url, version='3.0')
+            oa_feat = Features(self._resource.url,
+                               headers=self.get_request_headers())
         except Exception as err:
             result.set(False, '%s:%s' % (result.message, str(err)))
 
@@ -61,7 +101,7 @@ def perform_request(self):
         result = Result(True, 'conformance endpoint exists')
         result.start()
         try:
-            wfs3.conformance()
+            oa_feat.conformance()
         except Exception as err:
             result.set(False, str(err))
 
@@ -72,7 +112,7 @@ def perform_request(self):
         result = Result(True, 'Get collections')
         result.start()
         try:
-            collections = wfs3.collections()
+            collections = oa_feat.collections()['collections']
         except Exception as err:
             result.set(False, '%s:%s' % (result.message, str(err)))
 
@@ -84,11 +124,10 @@ def perform_request(self):
         result.start()
         try:
 
-            # TODO: OWSLib 0.17.1 has no call to '/api yet, upgrade when GHC
-            #  supports Py3 to 0.19+.
-            api = wfs3_api_doc(wfs3)
+            # OWSLib 0.20.0+ has call to '/api now.
+            api_doc = oa_feat.api()
             for attr in ['components', 'paths', 'openapi']:
-                val = api.get(attr, None)
+                val = api_doc.get(attr, None)
                 if val is None:
                     msg = 'missing attr: %s' % attr
                     result = push_result(
@@ -117,7 +156,7 @@ def perform_request(self):
                 coll_id = coll_id
 
                 try:
-                    coll = wfs3.collection(coll_id)
+                    coll = oa_feat.collection(coll_id)
 
                     # TODO: Maybe also add crs
                     for attr in ['id', 'links']:
@@ -136,7 +175,7 @@ def perform_request(self):
                     continue
 
                 try:
-                    items = wfs3.collection_items(coll_id, limit=1)
+                    items = oa_feat.collection_items(coll_id, limit=1)
                 except Exception as e:
                     msg = 'GetItems %s: OWSLib err: %s ' % (str(e), coll_id)
                     result = push_result(
@@ -162,7 +201,7 @@ def perform_request(self):
 
                     fid = items['features'][0]['id']
                     try:
-                        item = wfs3.collection_item(coll_id, fid)
+                        item = oa_feat.collection_item(coll_id, fid)
                     except Exception as e:
                         msg = 'GetItem %s: OWSLib err: %s' \
                               % (str(e), coll_id)
@@ -199,13 +238,13 @@ def perform_request(self):
 
 class WFS3OpenAPIValidator(Probe):
     """
-    Probe for WFS3 OpenAPI Spec Validation (/api endpoint).
+    Probe for WFS3 (OAFeat) OpenAPI Spec Validation (/api endpoint).
     Uses https://pypi.org/project/openapi-spec-validator/.
 
     """
 
-    NAME = 'WFS3 OpenAPI Validator'
-    DESCRIPTION = 'Validates WFS3 /api endpoint for OpenAPI compliance'
+    NAME = 'WFS3 (OAFeat) OpenAPI Validator'
+    DESCRIPTION = 'Validates WFS3 (OAFeat) api endpoint for OpenAPI compliance'
     RESOURCE_TYPE = 'OGC:WFS3'
 
     REQUEST_METHOD = 'GET'
@@ -227,16 +266,17 @@ def perform_request(self):
         result.start()
         api_doc = None
         try:
-            wfs3 = WebFeatureService(self._resource.url, version='3.0')
+            oa_feat = Features(self._resource.url,
+                               headers=self.get_request_headers())
 
-            # TODO: OWSLib 0.17.1 has no call to '/api yet.
-            api_doc = wfs3_api_doc(wfs3)
+            # OWSLib 0.20.0 has call to OpenAPI doc now.
+            api_doc = oa_feat.api()
 
             # Basic sanity check
             for attr in ['components', 'paths', 'openapi']:
                 val = api_doc.get(attr, None)
                 if val is None:
-                    msg = '/api: missing attr: %s' % attr
+                    msg = 'OpenAPI doc: missing attr: %s' % attr
                     result.set(False, msg)
                     break
         except Exception as err:
@@ -249,7 +289,7 @@ def perform_request(self):
         if api_doc is None or result.success is False:
             return
 
-        # ASSERTION: /api exists, next OpenAPI Validation
+        # ASSERTION: OpenAPI doc exists, next OpenAPI Validation
 
         # Step 2 detailed OpenAPI Compliance test
         result = Result(True, 'Validate OpenAPI Compliance')
@@ -270,64 +310,8 @@ def perform_request(self):
         # Add to overall Probe result
         self.result.add_result(result)
 
-
-def wfs3_api_doc(wfs3):
-    """
-    implements Requirement 3 (/req/core/api-definition-op)
-    @returns: OpenAPI definition object
-    """
-
-    api_url = None
-
-    for link in wfs3.links:
-        if link['rel'] == 'service-desc':
-            api_url = link['href']
-
-    if not api_url:
-        raise RuntimeError('Did not find service-desc link in landing page')
-
-    return requests.get(api_url).json()
-
-
-# class WFS3Caps(Probe):
-#     """Probe for OGC WFS3 API main endpoint url"""
-#
-#     NAME = 'OGC WFS3 API Capabilities'
-#     DESCRIPTION = 'Perform OGC WFS3 API Capabilities
-#     Operation and check validity'
-#     RESOURCE_TYPE = 'OGC:WFS3'
-#
-#     REQUEST_METHOD = 'GET'
-#
-#     # e.g. https://demo.pygeoapi.io/master/collections?f=json
-#     REQUEST_TEMPLATE = '/{endpoint}?f=json'
-#
-#     def __init__(self):
-#         Probe.__init__(self)
-#
-#     PARAM_DEFS = {
-#         'endpoint': {
-#             'type': 'string',
-#             'description': 'The OGC WFS3 API service endpoint type',
-#             'default': '/collections',
-#             'required': True,
-#             'range': ['collections', 'conformance', 'api']
-#         }
-#     }
-#     """Param defs"""
-#
-#     CHECKS_AVAIL = {
-#         'GeoHealthCheck.plugins.check.checks.HttpStatusNoError': {
-#             'default': True
-#         },
-#         'GeoHealthCheck.plugins.check.checks.JsonParse': {
-#             'default': True
-#         }
-#     }
-#     """Check for OGC WFS3 API OGC WFS3 API service endpoint
-#     availability"""
-#
-#
+# TODO implement: similar to single WMS layer,
+#  expand params with collection names etc.
 # class WFS3Collection(Probe):
 #     """Probe for OGC WFS3 API main endpoint url"""
 #

GeoHealthCheck/util.py

@@ -283,3 +283,16 @@ def create_requests_retry_session(
     session.mount('http://', adapter)
     session.mount('https://', adapter)
     return session
+
+
+# Optionally expand a URL with query clause like 'f=json'
+def expand_url(url, query_clause):
+    if query_clause in url:
+        # Already present
+        return url
+
+    # Determine concatenator char
+    concat = '?'
+    if concat in url:
+        concat = '&'
+    return '{}{}{}'.format(url, concat, query_clause)

requirements.txt

@@ -6,8 +6,8 @@ Flask-Script==2.0.6
 SQLAlchemy==1.3.8
 Flask-SQLAlchemy==2.4.0
 itsdangerous==1.1.0
-pyproj==1.9.6
-OWSLib==0.17.1  # update to new version when solution to OWSLIB #614 is released
+pyproj==2.4.1
+OWSLib==0.20.0
 jsonschema==3.0.2 # downgrade from 3.2.0 on sept 29, 2020, issue 331, consider better fix
 openapi-spec-validator==0.2.8
 Sphinx==2.2.0

tests/data/fixtures.json

@@ -92,7 +92,7 @@
       "resource_type": "OSGeo:WFS3",
       "active": true,
       "title": "pygeoapi - master",
-      "url": "https://demo.pygeoapi.io/stable",
+      "url": "https://demo.pygeoapi.io/master",
       "tags": [
         "ogc"
       ]