From 49ff08aa1fb8c3861407651b713cee9454dc6647 Mon Sep 17 00:00:00 2001 From: David Huard Date: Wed, 17 Feb 2021 09:10:58 -0500 Subject: [PATCH 01/18] added json responses from the 52N OGCAPI test server implementation for static testing --- tests/resources/ogcapi/processes/api_52n.json | 234 ++++++++++++++++++ .../ogcapi/processes/capabilities_52n.json | 1 + .../processes/conformancedeclaration_52n.json | 1 + ...2n_multireferencebinaryinputalgorithm.json | 1 + .../ogcapi/processes/processlists_52n.json | 1 + 5 files changed, 238 insertions(+) create mode 100644 tests/resources/ogcapi/processes/api_52n.json create mode 100644 tests/resources/ogcapi/processes/capabilities_52n.json create mode 100644 tests/resources/ogcapi/processes/conformancedeclaration_52n.json create mode 100644 tests/resources/ogcapi/processes/processdescription_52n_multireferencebinaryinputalgorithm.json create mode 100644 tests/resources/ogcapi/processes/processlists_52n.json diff --git a/tests/resources/ogcapi/processes/api_52n.json b/tests/resources/ogcapi/processes/api_52n.json new file mode 100644 index 000000000..6915f12b9 --- /dev/null +++ b/tests/resources/ogcapi/processes/api_52n.json @@ -0,0 +1,234 @@ +{ + "openapi" : "3.0.2", + "info" : { + "title" : "The OGC API - Processes", + "version" : "1.0-draft.4", + "description" : "WARNING - THIS IS WORK IN PROGRESS", + "contact" : { + "name" : "Benjamin Pross", + "email" : "b.pross@52north.org" + }, + "license" : { + "name" : "OGC license", + "url" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/LICENSE" + } + }, + "paths" : { + "/" : { + "get" : { + "summary" : "landing page of this API", + "description" : "The landing page provides links to the API definition, the conformance declaration and the metadata about the processes offered by this service.", + "operationId" : "getLandingPage", + "tags" : [ "Capabilities" ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/LandingPage.yaml" + }, + "500" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ServerError.yaml" + } + } + } + }, + "/conformance" : { + "get" : { + "summary" : "information about standards that this API conforms to", + "description" : "Lists all requirements classes specified in the standard (e.g., OGC API - Processes Part 1: Core) that the server conforms to", + "operationId" : "getConformanceClasses", + "tags" : [ "ConformanceDeclaration" ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ConformanceDeclaration.yaml" + }, + "500" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ServerError.yaml" + } + } + } + }, + "/processes" : { + "get" : { + "summary" : "retrieve available processes", + "description" : "Lists all available processes this server offers.", + "operationId" : "getProcesses", + "tags" : [ "ProcessList" ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ProcessList.yaml" + } + } + } + }, + "/processes/{processId}" : { + "get" : { + "summary" : "retrieve a process description", + "description" : "Describes a process.", + "operationId" : "getProcessDescription", + "tags" : [ "ProcessDescription" ], + "parameters" : [ { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/processId.yaml" + } ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ProcessDescription.yaml" + }, + "404" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/NotFound.yaml" + } + } + } + }, + "/processes/{processId}/jobs" : { + "get" : { + "summary" : "retrieve the list of jobs for a process.", + "description" : "Lists available jobs of a process.", + "operationId" : "getJobs", + "tags" : [ "JobList" ], + "parameters" : [ { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/processId.yaml" + } ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/JobList.yaml" + }, + "404" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/NotFound.yaml" + } + } + }, + "post" : { + "summary" : "execute a process.", + "description" : "Submits a new job.", + "operationId" : "execute", + "tags" : [ "Execute" ], + "parameters" : [ { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/processId.yaml" + } ], + "requestBody" : { + "description" : "Mandatory execute request JSON", + "required" : true, + "content" : { + "application/json" : { + "schema" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/execute.yaml" + } + } + } + }, + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ExecuteSync.yaml" + }, + "201" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ExecuteAsync.yaml" + }, + "404" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/NotFound.yaml" + }, + "500" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ServerError.yaml" + } + }, + "callbacks" : { + "jobCompleted" : { + "{$request.body#/subscriber/successUri}" : { + "post" : { + "requestBody" : { + "content" : { + "application/json" : { + "schema" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/schemas/result.yaml" + } + } + } + }, + "responses" : { + "200" : { + "description" : "Results received successfully" + } + } + } + } + } + } + } + }, + "/processes/{processId}/jobs/{jobId}" : { + "get" : { + "summary" : "retrieve the status of a job", + "description" : "Shows the status of a job.", + "operationId" : "getStatus", + "tags" : [ "Status" ], + "parameters" : [ { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/processId.yaml" + }, { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/jobId.yaml" + } ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/Status.yaml" + }, + "404" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/NotFound.yaml" + }, + "500" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ServerError.yaml" + } + } + }, + "delete" : { + "summary" : "cancel a job execution, remove a finished job", + "description" : "Cancel a job execution and remove it from the jobs list.", + "operationId" : "dismiss", + "tags" : [ "Dismiss" ], + "parameters" : [ { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/processId.yaml" + }, { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/jobId.yaml" + } ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/Status.yaml" + }, + "404" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/NotFound.yaml" + }, + "500" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ServerError.yaml" + } + } + } + }, + "/processes/{processId}/jobs/{jobId}/results" : { + "get" : { + "summary" : "retrieve the result(s) of a job", + "description" : "Lists available results of a job. In case of a failure, lists exceptions instead.", + "operationId" : "getResult", + "tags" : [ "Result" ], + "parameters" : [ { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/processId.yaml" + }, { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/parameters/jobId.yaml" + } ], + "responses" : { + "200" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/Results.yaml" + }, + "404" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/NotFound.yaml" + }, + "500" : { + "$ref" : "https://raw.githubusercontent.com/opengeospatial/wps-rest-binding/master/core/openapi/responses/ServerError.yaml" + } + } + } + } + }, + "servers" : [ { + "description" : "SwaggerHub API Auto Mocking", + "url" : "https://virtserver.swaggerhub.com/geoprocessing/WPS/0.01" + }, { + "description" : "52°North demo server", + "url" : "http://geoprocessing.demo.52north.org:8080/javaps/rest/" + } ] +} diff --git a/tests/resources/ogcapi/processes/capabilities_52n.json b/tests/resources/ogcapi/processes/capabilities_52n.json new file mode 100644 index 000000000..425ec2b9b --- /dev/null +++ b/tests/resources/ogcapi/processes/capabilities_52n.json @@ -0,0 +1 @@ +{"title":"52°North draft OGC API - Processes","description":"52°North draft OGC API - Processes, powered by javaPS","links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest","rel":"self","type":"application/json","title":"This document"},{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/api/","rel":"service","type":"application/openapi+json;version=3.0","title":"The API definition"},{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/conformance/","rel":"conformance","type":"application/json","title":"Conformance classes implemented by this server"},{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/","rel":"processes","type":"application/json","title":"The processes offered by this server"}]} diff --git a/tests/resources/ogcapi/processes/conformancedeclaration_52n.json b/tests/resources/ogcapi/processes/conformancedeclaration_52n.json new file mode 100644 index 000000000..090361aa2 --- /dev/null +++ b/tests/resources/ogcapi/processes/conformancedeclaration_52n.json @@ -0,0 +1 @@ +{"conformsTo":["http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core","http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30","http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json","http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/html","http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/job-list"]} diff --git a/tests/resources/ogcapi/processes/processdescription_52n_multireferencebinaryinputalgorithm.json b/tests/resources/ogcapi/processes/processdescription_52n_multireferencebinaryinputalgorithm.json new file mode 100644 index 000000000..b1d70bd85 --- /dev/null +++ b/tests/resources/ogcapi/processes/processdescription_52n_multireferencebinaryinputalgorithm.json @@ -0,0 +1 @@ +{"id":"org.n52.javaps.test.MultiReferenceBinaryInputAlgorithm","title":"for testing multiple binary inputs by reference","keywords":[],"metadata":[],"version":"1.1.0","jobControlOptions":["async-execute","sync-execute"],"outputTransmission":["value","reference"],"links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.MultiReferenceBinaryInputAlgorithm/jobs","rel":"execute","title":"Execute endpoint"}],"inputs":[{"id":"data","title":"data","keywords":[],"metadata":[],"input":{"formats":[{"default":true,"mimeType":"application/x-zipped-shp"},{"default":false,"mimeType":"application/x-zipped-shp"},{"default":false,"mimeType":"image/tiff","encoding":"base64"},{"default":false,"mimeType":"image/png"},{"default":false,"mimeType":"application/x-zipped-shp","encoding":"base64"},{"default":false,"mimeType":"image/tiff"},{"default":false,"mimeType":"image/png","encoding":"base64"},{"default":false,"mimeType":"image/geotiff","encoding":"base64"},{"default":false,"mimeType":"image/geotiff"}]},"minOccurs":1,"maxOccurs":2}],"outputs":[{"id":"result","title":"result","keywords":[],"metadata":[],"output":{"formats":[{"default":true,"mimeType":"application/x-zipped-shp"},{"default":false,"mimeType":"application/x-zipped-shp"},{"default":false,"mimeType":"image/tiff","encoding":"base64"},{"default":false,"mimeType":"image/png"},{"default":false,"mimeType":"application/x-zipped-shp","encoding":"base64"},{"default":false,"mimeType":"image/tiff"},{"default":false,"mimeType":"image/png","encoding":"base64"},{"default":false,"mimeType":"image/geotiff","encoding":"base64"},{"default":false,"mimeType":"image/geotiff"}]}}]} diff --git a/tests/resources/ogcapi/processes/processlists_52n.json b/tests/resources/ogcapi/processes/processlists_52n.json new file mode 100644 index 000000000..ae5778387 --- /dev/null +++ b/tests/resources/ogcapi/processes/processlists_52n.json @@ -0,0 +1 @@ +[{"id":"org.n52.javaps.test.MultiReferenceBinaryInputAlgorithm","title":"for testing multiple binary inputs by reference","keywords":[],"metadata":[],"version":"1.1.0","jobControlOptions":["async-execute","sync-execute"],"outputTransmission":["value","reference"],"links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.MultiReferenceBinaryInputAlgorithm","rel":"process-desc","type":"application/json","title":"Process description"}]},{"id":"org.n52.geoprocessing.geotools.algorithm.CoordinateTransformationAlgorithm","title":"org.n52.geoprocessing.geotools.algorithm.CoordinateTransformationAlgorithm","keywords":[],"metadata":[],"version":"1.1.0","jobControlOptions":["async-execute","sync-execute"],"outputTransmission":["value","reference"],"links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.geoprocessing.geotools.algorithm.CoordinateTransformationAlgorithm","rel":"process-desc","type":"application/json","title":"Process description"}]},{"id":"org.n52.wps.server.algorithm.SimpleBufferAlgorithm","title":"org.n52.wps.server.algorithm.SimpleBufferAlgorithm","keywords":[],"metadata":[],"version":"1.1.0","jobControlOptions":["async-execute","sync-execute"],"outputTransmission":["value","reference"],"links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.wps.server.algorithm.SimpleBufferAlgorithm","rel":"process-desc","type":"application/json","title":"Process description"}]},{"id":"org.n52.javaps.test.MultiReferenceInputAlgorithm","title":"for testing multiple inputs by reference","keywords":[],"metadata":[],"version":"1.1.0","jobControlOptions":["async-execute","sync-execute"],"outputTransmission":["value","reference"],"links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.MultiReferenceInputAlgorithm","rel":"process-desc","type":"application/json","title":"Process description"}]},{"id":"org.n52.javaps.test.EchoProcess","title":"org.n52.javaps.test.EchoProcess","keywords":[],"metadata":[],"version":"1.0.0","jobControlOptions":["async-execute","sync-execute"],"outputTransmission":["value","reference"],"links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess","rel":"process-desc","type":"application/json","title":"Process description"}]}] From 18e48c719a58e02d9e39a937e77af07212d9c3d2 Mon Sep 17 00:00:00 2001 From: David Huard Date: Wed, 17 Feb 2021 09:21:02 -0500 Subject: [PATCH 02/18] add echoprocess description --- .../ogcapi/processes/processdescription_52n_echoprocess.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/resources/ogcapi/processes/processdescription_52n_echoprocess.json diff --git a/tests/resources/ogcapi/processes/processdescription_52n_echoprocess.json b/tests/resources/ogcapi/processes/processdescription_52n_echoprocess.json new file mode 100644 index 000000000..eabbe2446 --- /dev/null +++ b/tests/resources/ogcapi/processes/processdescription_52n_echoprocess.json @@ -0,0 +1 @@ +{"id":"org.n52.javaps.test.EchoProcess","title":"org.n52.javaps.test.EchoProcess","keywords":[],"metadata":[],"version":"1.0.0","jobControlOptions":["async-execute","sync-execute"],"outputTransmission":["value","reference"],"links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess/jobs","rel":"execute","title":"Execute endpoint"}],"inputs":[{"id":"boundingboxInput","title":"boundingboxInput","keywords":[],"metadata":[],"input":{"supportedCRS":[{"default":true,"crs":"EPSG:4326"},{"default":false,"crs":"http://www.opengis.net/def/crs/EPSG/0/4326"}]},"minOccurs":0,"maxOccurs":1},{"id":"duration","title":"duration","keywords":[],"metadata":[],"input":{"literalDataDomains":[{"valueDefinition":{"anyValue":true},"defaultValue":"","dataType":{"name":"int","reference":"https://www.w3.org/2001/XMLSchema-datatypes#int"}}]},"minOccurs":0,"maxOccurs":1},{"id":"literalInput","title":"literalInput","keywords":[],"metadata":[],"input":{"literalDataDomains":[{"valueDefinition":{"anyValue":true},"defaultValue":"","dataType":{"name":"string","reference":"https://www.w3.org/2001/XMLSchema-datatypes#string"}}]},"minOccurs":0,"maxOccurs":1},{"id":"complexInput","title":"complexInput","keywords":[],"metadata":[],"input":{"formats":[{"default":true,"mimeType":"application/xml"},{"default":false,"mimeType":"application/xml"},{"default":false,"mimeType":"text/xml"}]},"minOccurs":0,"maxOccurs":2}],"outputs":[{"id":"boundingboxOutput","title":"boundingboxOutput","keywords":[],"metadata":[],"output":{"supportedCRS":[{"default":true,"crs":"EPSG:4326"}]}},{"id":"literalOutput","title":"literalOutput","keywords":[],"metadata":[],"output":{"literalDataDomains":[{"valueDefinition":{"anyValue":true},"dataType":{"name":"string","reference":"https://www.w3.org/2001/XMLSchema-datatypes#string"}}]}},{"id":"complexOutput","title":"complexOutput","keywords":[],"metadata":[],"output":{"formats":[{"default":true,"mimeType":"application/xml"},{"default":false,"mimeType":"application/xml"},{"default":false,"mimeType":"text/xml"}]}}]} From e721dc353ff8c32dcad792c043cce55a6aa5f5fd Mon Sep 17 00:00:00 2001 From: David Huard Date: Wed, 17 Feb 2021 09:37:01 -0500 Subject: [PATCH 03/18] add execute inputs, jobs responses and execute responses for echoprocess. Put all these into 52n directory. --- .../processes/{api_52n.json => 52n/api.json} | 0 .../capabilites.json} | 0 .../conformancedeclaration.json} | 0 .../52n/echoprocess_execute_input.json | 59 +++++++++++++++++++ .../52n/echoprocess_jobs_results.json | 1 + .../52n/echoprocess_jobs_status.json | 1 + .../processdescription_echoprocess.json} | 0 ...n_multireferencebinaryinputalgorithm.json} | 0 .../processlists.json} | 0 9 files changed, 61 insertions(+) rename tests/resources/ogcapi/processes/{api_52n.json => 52n/api.json} (100%) rename tests/resources/ogcapi/processes/{capabilities_52n.json => 52n/capabilites.json} (100%) rename tests/resources/ogcapi/processes/{conformancedeclaration_52n.json => 52n/conformancedeclaration.json} (100%) create mode 100644 tests/resources/ogcapi/processes/52n/echoprocess_execute_input.json create mode 100644 tests/resources/ogcapi/processes/52n/echoprocess_jobs_results.json create mode 100644 tests/resources/ogcapi/processes/52n/echoprocess_jobs_status.json rename tests/resources/ogcapi/processes/{processdescription_52n_echoprocess.json => 52n/processdescription_echoprocess.json} (100%) rename tests/resources/ogcapi/processes/{processdescription_52n_multireferencebinaryinputalgorithm.json => 52n/processdescription_multireferencebinaryinputalgorithm.json} (100%) rename tests/resources/ogcapi/processes/{processlists_52n.json => 52n/processlists.json} (100%) diff --git a/tests/resources/ogcapi/processes/api_52n.json b/tests/resources/ogcapi/processes/52n/api.json similarity index 100% rename from tests/resources/ogcapi/processes/api_52n.json rename to tests/resources/ogcapi/processes/52n/api.json diff --git a/tests/resources/ogcapi/processes/capabilities_52n.json b/tests/resources/ogcapi/processes/52n/capabilites.json similarity index 100% rename from tests/resources/ogcapi/processes/capabilities_52n.json rename to tests/resources/ogcapi/processes/52n/capabilites.json diff --git a/tests/resources/ogcapi/processes/conformancedeclaration_52n.json b/tests/resources/ogcapi/processes/52n/conformancedeclaration.json similarity index 100% rename from tests/resources/ogcapi/processes/conformancedeclaration_52n.json rename to tests/resources/ogcapi/processes/52n/conformancedeclaration.json diff --git a/tests/resources/ogcapi/processes/52n/echoprocess_execute_input.json b/tests/resources/ogcapi/processes/52n/echoprocess_execute_input.json new file mode 100644 index 000000000..bf1933221 --- /dev/null +++ b/tests/resources/ogcapi/processes/52n/echoprocess_execute_input.json @@ -0,0 +1,59 @@ +{ + "inputs": [ + { + "id": "complexInput", + "input": { + "format": { + "mimeType": "application/xml" + }, + "value": { + "inlineValue": "" + } + } + }, + { + "id": "literalInput", + "input": { + "dataType": "double", + "value": "0.05" + } + }, + { + "id": "boundingboxInput", + "input": { + "bbox": [ + 51.9, + 7, + 52, + 7.1 + ], + "crs": "EPSG:4326" + } + } + ], + "outputs": [ + { + "id": "literalOutput", + "format": { + "mimeType": "text/plain" + }, + "transmissionMode": "value" + }, + { + "id": "boundingboxOutput", + "format": { + "mimeType": "application/json" + }, + "transmissionMode": "value" + }, + { + "id": "complexOutput", + "format": { + "mimeType": "application/xml" + }, + "transmissionMode": "value" + } + ], + "response": "document", + "mode": "async" +} diff --git a/tests/resources/ogcapi/processes/52n/echoprocess_jobs_results.json b/tests/resources/ogcapi/processes/52n/echoprocess_jobs_results.json new file mode 100644 index 000000000..554cd9292 --- /dev/null +++ b/tests/resources/ogcapi/processes/52n/echoprocess_jobs_results.json @@ -0,0 +1 @@ +{"outputs":[{"id":"literalOutput","value":{"inlineValue":0.05}},{"id":"complexOutput","value":{"inlineValue":""}},{"id":"boundingboxOutput","value":{"inlineValue":{"bbox":[51.9,7.0,52.0,7.1],"crs":"EPSG:4326"}}}]} diff --git a/tests/resources/ogcapi/processes/52n/echoprocess_jobs_status.json b/tests/resources/ogcapi/processes/52n/echoprocess_jobs_status.json new file mode 100644 index 000000000..9ba4926e9 --- /dev/null +++ b/tests/resources/ogcapi/processes/52n/echoprocess_jobs_status.json @@ -0,0 +1 @@ +{"status":"successful","jobID":"b723ca9e-4a8a-4098-a752-0b80ffd3a8a4","links":[{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess/jobs/b723ca9e-4a8a-4098-a752-0b80ffd3a8a4","rel":"self","type":"application/json","title":"This document"},{"href":"http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess/jobs/b723ca9e-4a8a-4098-a752-0b80ffd3a8a4/results","rel":"results","type":"application/json","title":"Job results"}]} diff --git a/tests/resources/ogcapi/processes/processdescription_52n_echoprocess.json b/tests/resources/ogcapi/processes/52n/processdescription_echoprocess.json similarity index 100% rename from tests/resources/ogcapi/processes/processdescription_52n_echoprocess.json rename to tests/resources/ogcapi/processes/52n/processdescription_echoprocess.json diff --git a/tests/resources/ogcapi/processes/processdescription_52n_multireferencebinaryinputalgorithm.json b/tests/resources/ogcapi/processes/52n/processdescription_multireferencebinaryinputalgorithm.json similarity index 100% rename from tests/resources/ogcapi/processes/processdescription_52n_multireferencebinaryinputalgorithm.json rename to tests/resources/ogcapi/processes/52n/processdescription_multireferencebinaryinputalgorithm.json diff --git a/tests/resources/ogcapi/processes/processlists_52n.json b/tests/resources/ogcapi/processes/52n/processlists.json similarity index 100% rename from tests/resources/ogcapi/processes/processlists_52n.json rename to tests/resources/ogcapi/processes/52n/processlists.json From 6381e2c7d7395af11424ee4f34d6c89d05155393 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Wed, 17 Feb 2021 20:27:23 +0100 Subject: [PATCH 04/18] initial ogcapi-processes implementation --- owslib/ogcapi/processes.py | 128 ++++++++++++++++++++++++ tests/test_ogcapi_processes_52n.py | 43 ++++++++ tests/test_ogcapi_processes_pygeoapi.py | 47 +++++++++ tests/test_ogcapi_processes_weaver.py | 40 ++++++++ 4 files changed, 258 insertions(+) create mode 100644 owslib/ogcapi/processes.py create mode 100644 tests/test_ogcapi_processes_52n.py create mode 100644 tests/test_ogcapi_processes_pygeoapi.py create mode 100644 tests/test_ogcapi_processes_weaver.py diff --git a/owslib/ogcapi/processes.py b/owslib/ogcapi/processes.py new file mode 100644 index 000000000..4201d4302 --- /dev/null +++ b/owslib/ogcapi/processes.py @@ -0,0 +1,128 @@ +# ============================================================================= +# Copyright (c) 2021 Tom Kralidis +# +# Author: Tom Kralidis +# +# Contact email: tomkralidis@gmail.com +# ============================================================================= + +import logging + +import requests + +from owslib.ogcapi import API +from owslib.util import Authentication + +LOGGER = logging.getLogger(__name__) + + +class Processes(API): + """Abstraction for OGC API - Processes + + https://ogcapi.ogc.org/processes/overview.html + """ + + def __init__(self, url: str, json_: str = None, timeout: int = 30, + headers: dict = None, auth: Authentication = None): + __doc__ = API.__doc__ # noqa + super().__init__(url, json_, timeout, headers, auth) + + def processes(self) -> dict: + """ + implements: GET /processes + + Lists the processes this API offers. + + WPS 1.0.0: GetCapabilities + + @returns: `dict` of available processes. + """ + + path = 'processes' + return self._request(path) + + def process(self, process_id: str) -> dict: + """ + implements: GET /processes/{process-id} + + Returns a detailed description of a process. + + WPS 1.0.0: DescribeProcess + + @returns: `dict` of a process description. + """ + + path = f'processes/{process_id}' + return self._request(path) + + def process_jobs(self, process_id: str) -> dict: + """ + implements: GET /processes/{process-id}/jobs + + Returns the running and finished jobs for a process (optional). + + @returns: `dict` of .... + """ + + path = f'processes/{process_id}/jobs' + return self._request(path) + + def process_execute(self, process_id: str, json: dict) -> dict: + """ + implements: POST /processes/{process-id}/jobs + + Executes a process, i.e. creates a new job. Inputs and outputs will have + to be specified in a JSON document that needs to be send in the POST body. + + @returns: `dict` of ... + """ + + path = f'processes/{process_id}/jobs' + return self._request_post(path, json) + + def _request_post(self, path: str, json: dict) -> dict: + # TODO: needs to be implemented in base class + url = self._build_url(path) + + response = requests.post(url, json=json) + + if response.status_code != requests.codes.ok: + raise RuntimeError(response.text) + + return response.json() + + def process_status(self, process_id: str, job_id: str) -> dict: + """ + implements: GET /processes/{process-id}/jobs/{job-id} + + Returns the status of a job of a process. + + @returns: `dict` of ... + """ + + path = f'processes/{process_id}/jobs/{job_id}' + return self._request(path) + + def process_cancel(self, process_id: str, job_id: str) -> dict: + """ + implements: DELETE /processes/{process-id}/jobs/{job-id} + + Cancel a job execution. + + @returns: `dict` of ... + """ + + path = f'processes/{process_id}/jobs/{job_id}' + return self._request(path) + + def process_result(self, process_id: str, job_id: str) -> dict: + """ + implements: GET /processes/{process-id}/jobs/{job-id}/results + + Returns the result of a job of a process. + + @returns: `dict` of ... + """ + + path = f'processes/{process_id}/jobs/{job_id}/results' + return self._request(path) diff --git a/tests/test_ogcapi_processes_52n.py b/tests/test_ogcapi_processes_52n.py new file mode 100644 index 000000000..4f0454180 --- /dev/null +++ b/tests/test_ogcapi_processes_52n.py @@ -0,0 +1,43 @@ +from tests.utils import service_ok + +import pytest + +from owslib.ogcapi.processes import Processes + +SERVICE_URL = 'http://geoprocessing.demo.52north.org:8080/javaps/rest/' + + +@pytest.mark.online +@pytest.mark.skipif(not service_ok(SERVICE_URL), + reason='service is unreachable') +def test_ogcapi_processes_52n(): + w = Processes(SERVICE_URL) + + assert w.url == 'http://geoprocessing.demo.52north.org:8080/javaps/rest/' + assert w.url_query_string is None + + # TODO: RuntimeError: Did not find service-desc link + # api = w.api() + # assert api['components']['parameters'] is not None + # paths = api['paths'] + # assert paths is not None + # assert paths['/processes/hello-world'] is not None + + conformance = w.conformance() + assert len(conformance['conformsTo']) == 5 + + # list processes + processes = w.processes() + assert len(processes) > 0 + + # process description + echo = w.process('org.n52.javaps.test.EchoProcess') + assert echo['id'] == 'org.n52.javaps.test.EchoProcess' + assert echo['title'] == 'org.n52.javaps.test.EchoProcess' + # assert "An example process that takes a name as input" in echo['description'] + + # running jobs + jobs = w.process_jobs('org.n52.javaps.test.EchoProcess') + assert len(jobs) >= 0 + + # TODO: post request not allowed at 52n? diff --git a/tests/test_ogcapi_processes_pygeoapi.py b/tests/test_ogcapi_processes_pygeoapi.py new file mode 100644 index 000000000..e9e03359e --- /dev/null +++ b/tests/test_ogcapi_processes_pygeoapi.py @@ -0,0 +1,47 @@ +from tests.utils import service_ok + +import pytest + +from owslib.ogcapi.processes import Processes + +SERVICE_URL = 'https://demo.pygeoapi.io/master' + + +@pytest.mark.online +@pytest.mark.skipif(not service_ok(SERVICE_URL), + reason='service is unreachable') +def test_ogcapi_processes_pygeoapi(): + w = Processes(SERVICE_URL) + + assert w.url == 'https://demo.pygeoapi.io/master/' + assert w.url_query_string is None + + api = w.api() + assert api['components']['parameters'] is not None + paths = api['paths'] + assert paths is not None + assert paths['/processes/hello-world'] is not None + + conformance = w.conformance() + assert len(conformance['conformsTo']) == 9 + + # list processes + processes = w.processes() + assert len(processes) > 0 + + # process description + hello = w.process('hello-world') + assert hello['id'] == 'hello-world' + assert hello['title'] == 'Hello World' + assert "An example process that takes a name as input" in hello['description'] + + # running jobs + jobs = w.process_jobs('hello-world') + assert len(jobs) >= 0 + + # execute process in sync mode + request_json = {"inputs": [{"id": "name", "value": "hello"}], "mode": "sync"} + resp = w.process_execute('hello-world', json=request_json) + assert len(resp['outputs']) == 1 + assert resp['outputs'][0]['id'] == 'echo' + assert resp['outputs'][0]['value'] == 'Hello hello!' diff --git a/tests/test_ogcapi_processes_weaver.py b/tests/test_ogcapi_processes_weaver.py new file mode 100644 index 000000000..33f129e48 --- /dev/null +++ b/tests/test_ogcapi_processes_weaver.py @@ -0,0 +1,40 @@ +from tests.utils import service_ok + +import pytest + +from owslib.ogcapi.processes import Processes + +SERVICE_URL = 'https://ogc-ades.crim.ca/ADES/' + + +@pytest.mark.online +@pytest.mark.skipif(not service_ok(SERVICE_URL), + reason='service is unreachable') +def test_ogcapi_processes_weaver(): + w = Processes(SERVICE_URL) + + assert w.url == 'https://ogc-ades.crim.ca/ADES/' + assert w.url_query_string is None + + # TODO: RuntimeError: Did not find service-desc link + # api = w.api() + # assert api['components']['parameters'] is not None + # paths = api['paths'] + # assert paths is not None + # assert paths['/processes/hello-world'] is not None + + conformance = w.conformance() + assert len(conformance['conformsTo']) == 5 + + # list processes + processes = w.processes() + assert len(processes) > 0 + + # process description + # TODO: response not as expected {'process': {'id': 'ColibriFlyingpigeon_SubsetBbox'}} + # should be {'id': 'ColibriFlyingpigeon_SubsetBbox'} + # process = w.process('ColibriFlyingpigeon_SubsetBbox') + # print(process) + # assert process['process']['id'] == 'ColibriFlyingpigeon_SubsetBbox' + # assert process['process']['title'] == 'ColibriFlyingpigeon_SubsetBbox' + # assert "An example process that takes a name as input" in process['process']['description'] From cbd813ff1dddad9e1ab45592b408c3363c7871b4 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Wed, 17 Feb 2021 21:19:26 +0100 Subject: [PATCH 05/18] renamed processes methods --- owslib/ogcapi/processes.py | 20 +++++++++----------- tests/test_ogcapi_processes_52n.py | 4 ++-- tests/test_ogcapi_processes_pygeoapi.py | 6 +++--- tests/test_ogcapi_processes_weaver.py | 2 +- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/owslib/ogcapi/processes.py b/owslib/ogcapi/processes.py index 4201d4302..baad87632 100644 --- a/owslib/ogcapi/processes.py +++ b/owslib/ogcapi/processes.py @@ -19,7 +19,9 @@ class Processes(API): """Abstraction for OGC API - Processes - https://ogcapi.ogc.org/processes/overview.html + * https://ogcapi.ogc.org/processes/overview.html + * https://docs.opengeospatial.org/DRAFTS/18-062.html + * https://app.swaggerhub.com/apis/geoprocessing/WPS-all-in-one/1.0-draft.6-SNAPSHOT """ def __init__(self, url: str, json_: str = None, timeout: int = 30, @@ -33,29 +35,25 @@ def processes(self) -> dict: Lists the processes this API offers. - WPS 1.0.0: GetCapabilities - @returns: `dict` of available processes. """ path = 'processes' return self._request(path) - def process(self, process_id: str) -> dict: + def process_description(self, process_id: str) -> dict: """ implements: GET /processes/{process-id} Returns a detailed description of a process. - WPS 1.0.0: DescribeProcess - @returns: `dict` of a process description. """ path = f'processes/{process_id}' return self._request(path) - def process_jobs(self, process_id: str) -> dict: + def job_list(self, process_id: str) -> dict: """ implements: GET /processes/{process-id}/jobs @@ -67,7 +65,7 @@ def process_jobs(self, process_id: str) -> dict: path = f'processes/{process_id}/jobs' return self._request(path) - def process_execute(self, process_id: str, json: dict) -> dict: + def execute(self, process_id: str, json: dict) -> dict: """ implements: POST /processes/{process-id}/jobs @@ -91,7 +89,7 @@ def _request_post(self, path: str, json: dict) -> dict: return response.json() - def process_status(self, process_id: str, job_id: str) -> dict: + def status(self, process_id: str, job_id: str) -> dict: """ implements: GET /processes/{process-id}/jobs/{job-id} @@ -103,7 +101,7 @@ def process_status(self, process_id: str, job_id: str) -> dict: path = f'processes/{process_id}/jobs/{job_id}' return self._request(path) - def process_cancel(self, process_id: str, job_id: str) -> dict: + def cancel(self, process_id: str, job_id: str) -> dict: """ implements: DELETE /processes/{process-id}/jobs/{job-id} @@ -115,7 +113,7 @@ def process_cancel(self, process_id: str, job_id: str) -> dict: path = f'processes/{process_id}/jobs/{job_id}' return self._request(path) - def process_result(self, process_id: str, job_id: str) -> dict: + def result(self, process_id: str, job_id: str) -> dict: """ implements: GET /processes/{process-id}/jobs/{job-id}/results diff --git a/tests/test_ogcapi_processes_52n.py b/tests/test_ogcapi_processes_52n.py index 4f0454180..7ac82db59 100644 --- a/tests/test_ogcapi_processes_52n.py +++ b/tests/test_ogcapi_processes_52n.py @@ -31,13 +31,13 @@ def test_ogcapi_processes_52n(): assert len(processes) > 0 # process description - echo = w.process('org.n52.javaps.test.EchoProcess') + echo = w.process_description('org.n52.javaps.test.EchoProcess') assert echo['id'] == 'org.n52.javaps.test.EchoProcess' assert echo['title'] == 'org.n52.javaps.test.EchoProcess' # assert "An example process that takes a name as input" in echo['description'] # running jobs - jobs = w.process_jobs('org.n52.javaps.test.EchoProcess') + jobs = w.job_list('org.n52.javaps.test.EchoProcess') assert len(jobs) >= 0 # TODO: post request not allowed at 52n? diff --git a/tests/test_ogcapi_processes_pygeoapi.py b/tests/test_ogcapi_processes_pygeoapi.py index e9e03359e..2cf5dd9dd 100644 --- a/tests/test_ogcapi_processes_pygeoapi.py +++ b/tests/test_ogcapi_processes_pygeoapi.py @@ -30,18 +30,18 @@ def test_ogcapi_processes_pygeoapi(): assert len(processes) > 0 # process description - hello = w.process('hello-world') + hello = w.process_description('hello-world') assert hello['id'] == 'hello-world' assert hello['title'] == 'Hello World' assert "An example process that takes a name as input" in hello['description'] # running jobs - jobs = w.process_jobs('hello-world') + jobs = w.job_list('hello-world') assert len(jobs) >= 0 # execute process in sync mode request_json = {"inputs": [{"id": "name", "value": "hello"}], "mode": "sync"} - resp = w.process_execute('hello-world', json=request_json) + resp = w.execute('hello-world', json=request_json) assert len(resp['outputs']) == 1 assert resp['outputs'][0]['id'] == 'echo' assert resp['outputs'][0]['value'] == 'Hello hello!' diff --git a/tests/test_ogcapi_processes_weaver.py b/tests/test_ogcapi_processes_weaver.py index 33f129e48..f209b35e0 100644 --- a/tests/test_ogcapi_processes_weaver.py +++ b/tests/test_ogcapi_processes_weaver.py @@ -33,7 +33,7 @@ def test_ogcapi_processes_weaver(): # process description # TODO: response not as expected {'process': {'id': 'ColibriFlyingpigeon_SubsetBbox'}} # should be {'id': 'ColibriFlyingpigeon_SubsetBbox'} - # process = w.process('ColibriFlyingpigeon_SubsetBbox') + # process = w.process_description('ColibriFlyingpigeon_SubsetBbox') # print(process) # assert process['process']['id'] == 'ColibriFlyingpigeon_SubsetBbox' # assert process['process']['title'] == 'ColibriFlyingpigeon_SubsetBbox' From d26779bcb61dfd940140435508d31edd7f57b52d Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 17 Feb 2021 18:57:07 -0500 Subject: [PATCH 06/18] add implementation of wps-rest-client with ogc-api interface (#749) --- examples/wps-rest-client.py | 251 ++++++++++++++++++++++++++++++++++++ owslib/ogcapi/__init__.py | 8 +- owslib/ogcapi/processes.py | 50 +++++-- 3 files changed, 297 insertions(+), 12 deletions(-) create mode 100755 examples/wps-rest-client.py diff --git a/examples/wps-rest-client.py b/examples/wps-rest-client.py new file mode 100755 index 000000000..31dd1a987 --- /dev/null +++ b/examples/wps-rest-client.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python +# -*- coding: ISO-8859-15 -*- +# ============================================================================= +# +# Authors : Francis Charette-Migneault (@fmigneault) +# +# ============================================================================= + +import json +import sys +import getopt +import os +import yaml + +from owslib.ogcapi.processes import Processes + + +def usage(): + print(""" + +Usage: %s [parameters] + +Common Parameters for all request types +------------------- + + -u, --url=[URL] the base URL of the WPS - required + -r, --request=[REQUEST] the request type (GetCapabilities, DescribeProcess, Execute) - required + -v, --verbose set flag for verbose output - optional (defaults to False) + -o, --output=[FORMAT] format of the response to provide - optional {json, yaml} + +Request Specific Parameters +--------------------------- + + DescribeProcess + -i, --identifier=[ID] process identifier - required + Execute + -d, --data, --json JSON file containing pre-made request to be submitted - required + +Examples +-------- +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r GetCapabilities + +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Processes + +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r DescribeProcess -i las2tif + +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i las2tif -d payload.json +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i las2tif --json payload.json + +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r GetStatus -j + + with 'payload.json' contents: + + { + "mode": "async", + "response": "document", + "inputs": [ + { + "id": "input", + "href": "https://ogc-ems.crim.ca/wps-outputs/example-nc-array.json" + } + ], + "outputs": [ + { + "id": "output", + "transmissionMode": "reference" + } + ] + } + +""" % sys.argv[0]) + + +# check args +if len(sys.argv) == 1: + usage() + sys.exit(1) + +print('ARGV :', sys.argv[1:]) + +try: + # json == data + opts = ['url=', 'request=', 'json=', 'data=', 'identifier=', 'job=', 'output=', 'verbose'] + options, remainder = getopt.getopt(sys.argv[1:], 'u:r:d:i:j:o:v', opts) +except getopt.GetoptError as err: + print(str(err)) + usage() + sys.exit(2) + +print('OPTIONS :', options) + +url = None +request = None +identifier = None +job = None +data = None +verbose = False +output = None + +for opt, arg in options: + if opt in ('-u', '--url'): + url = arg + elif opt in ('-r', '--request'): + request = arg + elif opt in ('-d', '--data', '--json'): + data = arg + elif opt in ('-i', '--identifier'): + identifier = arg + elif opt in ('-j', '--job'): + job = arg + elif opt in ('-v', '--verbose'): + verbose = True + elif opt in ('-o', '--output'): + output = arg + else: + assert False, 'Unhandled option' + +# required arguments for all requests +if request is None or url is None: + usage() + sys.exit(3) + + +def print_content(output_format, output_content): + if output_format == "yaml": + print(yaml.safe_dump(output_content, indent=2, sort_keys=False, allow_unicode=True)) + elif output_format == "": + print(json.dumps(output_content, indent=2, ensure_ascii=False)) + else: + print(f"\nUnknown output format: {output_format}") + sys.exit(20) + + +# instantiate client +wps = Processes(url) + +if request == 'GetCapabilities': + links = wps.links() + if output: + print_content(output, links) + sys.exit(0) + + print('WPS Links:') + for link in wps.links(): + print(f' {link["title"]} ({link["rel"]})') + print(f' {link["href"]}') + +elif request == 'Processes': + processes = wps.processes() + if output: + print_content(output, processes) + sys.exit(0) + + print('WPS Processes:') + for process in processes: + print(f' identifier={process["id"]} title={process.get("title", "")}') + +elif request == 'DescribeProcess': + if identifier is None: + print('\nERROR: missing mandatory "-i (or --identifier)" argument') + usage() + sys.exit(4) + data = wps.process_description(identifier) + if output: + print_content(output, data) + sys.exit(0) + + process = data["process"] + print('WPS Process:') + process.setdefault("abstract", "") + print(f' id={process["id"]}') + for field in ["title", "version", "abstract"]: + print(f' {field}={process.get(field, "")}') + print(' inputs:') + for p_input in process["inputs"]: + minOccurs = p_input.get("minOccurs", 1) + maxOccurs = p_input.get("maxOccurs", 1) + _domains = p_input.get("input", {}).get("literalDataDomain", {}) + _formats = p_input.get("formats", p_input.get("input", {}).get("format", [])) # FIXME: depending on impl + print(f' - id={p_input["id"]} title={p_input.get("title", "")} minOccurs={minOccurs} maxOccurs={maxOccurs}') + if _formats: + print(f' formats:') + for fmt in p_input["formats"]: + _mime = fmt.get("mediaType", fmt.get("mimeType")) # FIXME: depending on impl + _default = " (default)" if fmt.get("default") else "" + print(f' - {_mime}{_default}') + if "dataType" in _domains: + print(f' dataType={_domains["dataType"]}') + # FIXME: support other variants (csr for bbox, literal value ranges, etc.) + + print(' outputs:') + for p_output in process["outputs"]: + _formats = p_output.get("formats", p_output.get("output", {}).get("format", [])) # FIXME: depending on impl + print(f' - id={p_output["id"]} title={p_output.get("title", "")}') + if _formats: + print(f' formats:') + for fmt in p_output["formats"]: + print(f' - {fmt}') + +elif request == 'Execute': + if identifier is None: + print('\nERROR: missing mandatory "-i (or --identifier)" argument') + usage() + sys.exit(5) + if data is None: + print('\nERROR: missing mandatory "-d (or --data/--json)" argument') + usage() + sys.exit(6) + data = os.path.abspath(data) + if not os.path.isfile(data): + print(f'\nERROR: File not found: {data}') + sys.exit(7) + with open(data, 'r') as f: + payload = json.load(f) + if not payload: + print(f'\nERROR: Empty JSON data from {data}') + sys.exit(7) + + status_location = wps.execute(identifier, payload) + data, success = wps.monitor_execution(location=status_location) + if output: + print_content(output, data) + sys.exit(0) + + print(f'Process execution: {data["status"]}') + job = data["jobID"] + path = f'{wps.url}/processes/{identifier}/jobs/{job}/results' + if success: + print(f'"Results location: {path}"') + +elif request == 'GetStatus': + if identifier is None: + print('\nERROR: missing mandatory "-i (or --identifier)" argument') + usage() + sys.exit(8) + if job is None: + print('\nERROR: missing mandatory "-j (or --job)" argument') + usage() + sys.exit(9) + + data = wps.status(process_id=identifier, job_id=job) + if output: + print_content(output, data) + sys.exit(0) + + print(f'Status: {data["status"]}') + +else: + print('\nERROR: Unknown request type') + usage() + sys.exit(6) diff --git a/owslib/ogcapi/__init__.py b/owslib/ogcapi/__init__.py index ddbdfe8e7..ca1e54f2f 100644 --- a/owslib/ogcapi/__init__.py +++ b/owslib/ogcapi/__init__.py @@ -172,7 +172,7 @@ def _build_url(self, path: str = None) -> str: return url def _request(self, path: str = None, as_dict: bool = True, - kwargs: dict = {}) -> dict: + kwargs: dict = None, url: str = None) -> dict: """ helper function for request/response patterns against OGC API endpoints @@ -185,8 +185,10 @@ def _request(self, path: str = None, as_dict: bool = True, @returns: response as JSON ``dict`` """ - - url = self._build_url(path) + if not kwargs: + kwargs = {} + if not url: + url = self._build_url(path) LOGGER.debug('Request: {}'.format(url)) LOGGER.debug('Params: {}'.format(kwargs)) diff --git a/owslib/ogcapi/processes.py b/owslib/ogcapi/processes.py index baad87632..356b8c318 100644 --- a/owslib/ogcapi/processes.py +++ b/owslib/ogcapi/processes.py @@ -7,6 +7,8 @@ # ============================================================================= import logging +import time +from typing import Tuple import requests @@ -39,7 +41,8 @@ def processes(self) -> dict: """ path = 'processes' - return self._request(path) + data = self._request(path) + return data["processes"] def process_description(self, process_id: str) -> dict: """ @@ -65,29 +68,31 @@ def job_list(self, process_id: str) -> dict: path = f'processes/{process_id}/jobs' return self._request(path) - def execute(self, process_id: str, json: dict) -> dict: + def execute(self, process_id: str, json: dict) -> str: """ implements: POST /processes/{process-id}/jobs Executes a process, i.e. creates a new job. Inputs and outputs will have to be specified in a JSON document that needs to be send in the POST body. - @returns: `dict` of ... + @returns: `str` of the status location """ path = f'processes/{process_id}/jobs' - return self._request_post(path, json) + resp = self._request_post(path, json) + data = resp.json() + return resp.headers.get("Location", data["location"]) - def _request_post(self, path: str, json: dict) -> dict: + def _request_post(self, path: str, json: dict) -> requests.Response: # TODO: needs to be implemented in base class url = self._build_url(path) - response = requests.post(url, json=json) + resp = requests.post(url, json=json) - if response.status_code != requests.codes.ok: - raise RuntimeError(response.text) + if resp.status_code != requests.codes.ok: + raise RuntimeError(resp.text) - return response.json() + return resp def status(self, process_id: str, job_id: str) -> dict: """ @@ -124,3 +129,30 @@ def result(self, process_id: str, job_id: str) -> dict: path = f'processes/{process_id}/jobs/{job_id}/results' return self._request(path) + + def monitor_execution(self, process_id: str = None, job_id: str = None, location: str = None, + timeout: int = 3600, delta: int = 10) -> Tuple[dict, bool]: + """ + Job polling of status URL until completion or timeout. + + If `location` is provided, it is used instead. + + @returns: results of the monitoring upon completion as `tuple` of (data, success?) + """ + time.sleep(1) # small delay to ensure process execution had a change to start before monitoring + left = timeout + once = True + data = None + while left >= 0 or once: + if location: + data = self._request(url=location) + else: + data = self.status(process_id, job_id) + if data['status'] in ['running', 'succeeded']: + break + if data['status'] == 'failed': + return data, False + time.sleep(delta) + once = False + left -= delta + return data, data['status'] == 'succeeded' From 1295787152257a7345b57029ec87665a296ee02e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 17 Feb 2021 19:16:27 -0500 Subject: [PATCH 07/18] fix doc of wps-rest-client --- examples/wps-rest-client.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/wps-rest-client.py b/examples/wps-rest-client.py index 31dd1a987..d29aa6e1d 100755 --- a/examples/wps-rest-client.py +++ b/examples/wps-rest-client.py @@ -24,17 +24,21 @@ def usage(): ------------------- -u, --url=[URL] the base URL of the WPS - required - -r, --request=[REQUEST] the request type (GetCapabilities, DescribeProcess, Execute) - required + -r, --request=[REQUEST] the request type - required + {GetCapabilities, DescribeProcess, Processes, Execute, GetStatus} -v, --verbose set flag for verbose output - optional (defaults to False) - -o, --output=[FORMAT] format of the response to provide - optional {json, yaml} + -o, --output=[FORMAT] format of the response to provide - optional {json, yaml} (default: parse and print items) + when specified, responses are returned directly with the given format Request Specific Parameters --------------------------- - DescribeProcess + DescribeProcess, Execute, GetStatus -i, --identifier=[ID] process identifier - required Execute -d, --data, --json JSON file containing pre-made request to be submitted - required + GetStatus + -j, --job JobID returned by the Execute request - required Examples -------- @@ -42,12 +46,12 @@ def usage(): python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Processes -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r DescribeProcess -i las2tif +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r DescribeProcess -i jsonarray2netcdf -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i las2tif -d payload.json -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i las2tif --json payload.json +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i jsonarray2netcdf -d payload.json +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i jsonarray2netcdf --json payload.json -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r GetStatus -j +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r GetStatus -i jsonarray2netcdf -j with 'payload.json' contents: @@ -246,6 +250,6 @@ def print_content(output_format, output_content): print(f'Status: {data["status"]}') else: - print('\nERROR: Unknown request type') + print(f'\nERROR: Unknown request type: {request}') usage() - sys.exit(6) + sys.exit(30) From 30ae777a27f41596e0eb44391f0bcc73eda10c51 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 17 Feb 2021 19:16:27 -0500 Subject: [PATCH 08/18] fix doc of wps-rest-client --- examples/wps-rest-client.py | 22 +++++++++++++--------- tests/test_ogcapi_processes_weaver.py | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/examples/wps-rest-client.py b/examples/wps-rest-client.py index 31dd1a987..d29aa6e1d 100755 --- a/examples/wps-rest-client.py +++ b/examples/wps-rest-client.py @@ -24,17 +24,21 @@ def usage(): ------------------- -u, --url=[URL] the base URL of the WPS - required - -r, --request=[REQUEST] the request type (GetCapabilities, DescribeProcess, Execute) - required + -r, --request=[REQUEST] the request type - required + {GetCapabilities, DescribeProcess, Processes, Execute, GetStatus} -v, --verbose set flag for verbose output - optional (defaults to False) - -o, --output=[FORMAT] format of the response to provide - optional {json, yaml} + -o, --output=[FORMAT] format of the response to provide - optional {json, yaml} (default: parse and print items) + when specified, responses are returned directly with the given format Request Specific Parameters --------------------------- - DescribeProcess + DescribeProcess, Execute, GetStatus -i, --identifier=[ID] process identifier - required Execute -d, --data, --json JSON file containing pre-made request to be submitted - required + GetStatus + -j, --job JobID returned by the Execute request - required Examples -------- @@ -42,12 +46,12 @@ def usage(): python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Processes -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r DescribeProcess -i las2tif +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r DescribeProcess -i jsonarray2netcdf -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i las2tif -d payload.json -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i las2tif --json payload.json +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i jsonarray2netcdf -d payload.json +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r Execute -i jsonarray2netcdf --json payload.json -python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r GetStatus -j +python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r GetStatus -i jsonarray2netcdf -j with 'payload.json' contents: @@ -246,6 +250,6 @@ def print_content(output_format, output_content): print(f'Status: {data["status"]}') else: - print('\nERROR: Unknown request type') + print(f'\nERROR: Unknown request type: {request}') usage() - sys.exit(6) + sys.exit(30) diff --git a/tests/test_ogcapi_processes_weaver.py b/tests/test_ogcapi_processes_weaver.py index f209b35e0..0164f8f7a 100644 --- a/tests/test_ogcapi_processes_weaver.py +++ b/tests/test_ogcapi_processes_weaver.py @@ -17,7 +17,7 @@ def test_ogcapi_processes_weaver(): assert w.url_query_string is None # TODO: RuntimeError: Did not find service-desc link - # api = w.api() + api = w.api() # assert api['components']['parameters'] is not None # paths = api['paths'] # assert paths is not None From fbec0bd6f54e07ce290e4577478e8942acba943c Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 18 Feb 2021 10:37:44 -0500 Subject: [PATCH 09/18] add minimal test case for weaver OGC-API example - missing conformace commented (crim-ca/weaver#200) --- tests/test_ogcapi_processes_weaver.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_ogcapi_processes_weaver.py b/tests/test_ogcapi_processes_weaver.py index 0164f8f7a..c4be7bea9 100644 --- a/tests/test_ogcapi_processes_weaver.py +++ b/tests/test_ogcapi_processes_weaver.py @@ -16,25 +16,27 @@ def test_ogcapi_processes_weaver(): assert w.url == 'https://ogc-ades.crim.ca/ADES/' assert w.url_query_string is None - # TODO: RuntimeError: Did not find service-desc link - api = w.api() + # TODO: RuntimeError: Did not find service-desc link (https://github.com/crim-ca/weaver/issues/200) + # api = w.api() # assert api['components']['parameters'] is not None # paths = api['paths'] # assert paths is not None # assert paths['/processes/hello-world'] is not None conformance = w.conformance() - assert len(conformance['conformsTo']) == 5 + assert len(conformance['conformsTo']) > 0 and all(isinstance(c, str) for c in conformance['conformsTo']) # list processes processes = w.processes() assert len(processes) > 0 # process description - # TODO: response not as expected {'process': {'id': 'ColibriFlyingpigeon_SubsetBbox'}} - # should be {'id': 'ColibriFlyingpigeon_SubsetBbox'} - # process = w.process_description('ColibriFlyingpigeon_SubsetBbox') - # print(process) - # assert process['process']['id'] == 'ColibriFlyingpigeon_SubsetBbox' - # assert process['process']['title'] == 'ColibriFlyingpigeon_SubsetBbox' - # assert "An example process that takes a name as input" in process['process']['description'] + # following are builtin processes and should always be available + for pid in ['file2string_array', 'jsonarray2netcdf', 'metalink2netcdf']: + process = w.process_description(pid) + proc_desc = process['process'] + assert proc_desc['id'] == pid + assert 'inputs' in proc_desc and isinstance(proc_desc['inputs'], list) and len(proc_desc['inputs']) + assert all('id' in p_in and isinstance(p_in['id'], str) for p_in in proc_desc['inputs']) + assert 'outputs' in proc_desc and isinstance(proc_desc['outputs'], list) and len(proc_desc['outputs']) + assert all('id' in p_out and isinstance(p_out['id'], str) for p_out in proc_desc['outputs']) From 8a870fd745550b378ff63c9973ba7417f85bc5a2 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 18 Feb 2021 11:23:55 -0500 Subject: [PATCH 10/18] added readme. --- .../resources/ogcapi/processes/52n/README.md | 31 +++++++++++++++++++ .../{processlists.json => processlist.json} | 0 2 files changed, 31 insertions(+) create mode 100644 tests/resources/ogcapi/processes/52n/README.md rename tests/resources/ogcapi/processes/52n/{processlists.json => processlist.json} (100%) diff --git a/tests/resources/ogcapi/processes/52n/README.md b/tests/resources/ogcapi/processes/52n/README.md new file mode 100644 index 000000000..589e3989f --- /dev/null +++ b/tests/resources/ogcapi/processes/52n/README.md @@ -0,0 +1,31 @@ +# 52 North OGC API Process server responses + +This directory contains static responses from the 52North OGC API Process server. +Note that calling `GET /process/{processid}/jobs` open a window where we can load an example demo input, execute the request and get the response URL. + +Demo input to the EchoProcess: + echoprocess_execute_input.json + +http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess/echoprocess_execute_input.json -> http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess/jobs/ + +## Responses + +The static files were obtained by making the following requests. + +- api.json + http://geoprocessing.demo.52north.org:8080/javaps/rest/api +- capabilities.json + http://geoprocessing.demo.52north.org:8080/javaps/rest/ +- conformancedeclaration.json: + http://geoprocessing.demo.52north.org:8080/javaps/rest/conformance +- echoprocess_jobs_results.json + http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess/jobs/{jobid}/results +- echoprocess_jobs_status.json + http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess/jobs/{jobid} +- processdescription_multireferencebinaryinputalgorithm.json + http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.MultiReferenceBinaryInputAlgorithm +- processdescription_echoprocess.json + http://geoprocessing.demo.52north.org:8080/javaps/rest/processes/org.n52.javaps.test.EchoProcess +- processlist.json + http://geoprocessing.demo.52north.org:8080/javaps/rest/processes + diff --git a/tests/resources/ogcapi/processes/52n/processlists.json b/tests/resources/ogcapi/processes/52n/processlist.json similarity index 100% rename from tests/resources/ogcapi/processes/52n/processlists.json rename to tests/resources/ogcapi/processes/52n/processlist.json From 914104492a3d287af0e1dab60706ec0266a15313 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Thu, 18 Feb 2021 21:15:57 +0100 Subject: [PATCH 11/18] fixed tests --- examples/wps-rest-client.py | 12 ++++++------ owslib/ogcapi/processes.py | 22 +++++++++++++++------- tests/test_ogcapi_processes_52n.py | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/examples/wps-rest-client.py b/examples/wps-rest-client.py index d29aa6e1d..808c34837 100755 --- a/examples/wps-rest-client.py +++ b/examples/wps-rest-client.py @@ -17,7 +17,7 @@ def usage(): print(""" - + Usage: %s [parameters] Common Parameters for all request types @@ -25,8 +25,8 @@ def usage(): -u, --url=[URL] the base URL of the WPS - required -r, --request=[REQUEST] the request type - required - {GetCapabilities, DescribeProcess, Processes, Execute, GetStatus} - -v, --verbose set flag for verbose output - optional (defaults to False) + {GetCapabilities, DescribeProcess, Processes, Execute, GetStatus} + -v, --verbose set flag for verbose output - optional (defaults to False) -o, --output=[FORMAT] format of the response to provide - optional {json, yaml} (default: parse and print items) when specified, responses are returned directly with the given format @@ -53,8 +53,8 @@ def usage(): python wps-rest-client.py -u https://ogc-ades.crim.ca/ADES -r GetStatus -i jsonarray2netcdf -j - with 'payload.json' contents: - + with 'payload.json' contents: + { "mode": "async", "response": "document", @@ -220,7 +220,7 @@ def print_content(output_format, output_content): print(f'\nERROR: Empty JSON data from {data}') sys.exit(7) - status_location = wps.execute(identifier, payload) + status_location = wps.execute(identifier, payload).get('location') data, success = wps.monitor_execution(location=status_location) if output: print_content(output, data) diff --git a/owslib/ogcapi/processes.py b/owslib/ogcapi/processes.py index 356b8c318..4fe0102b2 100644 --- a/owslib/ogcapi/processes.py +++ b/owslib/ogcapi/processes.py @@ -31,18 +31,21 @@ def __init__(self, url: str, json_: str = None, timeout: int = 30, __doc__ = API.__doc__ # noqa super().__init__(url, json_, timeout, headers, auth) - def processes(self) -> dict: + def processes(self) -> list: """ implements: GET /processes Lists the processes this API offers. - @returns: `dict` of available processes. + @returns: `list` of available processes. """ + processes_ = [] path = 'processes' data = self._request(path) - return data["processes"] + if 'processes' in data: + processes_.extend(data["processes"]) + return processes_ def process_description(self, process_id: str) -> dict: """ @@ -68,20 +71,25 @@ def job_list(self, process_id: str) -> dict: path = f'processes/{process_id}/jobs' return self._request(path) - def execute(self, process_id: str, json: dict) -> str: + def execute(self, process_id: str, json: dict) -> dict: """ implements: POST /processes/{process-id}/jobs Executes a process, i.e. creates a new job. Inputs and outputs will have to be specified in a JSON document that needs to be send in the POST body. - @returns: `str` of the status location + @returns: `dict` of the status location (async) or outputs (sync) """ + result = {} path = f'processes/{process_id}/jobs' resp = self._request_post(path, json) data = resp.json() - return resp.headers.get("Location", data["location"]) + if 'outputs' in data: + result['outputs'] = data['outputs'] + else: + result['location'] = resp.headers.get("Location", data.get("location")) + return result def _request_post(self, path: str, json: dict) -> requests.Response: # TODO: needs to be implemented in base class @@ -89,7 +97,7 @@ def _request_post(self, path: str, json: dict) -> requests.Response: resp = requests.post(url, json=json) - if resp.status_code != requests.codes.ok: + if resp.status_code not in [requests.codes.ok, 201]: raise RuntimeError(resp.text) return resp diff --git a/tests/test_ogcapi_processes_52n.py b/tests/test_ogcapi_processes_52n.py index 7ac82db59..ae93f86f0 100644 --- a/tests/test_ogcapi_processes_52n.py +++ b/tests/test_ogcapi_processes_52n.py @@ -7,6 +7,7 @@ SERVICE_URL = 'http://geoprocessing.demo.52north.org:8080/javaps/rest/' +@pytest.mark.xfail @pytest.mark.online @pytest.mark.skipif(not service_ok(SERVICE_URL), reason='service is unreachable') From b5d601b190bdd7ade052ec998c8eeae6e32aa2b1 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 18 Feb 2021 21:52:28 -0500 Subject: [PATCH 12/18] data models based on schemas --- owslib/ogcapi/models.py | 229 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 owslib/ogcapi/models.py diff --git a/owslib/ogcapi/models.py b/owslib/ogcapi/models.py new file mode 100644 index 000000000..190298319 --- /dev/null +++ b/owslib/ogcapi/models.py @@ -0,0 +1,229 @@ +from enum import Enum +from typing import Any, List, Union, Optional, Dict + +from pydantic import BaseModel, Extra, conint, Field, AnyUrl + +# ---- # +# Enum # +# ---- # + + +class Mode(Enum): + sync = 'sync' + async_ = 'async' + auto = 'auto' + + +class Response(Enum): + raw = 'raw' + document = 'document' + + +class RangeClosure(Enum): + closed = 'closed' + open = 'open' + open_closed = 'open-closed' + closed_open = 'closed-open' + + +class Status(Enum): + accepted = 'accepted' + running = 'running' + successful = 'successful' + failed = 'failed' + dismissed = 'dismissed' + + +class JobControlOptions(Enum): + sync_execute = 'sync-execute' + async_execute = 'async-execute' + + +class TransmissionMode(Enum): + value = 'value' + reference = 'reference' + +# ------ # +# Values # +# ------ # + + +class AllowedValues(BaseModel): + __root__: List[Any] + + +class Range(BaseModel): + minimumValue: Optional[str] = None + maximumValue: Optional[str] = None + spacing: Optional[str] = None + rangeClosure: Optional[RangeClosure] = None + + +class AnyValue(BaseModel): + anyValue: Optional[bool] = True + + +class ValuesReference(BaseModel): + __root__: AnyUrl + + +class Format(BaseModel): + mediaType: str + schema: Optional[str] = None + encoding: Optional[str] = None + + +class FormatDescription(Format): + maximumMegabytes: Optional[float] = None + default: Optional[bool] = False + + +class SupportedCRS(BaseModel): + crs: Optional[str] = None + default: Optional[bool] = False + + +class Metadata(BaseModel): + title: Optional[str] = None + role: Optional[str] = None + href: Optional[str] = None + + +# ---------- # +# Parameters # +# ---------- # + + +class InvalidParameter(BaseModel): + __root__: Any = Field(..., description='A query parameter has an invalid value.') + + +class AdditionalParameter(BaseModel): + name: str + value: List[Any] + + +class AdditionalParameters(Metadata): + parameters: Optional[List[AdditionalParameter]] = None + + +# ---------- # +# Data types # +# ---------- # + +class NameReferenceType(BaseModel): + name: str + reference: Optional[AnyUrl] = None + + +class LiteralDataDomain(BaseModel): + valueDefinition: Optional[Union[AllowedValues, AnyValue, ValuesReference]] = None + defaultValue: Optional[str] = None + dataType: Optional[NameReferenceType] = None + uom: Optional[NameReferenceType] = None + + +class LiteralDataType(BaseModel): + literalDataDomains: List[LiteralDataDomain] + + +class ComplexDataType(BaseModel): + formats: List[FormatDescription] + + +class BoundingBoxDataType(BaseModel): + supportedCRS: List[SupportedCRS] + + +# ---- # +# Data # +# ---- # + + +class BoundingBoxData(BaseModel): + crs: Optional[AnyUrl] = None + bbox: List[float] = Field(..., max_items=6, min_items=4) + + +class InlineOrRefData(BaseModel): + dataType: Optional[NameReferenceType] = None + uom: Optional[NameReferenceType] = None + format: Optional[Format] = None + href: Optional[AnyUrl] = None + value: Optional[Union[str, float, bool, Dict[str, Any]]] = None + + +# -- # +# IO # +# -- # + + +class Output(BaseModel): + __root__: Any + + +class Link(BaseModel): + href: str + rel: Optional[str] = Field(None, example='service') + type: Optional[str] = Field(None, example='application/json') + hreflang: Optional[str] = Field(None, example='en') + title: Optional[str] = None + + +class StatusInfo(BaseModel): + jobID: str + status: Status + message: Optional[str] = None + progress: Optional[conint(ge=0, le=100)] = None + links: Optional[List[Link]] = None + + +class ObservedProperty(BaseModel): + name: Optional[str] = None + uri: Optional[AnyUrl] = None + description: Optional[str] = None + + +class DescriptionType(BaseModel): + id: str + title: Optional[str] = None + description: Optional[str] = None + keywords: Optional[List[str]] = None + metadata: Optional[List[Metadata]] = None + additionalParameters: Optional[AdditionalParameters] = None + + +class MaxOccur(BaseModel): + pass + + +class InputDescription(DescriptionType): + input: Optional[ + Union[ + ComplexDataType, + LiteralDataType, + BoundingBoxDataType, + ] + ] = None + minOccurs: Optional[int] = None + maxOccurs: Optional[Union[int, MaxOccur]] = None + observedProperty: Optional[ObservedProperty] = None + + +class OutputDescription(DescriptionType): + output: Optional[ + Union[ComplexDataType, LiteralDataType, BoundingBoxDataType] + ] = None + observedProperty: Optional[ObservedProperty] = None + + +class ProcessSummary(DescriptionType): + version: str + jobControlOptions: Optional[List[JobControlOptions]] = None + outputTransmission: Optional[List[TransmissionMode]] = None + links: Optional[List[Link]] = None + + +class Process(ProcessSummary): + inputs: Optional[List[InputDescription]] = None + outputs: Optional[List[OutputDescription]] = None From 3a72a95e01aa6e303897bce14a5953aa0b937920 Mon Sep 17 00:00:00 2001 From: David Huard Date: Fri, 19 Feb 2021 00:30:46 -0500 Subject: [PATCH 13/18] add missing models. --- owslib/ogcapi/models.py | 50 +++++++++++++++++++++++-- tests/test_ogcapi_processes_pygeoapi.py | 17 +++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/owslib/ogcapi/models.py b/owslib/ogcapi/models.py index 190298319..f353fe7ff 100644 --- a/owslib/ogcapi/models.py +++ b/owslib/ogcapi/models.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Any, List, Union, Optional, Dict -from pydantic import BaseModel, Extra, conint, Field, AnyUrl +from pydantic import BaseModel, conint, Field, AnyUrl, Extra # ---- # # Enum # @@ -69,7 +69,7 @@ class ValuesReference(BaseModel): class Format(BaseModel): mediaType: str - schema: Optional[str] = None + schema_: Optional[str] = Field(alias='schema', default=None) encoding: Optional[str] = None @@ -123,8 +123,10 @@ class LiteralDataDomain(BaseModel): uom: Optional[NameReferenceType] = None +# TODO: Example in ogcapi-processes has `literalDataDomain`, but the schema says `literalDataDomains` class LiteralDataType(BaseModel): - literalDataDomains: List[LiteralDataDomain] + # literalDataDomains: List[LiteralDataDomain] + literalDataDomain: LiteralDataDomain class ComplexDataType(BaseModel): @@ -157,8 +159,17 @@ class InlineOrRefData(BaseModel): # IO # # -- # +class Input(BaseModel): + class Config: + extra = Extra.allow + + __root__: Union[InlineOrRefData, BoundingBoxData, Any, List[Union[InlineOrRefData, BoundingBoxData, Any]]] + class Output(BaseModel): + class Config: + extra = Extra.allow + __root__: Any @@ -227,3 +238,36 @@ class ProcessSummary(DescriptionType): class Process(ProcessSummary): inputs: Optional[List[InputDescription]] = None outputs: Optional[List[OutputDescription]] = None + + +# Experimenting with adding dunder methods +class ProcessList(BaseModel): + __root__: List[ProcessSummary] + + def __iter__(self): + return iter(self.__root__) + + def __getitem__(self, item): + return self.__root__[item] + + def __len__(self): + return len(self.__root__) + + +class ConfClasses(BaseModel): + conformsTo: List[str] + + +class Subscriber(BaseModel): + successUri: Optional[AnyUrl] = None + inProgressUri: Optional[AnyUrl] = None + failedUri: Optional[AnyUrl] = None + + +class Execute(BaseModel): + id: str + inputs: Optional[Input] = None + outputs: Output + mode: Mode + response: Response + subscriber: Optional[Subscriber] = None diff --git a/tests/test_ogcapi_processes_pygeoapi.py b/tests/test_ogcapi_processes_pygeoapi.py index 2cf5dd9dd..6cb045ab8 100644 --- a/tests/test_ogcapi_processes_pygeoapi.py +++ b/tests/test_ogcapi_processes_pygeoapi.py @@ -45,3 +45,20 @@ def test_ogcapi_processes_pygeoapi(): assert len(resp['outputs']) == 1 assert resp['outputs'][0]['id'] == 'echo' assert resp['outputs'][0]['value'] == 'Hello hello!' + + +@pytest.mark.online +@pytest.mark.skipif(not service_ok(SERVICE_URL), + reason='service is unreachable') +def test_ogcapi_models_pygeoapi(): + import owslib.ogcapi.models as m + w = Processes(SERVICE_URL) + + conformance = m.ConfClasses.parse_obj(w.conformance()) + + assert len(conformance.conformsTo) == 9 + + # list processes + processes = m.ProcessList.parse_obj(w.processes()) + assert len(processes) > 0 + From 443d28647cf57a15600bfa9dbf7a08f5cf3bcc8f Mon Sep 17 00:00:00 2001 From: David Huard Date: Fri, 19 Feb 2021 08:08:45 -0500 Subject: [PATCH 14/18] add pydantic to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index fcca5839b..5e404511a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ pytz requests>=1.0 pyproj >=2 pyyaml +pydantic From 2182a4ca8024583e6492fc4ec4af952770017eb9 Mon Sep 17 00:00:00 2001 From: David Huard Date: Fri, 19 Feb 2021 09:31:05 -0500 Subject: [PATCH 15/18] added test for some models based on examples. Need to set env variable. --- owslib/ogcapi/models.py | 12 +++++-- owslib/ogcapi/test_ogcapi_process_models.py | 39 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 owslib/ogcapi/test_ogcapi_process_models.py diff --git a/owslib/ogcapi/models.py b/owslib/ogcapi/models.py index f353fe7ff..9ce2be5df 100644 --- a/owslib/ogcapi/models.py +++ b/owslib/ogcapi/models.py @@ -1,6 +1,5 @@ from enum import Enum -from typing import Any, List, Union, Optional, Dict - +from typing import Any, List, Union, Optional, Dict, ForwardRef from pydantic import BaseModel, conint, Field, AnyUrl, Extra # ---- # @@ -158,12 +157,19 @@ class InlineOrRefData(BaseModel): # -- # # IO # # -- # +#Input = ForwardRef("Input") + class Input(BaseModel): class Config: extra = Extra.allow - __root__: Union[InlineOrRefData, BoundingBoxData, Any, List[Union[InlineOrRefData, BoundingBoxData, Any]]] + # The schema refers to Input. Self-referencing objects are supported with ForwardRef, but getting parsing errors. + # See https://pydantic-docs.helpmanual.io/usage/postponed_annotations/ + __root__: Union[InlineOrRefData, BoundingBoxData, List[Union[InlineOrRefData, BoundingBoxData]]] + + +# Input.update_forward_refs() class Output(BaseModel): diff --git a/owslib/ogcapi/test_ogcapi_process_models.py b/owslib/ogcapi/test_ogcapi_process_models.py new file mode 100644 index 000000000..2088c22ab --- /dev/null +++ b/owslib/ogcapi/test_ogcapi_process_models.py @@ -0,0 +1,39 @@ +"""Use the examples from ogcapi-processes/core/examples/json/ to check data models. + +TODO: Fetch the json files from github. Probably need caching to avoid hitting usage limits on github. A fixture? +TODO: For each json response, instantiate the corresponding class. +""" +import os +import pytest +import json +from pathlib import Path +import owslib.ogcapi.models as m + +# Path to the ogcapi-processes/core/examples/json directory +PATH = os.environ.get("PATH_OGCAPI_PROCESSES_EXAMPLES") +PATH = Path(PATH) if PATH is not None else PATH + + +@pytest.mark.skipif(PATH is None, reason="No json examples") +class TestModels: + + def test_confclasses(self): + fn = PATH / "ConfClasses.json" + m.ConfClasses.parse_file(fn) + + @pytest.mark.xfail + def test_execute(self): + fn = PATH / "Execute.json" + # m.Execute.parse_file(fn) + obj = json.loads(fn.read_text()) + m.Execute.parse_obj(obj) + + def test_input(self): + fn = PATH / "Execute.json" + obj = json.loads(fn.read_text()) + m.Input.parse_obj(obj["inputs"]["complexInputId"]) + m.Input.parse_obj(obj["inputs"]["complexInputsId"]) + m.Input.parse_obj(obj["inputs"]["literalInputId"]) + m.Input.parse_obj(obj["inputs"]["boundingboxInputId"]) + m.Input.parse_obj(obj["inputs"]) + From 3bf11bc61c582e49964d3a3af0a912bfee26c678 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Fri, 19 Feb 2021 20:22:03 +0100 Subject: [PATCH 16/18] using pydantic models in ogcapi.processes --- owslib/ogcapi/models.py | 26 ++++++++--- owslib/ogcapi/processes.py | 27 +++++++----- tests/test_ogcapi_features_pygeoapi.py | 2 +- tests/test_ogcapi_processes_52n.py | 44 ------------------- .../test_ogcapi_processes_models.py | 0 tests/test_ogcapi_processes_pygeoapi.py | 44 ++++++++----------- 6 files changed, 54 insertions(+), 89 deletions(-) delete mode 100644 tests/test_ogcapi_processes_52n.py rename owslib/ogcapi/test_ogcapi_process_models.py => tests/test_ogcapi_processes_models.py (100%) diff --git a/owslib/ogcapi/models.py b/owslib/ogcapi/models.py index 9ce2be5df..58b858039 100644 --- a/owslib/ogcapi/models.py +++ b/owslib/ogcapi/models.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, List, Union, Optional, Dict, ForwardRef +from typing import Any, List, Union, Optional, Dict from pydantic import BaseModel, conint, Field, AnyUrl, Extra # ---- # @@ -67,7 +67,8 @@ class ValuesReference(BaseModel): class Format(BaseModel): - mediaType: str + # mediaType: str + mimeType: str schema_: Optional[str] = Field(alias='schema', default=None) encoding: Optional[str] = None @@ -129,7 +130,7 @@ class LiteralDataType(BaseModel): class ComplexDataType(BaseModel): - formats: List[FormatDescription] + formats: Optional[List[FormatDescription]] class BoundingBoxDataType(BaseModel): @@ -157,7 +158,7 @@ class InlineOrRefData(BaseModel): # -- # # IO # # -- # -#Input = ForwardRef("Input") +# Input = ForwardRef("Input") class Input(BaseModel): @@ -271,9 +272,20 @@ class Subscriber(BaseModel): class Execute(BaseModel): - id: str + id: Optional[str] inputs: Optional[Input] = None - outputs: Output + outputs: Optional[Output] = None mode: Mode - response: Response + response: Optional[Response] = None subscriber: Optional[Subscriber] = None + + +class LiteralOutput(BaseModel): + id: str + value: str + + +class Result(BaseModel): + jobID: Optional[str] + location: Optional[str] = None + outputs: Optional[List[LiteralOutput]] = None diff --git a/owslib/ogcapi/processes.py b/owslib/ogcapi/processes.py index 4fe0102b2..6885e5b6a 100644 --- a/owslib/ogcapi/processes.py +++ b/owslib/ogcapi/processes.py @@ -14,6 +14,7 @@ from owslib.ogcapi import API from owslib.util import Authentication +from owslib.ogcapi import models as m LOGGER = logging.getLogger(__name__) @@ -40,11 +41,9 @@ def processes(self) -> list: @returns: `list` of available processes. """ - processes_ = [] path = 'processes' data = self._request(path) - if 'processes' in data: - processes_.extend(data["processes"]) + processes_ = m.ProcessList.parse_obj(data['processes']) return processes_ def process_description(self, process_id: str) -> dict: @@ -57,7 +56,8 @@ def process_description(self, process_id: str) -> dict: """ path = f'processes/{process_id}' - return self._request(path) + process_ = m.Process.parse_obj(self._request(path)) + return process_ def job_list(self, process_id: str) -> dict: """ @@ -69,7 +69,8 @@ def job_list(self, process_id: str) -> dict: """ path = f'processes/{process_id}/jobs' - return self._request(path) + data = self._request(path) + return data def execute(self, process_id: str, json: dict) -> dict: """ @@ -80,16 +81,17 @@ def execute(self, process_id: str, json: dict) -> dict: @returns: `dict` of the status location (async) or outputs (sync) """ + # validate input + m.Execute.parse_obj(json) - result = {} path = f'processes/{process_id}/jobs' resp = self._request_post(path, json) data = resp.json() - if 'outputs' in data: - result['outputs'] = data['outputs'] - else: - result['location'] = resp.headers.get("Location", data.get("location")) - return result + if 'location' not in data: + data['location'] = resp.headers.get("Location") + data['jobID'] = data['location'].split('/')[-1] + result_ = m.Result.parse_obj(data) + return result_ def _request_post(self, path: str, json: dict) -> requests.Response: # TODO: needs to be implemented in base class @@ -112,7 +114,8 @@ def status(self, process_id: str, job_id: str) -> dict: """ path = f'processes/{process_id}/jobs/{job_id}' - return self._request(path) + status_ = m.StatusInfo.parse_obj(self._request(path)) + return status_ def cancel(self, process_id: str, job_id: str) -> dict: """ diff --git a/tests/test_ogcapi_features_pygeoapi.py b/tests/test_ogcapi_features_pygeoapi.py index 6f42fe847..7973bd1a8 100644 --- a/tests/test_ogcapi_features_pygeoapi.py +++ b/tests/test_ogcapi_features_pygeoapi.py @@ -23,7 +23,7 @@ def test_ogcapi_features_pygeoapi(): assert paths['/collections/lakes'] is not None conformance = w.conformance() - assert len(conformance['conformsTo']) == 9 + assert len(conformance['conformsTo']) == 16 collections = w.collections() assert len(collections) > 0 diff --git a/tests/test_ogcapi_processes_52n.py b/tests/test_ogcapi_processes_52n.py deleted file mode 100644 index ae93f86f0..000000000 --- a/tests/test_ogcapi_processes_52n.py +++ /dev/null @@ -1,44 +0,0 @@ -from tests.utils import service_ok - -import pytest - -from owslib.ogcapi.processes import Processes - -SERVICE_URL = 'http://geoprocessing.demo.52north.org:8080/javaps/rest/' - - -@pytest.mark.xfail -@pytest.mark.online -@pytest.mark.skipif(not service_ok(SERVICE_URL), - reason='service is unreachable') -def test_ogcapi_processes_52n(): - w = Processes(SERVICE_URL) - - assert w.url == 'http://geoprocessing.demo.52north.org:8080/javaps/rest/' - assert w.url_query_string is None - - # TODO: RuntimeError: Did not find service-desc link - # api = w.api() - # assert api['components']['parameters'] is not None - # paths = api['paths'] - # assert paths is not None - # assert paths['/processes/hello-world'] is not None - - conformance = w.conformance() - assert len(conformance['conformsTo']) == 5 - - # list processes - processes = w.processes() - assert len(processes) > 0 - - # process description - echo = w.process_description('org.n52.javaps.test.EchoProcess') - assert echo['id'] == 'org.n52.javaps.test.EchoProcess' - assert echo['title'] == 'org.n52.javaps.test.EchoProcess' - # assert "An example process that takes a name as input" in echo['description'] - - # running jobs - jobs = w.job_list('org.n52.javaps.test.EchoProcess') - assert len(jobs) >= 0 - - # TODO: post request not allowed at 52n? diff --git a/owslib/ogcapi/test_ogcapi_process_models.py b/tests/test_ogcapi_processes_models.py similarity index 100% rename from owslib/ogcapi/test_ogcapi_process_models.py rename to tests/test_ogcapi_processes_models.py diff --git a/tests/test_ogcapi_processes_pygeoapi.py b/tests/test_ogcapi_processes_pygeoapi.py index 6cb045ab8..c10f3312f 100644 --- a/tests/test_ogcapi_processes_pygeoapi.py +++ b/tests/test_ogcapi_processes_pygeoapi.py @@ -23,17 +23,18 @@ def test_ogcapi_processes_pygeoapi(): assert paths['/processes/hello-world'] is not None conformance = w.conformance() - assert len(conformance['conformsTo']) == 9 + assert len(conformance['conformsTo']) == 16 # list processes processes = w.processes() assert len(processes) > 0 + assert processes[0].id == 'hello-world' # process description hello = w.process_description('hello-world') - assert hello['id'] == 'hello-world' - assert hello['title'] == 'Hello World' - assert "An example process that takes a name as input" in hello['description'] + assert hello.id == 'hello-world' + assert hello.title == 'Hello World' + assert "An example process that takes a name as input" in hello.description # running jobs jobs = w.job_list('hello-world') @@ -41,24 +42,17 @@ def test_ogcapi_processes_pygeoapi(): # execute process in sync mode request_json = {"inputs": [{"id": "name", "value": "hello"}], "mode": "sync"} - resp = w.execute('hello-world', json=request_json) - assert len(resp['outputs']) == 1 - assert resp['outputs'][0]['id'] == 'echo' - assert resp['outputs'][0]['value'] == 'Hello hello!' - - -@pytest.mark.online -@pytest.mark.skipif(not service_ok(SERVICE_URL), - reason='service is unreachable') -def test_ogcapi_models_pygeoapi(): - import owslib.ogcapi.models as m - w = Processes(SERVICE_URL) - - conformance = m.ConfClasses.parse_obj(w.conformance()) - - assert len(conformance.conformsTo) == 9 - - # list processes - processes = m.ProcessList.parse_obj(w.processes()) - assert len(processes) > 0 - + result = w.execute('hello-world', json=request_json) + assert len(result.outputs) == 1 + assert result.outputs[0].id == 'echo' + assert result.outputs[0].value == 'Hello hello!' + + # execute process in async mode + request_json = {"inputs": [{"id": "name", "value": "hello"}], "mode": "async"} + result = w.execute('hello-world', json=request_json) + assert result.outputs is None + assert 'processes/hello-world/jobs' in result.location + assert result.jobID is not None + + # get job status + # status = w.status('hello-world', result.job_id) From 4240d72dc6c3d667d497385de896f195b7841aac Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Fri, 19 Feb 2021 22:51:18 +0100 Subject: [PATCH 17/18] update model with default --- owslib/ogcapi/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owslib/ogcapi/models.py b/owslib/ogcapi/models.py index 58b858039..65dd6be3b 100644 --- a/owslib/ogcapi/models.py +++ b/owslib/ogcapi/models.py @@ -130,7 +130,7 @@ class LiteralDataType(BaseModel): class ComplexDataType(BaseModel): - formats: Optional[List[FormatDescription]] + formats: Optional[List[FormatDescription]] = None class BoundingBoxDataType(BaseModel): From df130f808c595053c9eb3d3c582f1be190c4ac21 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Fri, 19 Feb 2021 23:03:41 +0100 Subject: [PATCH 18/18] added job list --- owslib/ogcapi/models.py | 17 +++++++++++++++++ owslib/ogcapi/processes.py | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/owslib/ogcapi/models.py b/owslib/ogcapi/models.py index 65dd6be3b..097537d6b 100644 --- a/owslib/ogcapi/models.py +++ b/owslib/ogcapi/models.py @@ -2,6 +2,10 @@ from typing import Any, List, Union, Optional, Dict from pydantic import BaseModel, conint, Field, AnyUrl, Extra + +# Taken from the openapi schema +# https://github.com/opengeospatial/ogcapi-processes/tree/master/core/openapi/schemas + # ---- # # Enum # # ---- # @@ -196,6 +200,19 @@ class StatusInfo(BaseModel): links: Optional[List[Link]] = None +class JobList(BaseModel): + __root__: List[StatusInfo] + + def __iter__(self): + return iter(self.__root__) + + def __getitem__(self, item): + return self.__root__[item] + + def __len__(self): + return len(self.__root__) + + class ObservedProperty(BaseModel): name: Optional[str] = None uri: Optional[AnyUrl] = None diff --git a/owslib/ogcapi/processes.py b/owslib/ogcapi/processes.py index 6885e5b6a..d492dad8b 100644 --- a/owslib/ogcapi/processes.py +++ b/owslib/ogcapi/processes.py @@ -69,8 +69,8 @@ def job_list(self, process_id: str) -> dict: """ path = f'processes/{process_id}/jobs' - data = self._request(path) - return data + jobs_ = m.JobList.parse_obj(self._request(path)) + return jobs_ def execute(self, process_id: str, json: dict) -> dict: """