From 37835f383177ecf2eabd0470c90badeb04897d28 Mon Sep 17 00:00:00 2001 From: David Nichols Date: Fri, 30 Oct 2020 18:43:49 +0100 Subject: [PATCH] refs #4059 better base path handling fixes for REST and Swagger --- doxygen/lang/900_release_notes.dox.tmpl | 3 + qlib/ConnectionProvider/AbstractConnection.qc | 11 ++-- qlib/RestClient.qm | 58 +++++++++++++------ qlib/Swagger.qm | 49 +++++++++++++++- .../SwaggerRequestDataProvider.qc | 22 +++++-- qlib/ZeyosRestClient.qm | 46 ++++++++++++--- 6 files changed, 153 insertions(+), 36 deletions(-) diff --git a/doxygen/lang/900_release_notes.dox.tmpl b/doxygen/lang/900_release_notes.dox.tmpl index cf98535a47..f70a46017d 100644 --- a/doxygen/lang/900_release_notes.dox.tmpl +++ b/doxygen/lang/900_release_notes.dox.tmpl @@ -17,6 +17,9 @@ - WebSocketHandler module updates: - fixed a bug handling the case when a connection object is deleted in a callback method (issue 4063) + - RestClient module updates: + - fixed a bug where the base path in the Swagger schema was ignored in + (issue 4059) @section qore_096 Qore 0.9.6 diff --git a/qlib/ConnectionProvider/AbstractConnection.qc b/qlib/ConnectionProvider/AbstractConnection.qc index 5574a4f81b..c88a280298 100644 --- a/qlib/ConnectionProvider/AbstractConnection.qc +++ b/qlib/ConnectionProvider/AbstractConnection.qc @@ -409,19 +409,20 @@ scheme://user:pass@hostname:port/path # check for required options if (scheme_info.required_options) { list req_list = scheme_info.required_options.split("|"); - bool ok; + hash missing; foreach string req in (req_list) { list req_opt = req.split(","); # check which options are missing *hash have = options{req_opt}; - if (have.size() == req_opt.size()) { - ok = True; + if (have.size() < req_opt.size()) { + missing = map {$1: True}, req_opt; + missing -= keys have; break; } } - if (!ok) { + if (missing) { throw "CONNECTION-OPTION-ERROR", sprintf("missing required options %y; options provided: %y", - scheme_info.required_options, options); + keys missing, options); } } return options; diff --git a/qlib/RestClient.qm b/qlib/RestClient.qm index bd15545ed6..0f7aad3342 100644 --- a/qlib/RestClient.qm +++ b/qlib/RestClient.qm @@ -23,7 +23,7 @@ */ # minimum qore version -%requires qore >= 0.9.6 +%requires qore >= 0.9.7 # require type definitions everywhere %require-types @@ -54,7 +54,7 @@ %endtry module RestClient { - version = "1.7.1"; + version = "1.7.2"; desc = "user module for calling REST services"; author = "David Nichols "; url = "http://qore.org"; @@ -126,6 +126,10 @@ printf("%N\n", ans.body); @section restclientrelnotes Release Notes + @subsection restclientv1_7_2 RestClient v1.7.2 + - better fixes to REST URI path handling with schema validation + (issue 4059) + @subsection restclientv1_7_1 RestClient v1.7.1 - added the \a swagger_base_path option to REST clients and connections to allow for Swagger schemas to have their base path overridden @@ -446,14 +450,6 @@ RestClient rest(("url": "http://localhost:8001/rest")); validator = new NullRestSchemaValidator(); } } - if (validator) { - # issue #4059: strip the URI path from the URL if it matches the prefix of the REST validator's base path - # otherwise we will run into compatibility problems with the fix - string base_path = validator.getBasePath(); - if (base_path == getConnectionPath()) { - setConnectionPath(); - } - } if (!do_not_connect) connect(); @@ -870,18 +866,27 @@ hash ans = rest.del("/orders/1"); #! sets up the path for the HTTP request URI private nothing preparePath(reference path) { - # prepare path - *string p = getConnectionPath(); - - # strip trailing "/" off the connection path - p =~ s/\/+$//; # strip leading "/" off the given path path =~ s/^\/+//; - if (p.val()) + + # prepare path = connection path + base path + path + *string p = getConnectionPath(); + string base_path = validator.getBasePath(); + if (base_path.val() && base_path != "/" && p != base_path) { + # strip trailing "/" off the base path + base_path =~ s/\/+$//; + path = base_path + (path.val() ? ("/" + path) : ""); + } + + if (exists p) { + # strip trailing "/" off the connection path + p =~ s/\/+$//; path = p + (path.val() ? ("/" + path) : ""); + } # ensure path is absolute - if (path !~ /^\//) + if (path !~ /^\//) { splice path, 0, 0, "/"; + } path = encode_uri_request(path); } @@ -963,6 +968,23 @@ hash ans = rest.doRequest("DELETE", "/orders/1"); return sendAndDecodeResponse(body, m, path, hdr, \info, decode_errors); } + #! The same as doRequest() except no schema validation is performed on the request + /** @since RestClient 1.7.2 + */ + hash doValidatedRequest(string m, string path, auto body, *reference> info, softbool decode_errors = True, *hash hdr) { + # use {} + ... here to ensure that hdr stays "hash" + hdr = {} + headers + hdr; + + on_exit if (exists body) { + info += { + "request-body": body, + "request-serialization": ds, + }; + } + + return sendAndDecodeResponse(body, m, path, hdr, \info, decode_errors); + } + #! sends the outgoing HTTP message and recodes the response to data private hash sendAndDecodeResponse(*data body, string m, string path, hash hdr, *reference> info, *softbool decode_errors) { hash h; @@ -1284,7 +1306,7 @@ public class RestConnection inherits ConnectionProvider::HttpBasedConnection { *AbstractRestSchemaValidator validator = rest.getValidator(); if (validator) { # if there's a validator, return the provider - return validator.getDataProvider(get()); + return validator.getDataProvider(rest); } } throw "DATA-PROVIDER-ERROR", sprintf("there is no validator object in the %s object to use to return a " diff --git a/qlib/Swagger.qm b/qlib/Swagger.qm index d2287efe04..60bb4b4ca6 100644 --- a/qlib/Swagger.qm +++ b/qlib/Swagger.qm @@ -1451,7 +1451,13 @@ public class SwaggerSchema inherits ObjectBase, AbstractRestSchemaValidator { /** @param basePath the new base path value; use an empty string here to clear the basePath */ private setBasePathImpl(string basePath) { - self.basePath = basePath; + # remove any trailing '/'s from path + basePath =~ s/\/+$//; + if (basePath.val()) { + self.basePath = basePath; + } else { + remove basePath; + } } #! returns example Qore code for the given request @@ -2491,6 +2497,47 @@ public class OperationObject inherits ObjectBase { } } + #! Processes a generated request + *data getRequestBody(PathItemObject pio, auto body, reference> headers) { + *AbstractParameterObject body_po = self.body ?? pio.body; + if (body_po) { + body_po.check(True, True, path, method, "body", \body); + } else if (formData) { + map $1.value.check(True, True, path, method, "formData." + $1.key, \body{$1.key}), + formData.pairIterator(); + } else if (body) { + error("SCHEMA-VALIDATION-ERROR", "No message body is accepted; body keys passed: %y", keys body); + } + + if (!exists body) { + return; + } + + # get content type + string ct = consumes.firstKey(); + + if (ct == MimeTypeMultipartFormData) { + hash> parts = cast>>(map {$1.key: cast>(( + "name": $1.key, + "filename": $1.key, + "hdr": ("Content-Type": MimeTypeText), + "body": $1.value, + ))}, body.pairIterator()); + + hash mh = MultiPartFormDataMessage::makeMessage(parts).getMsgAndHeaders(); + headers."Content-Type" = mh.hdr."Content-Type"; + return mh.body; + } + + if (MimeDataTypes{ct}) { + headers."Content-Type" = ct; + return MimeDataTypes{ct}.serialize(body); + } + + throw "SERIALIZATION-ERROR", sprintf("%s %s: message body cannot be serialized; available MIME type(s): %y; " + "available serialization modules: %y", method.upr(), path, keys consumes, SerializationModules); + } + #! processes a REST API client-side request to the operation /** @param serialize if request arguments should be processed for serialization (client-side) or not (server-side) @param pio the PathItemObject corresponding to the URI path diff --git a/qlib/SwaggerDataProvider/SwaggerRequestDataProvider.qc b/qlib/SwaggerDataProvider/SwaggerRequestDataProvider.qc index b850bdeb6b..71d9409fdc 100644 --- a/qlib/SwaggerDataProvider/SwaggerRequestDataProvider.qc +++ b/qlib/SwaggerDataProvider/SwaggerRequestDataProvider.qc @@ -75,7 +75,12 @@ public class SwaggerRequestDataProvider inherits SwaggerDataProviderBase { #! Returns the data provider name string getName() { - return schema.info.title + getUriPath() + "/" + op.method.upr(); + string base_path = schema.getBasePath(); + string rv = schema.info.title; + if (base_path.val() && base_path != "/") { + rv += base_path; + } + return rv + getUriPath() + "/" + op.method.upr(); } #! Returns data provider info @@ -247,7 +252,17 @@ public class SwaggerRequestDataProvider inherits SwaggerDataProviderBase { # make the call hash info; try { - hash resp = rest.doRequest(op.method, uri_path, req.body, \info, NOTHING, req.header + options.hdr); + *hash hdr = req.header + options.hdr; + *data body = op.getRequestBody(pio, req.body, \hdr); + # prepend base path to request, if any + string base_path = schema.getBasePath(); + if (base_path.val() && base_path != "/") { + if (uri_path !~ /^\//) { + base_path += "/"; + } + uri_path = base_path + uri_path; + } + hash resp = rest.doValidatedRequest(op.method, uri_path, body, \info, NOTHING, hdr); return resp + {"info": info}; } catch (hash ex) { #printf("XXX %s\n", get_exception_string(ex)); @@ -268,9 +283,6 @@ public class SwaggerRequestDataProvider inherits SwaggerDataProviderBase { #! Returns the URI path to use in requests private string getUriPath() { string uri_path = self.uri_path; - if ((string base_path = schema.getBasePath()).val() && base_path != "/") { - uri_path = base_path + uri_path; - } if (resolve_uri) { uri_path = cb_resolve_value(uri_path); } diff --git a/qlib/ZeyosRestClient.qm b/qlib/ZeyosRestClient.qm index 2e036d945b..723d83eee7 100644 --- a/qlib/ZeyosRestClient.qm +++ b/qlib/ZeyosRestClient.qm @@ -32,6 +32,8 @@ %requires(reexport) RestClient >= 1.3.1 %requires(reexport) ConnectionProvider >= 1.4 +%requires json + module ZeyosRestClient { version = "1.1"; desc = "user module for calling zeyos.com REST services"; @@ -113,7 +115,8 @@ public namespace ZeyosRestClient { public class ZeyosRestClient inherits RestClient::RestClient { public { const RequiredOptions = ( - "appsecret" + "appsecret", + "instance", ); const AuthPath = "/auth/v1/login"; @@ -124,6 +127,10 @@ public class ZeyosRestClient inherits RestClient::RestClient { string password; string identifier; string appsecret; + string instance; + + # the token acquired in the login call + string token; } #! creates the object with the given options @@ -140,7 +147,6 @@ ZeyosRestClient zeyos( @param options valid options are: - \c appsecret (optional): the zeyos.com appsecret - - \c identifier (optional): the zeyos.com identifier - \c additional_methods: Optional hash with more but not-HTTP-standardized methods to handle. It allows to create various HTTP extensions like e.g. WebDAV. The hash takes the method name as a key, and the value is a boolean @ref True "True" or @ref False "False": indicating if the method requires a message body as well. Example: @code{.py} # add new HTTP methods for WebDAV. Both of them require body posting to the server @@ -156,6 +162,8 @@ ZeyosRestClient zeyos( like any other response - \c headers: an optional hash of headers to send with every request, these can also be overridden in request method calls - \c http_version: Either '1.0' or '1.1' for the claimed HTTP protocol version compliancy in outgoing message headers + - \c identifier (optional): the zeyos.com identifier + - \c instance (required): the zeyos.com instance name (first segment of URI path) - \c max_redirects: The maximum number of redirects before throwing an exception (the default is 5) - \c proxy: The proxy URL for connecting through a proxy - \c send_encoding: a @ref EncodingSupport "send data encoding option" or the value \c "auto" which means to use automatic encoding; if not present defaults to no content-encoding on sent message bodies (note that the @ref RestClient::RestClient "RestClient" class will only compress outgoing message bodies over @ref RestClient::RestClient::CompressionThreshold "CompressionThreshold" bytes in size) @@ -200,7 +208,26 @@ ZeyosRestClient zeyos( "without both a username and password"); } - getToken(); + # make sure there is no connection path + setConnectionPath(); + + if (!do_not_connect) { + getToken(); + } + } + + hash doRequest(string m, string path, auto body, *reference info, softbool decode_errors = True, *hash hdr) { + if (!exists token) { + getToken(\info); + } + return RestClient::doRequest(m, path, body, \info, decode_errors, hdr); + } + + hash doValidatedRequest(string m, string path, auto body, *reference> info, softbool decode_errors = True, *hash hdr) { + if (!exists token) { + getToken(\info); + } + return RestClient::doValidatedRequest(m, path, body, \info, decode_errors, hdr); } #! returns options for the @ref RestClient::RestClient::constructor() "RestClient::constructor()" @@ -226,14 +253,15 @@ ZeyosRestClient zeyos( } #! logs in an sets the token for further communication - private getToken() { - hash result = RestClient::post(AuthPath, { + private getToken(*reference info) { + hash result = RestClient::doValidatedRequest("POST", "/" + instance + AuthPath, make_json({ "name": username, "password": password, "identifier": identifier ?? self.uniqueHash(), "appsecret": appsecret, - }); - headers{"Authorization"} = "Bearer " + result.body.token; + }), \info, True, {"Content-Type": MimeTypeJson}); + token = result.body.token; + headers{"Authorization"} = "Bearer " + token; } } @@ -274,6 +302,10 @@ public class ZeyosRestConnection inherits RestClient::RestConnection { "type": "string", "desc": "the Zeyos identifier;if none is provided, a random identifier string is used", }, + "instance": { + "type": "string", + "desc": "the Zeyos instance name (first segment of URI path)", + }, }, "required_options": foldl $1 + "," + $2, ZeyosRestClient::RequiredOptions, };