forked from npshub/mantid
-
Notifications
You must be signed in to change notification settings - Fork 0
/
InternetHelper.cpp
583 lines (502 loc) · 19.5 KB
/
InternetHelper.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
// NScD Oak Ridge National Laboratory, European Spallation Source,
// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidKernel/InternetHelper.h"
#include "MantidKernel/ConfigService.h"
#include "MantidKernel/DateAndTime.h"
#include "MantidKernel/Exception.h"
#include "MantidKernel/Logger.h"
#include "MantidKernel/MantidVersion.h"
// Poco
#include <Poco/Net/AcceptCertificateHandler.h>
#include <Poco/Net/HTMLForm.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Net/HTTPResponse.h>
#include <Poco/Net/HTTPSClientSession.h>
#include <Poco/Net/NetException.h>
#include <Poco/Net/PrivateKeyPassphraseHandler.h>
#include <Poco/Net/SSLManager.h>
#include <Poco/StreamCopier.h>
#include <Poco/TemporaryFile.h>
#include <Poco/URI.h>
#include <Poco/Exception.h>
#include <Poco/File.h>
#include <Poco/FileStream.h>
#include <Poco/Net/Context.h>
#include <Poco/Net/HTTPClientSession.h>
#include <Poco/Net/HTTPMessage.h>
#include <Poco/Net/InvalidCertificateHandler.h>
#include <Poco/SharedPtr.h>
#include <Poco/Timespan.h>
#include <Poco/Types.h>
#if defined(_WIN32) || defined(_WIN64)
#include <Winhttp.h>
#endif
#include <boost/lexical_cast.hpp>
// std
#include <fstream>
#include <mutex>
#include <utility>
namespace Mantid {
using namespace Types::Core;
namespace Kernel {
using namespace Poco::Net;
using std::map;
using std::string;
namespace {
// anonymous namespace for some utility functions
/// static Logger object
Logger g_log("InternetHelper");
/// Flag to protect SSL initialization
std::once_flag SSL_INIT_FLAG;
/**
* Perform initialization of SSL context. Implementation
* designed to be called by std::call_once
*/
void doSSLInit() {
// initialize ssl
Poco::SharedPtr<InvalidCertificateHandler> certificateHandler = new AcceptCertificateHandler(true);
// Currently do not use any means of authentication. This should be updated
// IDS has signed certificate.
const Context::Ptr context = new Context(Context::CLIENT_USE, "", "", "", Context::VERIFY_NONE);
// Create a singleton for holding the default context.
// e.g. any future requests to publish are made to this certificate and
// context.
SSLManager::instance().initializeClient(nullptr, certificateHandler, context);
}
/**
* Entry function to initialize SSL context for the process. It ensures the
* initialization only happens once per process.
*/
void initializeSSL() { std::call_once(SSL_INIT_FLAG, doSSLInit); }
} // namespace
//----------------------------------------------------------------------------------------------
/** Constructor
*/
InternetHelper::InternetHelper()
: m_proxyInfo(), m_isProxySet(false), m_timeout(30), m_isTimeoutSet(false), m_contentLength(0),
m_method(HTTPRequest::HTTP_GET), m_contentType("application/json"), m_body(), m_headers(), m_request(nullptr),
m_response(nullptr) {}
//----------------------------------------------------------------------------------------------
/** Constructor
*/
InternetHelper::InternetHelper(const Kernel::ProxyInfo &proxy)
: m_proxyInfo(proxy), m_isProxySet(true), m_timeout(30), m_isTimeoutSet(false), m_contentLength(0),
m_method(HTTPRequest::HTTP_GET), m_contentType("application/json"), m_body(), m_headers(), m_request(nullptr),
m_response(nullptr) {}
//----------------------------------------------------------------------------------------------
/** Destructor
*/
InternetHelper::~InternetHelper() = default;
void InternetHelper::setupProxyOnSession(HTTPClientSession &session, const std::string &proxyUrl) {
auto proxy = this->getProxy(proxyUrl);
if (!proxy.emptyProxy()) {
session.setProxyHost(proxy.host());
session.setProxyPort(static_cast<Poco::UInt16>(proxy.port()));
}
}
void InternetHelper::createRequest(Poco::URI &uri) {
m_request = std::make_unique<HTTPRequest>(m_method, uri.getPathAndQuery(), HTTPMessage::HTTP_1_1);
m_response = std::make_unique<HTTPResponse>();
if (!m_contentType.empty()) {
m_request->setContentType(m_contentType);
}
m_request->set("User-Agent",
// Use standard User-Agent format as per MDN documentation.
std::string("Mantid/") + MantidVersion::version());
if (m_method == "POST") {
// HTTP states that the 'Content-Length' header should not be included
// if the 'Transfer-Encoding' header is set. UNKNOWN_CONTENT_LENGTH
// indicates to Poco to remove the header field
m_request->setContentLength(HTTPMessage::UNKNOWN_CONTENT_LENGTH);
m_request->setChunkedTransferEncoding(true);
} else if (m_contentLength > 0) {
m_request->setContentLength(m_contentLength);
}
for (auto &header : m_headers) {
m_request->set(header.first, header.second);
}
}
int InternetHelper::sendRequestAndProcess(HTTPClientSession &session, Poco::URI &uri, std::ostream &responseStream) {
// create a request
this->createRequest(uri);
session.sendRequest(*m_request) << m_body;
std::istream &rs = session.receiveResponse(*m_response);
int retStatus = m_response->getStatus();
g_log.debug() << "Answer from web: " << retStatus << " " << m_response->getReason() << '\n';
if (retStatus == HTTP_OK || (retStatus == HTTP_CREATED && m_method == HTTPRequest::HTTP_POST)) {
Poco::StreamCopier::copyStream(rs, responseStream);
if (m_response)
processResponseHeaders(*m_response);
else
g_log.warning("Response is null pointer");
return retStatus;
} else if (isRelocated(retStatus)) {
return this->processRelocation(*m_response, responseStream);
} else {
Poco::StreamCopier::copyStream(rs, responseStream);
return processErrorStates(*m_response, rs, uri.toString());
}
}
int InternetHelper::processRelocation(const HTTPResponse &response, std::ostream &responseStream) {
std::string newLocation = response.get("location", "");
if (!newLocation.empty()) {
g_log.information() << "url relocated to " << newLocation << "\n";
return this->sendRequest(newLocation, responseStream);
} else {
g_log.warning("Apparent relocation did not give new location\n");
return response.getStatus();
}
}
/** Performs a request using http or https depending on the url
* @param url the address to the network resource
* @param responseStream The stream to fill with the reply on success
**/
int InternetHelper::sendRequest(const std::string &url, std::ostream &responseStream) {
// send the request
Poco::URI uri(url);
if (uri.getPath().empty())
uri = url + "/";
int retval;
if ((uri.getScheme() == "https") || (uri.getPort() == 443)) {
retval = sendHTTPSRequest(uri.toString(), responseStream);
} else {
retval = sendHTTPRequest(uri.toString(), responseStream);
}
return retval;
}
/**
* Helper to log (debug level) the request being sent (careful not to
* print blatant passwords, etc.).
*
* @param schemeName Normally "http" or "https"
* @param url url being sent (will be logged)
*/
void InternetHelper::logDebugRequestSending(const std::string &schemeName, const std::string &url) const {
const std::string insecString = "password=";
if (std::string::npos == url.find(insecString)) {
g_log.debug() << "Sending " << schemeName << " " << m_method << " request to: " << url << "\n";
} else {
g_log.debug() << "Sending " << schemeName << " " << m_method
<< " request to an url where the query string seems to contain a "
"password! (not shown for security reasons)."
<< "\n";
}
}
/** Performs a request using http
* @param url the address to the network resource
* @param responseStream The stream to fill with the reply on success
**/
int InternetHelper::sendHTTPRequest(const std::string &url, std::ostream &responseStream) {
int retStatus = 0;
logDebugRequestSending("http", url);
Poco::URI uri(url);
// Configure Poco HTTP Client Session
try {
Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort());
session.setTimeout(Poco::Timespan(getTimeout(), 0));
// configure proxy
setupProxyOnSession(session, url);
// low level sending the request
retStatus = this->sendRequestAndProcess(session, uri, responseStream);
} catch (HostNotFoundException &ex) {
throwNotConnected(url, ex);
} catch (Poco::Exception &ex) {
throw Exception::InternetError("Connection and request failed " + ex.displayText());
}
return retStatus;
}
/** Performs a request using https
* @param url the address to the network resource
* @param responseStream The stream to fill with the reply on success
**/
int InternetHelper::sendHTTPSRequest(const std::string &url, std::ostream &responseStream) {
int retStatus = 0;
logDebugRequestSending("https", url);
Poco::URI uri(url);
try {
initializeSSL();
// Create the session
HTTPSClientSession session(uri.getHost(), static_cast<Poco::UInt16>(uri.getPort()));
session.setTimeout(Poco::Timespan(getTimeout(), 0));
// HACK:: Currently the automatic proxy detection only supports http proxy
// detection
// most locations use the same proxy for http and https, so force it to use
// the http proxy
std::string urlforProxy = ConfigService::Instance().getString("proxy.httpsTargetUrl");
if (urlforProxy.empty()) {
urlforProxy = "http://" + uri.getHost();
}
setupProxyOnSession(session, urlforProxy);
// low level sending the request
retStatus = this->sendRequestAndProcess(session, uri, responseStream);
} catch (HostNotFoundException &ex) {
throwNotConnected(url, ex);
} catch (Poco::Exception &ex) {
throw Exception::InternetError("Connection and request failed " + ex.displayText());
}
return retStatus;
}
/** Gets proxy details for a system and a url.
@param url : The url to be called
*/
Kernel::ProxyInfo &InternetHelper::getProxy(const std::string &url) {
// set the proxy
if (!m_isProxySet) {
setProxy(ConfigService::Instance().getProxy(url));
}
return m_proxyInfo;
}
/** Clears cached proxy details.
*/
void InternetHelper::clearProxy() { m_isProxySet = false; }
/** sets the proxy details.
@param proxy the proxy information to use
*/
void InternetHelper::setProxy(const Kernel::ProxyInfo &proxy) {
m_proxyInfo = proxy;
m_isProxySet = true;
}
/** Process any headers from the response stream
Basic implementation does nothing.
*/
void InternetHelper::processResponseHeaders(const Poco::Net::HTTPResponse & /*unused*/) {}
/** Process any HTTP errors states.
@param res : The http response
@param rs : The iutput stream from the response
@param url : The url originally called
@exception Mantid::Kernel::Exception::InternetError : Coded for the failure
state.
*/
int InternetHelper::processErrorStates(const Poco::Net::HTTPResponse &res, std::istream &rs, const std::string &url) {
int retStatus = res.getStatus();
g_log.debug() << "Answer from web: " << res.getStatus() << " " << res.getReason() << '\n';
// get github api rate limit information if available;
int rateLimitRemaining;
DateAndTime rateLimitReset;
try {
rateLimitRemaining = boost::lexical_cast<int>(res.get("X-RateLimit-Remaining", "-1"));
rateLimitReset.set_from_time_t(boost::lexical_cast<int>(res.get("X-RateLimit-Reset", "0")));
} catch (boost::bad_lexical_cast const &) {
rateLimitRemaining = -1;
}
if (retStatus == HTTP_OK) {
throw Exception::InternetError("Response was ok, processing should never "
"have entered processErrorStates",
retStatus);
} else if (retStatus == HTTP_FOUND) {
throw Exception::InternetError("Response was HTTP_FOUND, processing should "
"never have entered processErrorStates",
retStatus);
} else if (retStatus == HTTP_MOVED_PERMANENTLY) {
throw Exception::InternetError("Response was HTTP_MOVED_PERMANENTLY, "
"processing should never have entered "
"processErrorStates",
retStatus);
} else if (retStatus == HTTP_NOT_MODIFIED) {
throw Exception::InternetError("Not modified since provided date" + rateLimitReset.toSimpleString(), retStatus);
} else if ((retStatus == HTTP_FORBIDDEN) && (rateLimitRemaining == 0)) {
throw Exception::InternetError("The Github API rate limit has been reached, try again after " +
rateLimitReset.toSimpleString() + " GMT",
retStatus);
} else {
std::stringstream info;
std::stringstream ss;
Poco::StreamCopier::copyStream(rs, ss);
if (retStatus == HTTP_NOT_FOUND)
info << "Failed to download " << url << " with the link "
<< "<a href=\"" << url << "\">.\n"
<< "Hint. Check that link is correct</a>";
else {
// show the error
info << res.getReason();
info << ss.str();
g_log.debug() << ss.str();
}
throw Exception::InternetError(info.str() + ss.str(), retStatus);
}
}
/** Download a url and fetch it inside the local path given.
@param urlFile: Define a valid URL for the file to be downloaded. Eventually, it
may give
any valid https path. For example:
url_file = "http://www.google.com"
url_file = "https://mantidweb/repository/README.md"
The result is to connect to the http server, and request the path given.
The answer, will be inserted at the local_file_path.
@param localFilePath : Provide the destination of the file downloaded at the
url_file.
@exception Mantid::Kernel::Exception::InternetError : For any unexpected
behaviour.
*/
int InternetHelper::downloadFile(const std::string &urlFile, const std::string &localFilePath) {
int retStatus = 0;
g_log.debug() << "DownloadFile from \"" << urlFile << "\" to file: \"" << localFilePath << "\"\n";
Poco::TemporaryFile tempFile;
Poco::FileStream tempFileStream(tempFile.path());
retStatus = sendRequest(urlFile, tempFileStream);
tempFileStream.close();
// if there have been no errors move it to the final location, and turn off
// automatic deletion.
// clear the way if the target file path is already in use
Poco::File file(localFilePath);
if (file.exists()) {
file.remove();
}
tempFile.moveTo(localFilePath);
tempFile.keep();
return retStatus;
}
/** Sets the timeout in seconds
* @param seconds The value in seconds for the timeout
**/
void InternetHelper::setTimeout(int seconds) {
m_timeout = seconds;
m_isTimeoutSet = true;
}
/// Checks the HTTP status to decide if this is a relocation
/// @param response the HTTP status
/// @returns true if the return code is considered a relocation
bool InternetHelper::isRelocated(const int response) {
return ((response == HTTP_FOUND) || (response == HTTP_MOVED_PERMANENTLY) || (response == HTTP_TEMPORARY_REDIRECT) ||
(response == HTTP_SEE_OTHER));
}
/// Throw an exception occurs when the computer
/// is not connected to the internet
/// @param url The url that was use
/// @param ex The exception generated by Poco
void InternetHelper::throwNotConnected(const std::string &url, const HostNotFoundException &ex) {
std::stringstream info;
info << "Failed to access " << url << " because there is no connection to the host " << ex.message()
<< ".\nHint: Check your connection following this link: <a href=\"" << url << "\">" << url << "</a> ";
throw Exception::InternetError(info.str() + ex.displayText());
}
/** Gets the timeout in seconds
* @returns The value in seconds for the timeout
**/
int InternetHelper::getTimeout() {
if (!m_isTimeoutSet) {
auto m_timeout = ConfigService::Instance().getValue<int>("network.default.timeout");
if (!m_timeout.is_initialized()) {
m_timeout = 30; // the default value if the key is not found
}
}
return m_timeout;
}
/** Sets the Method
* @param method A string of GET or POST, anything other than POST is considered
*GET
**/
void InternetHelper::setMethod(const std::string &method) {
if (method == "POST") {
m_method = method;
} else {
m_method = "GET";
}
}
/** Gets the method
* @returns either "GET" or "POST"
**/
const std::string &InternetHelper::getMethod() { return m_method; }
/** Sets the Content Type
* @param contentType A string of the content type
**/
void InternetHelper::setContentType(const std::string &contentType) { m_contentType = contentType; }
/** Gets the Content Type
* @returns A string of the content type
**/
const std::string &InternetHelper::getContentType() { return m_contentType; }
/** Sets the content length
* @param length The content length in bytes
**/
void InternetHelper::setContentLength(std::streamsize length) { m_contentLength = length; }
/** Gets the content length
* @returns The content length in bytes
**/
std::streamsize InternetHelper::getContentLength() { return m_contentLength; }
/** Sets the body & content length for future requests, this will also
* set the method to POST is the body is not empty
* and GET if it is.
* @param body A string of the body
**/
void InternetHelper::setBody(const std::string &body) {
m_body = body;
if (m_body.empty()) {
m_method = "GET";
} else {
m_method = "POST";
}
setContentLength(m_body.size());
}
/** Sets the body & content length for future requests, this will also
* set the method to POST is the body is not empty
* and GET if it is.
* @param body A stringstream of the body
**/
void InternetHelper::setBody(const std::ostringstream &body) { setBody(body.str()); }
/** Sets the body & content length for future requests, this will also
* set the method to POST is the body is not empty
* and GET if it is.
* @param form A HTMLform
**/
void InternetHelper::setBody(Poco::Net::HTMLForm &form) {
setMethod("POST");
if (m_request == nullptr) {
Poco::URI uri("http://www.mantidproject.org");
createRequest(uri);
}
form.prepareSubmit(*m_request);
setContentType(m_request->getContentType());
std::ostringstream ss;
form.write(ss);
m_body = ss.str();
setContentLength(m_body.size());
}
/** Gets the body set for future requests
* @returns A string of the content type
**/
const std::string &InternetHelper::getBody() { return m_body; }
/** Gets the body set for future requests
* @returns A string of the content type
**/
int InternetHelper::getResponseStatus() { return m_response->getStatus(); }
/** Gets the body set for future requests
* @returns A string of the content type
**/
const std::string &InternetHelper::getResponseReason() { return m_response->getReason(); }
/** Adds a header
* @param key The key to refer to the value
* @param value The value in seconds for the timeout
**/
void InternetHelper::addHeader(const std::string &key, const std::string &value) { m_headers.emplace(key, value); }
/** Removes a header
* @param key The key to refer to the value
**/
void InternetHelper::removeHeader(const std::string &key) { m_headers.erase(key); }
/** Gets the value of a header
* @param key The key to refer to the value
* @returns the value as a string
**/
const std::string &InternetHelper::getHeader(const std::string &key) { return m_headers[key]; }
/** Clears all headers
**/
void InternetHelper::clearHeaders() { m_headers.clear(); }
/** Returns a reference to the headers map
**/
std::map<std::string, std::string> &InternetHelper::headers() { return m_headers; }
/** Resets properties to defaults (except the proxy)
**/
void InternetHelper::reset() {
m_headers.clear();
m_timeout = 30;
m_isTimeoutSet = false;
m_body = "";
m_method = HTTPRequest::HTTP_GET;
m_contentType = "application/json";
m_request = nullptr;
}
} // namespace Kernel
} // namespace Mantid