Skip to content

Commit

Permalink
NETWORKING: Enter Session
Browse files Browse the repository at this point in the history
Session allows to reuse SessionRequests to the same host by making them
keeping alive connection. Turns out, though, that libcurl already does
that for us, and we didn't gain any speedup we thought we'd get.

Usage:
```
Networking::Session s;
Networking::SessionRequest *request = s.get(url);
request->startAndWait();
warning("HTTP GET: %s", request->text());
s.close();
```

You can still use SessionRequest without Session (but you can't put them
on stack!):
```
Networking::SessionRequest *request = new
Networking::SessionRequest(url);
request->startAndWait();
warning("HTTP GET: %s", request->text());
request->close();
```
  • Loading branch information
Tkachov committed Nov 4, 2019
1 parent 1f254ee commit e8b4645
Show file tree
Hide file tree
Showing 8 changed files with 302 additions and 80 deletions.
18 changes: 14 additions & 4 deletions backends/networking/curl/curlrequest.cpp
Expand Up @@ -32,7 +32,7 @@ namespace Networking {

CurlRequest::CurlRequest(DataCallback cb, ErrorCallback ecb, Common::String url):
Request(cb, ecb), _url(url), _stream(nullptr), _headersList(nullptr), _bytesBuffer(nullptr),
_bytesBufferSize(0), _uploading(false), _usingPatch(false) {}
_bytesBufferSize(0), _uploading(false), _usingPatch(false), _keepAlive(false), _keepAliveIdle(120), _keepAliveInterval(60) {}

CurlRequest::~CurlRequest() {
delete _stream;
Expand All @@ -41,10 +41,10 @@ CurlRequest::~CurlRequest() {

NetworkReadStream *CurlRequest::makeStream() {
if (_bytesBuffer)
return new NetworkReadStream(_url.c_str(), _headersList, _bytesBuffer, _bytesBufferSize, _uploading, _usingPatch, true);
return new NetworkReadStream(_url.c_str(), _headersList, _bytesBuffer, _bytesBufferSize, _uploading, _usingPatch, true, _keepAlive, _keepAliveIdle, _keepAliveInterval);
if (!_formFields.empty() || !_formFiles.empty())
return new NetworkReadStream(_url.c_str(), _headersList, _formFields, _formFiles);
return new NetworkReadStream(_url.c_str(), _headersList, _postFields, _uploading, _usingPatch);
return new NetworkReadStream(_url.c_str(), _headersList, _formFields, _formFiles, _keepAlive, _keepAliveIdle, _keepAliveInterval);
return new NetworkReadStream(_url.c_str(), _headersList, _postFields, _uploading, _usingPatch, _keepAlive, _keepAliveIdle, _keepAliveInterval);
}

void CurlRequest::handle() {
Expand Down Expand Up @@ -137,6 +137,16 @@ void CurlRequest::usePut() { _uploading = true; }

void CurlRequest::usePatch() { _usingPatch = true; }

void CurlRequest::connectionKeepAlive(long idle, long interval) {
_keepAlive = true;
_keepAliveIdle = idle;
_keepAliveInterval = interval;
}

void CurlRequest::connectionClose() {
_keepAlive = false;
}

NetworkReadStreamResponse CurlRequest::execute() {
if (!_stream) {
_stream = makeStream();
Expand Down
6 changes: 6 additions & 0 deletions backends/networking/curl/curlrequest.h
Expand Up @@ -50,6 +50,8 @@ class CurlRequest: public Request {
uint32 _bytesBufferSize;
bool _uploading; //using PUT method
bool _usingPatch; //using PATCH method
bool _keepAlive;
long _keepAliveIdle, _keepAliveInterval;

virtual NetworkReadStream *makeStream();

Expand Down Expand Up @@ -85,6 +87,10 @@ class CurlRequest: public Request {
/** Remembers to use PATCH method when it would create NetworkReadStream. */
virtual void usePatch();

/** Remembers to use Connection: keep-alive or close. */
virtual void connectionKeepAlive(long idle = 120, long interval = 60);
virtual void connectionClose();

/**
* Starts this Request with ConnMan.
* @return its NetworkReadStream in NetworkReadStreamResponse.
Expand Down
123 changes: 71 additions & 52 deletions backends/networking/curl/networkreadstream.cpp
Expand Up @@ -63,13 +63,18 @@ int NetworkReadStream::curlProgressCallbackOlder(void *p, double dltotal, double
return curlProgressCallback(p, (curl_off_t)dltotal, (curl_off_t)dlnow, (curl_off_t)ultotal, (curl_off_t)ulnow);
}

void NetworkReadStream::init(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading, bool usingPatch, bool post) {
void NetworkReadStream::resetStream() {
_eos = _requestComplete = false;
_errorBuffer = (char *)calloc(CURL_ERROR_SIZE, 1);
if (!_errorBuffer)
_errorBuffer = (char *)calloc(CURL_ERROR_SIZE, 1);
_sendingContentsBuffer = nullptr;
_sendingContentsSize = _sendingContentsPos = 0;
_progressDownloaded = _progressTotal = 0;
_bufferCopy = nullptr;
}

void NetworkReadStream::initCurl(const char *url, curl_slist *headersList) {
resetStream();

_easy = curl_easy_init();
curl_easy_setopt(_easy, CURLOPT_WRITEFUNCTION, curlDataCallback);
Expand Down Expand Up @@ -102,6 +107,30 @@ void NetworkReadStream::init(const char *url, curl_slist *headersList, const byt
curl_easy_setopt(_easy, CURLOPT_XFERINFOFUNCTION, curlProgressCallback);
curl_easy_setopt(_easy, CURLOPT_XFERINFODATA, this);
#endif

if (_keepAlive) {
curl_easy_setopt(_easy, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(_easy, CURLOPT_TCP_KEEPIDLE, _keepAliveIdle);
curl_easy_setopt(_easy, CURLOPT_TCP_KEEPINTVL, _keepAliveInterval);
}
}

bool NetworkReadStream::reuseCurl(const char *url, curl_slist *headersList) {
if (!_keepAlive) {
warning("NetworkReadStream: Can't reuse curl handle (was not setup as keep-alive)");
return false;
}

resetStream();

curl_easy_setopt(_easy, CURLOPT_URL, url);
curl_easy_setopt(_easy, CURLOPT_HTTPHEADER, headersList);
curl_easy_setopt(_easy, CURLOPT_USERAGENT, gScummVMFullVersion); // in case headersList rewrites it

return true;
}

void NetworkReadStream::setupBufferContents(const byte *buffer, uint32 bufferSize, bool uploading, bool usingPatch, bool post) {
if (uploading) {
curl_easy_setopt(_easy, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(_easy, CURLOPT_READDATA, this);
Expand All @@ -126,46 +155,7 @@ void NetworkReadStream::init(const char *url, curl_slist *headersList, const byt
ConnMan.registerEasyHandle(_easy);
}

void NetworkReadStream::init(const char *url, curl_slist *headersList, Common::HashMap<Common::String, Common::String> formFields, Common::HashMap<Common::String, Common::String> formFiles) {
_eos = _requestComplete = false;
_errorBuffer = (char *)calloc(CURL_ERROR_SIZE, 1);
_sendingContentsBuffer = nullptr;
_sendingContentsSize = _sendingContentsPos = 0;
_progressDownloaded = _progressTotal = 0;
_bufferCopy = nullptr;

_easy = curl_easy_init();
curl_easy_setopt(_easy, CURLOPT_WRITEFUNCTION, curlDataCallback);
curl_easy_setopt(_easy, CURLOPT_WRITEDATA, this); //so callback can call us
curl_easy_setopt(_easy, CURLOPT_PRIVATE, this); //so ConnectionManager can call us when request is complete
curl_easy_setopt(_easy, CURLOPT_HEADER, 0L);
curl_easy_setopt(_easy, CURLOPT_HEADERDATA, this);
curl_easy_setopt(_easy, CURLOPT_HEADERFUNCTION, curlHeadersCallback);
curl_easy_setopt(_easy, CURLOPT_URL, url);
curl_easy_setopt(_easy, CURLOPT_ERRORBUFFER, _errorBuffer);
curl_easy_setopt(_easy, CURLOPT_VERBOSE, 0L);
curl_easy_setopt(_easy, CURLOPT_FOLLOWLOCATION, 1L); //probably it's OK to have it always on
curl_easy_setopt(_easy, CURLOPT_HTTPHEADER, headersList);
curl_easy_setopt(_easy, CURLOPT_USERAGENT, gScummVMFullVersion);
curl_easy_setopt(_easy, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(_easy, CURLOPT_PROGRESSFUNCTION, curlProgressCallbackOlder);
curl_easy_setopt(_easy, CURLOPT_PROGRESSDATA, this);
#if defined NINTENDO_SWITCH || defined ANDROID_PLAIN_PORT
curl_easy_setopt(_easy, CURLOPT_SSL_VERIFYPEER, 0);
#endif

const char *caCertPath = ConnMan.getCaCertPath();
if (caCertPath) {
curl_easy_setopt(_easy, CURLOPT_CAINFO, caCertPath);
}

#if LIBCURL_VERSION_NUM >= 0x072000
// CURLOPT_XFERINFOFUNCTION introduced in libcurl 7.32.0
// CURLOPT_PROGRESSFUNCTION is used as a backup plan in case older version is used
curl_easy_setopt(_easy, CURLOPT_XFERINFOFUNCTION, curlProgressCallback);
curl_easy_setopt(_easy, CURLOPT_XFERINFODATA, this);
#endif

void NetworkReadStream::setupFormMultipart(Common::HashMap<Common::String, Common::String> formFields, Common::HashMap<Common::String, Common::String> formFiles) {
// set POST multipart upload form fields/files
struct curl_httppost *formpost = nullptr;
struct curl_httppost *lastptr = nullptr;
Expand Down Expand Up @@ -197,23 +187,52 @@ void NetworkReadStream::init(const char *url, curl_slist *headersList, Common::H
}

curl_easy_setopt(_easy, CURLOPT_HTTPPOST, formpost);

ConnMan.registerEasyHandle(_easy);
}

NetworkReadStream::NetworkReadStream(const char *url, curl_slist *headersList, Common::String postFields, bool uploading, bool usingPatch) :
_backingStream(DisposeAfterUse::YES) {
init(url, headersList, (const byte *)postFields.c_str(), postFields.size(), uploading, usingPatch, false);
NetworkReadStream::NetworkReadStream(const char *url, curl_slist *headersList, Common::String postFields, bool uploading, bool usingPatch, bool keepAlive, long keepAliveIdle, long keepAliveInterval):
_backingStream(DisposeAfterUse::YES), _keepAlive(keepAlive), _keepAliveIdle(keepAliveIdle), _keepAliveInterval(keepAliveInterval), _errorBuffer(nullptr) {
initCurl(url, headersList);
setupBufferContents((const byte *)postFields.c_str(), postFields.size(), uploading, usingPatch, false);
}

NetworkReadStream::NetworkReadStream(const char *url, curl_slist *headersList, Common::HashMap<Common::String, Common::String> formFields, Common::HashMap<Common::String, Common::String> formFiles, bool keepAlive, long keepAliveIdle, long keepAliveInterval):
_backingStream(DisposeAfterUse::YES), _keepAlive(keepAlive), _keepAliveIdle(keepAliveIdle), _keepAliveInterval(keepAliveInterval), _errorBuffer(nullptr) {
initCurl(url, headersList);
setupFormMultipart(formFields, formFiles);
}

NetworkReadStream::NetworkReadStream(const char *url, curl_slist *headersList, Common::HashMap<Common::String, Common::String> formFields, Common::HashMap<Common::String, Common::String> formFiles) :
_backingStream(DisposeAfterUse::YES) {
init(url, headersList, formFields, formFiles);
NetworkReadStream::NetworkReadStream(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading, bool usingPatch, bool post, bool keepAlive, long keepAliveIdle, long keepAliveInterval):
_backingStream(DisposeAfterUse::YES), _keepAlive(keepAlive), _keepAliveIdle(keepAliveIdle), _keepAliveInterval(keepAliveInterval), _errorBuffer(nullptr) {
initCurl(url, headersList);
setupBufferContents(buffer, bufferSize, uploading, usingPatch, post);
}

NetworkReadStream::NetworkReadStream(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading, bool usingPatch, bool post) :
_backingStream(DisposeAfterUse::YES) {
init(url, headersList, buffer, bufferSize, uploading, usingPatch, post);
bool NetworkReadStream::reuse(const char *url, curl_slist *headersList, Common::String postFields, bool uploading, bool usingPatch) {
if (!reuseCurl(url, headersList))
return false;

_backingStream = Common::MemoryReadWriteStream(DisposeAfterUse::YES);
setupBufferContents((const byte *)postFields.c_str(), postFields.size(), uploading, usingPatch, false);
return true;
}

bool NetworkReadStream::reuse(const char *url, curl_slist *headersList, Common::HashMap<Common::String, Common::String> formFields, Common::HashMap<Common::String, Common::String> formFiles) {
if (!reuseCurl(url, headersList))
return false;

_backingStream = Common::MemoryReadWriteStream(DisposeAfterUse::YES);
setupFormMultipart(formFields, formFiles);
return true;
}

bool NetworkReadStream::reuse(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading, bool usingPatch, bool post) {
if (!reuseCurl(url, headersList))
return false;

_backingStream = Common::MemoryReadWriteStream(DisposeAfterUse::YES);
setupBufferContents(buffer, bufferSize, uploading, usingPatch, post);
return true;
}

NetworkReadStream::~NetworkReadStream() {
Expand Down
33 changes: 26 additions & 7 deletions backends/networking/curl/networkreadstream.h
Expand Up @@ -37,16 +37,22 @@ namespace Networking {
class NetworkReadStream: public Common::ReadStream {
CURL *_easy;
Common::MemoryReadWriteStream _backingStream;
bool _keepAlive;
long _keepAliveIdle, _keepAliveInterval;
bool _eos, _requestComplete;
char *_errorBuffer;
const byte *_sendingContentsBuffer;
uint32 _sendingContentsSize;
uint32 _sendingContentsPos;
byte* _bufferCopy; // To use with old curl version where CURLOPT_COPYPOSTFIELDS is not available
byte *_bufferCopy; // To use with old curl version where CURLOPT_COPYPOSTFIELDS is not available
Common::String _responseHeaders;
uint64 _progressDownloaded, _progressTotal;
void init(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading, bool usingPatch, bool post);
void init(const char *url, curl_slist *headersList, Common::HashMap<Common::String, Common::String> formFields, Common::HashMap<Common::String, Common::String> formFiles);

void resetStream();
void initCurl(const char *url, curl_slist *headersList);
bool reuseCurl(const char *url, curl_slist *headersList);
void setupBufferContents(const byte *buffer, uint32 bufferSize, bool uploading, bool usingPatch, bool post);
void setupFormMultipart(Common::HashMap<Common::String, Common::String> formFields, Common::HashMap<Common::String, Common::String> formFiles);

/**
* Fills the passed buffer with _sendingContentsBuffer contents.
Expand All @@ -70,16 +76,27 @@ class NetworkReadStream: public Common::ReadStream {
static int curlProgressCallbackOlder(void *p, double dltotal, double dlnow, double ultotal, double ulnow);
public:
/** Send <postFields>, using POST by default. */
NetworkReadStream(const char *url, curl_slist *headersList, Common::String postFields, bool uploading = false, bool usingPatch = false);
NetworkReadStream(const char *url, curl_slist *headersList, Common::String postFields, bool uploading = false, bool usingPatch = false, bool keepAlive = false, long keepAliveIdle = 120, long keepAliveInterval = 60);
/** Send <formFields>, <formFiles>, using POST multipart/form. */
NetworkReadStream(
const char *url, curl_slist *headersList,
Common::HashMap<Common::String, Common::String> formFields,
Common::HashMap<Common::String, Common::String> formFiles);
/** Send <buffer, using POST by default. */
NetworkReadStream(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading = false, bool usingPatch = false, bool post = true);
Common::HashMap<Common::String, Common::String> formFiles,
bool keepAlive = false, long keepAliveIdle = 120, long keepAliveInterval = 60);
/** Send <buffer>, using POST by default. */
NetworkReadStream(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading = false, bool usingPatch = false, bool post = true, bool keepAlive = false, long keepAliveIdle = 120, long keepAliveInterval = 60);
virtual ~NetworkReadStream();

/** Send <postFields>, using POST by default. */
bool reuse(const char *url, curl_slist *headersList, Common::String postFields, bool uploading = false, bool usingPatch = false);
/** Send <formFields>, <formFiles>, using POST multipart/form. */
bool reuse(
const char *url, curl_slist *headersList,
Common::HashMap<Common::String, Common::String> formFields,
Common::HashMap<Common::String, Common::String> formFiles);
/** Send <buffer>, using POST by default. */
bool reuse(const char *url, curl_slist *headersList, const byte *buffer, uint32 bufferSize, bool uploading = false, bool usingPatch = false, bool post = true);

/**
* Returns true if a read failed because the stream end has been reached.
* This flag is cleared by clearErr().
Expand Down Expand Up @@ -150,6 +167,8 @@ class NetworkReadStream: public Common::ReadStream {

/** Used in curl progress callback to pass current downloaded/total values. */
void setProgress(uint64 downloaded, uint64 total);

bool keepAlive() const { return _keepAlive; }
};

} // End of namespace Networking
Expand Down
77 changes: 77 additions & 0 deletions backends/networking/curl/session.cpp
@@ -0,0 +1,77 @@
/* ScummVM - Graphic Adventure Engine
*
* ScummVM is the legal property of its developers, whose names
* are too numerous to list here. Please refer to the COPYRIGHT
* file distributed with this source distribution.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/

#define FORBIDDEN_SYMBOL_ALLOW_ALL

#include "backends/networking/curl/session.h"

namespace Networking {

Session::Session(Common::String prefix):
_prefix(prefix), _request(nullptr) {}

Session::~Session() {
close();
}

SessionRequest *Session::get(Common::String url, DataCallback cb, ErrorCallback ecb) {
// check url prefix
if (!_prefix.empty()) {
if (url.contains("://")) {
if (url.size() < _prefix.size() || url.find(_prefix) != 0) {
warning("Session: given URL does not match the prefix!\n\t%s\n\t%s", url.c_str(), _prefix.c_str());
return nullptr;
}
} else {
// if no schema given, just append <url> to <_prefix>
Common::String newUrl = _prefix;
if (newUrl.lastChar() != '/' && (url.size() > 0 && url.firstChar() != '/'))
newUrl += "/";
newUrl += url;
url = newUrl;
}
}

// check if request has finished (ready to be replaced)
if (_request) {
if (!_request->complete()) {
warning("Session: can't reuse Request that is being processed");
return nullptr;
}
}

if (!_request) {
_request = new Networking::SessionRequest(url, cb, ecb); // automatically added to ConnMan
_request->connectionKeepAlive();
} else {
_request->reuse(url, cb, ecb);
}

return _request;
}

void Session::close() {
if (_request)
_request->close();
}

} // End of namespace Networking

0 comments on commit e8b4645

Please sign in to comment.