Test using DownloadManager from my mod instead of RequestBroker

The crash when opening saves on mac is definitely something to do with the threading, and i'd rather just test this first to see if it fixes it

Copied from my mod almost entirely as-is, with the changes to HTTP.cpp included, also added locks into Download.cpp even though it is probably overkill

src/client/Client.h

@@ -6,7 +6,7 @@
 #include <list>
 
 #include "Config.h"
-#include "Singleton.h"
+#include "common/Singleton.h"
 
 #include "User.h"
 #include "UserInfo.h"

src/client/Download.cpp

@@ -0,0 +1,151 @@
+#include <stdlib.h>
+#include "Download.h"
+#include "DownloadManager.h"
+#include "http.h"
+
+Download::Download(std::string uri_, bool keepAlive):
+	http(NULL),
+	keepAlive(keepAlive),
+	downloadData(NULL),
+	downloadSize(0),
+	downloadStatus(0),
+	downloadFinished(false),
+	downloadCanceled(false),
+	downloadStarted(false),
+	postData(""),
+	postDataBoundary(""),
+	userID(""),
+	userSession("")
+{
+	uri = std::string(uri_);
+	DownloadManager::Ref().AddDownload(this);
+}
+
+// called by download thread itself if download was canceled
+Download::~Download()
+{
+	if (http && (keepAlive || downloadCanceled))
+		http_async_req_close(http);
+	if (downloadData)
+		free(downloadData);
+}
+
+// add post data to a request
+void Download::AddPostData(std::map<std::string, std::string> data)
+{
+	postDataBoundary = FindBoundary(data, "");
+	postData = GetMultipartMessage(data, postDataBoundary);
+}
+void Download::AddPostData(std::pair<std::string, std::string> data)
+{
+	std::map<std::string, std::string> postData;
+	postData.insert(data);
+	AddPostData(postData);
+}
+
+// add userID and sessionID headers to the download. Must be done after download starts for some reason
+void Download::AuthHeaders(std::string ID, std::string session)
+{
+	if (ID != "0")
+		userID = ID;
+	userSession = session;
+}
+
+// start the download thread
+void Download::Start()
+{
+	if (CheckStarted() || CheckDone())
+		return;
+	http = http_async_req_start(http, uri.c_str(), postData.c_str(), postData.length(), keepAlive ? 1 : 0);
+	// add the necessary headers
+	if (userID.length() || userSession.length())
+		http_auth_headers(http, userID.c_str(), NULL, userSession.c_str());
+	if (postDataBoundary.length())
+		http_add_multipart_header(http, postDataBoundary);
+	DownloadManager::Ref().Lock();
+	downloadStarted = true;
+	DownloadManager::Ref().Unlock();
+}
+
+// for persistent connections (keepAlive = true), reuse the open connection to make another request
+bool Download::Reuse(std::string newuri)
+{
+	if (!keepAlive || !CheckDone() || CheckCanceled())
+	{
+		return false;
+	}
+	uri = std::string(newuri);
+	DownloadManager::Ref().Lock();
+	downloadFinished = false;
+	DownloadManager::Ref().Unlock();
+	Start();
+	DownloadManager::Ref().EnsureRunning();
+	return true;
+}
+
+// finish the download (if called before the download is done, this will block)
+char* Download::Finish(int *length, int *status)
+{
+	if (CheckCanceled())
+		return NULL; // shouldn't happen but just in case
+	while (!CheckDone()); // block
+	DownloadManager::Ref().Lock();
+	downloadStarted = false;
+	if (length)
+		*length = downloadSize;
+	if (status)
+		*status = downloadStatus;
+	char *ret = downloadData;
+	downloadData = NULL;
+	if (!keepAlive)
+		downloadCanceled = true;
+	DownloadManager::Ref().Unlock();
+	return ret;
+}
+
+// returns the download size and progress (if the download has the correct length headers)
+void Download::CheckProgress(int *total, int *done)
+{
+	DownloadManager::Ref().Lock();
+	if (!downloadFinished && http)
+		http_async_get_length(http, total, done);
+	else
+		*total = *done = 0;
+	DownloadManager::Ref().Unlock();
+}
+
+// returns true if the download has finished
+bool Download::CheckDone()
+{
+	DownloadManager::Ref().Lock();
+	bool ret = downloadFinished;
+	DownloadManager::Ref().Unlock();
+	return ret;
+}
+
+// returns true if the download was canceled
+bool Download::CheckCanceled()
+{
+	DownloadManager::Ref().Lock();
+	bool ret = downloadCanceled;
+	DownloadManager::Ref().Unlock();
+	return ret;
+}
+
+// returns true if the download is running
+bool Download::CheckStarted()
+{
+	DownloadManager::Ref().Lock();
+	bool ret = downloadStarted;
+	DownloadManager::Ref().Unlock();
+	return ret;
+
+}
+
+// cancels the download, the download thread will delete the Download* when it finishes (do not use Download in any way after canceling)
+void Download::Cancel()
+{
+	DownloadManager::Ref().Lock();
+	downloadCanceled = true;
+	DownloadManager::Ref().Unlock();
+}

src/client/Download.h

@@ -0,0 +1,47 @@
+#ifndef DOWNLOAD_H
+#define DOWNLOAD_H
+#include <map>
+#include <string>
+
+class DownloadManager;
+class Download
+{
+	std::string uri;
+	void *http;
+	bool keepAlive;
+
+	char *downloadData;
+	int downloadSize;
+	int downloadStatus;
+
+	std::string postData;
+	std::string postDataBoundary;
+
+	std::string userID;
+	std::string userSession;
+
+	volatile bool downloadFinished;
+	volatile bool downloadCanceled;
+	volatile bool downloadStarted;
+
+public:
+	Download(std::string uri, bool keepAlive = false);
+	~Download();
+
+	void AddPostData(std::map<std::string, std::string> data);
+	void AddPostData(std::pair<std::string, std::string> data);
+	void AuthHeaders(std::string ID, std::string session);
+	void Start();
+	bool Reuse(std::string newuri);
+	char* Finish(int *length, int *status);
+	void Cancel();
+
+	void CheckProgress(int *total, int *done);
+	bool CheckDone();
+	bool CheckCanceled();
+	bool CheckStarted();
+
+	friend class DownloadManager;
+};
+
+#endif

src/client/DownloadManager.cpp

@@ -0,0 +1,145 @@
+#include "DownloadManager.h"
+#include "Download.h"
+#include "http.h"
+#include "Config.h"
+#include "Platform.h"
+
+DownloadManager::DownloadManager():
+	threadStarted(false),
+	lastUsed(time(NULL)),
+	managerRunning(false),
+	managerShutdown(false),
+	downloads(NULL),
+	downloadsAddQueue(NULL)
+{
+	pthread_mutex_init(&downloadLock, NULL);
+	pthread_mutex_init(&downloadAddLock, NULL);
+}
+
+DownloadManager::~DownloadManager()
+{
+
+}
+
+void DownloadManager::Shutdown()
+{
+	pthread_mutex_lock(&downloadLock);
+	pthread_mutex_lock(&downloadAddLock);
+	for (std::vector<Download*>::iterator iter = downloads.begin(); iter != downloads.end(); ++iter)
+	{
+		Download *download = (*iter);
+		if (download->http)
+			http_force_close(download->http);
+		download->downloadCanceled = true;
+		delete download;
+	}
+	downloads.clear();
+	downloadsAddQueue.clear();
+	managerShutdown = true;
+	pthread_mutex_unlock(&downloadAddLock);
+	pthread_mutex_unlock(&downloadLock);
+	pthread_join(downloadThread, NULL);
+}
+
+//helper function for download
+TH_ENTRY_POINT void* DownloadManagerHelper(void* obj)
+{
+	DownloadManager *temp = (DownloadManager*)obj;
+	temp->Update();
+	return NULL;
+}
+
+void DownloadManager::Start()
+{
+	managerRunning = true;
+	lastUsed = time(NULL);
+	pthread_create(&downloadThread, NULL, &DownloadManagerHelper, this);
+}
+
+void DownloadManager::Update()
+{
+	unsigned int numActiveDownloads;
+	while (!managerShutdown)
+	{
+		pthread_mutex_lock(&downloadAddLock);
+		if (downloadsAddQueue.size())
+		{
+			for (size_t i = 0; i < downloadsAddQueue.size(); i++)
+			{
+				downloads.push_back(downloadsAddQueue[i]);
+			}
+			downloadsAddQueue.clear();
+		}
+		pthread_mutex_unlock(&downloadAddLock);
+		if (downloads.size())
+		{
+			numActiveDownloads = 0;
+			pthread_mutex_lock(&downloadLock);
+			for (size_t i = 0; i < downloads.size(); i++)
+			{
+				Download *download = downloads[i];
+				if (download->downloadCanceled)
+				{
+					if (download->http && download->downloadStarted)
+						http_force_close(download->http);
+					delete download;
+					downloads.erase(downloads.begin()+i);
+					i--;
+				}
+				else if (download->downloadStarted && !download->downloadFinished)
+				{
+					if (http_async_req_status(download->http) != 0)
+					{
+						download->downloadData = http_async_req_stop(download->http, &download->downloadStatus, &download->downloadSize);
+						download->downloadFinished = true;
+						if (!download->keepAlive)
+							download->http = NULL;
+					}
+					lastUsed = time(NULL);
+					numActiveDownloads++;
+				}
+			}
+			pthread_mutex_unlock(&downloadLock);
+		}
+		if (time(NULL) > lastUsed+http_timeout*2 && !numActiveDownloads)
+		{
+			pthread_mutex_lock(&downloadLock);
+			managerRunning = false;
+			pthread_mutex_unlock(&downloadLock);
+			return;
+		}
+		Platform::Millisleep(1);
+	}
+}
+
+void DownloadManager::EnsureRunning()
+{
+	pthread_mutex_lock(&downloadLock);
+	if (!managerRunning)
+	{
+		if (threadStarted)
+			pthread_join(downloadThread, NULL);
+		else
+			threadStarted = true;
+		Start();
+	}
+	pthread_mutex_unlock(&downloadLock);
+}
+
+void DownloadManager::AddDownload(Download *download)
+{
+	pthread_mutex_lock(&downloadAddLock);
+	downloadsAddQueue.push_back(download);
+	pthread_mutex_unlock(&downloadAddLock);
+	EnsureRunning();
+}
+
+void DownloadManager::Lock()
+{
+	pthread_mutex_lock(&downloadAddLock);
+}
+
+void DownloadManager::Unlock()
+{
+	pthread_mutex_unlock(&downloadAddLock);
+}

src/client/DownloadManager.h

@@ -0,0 +1,39 @@
+#ifndef DOWNLOADMANAGER_H
+#define DOWNLOADMANAGER_H
+#include "common/tpt-thread.h"
+#include <time.h>
+#include <vector>
+#include "common/Singleton.h"
+
+class Download;
+class DownloadManager : public Singleton<DownloadManager>
+{
+private:
+	pthread_t downloadThread;
+	pthread_mutex_t downloadLock;
+	pthread_mutex_t downloadAddLock;
+	bool threadStarted;
+
+	int lastUsed;
+	volatile bool managerRunning;
+	volatile bool managerShutdown;
+	std::vector<Download*> downloads;
+	std::vector<Download*> downloadsAddQueue;
+
+	void Start();
+public:
+	DownloadManager();
+	~DownloadManager();
+
+	void Shutdown();
+	void Update();
+	void EnsureRunning();
+
+	void AddDownload(Download *download);
+	void RemoveDownload(int id);
+
+	void Lock();
+	void Unlock();
+};
+
+#endif // DOWNLOADMANAGER_H