313 changes: 83 additions & 230 deletions src/client.cpp

Large diffs are not rendered by default.

67 changes: 15 additions & 52 deletions src/client.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,32 +31,21 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "clientobject.h"
#include "gamedef.h"
#include "inventorymanager.h"
#include "filecache.h"
#include "localplayer.h"
#include "server.h"
#include "hud.h"
#include "particles.h"
#include "util/pointedthing.h"
#include <algorithm>

struct MeshMakeData;
class MapBlockMesh;
class IGameDef;
class IWritableTextureSource;
class IWritableShaderSource;
class IWritableItemDefManager;
class IWritableNodeDefManager;
//class IWritableCraftDefManager;
class ClientEnvironment;
class ClientMediaDownloader;
struct MapDrawControl;
class MtEventManager;

class ClientNotReadyException : public BaseException
{
public:
ClientNotReadyException(const char *s):
BaseException(s)
{}
};
struct PointedThing;

struct QueuedMeshUpdate
{
Expand Down Expand Up @@ -132,31 +121,12 @@ class MeshUpdateThread : public SimpleThread
IGameDef *m_gamedef;
};

class MediaFetchThread : public SimpleThread
{
public:

MediaFetchThread(IGameDef *gamedef):
m_gamedef(gamedef)
{
}

void * Thread();

std::list<MediaRequest> m_file_requests;
MutexedQueue<std::pair<std::string, std::string> > m_file_data;
std::list<MediaRequest> m_failed;
std::string m_remote_url;
IGameDef *m_gamedef;
};

enum ClientEventType
{
CE_NONE,
CE_PLAYER_DAMAGE,
CE_PLAYER_FORCE_MOVE,
CE_DEATHSCREEN,
CE_TEXTURES_UPDATED,
CE_SHOW_FORMSPEC,
CE_SPAWN_PARTICLE,
CE_ADD_PARTICLESPAWNER,
Expand Down Expand Up @@ -426,19 +396,15 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
std::wstring accessDeniedReason()
{ return m_access_denied_reason; }

float mediaReceiveProgress()
{
if (!m_media_receive_started) return 0;
return 1.0 * m_media_received_count / m_media_count;
}

bool texturesReceived()
{ return m_media_receive_started && m_media_received_count == m_media_count; }
bool itemdefReceived()
{ return m_itemdef_received; }
bool nodedefReceived()
{ return m_nodedef_received; }

bool mediaReceived()
{ return m_media_downloader == NULL; }

float mediaReceiveProgress();

void afterContentReceived(IrrlichtDevice *device, gui::IGUIFont* font);

float getRTT(void);
Expand All @@ -455,12 +421,15 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
virtual bool checkLocalPrivilege(const std::string &priv)
{ return checkPrivilege(priv); }

private:

// The following set of functions is used by ClientMediaDownloader
// Insert a media file appropriately into the appropriate manager
bool loadMedia(const std::string &data, const std::string &filename);
// Send a request for conventional media transfer
void request_media(const std::list<std::string> &file_requests);
// Send a notification that no conventional media transfer is needed
void received_media();

void request_media(const std::list<MediaRequest> &file_requests);
private:

// Virtual methods from con::PeerHandler
void peerAdded(con::Peer *peer);
Expand Down Expand Up @@ -488,7 +457,6 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
MtEventManager *m_event;

MeshUpdateThread m_mesh_update_thread;
std::list<MediaFetchThread*> m_media_fetch_threads;
ClientEnvironment m_env;
con::Connection m_con;
IrrlichtDevice *m_device;
Expand All @@ -514,14 +482,9 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
bool m_access_denied;
std::wstring m_access_denied_reason;
Queue<ClientEvent> m_client_event_queue;
FileCache m_media_cache;
// Mapping from media file name to SHA1 checksum
std::map<std::string, std::string> m_media_name_sha1_map;
bool m_media_receive_started;
u32 m_media_count;
u32 m_media_received_count;
bool m_itemdef_received;
bool m_nodedef_received;
ClientMediaDownloader *m_media_downloader;

// time_of_day speed approximation for old protocol
bool m_time_of_day_set;
Expand Down
656 changes: 656 additions & 0 deletions src/clientmedia.cpp

Large diffs are not rendered by default.

150 changes: 150 additions & 0 deletions src/clientmedia.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/

#ifndef CLIENTMEDIA_HEADER
#define CLIENTMEDIA_HEADER

#include "irrlichttypes.h"
#include "filecache.h"
#include <ostream>
#include <map>
#include <set>
#include <vector>

class Client;
struct HTTPFetchResult;

#define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS'
#define MTHASHSET_FILE_NAME "index.mth"

class ClientMediaDownloader
{
public:
ClientMediaDownloader();
~ClientMediaDownloader();

float getProgress() const {
if (m_uncached_count >= 1)
return 1.0 * m_uncached_received_count /
m_uncached_count;
else
return 0.0;
}

bool isStarted() const {
return m_initial_step_done;
}

// If this returns true, the downloader is done and can be deleted
bool isDone() const {
return m_initial_step_done &&
m_uncached_received_count == m_uncached_count;
}

// Add a file to the list of required file (but don't fetch it yet)
void addFile(std::string name, std::string sha1);

// Add a remote server to the list; ignored if not built with cURL
void addRemoteServer(std::string baseurl);

// Steps the media downloader:
// - May load media into client by calling client->loadMedia()
// - May check media cache for files
// - May add files to media cache
// - May start remote transfers by calling httpfetch_async
// - May check for completion of current remote transfers
// - May start conventional transfers by calling client->request_media()
// - May inform server that all media has been loaded
// by calling client->received_media()
// After step has been called once, don't call addFile/addRemoteServer.
void step(Client *client);

// Must be called for each file received through TOCLIENT_MEDIA
void conventionalTransferDone(
const std::string &name,
const std::string &data,
Client *client);

private:
struct FileStatus {
bool received;
std::string sha1;
s32 current_remote;
std::vector<s32> available_remotes;
};

struct RemoteServerStatus {
std::string baseurl;
s32 active_count;
bool request_by_filename;
};

void initialStep(Client *client);
void remoteHashSetReceived(const HTTPFetchResult &fetchresult);
void remoteMediaReceived(const HTTPFetchResult &fetchresult,
Client *client);
s32 selectRemoteServer(FileStatus *filestatus);
void startRemoteMediaTransfers();
void startConventionalTransfers(Client *client);

bool checkAndLoad(const std::string &name, const std::string &sha1,
const std::string &data, bool is_from_cache,
Client *client);

std::string serializeRequiredHashSet();
static void deSerializeHashSet(const std::string &data,
std::set<std::string> &result);

// Maps filename to file status
std::map<std::string, FileStatus*> m_files;

// Array of remote media servers
std::vector<RemoteServerStatus*> m_remotes;

// Filesystem-based media cache
FileCache m_media_cache;

// Has an attempt been made to load media files from the file cache?
// Have hash sets been requested from remote servers?
bool m_initial_step_done;

// Total number of media files to load
s32 m_uncached_count;

// Number of media files that have been received
s32 m_uncached_received_count;

// Status of remote transfers
unsigned long m_httpfetch_caller;
unsigned long m_httpfetch_next_id;
long m_httpfetch_timeout;
s32 m_httpfetch_active;
s32 m_httpfetch_active_limit;
s32 m_outstanding_hash_sets;
std::map<unsigned long, std::string> m_remote_file_transfers;

// All files up to this name have either been received from a
// remote server or failed on all remote servers, so those files
// don't need to be looked at again
// (use m_files.upper_bound(m_name_bound) to get an iterator)
std::string m_name_bound;

};

#endif // !CLIENTMEDIA_HEADER
3 changes: 1 addition & 2 deletions src/defaultsettings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ void set_default_settings(Settings *settings)
settings->setDefault("repeat_rightclick_time", "0.25");
settings->setDefault("enable_particles", "true");

settings->setDefault("media_fetch_threads", "8");
settings->setDefault("curl_timeout", "5000");
settings->setDefault("curl_parallel_limit", "8");

settings->setDefault("serverlist_url", "servers.minetest.net");
Expand Down Expand Up @@ -284,7 +284,6 @@ void set_default_settings(Settings *settings)

settings->setDefault("mgmath_generator", "mandelbox");

settings->setDefault("curl_timeout", "5000");

// IPv6
settings->setDefault("enable_ipv6", "true");
Expand Down
31 changes: 0 additions & 31 deletions src/filecache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "clientserver.h"
#include "log.h"
#include "filesys.h"
#include "hex.h"
#include "sha1.h"
#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <stdlib.h>

bool FileCache::loadByPath(const std::string &path, std::ostream &os)
Expand Down Expand Up @@ -85,36 +82,8 @@ bool FileCache::update(const std::string &name, const std::string &data)
std::string path = m_dir + DIR_DELIM + name;
return updateByPath(path, data);
}
bool FileCache::update_sha1(const std::string &data)
{
SHA1 sha1;
sha1.addBytes(data.c_str(), data.size());
unsigned char *digest = sha1.getDigest();
std::string sha1_raw((char*)digest, 20);
free(digest);
std::string sha1_hex = hex_encode(sha1_raw);
return update(sha1_hex, data);
}
bool FileCache::load(const std::string &name, std::ostream &os)
{
std::string path = m_dir + DIR_DELIM + name;
return loadByPath(path, os);
}
bool FileCache::load_sha1(const std::string &sha1_raw, std::ostream &os)
{
std::ostringstream tmp_os(std::ios_base::binary);
if(!load(hex_encode(sha1_raw), tmp_os))
return false;
SHA1 sha1;
sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
unsigned char *digest = sha1.getDigest();
std::string sha1_real_raw((char*)digest, 20);
free(digest);
if(sha1_real_raw != sha1_raw){
verbosestream<<"FileCache["<<m_dir<<"]: filename "<<sha1_real_raw
<<" mismatches actual checksum"<<std::endl;
return false;
}
os<<tmp_os.str();
return true;
}
2 changes: 0 additions & 2 deletions src/filecache.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,7 @@ class FileCache
}

bool update(const std::string &name, const std::string &data);
bool update_sha1(const std::string &data);
bool load(const std::string &name, std::ostream &os);
bool load_sha1(const std::string &sha1_raw, std::ostream &os);
private:
std::string m_dir;

Expand Down
9 changes: 3 additions & 6 deletions src/game.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <iomanip>
#include <list>
#include "util/directiontables.h"
#include "util/pointedthing.h"

/*
Text input system
Expand Down Expand Up @@ -1207,7 +1208,7 @@ void the_game(
server->step(dtime);

// End condition
if(client.texturesReceived() &&
if(client.mediaReceived() &&
client.itemdefReceived() &&
client.nodedefReceived()){
got_content = true;
Expand Down Expand Up @@ -1422,7 +1423,7 @@ void the_game(
bool invert_mouse = g_settings->getBool("invert_mouse");

bool respawn_menu_active = false;
bool update_wielded_item_trigger = false;
bool update_wielded_item_trigger = true;

bool show_hud = true;
bool show_chat = true;
Expand Down Expand Up @@ -2290,10 +2291,6 @@ void the_game(
delete(event.show_formspec.formspec);
delete(event.show_formspec.formname);
}
else if(event.type == CE_TEXTURES_UPDATED)
{
update_wielded_item_trigger = true;
}
else if(event.type == CE_SPAWN_PARTICLE)
{
LocalPlayer* player = client.getEnv().getLocalPlayer();
Expand Down
20 changes: 11 additions & 9 deletions src/server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2745,7 +2745,7 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
std::string datastring((char*)&data[2], datasize-2);
std::istringstream is(datastring, std::ios_base::binary);

std::list<MediaRequest> tosend;
std::list<std::string> tosend;
u16 numfiles = readU16(is);

infostream<<"Sending "<<numfiles<<" files to "
Expand All @@ -2754,7 +2754,7 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)

for(int i = 0; i < numfiles; i++) {
std::string name = deSerializeString(is);
tosend.push_back(MediaRequest(name));
tosend.push_back(name);
verbosestream<<"TOSERVER_REQUEST_MEDIA: requested file "
<<name<<std::endl;
}
Expand Down Expand Up @@ -4453,7 +4453,7 @@ struct SendableMedia
};

void Server::sendRequestedMedia(u16 peer_id,
const std::list<MediaRequest> &tosend)
const std::list<std::string> &tosend)
{
DSTACK(__FUNCTION_NAME);

Expand All @@ -4470,17 +4470,19 @@ void Server::sendRequestedMedia(u16 peer_id,

u32 file_size_bunch_total = 0;

for(std::list<MediaRequest>::const_iterator i = tosend.begin();
for(std::list<std::string>::const_iterator i = tosend.begin();
i != tosend.end(); ++i)
{
if(m_media.find(i->name) == m_media.end()){
const std::string &name = *i;

if(m_media.find(name) == m_media.end()){
errorstream<<"Server::sendRequestedMedia(): Client asked for "
<<"unknown file \""<<(i->name)<<"\""<<std::endl;
<<"unknown file \""<<(name)<<"\""<<std::endl;
continue;
}

//TODO get path + name
std::string tpath = m_media[(*i).name].path;
std::string tpath = m_media[name].path;

// Read data
std::ifstream fis(tpath.c_str(), std::ios_base::binary);
Expand All @@ -4506,14 +4508,14 @@ void Server::sendRequestedMedia(u16 peer_id,
}
if(bad){
errorstream<<"Server::sendRequestedMedia(): Failed to read \""
<<(*i).name<<"\""<<std::endl;
<<name<<"\""<<std::endl;
continue;
}
/*infostream<<"Server::sendRequestedMedia(): Loaded \""
<<tname<<"\""<<std::endl;*/
// Put in list
file_bunches[file_bunches.size()-1].push_back(
SendableMedia((*i).name, tpath, tmp_os.str()));
SendableMedia(name, tpath, tmp_os.str()));

// Start next bunch if got enough data
if(file_size_bunch_total >= bytes_per_bunch){
Expand Down
11 changes: 1 addition & 10 deletions src/server.h
Original file line number Diff line number Diff line change
Expand Up @@ -153,15 +153,6 @@ struct PrioritySortedBlockTransfer
u16 peer_id;
};

struct MediaRequest
{
std::string name;

MediaRequest(const std::string &name_=""):
name(name_)
{}
};

struct MediaInfo
{
std::string path;
Expand Down Expand Up @@ -569,7 +560,7 @@ class Server : public con::PeerHandler, public MapEventReceiver,
void fillMediaCache();
void sendMediaAnnouncement(u16 peer_id);
void sendRequestedMedia(u16 peer_id,
const std::list<MediaRequest> &tosend);
const std::list<std::string> &tosend);

void sendDetachedInventory(const std::string &name, u16 peer_id);
void sendDetachedInventoryToAll(const std::string &name);
Expand Down