Server pushing media at runtime (#9961)

doc/lua_api.txt

@@ -5217,6 +5217,20 @@ Server
     * Returns a code (0: successful, 1: no such player, 2: player is connected)
 * `minetest.remove_player_auth(name)`: remove player authentication data
     * Returns boolean indicating success (false if player nonexistant)
+* `minetest.dynamic_add_media(filepath)`
+    * Adds the file at the given path to the media sent to clients by the server
+      on startup and also pushes this file to already connected clients.
+      The file must be a supported image, sound or model format. It must not be
+      modified, deleted, moved or renamed after calling this function.
+      The list of dynamically added media is not persisted.
+    * Returns boolean indicating success (duplicate files count as error)
+    * The media will be ready to use (in e.g. entity textures, sound_play)
+      immediately after calling this function.
+      Old clients that lack support for this feature will not see the media
+      unless they reconnect to the server.
+    * Since media transferred this way does not use client caching or HTTP
+      transfers, dynamic media should not be used with big files or performance
+      will suffer.
 
 Bans
 ----

src/client/client.cpp

@@ -670,11 +670,9 @@ void Client::step(float dtime)
 	}
 }
 
-bool Client::loadMedia(const std::string &data, const std::string &filename)
+bool Client::loadMedia(const std::string &data, const std::string &filename,
+	bool from_media_push)
 {
-	// Silly irrlicht's const-incorrectness
-	Buffer<char> data_rw(data.c_str(), data.size());
-
 	std::string name;
 
 	const char *image_ext[] = {
@@ -690,6 +688,9 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
 		io::IFileSystem *irrfs = RenderingEngine::get_filesystem();
 		video::IVideoDriver *vdrv = RenderingEngine::get_video_driver();
 
+		// Silly irrlicht's const-incorrectness
+		Buffer<char> data_rw(data.c_str(), data.size());
+
 		// Create an irrlicht memory file
 		io::IReadFile *rfile = irrfs->createMemoryReadFile(
 				*data_rw, data_rw.getSize(), "_tempreadfile");
@@ -727,7 +728,6 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
 		".x", ".b3d", ".md2", ".obj",
 		NULL
 	};
-
 	name = removeStringEnd(filename, model_ext);
 	if (!name.empty()) {
 		verbosestream<<"Client: Storing model into memory: "
@@ -744,6 +744,8 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
 	};
 	name = removeStringEnd(filename, translate_ext);
 	if (!name.empty()) {
+		if (from_media_push)
+			return false;
 		TRACESTREAM(<< "Client: Loading translation: "
 				<< "\"" << filename << "\"" << std::endl);
 		g_client_translations->loadTranslation(data);

src/client/client.h

@@ -222,6 +222,7 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
 	void handleCommand_FormspecPrepend(NetworkPacket *pkt);
 	void handleCommand_CSMRestrictionFlags(NetworkPacket *pkt);
 	void handleCommand_PlayerSpeed(NetworkPacket *pkt);
+	void handleCommand_MediaPush(NetworkPacket *pkt);
 
 	void ProcessData(NetworkPacket *pkt);
 
@@ -376,7 +377,8 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
 
 	// The following set of functions is used by ClientMediaDownloader
 	// Insert a media file appropriately into the appropriate manager
-	bool loadMedia(const std::string &data, const std::string &filename);
+	bool loadMedia(const std::string &data, const std::string &filename,
+		bool from_media_push = false);
 	// Send a request for conventional media transfer
 	void request_media(const std::vector<std::string> &file_requests);
 
@@ -488,6 +490,7 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
 	Camera *m_camera = nullptr;
 	Minimap *m_minimap = nullptr;
 	bool m_minimap_disabled_by_server = false;
+
 	// Server serialization version
 	u8 m_server_ser_ver;
 
@@ -529,7 +532,6 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
 	AuthMechanism m_chosen_auth_mech;
 	void *m_auth_data = nullptr;
 
-
 	bool m_access_denied = false;
 	bool m_access_denied_reconnect = false;
 	std::string m_access_denied_reason = "";
@@ -538,7 +540,10 @@ class Client : public con::PeerHandler, public InventoryManager, public IGameDef
 	bool m_nodedef_received = false;
 	bool m_activeobjects_received = false;
 	bool m_mods_loaded = false;
+
 	ClientMediaDownloader *m_media_downloader;
+	// Set of media filenames pushed by server at runtime
+	std::unordered_set<std::string> m_media_pushed_files;
 
 	// time_of_day speed approximation for old protocol
 	bool m_time_of_day_set = false;

src/client/clientmedia.cpp

@@ -35,6 +35,15 @@ static std::string getMediaCacheDir()
 	return porting::path_cache + DIR_DELIM + "media";
 }
 
+bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &filedata)
+{
+	FileCache media_cache(getMediaCacheDir());
+	std::string sha1_hex = hex_encode(raw_hash);
+	if (!media_cache.exists(sha1_hex))
+		return media_cache.update(sha1_hex, filedata);
+	return true;
+}
+
 /*
 	ClientMediaDownloader
 */
@@ -559,7 +568,6 @@ bool ClientMediaDownloader::checkAndLoad(
 	return true;
 }
 
-
 /*
 	Minetest Hashset File Format
 

src/client/clientmedia.h

@@ -33,6 +33,11 @@ struct HTTPFetchResult;
 #define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS'
 #define MTHASHSET_FILE_NAME "index.mth"
 
+// Store file into media cache (unless it exists already)
+// Validating the hash is responsibility of the caller
+bool clientMediaUpdateCache(const std::string &raw_hash,
+	const std::string &filedata);
+
 class ClientMediaDownloader
 {
 public:

src/client/filecache.cpp

@@ -82,8 +82,16 @@ bool FileCache::update(const std::string &name, const std::string &data)
 	std::string path = m_dir + DIR_DELIM + name;
 	return updateByPath(path, data);
 }
+
 bool FileCache::load(const std::string &name, std::ostream &os)
 {
 	std::string path = m_dir + DIR_DELIM + name;
 	return loadByPath(path, os);
 }
+
+bool FileCache::exists(const std::string &name)
+{
+	std::string path = m_dir + DIR_DELIM + name;
+	std::ifstream fis(path.c_str(), std::ios_base::binary);
+	return fis.good();
+}

src/client/filecache.h

@@ -33,6 +33,7 @@ class FileCache
 
 	bool update(const std::string &name, const std::string &data);
 	bool load(const std::string &name, std::ostream &os);
+	bool exists(const std::string &name);
 
 private:
 	std::string m_dir;

src/filesys.cpp

@@ -691,6 +691,12 @@ std::string AbsolutePath(const std::string &path)
 const char *GetFilenameFromPath(const char *path)
 {
 	const char *filename = strrchr(path, DIR_DELIM_CHAR);
+	// Consistent with IsDirDelimiter this function handles '/' too
+	if (DIR_DELIM_CHAR != '/') {
+		const char *tmp = strrchr(path, '/');
+		if (tmp && tmp > filename)
+			filename = tmp;
+	}
 	return filename ? filename + 1 : path;
 }
 

src/network/clientopcodes.cpp

@@ -68,7 +68,7 @@ const ToClientCommandHandler toClientCommandTable[TOCLIENT_NUM_MSG_TYPES] =
 	{ "TOCLIENT_TIME_OF_DAY",              TOCLIENT_STATE_CONNECTED, &Client::handleCommand_TimeOfDay }, // 0x29
 	{ "TOCLIENT_CSM_RESTRICTION_FLAGS",    TOCLIENT_STATE_CONNECTED, &Client::handleCommand_CSMRestrictionFlags }, // 0x2A
 	{ "TOCLIENT_PLAYER_SPEED",             TOCLIENT_STATE_CONNECTED, &Client::handleCommand_PlayerSpeed }, // 0x2B
-	null_command_handler,
+	{ "TOCLIENT_MEDIA_PUSH",               TOCLIENT_STATE_CONNECTED, &Client::handleCommand_MediaPush }, // 0x2C
 	null_command_handler,
 	null_command_handler,
 	{ "TOCLIENT_CHAT_MESSAGE",             TOCLIENT_STATE_CONNECTED, &Client::handleCommand_ChatMessage }, // 0x2F

src/network/clientpackethandler.cpp

@@ -39,6 +39,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "script/scripting_client.h"
 #include "util/serialize.h"
 #include "util/srp.h"
+#include "util/sha1.h"
 #include "tileanimation.h"
 #include "gettext.h"
 #include "skyparams.h"
@@ -1471,6 +1472,51 @@ void Client::handleCommand_PlayerSpeed(NetworkPacket *pkt)
 	player->addVelocity(added_vel);
 }
 
+void Client::handleCommand_MediaPush(NetworkPacket *pkt)
+{
+	std::string raw_hash, filename, filedata;
+	bool cached;
+
+	*pkt >> raw_hash >> filename >> cached;
+	filedata = pkt->readLongString();
+
+	if (raw_hash.size() != 20 || filedata.empty() || filename.empty() ||
+			!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
+		throw PacketError("Illegal filename, data or hash");
+	}
+
+	verbosestream << "Server pushes media file \"" << filename << "\" with "
+		<< filedata.size() << " bytes of data (cached=" << cached
+		<< ")" << std::endl;
+
+	if (m_media_pushed_files.count(filename) != 0) {
+		// Silently ignore for synchronization purposes
+		return;
+	}
+
+	// Compute and check checksum of data
+	std::string computed_hash;
+	{
+		SHA1 ctx;
+		ctx.addBytes(filedata.c_str(), filedata.size());
+		unsigned char *buf = ctx.getDigest();
+		computed_hash.assign((char*) buf, 20);
+		free(buf);
+	}
+	if (raw_hash != computed_hash) {
+		verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
+		return;
+	}
+
+	// Actually load media
+	loadMedia(filedata, filename, true);
+	m_media_pushed_files.insert(filename);
+
+	// Cache file for the next time when this client joins the same server
+	if (cached)
+		clientMediaUpdateCache(raw_hash, filedata);
+}
+
 /*
  * Mod channels
  */

src/network/networkprotocol.h

@@ -323,6 +323,15 @@ enum ToClientCommand
 		v3f added_vel
 	 */
 
+	TOCLIENT_MEDIA_PUSH = 0x2C,
+	/*
+		std::string raw_hash
+		std::string filename
+		bool should_be_cached
+		u32 len
+		char filedata[len]
+	*/
+
 	// (oops, there is some gap here)
 
 	TOCLIENT_CHAT_MESSAGE = 0x2F,

src/network/serveropcodes.cpp

@@ -167,7 +167,7 @@ const ClientCommandFactory clientCommandFactoryTable[TOCLIENT_NUM_MSG_TYPES] =
 	{ "TOCLIENT_TIME_OF_DAY",              0, true }, // 0x29
 	{ "TOCLIENT_CSM_RESTRICTION_FLAGS",    0, true }, // 0x2A
 	{ "TOCLIENT_PLAYER_SPEED",             0, true }, // 0x2B
-	null_command_factory, // 0x2C
+	{ "TOCLIENT_MEDIA_PUSH",               0, true }, // 0x2C (sent over channel 1 too)
 	null_command_factory, // 0x2D
 	null_command_factory, // 0x2E
 	{ "TOCLIENT_CHAT_MESSAGE",             0, true }, // 0x2F

src/script/lua_api/l_server.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "common/c_converter.h"
 #include "common/c_content.h"
 #include "cpp_api/s_base.h"
+#include "cpp_api/s_security.h"
 #include "server.h"
 #include "environment.h"
 #include "remoteplayer.h"
@@ -412,9 +413,6 @@ int ModApiServer::l_get_modnames(lua_State *L)
 	std::vector<std::string> modlist;
 	getServer(L)->getModNames(modlist);
 
-	// Take unsorted items from mods_unsorted and sort them into
-	// mods_sorted; not great performance but the number of mods on a
-	// server will likely be small.
 	std::sort(modlist.begin(), modlist.end());
 
 	// Package them up for Lua
@@ -474,6 +472,23 @@ int ModApiServer::l_sound_fade(lua_State *L)
 	return 0;
 }
 
+// dynamic_add_media(filepath)
+int ModApiServer::l_dynamic_add_media(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	// Reject adding media before the server has started up
+	if (!getEnv(L))
+		throw LuaError("Dynamic media cannot be added before server has started up");
+
+	std::string filepath = readParam<std::string>(L, 1);
+	CHECK_SECURE_PATH(L, filepath.c_str(), false);
+
+	bool ok = getServer(L)->dynamicAddMedia(filepath);
+	lua_pushboolean(L, ok);
+	return 1;
+}
+
 // is_singleplayer()
 int ModApiServer::l_is_singleplayer(lua_State *L)
 {
@@ -538,6 +553,7 @@ void ModApiServer::Initialize(lua_State *L, int top)
 	API_FCT(sound_play);
 	API_FCT(sound_stop);
 	API_FCT(sound_fade);
+	API_FCT(dynamic_add_media);
 
 	API_FCT(get_player_information);
 	API_FCT(get_player_privs);

src/script/lua_api/l_server.h

@@ -70,6 +70,9 @@ class ModApiServer : public ModApiBase
 	// sound_fade(handle, step, gain)
 	static int l_sound_fade(lua_State *L);
 
+	// dynamic_add_media(filepath)
+	static int l_dynamic_add_media(lua_State *L);
+
 	// get_player_privs(name, text)
 	static int l_get_player_privs(lua_State *L);