Add server side translations capability (#9733)

* Add server side translations capability

doc/lua_api.txt

@@ -3176,8 +3176,22 @@ Strings that need to be translated can contain several escapes, preceded by `@`.
   `minetest.translate`, but is in translation files.
 * `@n` acts as a literal newline as well.
 
+Server side translations
+------------------------
+
+On some specific cases, server translation could be useful. For example, filter
+a list on labels and send results to client. A method is supplied to achieve
+that:
+
+`minetest.get_translated_string(lang_code, string)`: Translates `string` using
+translations for `lang_code` language. It gives the same result as if the string
+was translated by the client.
 
+The `lang_code` to use for a given player can be retrieved from
+the table returned by `minetest.get_player_information(name)`.
 
+IMPORTANT: This functionality should only be used for sorting, filtering or similar purposes.
+You do not need to use this to get translated strings to show up on the client.
 
 Perlin noise
 ============
@@ -4153,6 +4167,7 @@ Utilities
           connection_uptime = 200,   -- seconds since client connected
           protocol_version = 32,     -- protocol version used by client
           formspec_version = 2,      -- supported formspec version
+          lang_code = "fr"           -- Language code used for translation
           -- following information is available on debug build only!!!
           -- DO NOT USE IN MODS
           --ser_vers = 26,             -- serialization version used by client

src/client/client.cpp

@@ -736,7 +736,7 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
 	if (!name.empty()) {
 		TRACESTREAM(<< "Client: Loading translation: "
 				<< "\"" << filename << "\"" << std::endl);
-		g_translations->loadTranslation(data);
+		g_client_translations->loadTranslation(data);
 		return true;
 	}
 

src/client/game.cpp

@@ -1055,7 +1055,7 @@ bool Game::startup(bool *kill,
 	m_invert_mouse = g_settings->getBool("invert_mouse");
 	m_first_loop_after_window_activation = true;
 
-	g_translations->clear();
+	g_client_translations->clear();
 
 	if (!init(map_dir, address, port, gamespec))
 		return false;

src/clientiface.h

@@ -339,12 +339,18 @@ class RemoteClient
 	u8 getMinor() const { return m_version_minor; }
 	u8 getPatch() const { return m_version_patch; }
 	const std::string &getFull() const { return m_full_version; }
+
+	void setLangCode(const std::string &code) { m_lang_code = code; }
+	const std::string &getLangCode() const { return m_lang_code; }
 private:
 	// Version is stored in here after INIT before INIT2
 	u8 m_pending_serialization_version = SER_FMT_VER_INVALID;
 
 	/* current state of client */
 	ClientState m_state = CS_Created;
+
+	// Client sent language code
+	std::string m_lang_code;
 
 	/*
 		Blocks that have been sent to client.

src/network/serverpackethandler.cpp

@@ -311,6 +311,9 @@ void Server::handleCommand_Init2(NetworkPacket* pkt)
 
 	RemoteClient *client = getClient(peer_id, CS_InitDone);
 
+	// Keep client language for server translations
+	client->setLangCode(lang);
+
 	// Send active objects
 	{
 		PlayerSAO *sao = getPlayerSAO(peer_id);

src/script/lua_api/l_env.cpp

@@ -40,6 +40,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "remoteplayer.h"
 #include "server/luaentity_sao.h"
 #include "server/player_sao.h"
+#include "util/string.h"
+#include "translation.h"
 #ifndef SERVER
 #include "client/client.h"
 #endif
@@ -1302,6 +1304,19 @@ int ModApiEnvMod::l_forceload_free_block(lua_State *L)
 	return 0;
 }
 
+// get_translated_string(lang_code, string)
+int ModApiEnvMod::l_get_translated_string(lua_State * L)
+{
+	GET_ENV_PTR;
+	std::string lang_code = luaL_checkstring(L, 1);
+	std::string string = luaL_checkstring(L, 2);
+	getServer(L)->loadTranslationLanguage(lang_code);
+	string = wide_to_utf8(translate_string(utf8_to_wide(string),
+			&(*g_server_translations)[lang_code]));
+	lua_pushstring(L, string.c_str());
+	return 1;
+}
+
 void ModApiEnvMod::Initialize(lua_State *L, int top)
 {
 	API_FCT(set_node);
@@ -1349,6 +1364,7 @@ void ModApiEnvMod::Initialize(lua_State *L, int top)
 	API_FCT(transforming_liquid_add);
 	API_FCT(forceload_block);
 	API_FCT(forceload_free_block);
+	API_FCT(get_translated_string);
 }
 
 void ModApiEnvMod::InitializeClient(lua_State *L, int top)

src/script/lua_api/l_env.h

@@ -187,6 +187,9 @@ class ModApiEnvMod : public ModApiBase {
 	// stops forceloading a position
 	static int l_forceload_free_block(lua_State *L);
 
+	// Get a string translated server side
+	static int l_get_translated_string(lua_State * L);
+
 public:
 	static void Initialize(lua_State *L, int top);
 	static void InitializeClient(lua_State *L, int top);

src/script/lua_api/l_server.cpp

@@ -163,6 +163,7 @@ int ModApiServer::l_get_player_information(lua_State *L)
 	u16 prot_vers;
 	u8 ser_vers,major,minor,patch;
 	std::string vers_string;
+	std::string lang_code;
 
 #define ERET(code)                                                             \
 	if (!(code)) {                                                             \
@@ -182,7 +183,7 @@ int ModApiServer::l_get_player_information(lua_State *L)
 		&avg_jitter))
 
 	ERET(getServer(L)->getClientInfo(player->getPeerId(), &state, &uptime, &ser_vers,
-		&prot_vers, &major, &minor, &patch, &vers_string))
+		&prot_vers, &major, &minor, &patch, &vers_string, &lang_code))
 
 	lua_newtable(L);
 	int table = lua_gettop(L);
@@ -237,6 +238,10 @@ int ModApiServer::l_get_player_information(lua_State *L)
 	lua_pushnumber(L, player->formspec_version);
 	lua_settable(L, table);
 
+	lua_pushstring(L, "lang_code");
+	lua_pushstring(L, lang_code.c_str());
+	lua_settable(L, table);
+
 #ifndef NDEBUG
 	lua_pushstring(L,"serialization_version");
 	lua_pushnumber(L, ser_vers);

src/server.cpp

@@ -64,6 +64,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "chat_interface.h"
 #include "remoteplayer.h"
 #include "server/player_sao.h"
+#include "translation.h"
 
 class ClientNotFoundException : public BaseException
 {
@@ -1266,7 +1267,8 @@ bool Server::getClientInfo(
 		u8*          major,
 		u8*          minor,
 		u8*          patch,
-		std::string* vers_string
+		std::string* vers_string,
+		std::string* lang_code
 	)
 {
 	*state = m_clients.getClientState(peer_id);
@@ -1286,6 +1288,7 @@ bool Server::getClientInfo(
 	*minor = client->getMinor();
 	*patch = client->getPatch();
 	*vers_string = client->getFull();
+	*lang_code = client->getLangCode();
 
 	m_clients.unlock();
 
@@ -3937,3 +3940,20 @@ void Server::broadcastModChannelMessage(const std::string &channel,
 		m_script->on_modchannel_message(channel, sender, message);
 	}
 }
+
+void Server::loadTranslationLanguage(const std::string &lang_code)
+{
+	if (g_server_translations->count(lang_code))
+		return; // Already loaded
+
+	std::string suffix = "." + lang_code + ".tr";
+	for (const auto &i : m_media) {
+		if (str_ends_with(i.first, suffix)) {
+			std::ifstream t(i.second.path);
+			std::string data((std::istreambuf_iterator<char>(t)),
+			std::istreambuf_iterator<char>());
+
+			(*g_server_translations)[lang_code].loadTranslation(data);
+		}
+	}
+}

src/server.h

@@ -334,7 +334,7 @@ class Server : public con::PeerHandler, public MapEventReceiver,
 	bool getClientConInfo(session_t peer_id, con::rtt_stat_type type, float *retval);
 	bool getClientInfo(session_t peer_id, ClientState *state, u32 *uptime,
 			u8* ser_vers, u16* prot_vers, u8* major, u8* minor, u8* patch,
-			std::string* vers_string);
+			std::string* vers_string, std::string* lang_code);
 
 	void printToConsoleOnly(const std::string &text);
 
@@ -358,6 +358,9 @@ class Server : public con::PeerHandler, public MapEventReceiver,
 	// Send block to specific player only
 	bool SendBlock(session_t peer_id, const v3s16 &blockpos);
 
+	// Load translations for a language
+	void loadTranslationLanguage(const std::string &lang_code);
+
 	// Bind address
 	Address m_bind_addr;
 

src/translation.cpp

@@ -20,9 +20,18 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "translation.h"
 #include "log.h"
 #include "util/string.h"
+#include <unordered_map>
 
-static Translations main_translations;
-Translations *g_translations = &main_translations;
+
+#ifndef SERVER
+// Client translations
+Translations client_translations;
+Translations *g_client_translations = &client_translations;
+#endif
+
+// Per language server translations
+std::unordered_map<std::string,Translations> server_translations;
+std::unordered_map<std::string,Translations> *g_server_translations = &server_translations;
 
 Translations::~Translations()
 {

src/translation.h

@@ -23,7 +23,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <string>
 
 class Translations;
-extern Translations *g_translations;
+extern std::unordered_map<std::string, Translations> *g_server_translations;
+#ifndef SERVER
+extern Translations *g_client_translations;
+#endif
 
 class Translations
 {

src/util/string.cpp

@@ -693,10 +693,12 @@ void str_replace(std::string &str, char from, char to)
  * before filling it again.
  */
 
-void translate_all(const std::wstring &s, size_t &i, std::wstring &res);
+void translate_all(const std::wstring &s, size_t &i,
+		Translations *translations, std::wstring &res);
 
-void translate_string(const std::wstring &s, const std::wstring &textdomain,
-		size_t &i, std::wstring &res) {
+void translate_string(const std::wstring &s, Translations *translations,
+		const std::wstring &textdomain, size_t &i, std::wstring &res)
+{
 	std::wostringstream output;
 	std::vector<std::wstring> args;
 	int arg_number = 1;
@@ -750,15 +752,15 @@ void translate_string(const std::wstring &s, const std::wstring &textdomain,
 			if (arg_number >= 10) {
 				errorstream << "Ignoring too many arguments to translation" << std::endl;
 				std::wstring arg;
-				translate_all(s, i, arg);
+				translate_all(s, i, translations, arg);
 				args.push_back(arg);
 				continue;
 			}
 			output.put(L'@');
 			output << arg_number;
 			++arg_number;
 			std::wstring arg;
-			translate_all(s, i, arg);
+			translate_all(s, i, translations, arg);
 			args.push_back(arg);
 		} else {
 			// This is an escape sequence *inside* the template string to translate itself.
@@ -767,8 +769,13 @@ void translate_string(const std::wstring &s, const std::wstring &textdomain,
 		}
 	}
 
+	std::wstring toutput;
 	// Translate the template.
-	std::wstring toutput = g_translations->getTranslation(textdomain, output.str());
+	if (translations != nullptr)
+		toutput = translations->getTranslation(
+				textdomain, output.str());
+	else
+		toutput = output.str();
 
 	// Put back the arguments in the translated template.
 	std::wostringstream result;
@@ -802,7 +809,9 @@ void translate_string(const std::wstring &s, const std::wstring &textdomain,
 	res = result.str();
 }
 
-void translate_all(const std::wstring &s, size_t &i, std::wstring &res) {
+void translate_all(const std::wstring &s, size_t &i,
+		Translations *translations, std::wstring &res)
+{
 	std::wostringstream output;
 	while (i < s.length()) {
 		// Not an escape sequence: just add the character.
@@ -851,7 +860,7 @@ void translate_all(const std::wstring &s, size_t &i, std::wstring &res) {
 			if (parts.size() > 1)
 				textdomain = parts[1];
 			std::wstring translated;
-			translate_string(s, textdomain, i, translated);
+			translate_string(s, translations, textdomain, i, translated);
 			output << translated;
 		} else {
 			// Another escape sequence, such as colors. Preserve it.
@@ -862,9 +871,21 @@ void translate_all(const std::wstring &s, size_t &i, std::wstring &res) {
 	res = output.str();
 }
 
-std::wstring translate_string(const std::wstring &s) {
+// Translate string server side
+std::wstring translate_string(const std::wstring &s, Translations *translations)
+{
 	size_t i = 0;
 	std::wstring res;
-	translate_all(s, i, res);
+	translate_all(s, i, translations, res);
 	return res;
 }
+
+// Translate string client side
+std::wstring translate_string(const std::wstring &s)
+{
+#ifdef SERVER
+	return translate_string(s, nullptr);
+#else
+	return translate_string(s, g_client_translations);
+#endif
+}

src/util/string.h

@@ -31,6 +31,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <cctype>
 #include <unordered_map>
 
+class Translations;
+
 #define STRINGIFY(x) #x
 #define TOSTRING(x) STRINGIFY(x)
 
@@ -650,6 +652,8 @@ std::vector<std::basic_string<T> > split(const std::basic_string<T> &s, T delim)
 	return tokens;
 }
 
+std::wstring translate_string(const std::wstring &s, Translations *translations);
+
 std::wstring translate_string(const std::wstring &s);
 
 inline std::wstring unescape_translate(const std::wstring &s) {