rewrite localization; use .po and .pot files

schemes/texts.scheme

@@ -0,0 +1,8 @@
+[scheme]
+processor = cage-asset-processor texts
+index = 2
+
+[multilingual]
+display = multilingual
+type = bool
+default = true

sources/asset-processor/main.cpp

@@ -216,8 +216,8 @@ int main(int argc, const char *args[])
 			func.bind<&processSkeleton>();
 		else if (component == "font")
 			func.bind<&processFont>();
-		else if (component == "textpack")
-			func.bind<&processTextpack>();
+		else if (component == "texts")
+			func.bind<&processTexts>();
 		else if (component == "sound")
 			func.bind<&processSound>();
 		else if (component == "collider")

sources/asset-processor/processor.h

@@ -35,7 +35,7 @@ void processModel();
 void processSkeleton();
 void processAnimation();
 void processFont();
-void processTextpack();
+void processTexts();
 void processSound();
 void processCollider();
 void processRaw();

sources/asset-processor/texts.cpp

@@ -0,0 +1,150 @@
+#include "processor.h"
+
+#include <cage-core/texts.h>
+
+namespace
+{
+	LanguageCode extractLanguage(const String &pth)
+	{
+		// eg. pth = dir/gui.en_US.po
+		const String n = pathExtractFilenameNoExtension(pth); // -> gui.en_US
+		const String l = subString(pathExtractExtension(n), 1, m); // -> en_US
+		// eg. inputFile = dir/gui.pot
+		const String p = pathExtractFilenameNoExtension(inputFile); // -> gui
+		if (p + "." + l == n)
+			return l;
+		return "";
+	}
+
+	void parsePO(Holder<File> f, const LanguageCode &lang, Texts *txt)
+	{
+		String id, str, l;
+
+		const auto &quotes = [](const String &s) -> String
+		{
+			if (s.length() >= 2 && s[0] == '\"' && s[s.length() - 1] == '\"')
+				return subString(s, 1, s.length() - 2);
+			return s;
+		};
+
+		const auto &add = [&]()
+		{
+			if (!id.empty() && !str.empty())
+				txt->set(quotes(id), quotes(str), lang);
+			else if (!id.empty() || !str.empty())
+			{
+				CAGE_LOG(SeverityEnum::Note, "assetProcessor", id);
+				CAGE_LOG(SeverityEnum::Note, "assetProcessor", str);
+				CAGE_THROW_ERROR(Exception, "missing msgstr or msgid");
+			}
+			id = str = "";
+		};
+
+		while (f->readLine(l))
+		{
+			l = trim(l);
+			if (l.empty())
+				add();
+			else if (l[0] != '#')
+			{
+				const String p = l;
+				String k = split(l, "\t ");
+				k = toLower(k);
+				if (k == "msgid")
+				{
+					if (!id.empty())
+					{
+						CAGE_LOG(SeverityEnum::Note, "assetProcessor", id);
+						CAGE_LOG(SeverityEnum::Note, "assetProcessor", p);
+						CAGE_THROW_ERROR(Exception, "multiple msgid");
+					}
+					id = l;
+				}
+				else if (k == "msgstr")
+				{
+					if (!str.empty())
+					{
+						CAGE_LOG(SeverityEnum::Note, "assetProcessor", id);
+						CAGE_LOG(SeverityEnum::Note, "assetProcessor", p);
+						CAGE_THROW_ERROR(Exception, "multiple msgstr");
+					}
+					str = l;
+				}
+				else
+				{
+					CAGE_LOG(SeverityEnum::Note, "assetProcessor", p);
+					CAGE_THROW_ERROR(Exception, "unknown command");
+				}
+			}
+		}
+		add();
+	}
+}
+
+void processTexts()
+{
+	writeLine(String("use=") + inputFile);
+
+	{
+		const String ext = pathExtractExtension(inputFile);
+		if (ext != ".pot" && ext != ".po")
+			CAGE_THROW_ERROR(Exception, "input file must have .pot or .po extension");
+	}
+
+	Holder<Texts> txt = newTexts();
+
+	if (toBool(properties("multilingual")))
+	{
+		{ // validate template file
+			CAGE_LOG(SeverityEnum::Info, "assetProcessor", "validating template");
+			Holder<Texts> tmp = newTexts();
+			parsePO(readFile(inputFileName), "", +tmp);
+		}
+
+		for (const String &pth : pathListDirectory(pathExtractDirectory(inputFileName)))
+		{
+			if (!pathIsFile(pth))
+				continue;
+			if (pathExtractExtension(pth) != ".po")
+				continue;
+			const LanguageCode lang = extractLanguage(pth);
+			if (lang.empty())
+				continue;
+			writeLine(String("use=") + pathToRel(pth, inputDirectory));
+			CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "loading language: " + lang);
+			parsePO(readFile(pth), lang, +txt);
+		}
+	}
+	else
+	{
+		parsePO(readFile(inputFileName), "", +txt);
+	}
+
+	{
+		const auto l = txt->allLanguages();
+		if (l.empty())
+			CAGE_THROW_ERROR(Exception, "loaded no languages");
+		for (const String &n : l)
+			CAGE_LOG(SeverityEnum::Info, "language", n);
+	}
+	{
+		const auto l = txt->allNames();
+		if (l.empty())
+			CAGE_THROW_ERROR(Exception, "loaded no texts");
+		for (const String &n : l)
+			CAGE_LOG(SeverityEnum::Info, "name", n);
+	}
+
+	Holder<PointerRange<char>> buff = txt->exportBuffer();
+	CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "buffer size (before compression): " + buff.size());
+	Holder<PointerRange<char>> comp = compress(buff);
+	CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "buffer size (after compression): " + comp.size());
+
+	AssetHeader h = initializeAssetHeader();
+	h.originalSize = buff.size();
+	h.compressedSize = comp.size();
+	Holder<File> f = writeFile(outputFileName);
+	f->write(bufferView(h));
+	f->write(comp);
+	f->close();
+}

sources/include/cage-core/assetContext.h

@@ -11,7 +11,7 @@ namespace cage
 
 	struct CAGE_CORE_API AssetContext : private Immovable
 	{
-		detail::StringBase<64> textName;
+		detail::StringBase<64> textId;
 		Holder<void> customData;
 		Holder<PointerRange<uint32>> dependencies;
 		Holder<PointerRange<char>> compressedData;

sources/include/cage-core/assetHeader.h

@@ -12,7 +12,7 @@ namespace cage
 		char cageName[8] = "cageAss";
 		uint32 version = 0;
 		uint32 flags = 0;
-		char textName[64] = {};
+		char textId[64] = {};
 		uint64 compressedSize = 0;
 		uint64 originalSize = 0;
 		uint16 scheme = m;

sources/include/cage-core/assetManager.h

@@ -26,17 +26,17 @@ namespace cage
 		uint32 generateUniqueName();
 
 		template<uint32 Scheme, class T>
-		void loadValue(uint32 assetName, Holder<T> &&value, const String &textName = "")
+		void loadValue(uint32 assetName, Holder<T> &&value, const String &textId = "")
 		{
 			CAGE_ASSERT(detail::typeHash<T>() == schemeTypeHash_(Scheme))
-			load_(Scheme, assetName, textName, std::move(value).template cast<void>());
+			load_(Scheme, assetName, textId, std::move(value).template cast<void>());
 		}
 
 		template<uint32 Scheme, class T>
-		void loadCustom(uint32 assetName, const AssetScheme &customScheme, Holder<void> &&customData, const String &textName = "")
+		void loadCustom(uint32 assetName, const AssetScheme &customScheme, Holder<void> &&customData, const String &textId = "")
 		{
 			CAGE_ASSERT(detail::typeHash<T>() == schemeTypeHash_(Scheme))
-			load_(Scheme, assetName, textName, customScheme, std::move(customData));
+			load_(Scheme, assetName, textId, customScheme, std::move(customData));
 		}
 
 		// returns null if the asset is not yet loaded or has different scheme
@@ -61,8 +61,8 @@ namespace cage
 
 	private:
 		void defineScheme_(uint32 typeHash, uint32 scheme, const AssetScheme &value);
-		void load_(uint32 scheme, uint32 assetName, const String &textName, Holder<void> &&value);
-		void load_(uint32 scheme, uint32 assetName, const String &textName, const AssetScheme &customScheme, Holder<void> &&customData);
+		void load_(uint32 scheme, uint32 assetName, const String &textId, Holder<void> &&value);
+		void load_(uint32 scheme, uint32 assetName, const String &textId, const AssetScheme &customScheme, Holder<void> &&customData);
 		Holder<void> get_(uint32 scheme, uint32 assetName) const;
 		uint32 schemeTypeHash_(uint32 scheme) const;
 		friend class AssetOnDemand;

sources/include/cage-core/texts.h

@@ -0,0 +1,44 @@
+#ifndef guard_texts_h_B436B597745C461DAA266CE6FBBE10D1
+#define guard_texts_h_B436B597745C461DAA266CE6FBBE10D1
+
+#include <cage-core/core.h>
+
+namespace cage
+{
+	using LanguageCode = detail::StringBase<10>;
+
+	class CAGE_CORE_API Texts : private Immovable
+	{
+	public:
+		void clear();
+		Holder<Texts> copy() const;
+
+		void importBuffer(PointerRange<const char> buffer);
+		Holder<PointerRange<char>> exportBuffer() const;
+
+		void set(uint32 id, const String &text, const LanguageCode &language);
+		uint32 set(const String &name, const String &text, const LanguageCode &language);
+
+		Holder<PointerRange<LanguageCode>> allLanguages() const;
+		Holder<PointerRange<String>> allNames() const;
+		Holder<PointerRange<uint32>> allIds() const;
+
+		// returns empty string if not found
+		String get(uint32 id, const LanguageCode &language) const;
+	};
+
+	CAGE_CORE_API Holder<Texts> newTexts();
+
+	CAGE_CORE_API AssetScheme genAssetSchemeTexts();
+	constexpr uint32 AssetSchemeIndexTexts = 2;
+
+	CAGE_CORE_API void textsSetLanguages(PointerRange<const LanguageCode> languages);
+	CAGE_CORE_API void textsSetLanguages(const String &languages); // list of languages separated by ;
+	CAGE_CORE_API void textsAdd(const Texts *txt);
+	CAGE_CORE_API void textsRemove(const Texts *txt);
+	CAGE_CORE_API String textsGet(uint32 id, String params = "");
+
+	CAGE_CORE_API String textFormat(String format, const String &params);
+}
+
+#endif // guard_texts_h_B436B597745C461DAA266CE6FBBE10D1

sources/include/cage-engine/guiBuilder.h

@@ -25,7 +25,7 @@ namespace cage
 
 			BuilderItem text(const GuiTextComponent &txt);
 			BuilderItem text(const String &txt);
-			BuilderItem text(uint32 assetName, uint32 textName, const String &parameters = "");
+			BuilderItem text(uint32 textId, const String &parameters = "");
 			BuilderItem textFormat(const GuiTextFormatComponent &textFormat);
 			BuilderItem textColor(Vec3 color);
 			BuilderItem textSize(Real size);
@@ -46,10 +46,10 @@ namespace cage
 			BuilderItem update(Delegate<void(Entity *)> u);
 
 			BuilderItem tooltip(const GuiTooltipComponent &t);
-			template<StringLiteral Text, uint32 AssetName = 0, uint32 TextName = 0>
+			template<StringLiteral Text, uint32 TextId = 0>
 			BuilderItem tooltip(uint64 delay = GuiTooltipComponent().delay)
 			{
-				(*this)->template value<GuiTooltipComponent>().tooltip = detail::guiTooltipText<Text, AssetName, TextName>();
+				(*this)->template value<GuiTooltipComponent>().tooltip = detail::guiTooltipText<Text, TextId>();
 				(*this)->template value<GuiTooltipComponent>().delay = delay;
 				return *this;
 			}