From 2f5ba2d72c52fb8d39967d15df346b5d066832dd Mon Sep 17 00:00:00 2001 From: Tomas Maly Date: Sat, 4 May 2024 13:11:20 +0200 Subject: [PATCH] rewrite localization; use .po and .pot files --- schemes/textpack.scheme | 3 - schemes/texts.scheme | 8 + sources/asset-processor/main.cpp | 4 +- sources/asset-processor/processor.h | 2 +- sources/asset-processor/textPack.cpp | 55 ---- sources/asset-processor/texts.cpp | 150 +++++++++ sources/include/cage-core/assetContext.h | 2 +- sources/include/cage-core/assetHeader.h | 2 +- sources/include/cage-core/assetManager.h | 12 +- sources/include/cage-core/textPack.h | 34 -- sources/include/cage-core/texts.h | 44 +++ sources/include/cage-engine/guiBuilder.h | 6 +- sources/include/cage-engine/guiComponents.h | 7 +- sources/include/cage-engine/scene.h | 5 +- sources/libcore/assets/assetManager.cpp | 36 +-- sources/libcore/assets/assetStructs.cpp | 45 --- sources/libcore/assets/texts.cpp | 290 ++++++++++++++++++ sources/libcore/textPack.cpp | 119 ------- sources/libengine/assets/font.cpp | 2 +- sources/libengine/assets/model.cpp | 2 +- sources/libengine/assets/renderObject.cpp | 2 +- sources/libengine/assets/shaderProgram.cpp | 2 +- sources/libengine/assets/sound.cpp | 2 +- sources/libengine/assets/texture.cpp | 2 +- sources/libengine/graphics/renderPipeline.cpp | 4 +- sources/libengine/gui/guiBuilder.cpp | 4 +- sources/libengine/gui/items.cpp | 4 +- sources/libsimple/gameloop.cpp | 4 +- sources/test-core/assetManager.cpp | 4 +- sources/test-core/strings.cpp | 15 + 30 files changed, 560 insertions(+), 311 deletions(-) delete mode 100644 schemes/textpack.scheme create mode 100644 schemes/texts.scheme delete mode 100644 sources/asset-processor/textPack.cpp create mode 100644 sources/asset-processor/texts.cpp delete mode 100644 sources/include/cage-core/textPack.h create mode 100644 sources/include/cage-core/texts.h create mode 100644 sources/libcore/assets/texts.cpp delete mode 100644 sources/libcore/textPack.cpp diff --git a/schemes/textpack.scheme b/schemes/textpack.scheme deleted file mode 100644 index a572b1aa..00000000 --- a/schemes/textpack.scheme +++ /dev/null @@ -1,3 +0,0 @@ -[scheme] -processor = cage-asset-processor textpack -index = 2 diff --git a/schemes/texts.scheme b/schemes/texts.scheme new file mode 100644 index 00000000..60449113 --- /dev/null +++ b/schemes/texts.scheme @@ -0,0 +1,8 @@ +[scheme] +processor = cage-asset-processor texts +index = 2 + +[multilingual] +display = multilingual +type = bool +default = true diff --git a/sources/asset-processor/main.cpp b/sources/asset-processor/main.cpp index 44622eba..edf059be 100644 --- a/sources/asset-processor/main.cpp +++ b/sources/asset-processor/main.cpp @@ -216,8 +216,8 @@ int main(int argc, const char *args[]) func.bind<&processSkeleton>(); else if (component == "font") func.bind<&processFont>(); - else if (component == "textpack") - func.bind<&processTextpack>(); + else if (component == "texts") + func.bind<&processTexts>(); else if (component == "sound") func.bind<&processSound>(); else if (component == "collider") diff --git a/sources/asset-processor/processor.h b/sources/asset-processor/processor.h index 7f43b399..aa2fc2a9 100644 --- a/sources/asset-processor/processor.h +++ b/sources/asset-processor/processor.h @@ -35,7 +35,7 @@ void processModel(); void processSkeleton(); void processAnimation(); void processFont(); -void processTextpack(); +void processTexts(); void processSound(); void processCollider(); void processRaw(); diff --git a/sources/asset-processor/textPack.cpp b/sources/asset-processor/textPack.cpp deleted file mode 100644 index b08226a0..00000000 --- a/sources/asset-processor/textPack.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include - -#include "processor.h" - -#include -#include -#include - -void processTextpack() -{ - writeLine(String("use=") + inputFile); - - Holder ini = newIni(); - ini->importFile(inputFileName); - - std::map texts; - for (const String §ion : ini->sections()) - { - for (String n : ini->items(section)) - { - String v = ini->get(section, n); - if (!isDigitsOnly(section)) - n = section + "/" + n; - texts[n] = v; - } - } - CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "loaded " + texts.size() + " texts"); - - if (configGetBool("cage-asset-processor/textpack/preview")) - { - String dbgName = pathJoin(configGetString("cage-asset-processor/textpack/path", "asset-preview"), pathReplaceInvalidCharacters(inputName) + ".txt"); - FileMode fm(false, true); - fm.textual = true; - Holder f = newFile(dbgName, fm); - for (const auto &it : texts) - f->writeLine(fill(String(Stringizer() + HashString(it.first)), 10) + " " + it.first + " = " + it.second); - } - - Holder pack = newTextPack(); - for (const auto &it : texts) - pack->set(HashString(it.first), it.second); - - Holder> buff = pack->exportBuffer(); - CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "buffer size (before compression): " + buff.size()); - Holder> comp = compress(buff); - CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "buffer size (after compression): " + comp.size()); - - AssetHeader h = initializeAssetHeader(); - h.originalSize = buff.size(); - h.compressedSize = comp.size(); - Holder f = writeFile(outputFileName); - f->write(bufferView(h)); - f->write(comp); - f->close(); -} diff --git a/sources/asset-processor/texts.cpp b/sources/asset-processor/texts.cpp new file mode 100644 index 00000000..3aac456d --- /dev/null +++ b/sources/asset-processor/texts.cpp @@ -0,0 +1,150 @@ +#include "processor.h" + +#include + +namespace +{ + LanguageCode extractLanguage(const String &pth) + { + // eg. pth = dir/gui.en_US.po + const String n = pathExtractFilenameNoExtension(pth); // -> gui.en_US + const String l = subString(pathExtractExtension(n), 1, m); // -> en_US + // eg. inputFile = dir/gui.pot + const String p = pathExtractFilenameNoExtension(inputFile); // -> gui + if (p + "." + l == n) + return l; + return ""; + } + + void parsePO(Holder f, const LanguageCode &lang, Texts *txt) + { + String id, str, l; + + const auto "es = [](const String &s) -> String + { + if (s.length() >= 2 && s[0] == '\"' && s[s.length() - 1] == '\"') + return subString(s, 1, s.length() - 2); + return s; + }; + + const auto &add = [&]() + { + if (!id.empty() && !str.empty()) + txt->set(quotes(id), quotes(str), lang); + else if (!id.empty() || !str.empty()) + { + CAGE_LOG(SeverityEnum::Note, "assetProcessor", id); + CAGE_LOG(SeverityEnum::Note, "assetProcessor", str); + CAGE_THROW_ERROR(Exception, "missing msgstr or msgid"); + } + id = str = ""; + }; + + while (f->readLine(l)) + { + l = trim(l); + if (l.empty()) + add(); + else if (l[0] != '#') + { + const String p = l; + String k = split(l, "\t "); + k = toLower(k); + if (k == "msgid") + { + if (!id.empty()) + { + CAGE_LOG(SeverityEnum::Note, "assetProcessor", id); + CAGE_LOG(SeverityEnum::Note, "assetProcessor", p); + CAGE_THROW_ERROR(Exception, "multiple msgid"); + } + id = l; + } + else if (k == "msgstr") + { + if (!str.empty()) + { + CAGE_LOG(SeverityEnum::Note, "assetProcessor", id); + CAGE_LOG(SeverityEnum::Note, "assetProcessor", p); + CAGE_THROW_ERROR(Exception, "multiple msgstr"); + } + str = l; + } + else + { + CAGE_LOG(SeverityEnum::Note, "assetProcessor", p); + CAGE_THROW_ERROR(Exception, "unknown command"); + } + } + } + add(); + } +} + +void processTexts() +{ + writeLine(String("use=") + inputFile); + + { + const String ext = pathExtractExtension(inputFile); + if (ext != ".pot" && ext != ".po") + CAGE_THROW_ERROR(Exception, "input file must have .pot or .po extension"); + } + + Holder txt = newTexts(); + + if (toBool(properties("multilingual"))) + { + { // validate template file + CAGE_LOG(SeverityEnum::Info, "assetProcessor", "validating template"); + Holder tmp = newTexts(); + parsePO(readFile(inputFileName), "", +tmp); + } + + for (const String &pth : pathListDirectory(pathExtractDirectory(inputFileName))) + { + if (!pathIsFile(pth)) + continue; + if (pathExtractExtension(pth) != ".po") + continue; + const LanguageCode lang = extractLanguage(pth); + if (lang.empty()) + continue; + writeLine(String("use=") + pathToRel(pth, inputDirectory)); + CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "loading language: " + lang); + parsePO(readFile(pth), lang, +txt); + } + } + else + { + parsePO(readFile(inputFileName), "", +txt); + } + + { + const auto l = txt->allLanguages(); + if (l.empty()) + CAGE_THROW_ERROR(Exception, "loaded no languages"); + for (const String &n : l) + CAGE_LOG(SeverityEnum::Info, "language", n); + } + { + const auto l = txt->allNames(); + if (l.empty()) + CAGE_THROW_ERROR(Exception, "loaded no texts"); + for (const String &n : l) + CAGE_LOG(SeverityEnum::Info, "name", n); + } + + Holder> buff = txt->exportBuffer(); + CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "buffer size (before compression): " + buff.size()); + Holder> comp = compress(buff); + CAGE_LOG(SeverityEnum::Info, "assetProcessor", Stringizer() + "buffer size (after compression): " + comp.size()); + + AssetHeader h = initializeAssetHeader(); + h.originalSize = buff.size(); + h.compressedSize = comp.size(); + Holder f = writeFile(outputFileName); + f->write(bufferView(h)); + f->write(comp); + f->close(); +} diff --git a/sources/include/cage-core/assetContext.h b/sources/include/cage-core/assetContext.h index 48745896..3afc5a9f 100644 --- a/sources/include/cage-core/assetContext.h +++ b/sources/include/cage-core/assetContext.h @@ -11,7 +11,7 @@ namespace cage struct CAGE_CORE_API AssetContext : private Immovable { - detail::StringBase<64> textName; + detail::StringBase<64> textId; Holder customData; Holder> dependencies; Holder> compressedData; diff --git a/sources/include/cage-core/assetHeader.h b/sources/include/cage-core/assetHeader.h index c4734035..f5a6fce0 100644 --- a/sources/include/cage-core/assetHeader.h +++ b/sources/include/cage-core/assetHeader.h @@ -12,7 +12,7 @@ namespace cage char cageName[8] = "cageAss"; uint32 version = 0; uint32 flags = 0; - char textName[64] = {}; + char textId[64] = {}; uint64 compressedSize = 0; uint64 originalSize = 0; uint16 scheme = m; diff --git a/sources/include/cage-core/assetManager.h b/sources/include/cage-core/assetManager.h index 110772a9..38b47248 100644 --- a/sources/include/cage-core/assetManager.h +++ b/sources/include/cage-core/assetManager.h @@ -26,17 +26,17 @@ namespace cage uint32 generateUniqueName(); template - void loadValue(uint32 assetName, Holder &&value, const String &textName = "") + void loadValue(uint32 assetName, Holder &&value, const String &textId = "") { CAGE_ASSERT(detail::typeHash() == schemeTypeHash_(Scheme)) - load_(Scheme, assetName, textName, std::move(value).template cast()); + load_(Scheme, assetName, textId, std::move(value).template cast()); } template - void loadCustom(uint32 assetName, const AssetScheme &customScheme, Holder &&customData, const String &textName = "") + void loadCustom(uint32 assetName, const AssetScheme &customScheme, Holder &&customData, const String &textId = "") { CAGE_ASSERT(detail::typeHash() == schemeTypeHash_(Scheme)) - load_(Scheme, assetName, textName, customScheme, std::move(customData)); + load_(Scheme, assetName, textId, customScheme, std::move(customData)); } // returns null if the asset is not yet loaded or has different scheme @@ -61,8 +61,8 @@ namespace cage private: void defineScheme_(uint32 typeHash, uint32 scheme, const AssetScheme &value); - void load_(uint32 scheme, uint32 assetName, const String &textName, Holder &&value); - void load_(uint32 scheme, uint32 assetName, const String &textName, const AssetScheme &customScheme, Holder &&customData); + void load_(uint32 scheme, uint32 assetName, const String &textId, Holder &&value); + void load_(uint32 scheme, uint32 assetName, const String &textId, const AssetScheme &customScheme, Holder &&customData); Holder get_(uint32 scheme, uint32 assetName) const; uint32 schemeTypeHash_(uint32 scheme) const; friend class AssetOnDemand; diff --git a/sources/include/cage-core/textPack.h b/sources/include/cage-core/textPack.h deleted file mode 100644 index bdcfea85..00000000 --- a/sources/include/cage-core/textPack.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef guard_textPack_h_B436B597745C461DAA266CE6FBBE10D1 -#define guard_textPack_h_B436B597745C461DAA266CE6FBBE10D1 - -#include - -namespace cage -{ - class CAGE_CORE_API TextPack : private Immovable - { - public: - void clear(); - Holder copy() const; - - void importBuffer(PointerRange buffer); - Holder> exportBuffer() const; - - void set(uint32 name, const String &text); - void erase(uint32 name); - - String get(uint32 name) const; - String format(uint32 name, PointerRange params) const; - - static String format(const String &format, PointerRange params); - }; - - CAGE_CORE_API Holder newTextPack(); - - CAGE_CORE_API AssetScheme genAssetSchemeTextPack(); - constexpr uint32 AssetSchemeIndexTextPack = 2; - - CAGE_CORE_API String loadFormattedString(AssetManager *assets, uint32 asset, uint32 text, String params); -} - -#endif // guard_textPack_h_B436B597745C461DAA266CE6FBBE10D1 diff --git a/sources/include/cage-core/texts.h b/sources/include/cage-core/texts.h new file mode 100644 index 00000000..8235d05c --- /dev/null +++ b/sources/include/cage-core/texts.h @@ -0,0 +1,44 @@ +#ifndef guard_texts_h_B436B597745C461DAA266CE6FBBE10D1 +#define guard_texts_h_B436B597745C461DAA266CE6FBBE10D1 + +#include + +namespace cage +{ + using LanguageCode = detail::StringBase<10>; + + class CAGE_CORE_API Texts : private Immovable + { + public: + void clear(); + Holder copy() const; + + void importBuffer(PointerRange buffer); + Holder> exportBuffer() const; + + void set(uint32 id, const String &text, const LanguageCode &language); + uint32 set(const String &name, const String &text, const LanguageCode &language); + + Holder> allLanguages() const; + Holder> allNames() const; + Holder> allIds() const; + + // returns empty string if not found + String get(uint32 id, const LanguageCode &language) const; + }; + + CAGE_CORE_API Holder newTexts(); + + CAGE_CORE_API AssetScheme genAssetSchemeTexts(); + constexpr uint32 AssetSchemeIndexTexts = 2; + + CAGE_CORE_API void textsSetLanguages(PointerRange languages); + CAGE_CORE_API void textsSetLanguages(const String &languages); // list of languages separated by ; + CAGE_CORE_API void textsAdd(const Texts *txt); + CAGE_CORE_API void textsRemove(const Texts *txt); + CAGE_CORE_API String textsGet(uint32 id, String params = ""); + + CAGE_CORE_API String textFormat(String format, const String ¶ms); +} + +#endif // guard_texts_h_B436B597745C461DAA266CE6FBBE10D1 diff --git a/sources/include/cage-engine/guiBuilder.h b/sources/include/cage-engine/guiBuilder.h index 07e158f2..2ff669c4 100644 --- a/sources/include/cage-engine/guiBuilder.h +++ b/sources/include/cage-engine/guiBuilder.h @@ -25,7 +25,7 @@ namespace cage BuilderItem text(const GuiTextComponent &txt); BuilderItem text(const String &txt); - BuilderItem text(uint32 assetName, uint32 textName, const String ¶meters = ""); + BuilderItem text(uint32 textId, const String ¶meters = ""); BuilderItem textFormat(const GuiTextFormatComponent &textFormat); BuilderItem textColor(Vec3 color); BuilderItem textSize(Real size); @@ -46,10 +46,10 @@ namespace cage BuilderItem update(Delegate u); BuilderItem tooltip(const GuiTooltipComponent &t); - template + template BuilderItem tooltip(uint64 delay = GuiTooltipComponent().delay) { - (*this)->template value().tooltip = detail::guiTooltipText(); + (*this)->template value().tooltip = detail::guiTooltipText(); (*this)->template value().delay = delay; return *this; } diff --git a/sources/include/cage-engine/guiComponents.h b/sources/include/cage-engine/guiComponents.h index 04b6cf75..67041d6d 100644 --- a/sources/include/cage-engine/guiComponents.h +++ b/sources/include/cage-engine/guiComponents.h @@ -39,8 +39,7 @@ namespace cage struct CAGE_ENGINE_API GuiTextComponent { String value; // list of parameters separated by '|' when formatted, otherwise the string as is - uint32 assetName = 0; - uint32 textName = 0; + uint32 textId = 0; }; struct CAGE_ENGINE_API GuiTextFormatComponent @@ -306,10 +305,10 @@ namespace cage CAGE_ENGINE_API void guiDestroyChildrenRecursively(Entity *e); CAGE_ENGINE_API void guiDestroyEntityRecursively(Entity *e); - template + template GuiTooltipComponent::TooltipCallback guiTooltipText() noexcept { - static constexpr GuiTextComponent txt{ Text.value, AssetName, TextName }; + static constexpr GuiTextComponent txt{ Text.value, TextId }; return privat::guiTooltipText(&txt); } } diff --git a/sources/include/cage-engine/scene.h b/sources/include/cage-engine/scene.h index 1e366055..4ff59080 100644 --- a/sources/include/cage-engine/scene.h +++ b/sources/include/cage-engine/scene.h @@ -71,12 +71,11 @@ namespace cage Vec3 color = Vec3(1); // sRGB Real intensity = 1; // real opacity; // todo - uint32 assetName = 0; - uint32 textName = 0; + uint32 textId = 0; uint32 font = 0; uint32 sceneMask = 1; Real lineSpacing = 1; - TextAlignEnum align = TextAlignEnum::Left; + TextAlignEnum align = TextAlignEnum::Center; }; enum class CameraTypeEnum : uint32 diff --git a/sources/libcore/assets/assetManager.cpp b/sources/libcore/assets/assetManager.cpp index 4d54473f..ec34d266 100644 --- a/sources/libcore/assets/assetManager.cpp +++ b/sources/libcore/assets/assetManager.cpp @@ -50,8 +50,8 @@ namespace cage bool unloading = false; explicit Asset(AssetManagerImpl *impl, uint32 realName); - explicit Asset(AssetManagerImpl *impl, uint32 scheme, uint32 realName, const String &textName, Holder &&value); - explicit Asset(AssetManagerImpl *impl, uint32 scheme, uint32 realName, const String &textName, const AssetScheme &customScheme, Holder &&customData); + explicit Asset(AssetManagerImpl *impl, uint32 scheme, uint32 realName, const String &textId, Holder &&value); + explicit Asset(AssetManagerImpl *impl, uint32 scheme, uint32 realName, const String &textId, const AssetScheme &customScheme, Holder &&customData); ~Asset(); }; @@ -330,14 +330,14 @@ namespace cage Asset::Asset(AssetManagerImpl *impl, uint32 realName) : AssetContext(realName), impl(impl) { - textName = Stringizer() + "<" + realName + ">"; + textId = Stringizer() + "<" + realName + ">"; impl->existsCounter++; fetch = defaultFetch; } Asset::Asset(AssetManagerImpl *impl, uint32 scheme_, uint32 realName, const String &textName_, Holder &&value_) : AssetContext(realName), impl(impl) { - textName = textName_.empty() ? (Stringizer() + "<" + realName + "> with value") : textName_; + textId = textName_.empty() ? (Stringizer() + "<" + realName + "> with value") : textName_; scheme = scheme_; impl->existsCounter++; assetHolder = std::move(value_); @@ -345,7 +345,7 @@ namespace cage Asset::Asset(AssetManagerImpl *impl, uint32 scheme_, uint32 realName, const String &textName_, const AssetScheme &customScheme_, Holder &&customData_) : AssetContext(realName), impl(impl) { - textName = textName_.empty() ? (Stringizer() + "<" + realName + "> with custom scheme") : textName_; + textId = textName_.empty() ? (Stringizer() + "<" + realName + "> with custom scheme") : textName_; scheme = scheme_; impl->existsCounter++; *(AssetScheme *)this = customScheme_; @@ -374,7 +374,7 @@ namespace cage ScopeExit exit([&] { finish(); }); ProfilingScope profiling("asset fetching"); - profiling.set(asset->textName); + profiling.set(asset->textId); CAGE_ASSERT(asset->fetch); try @@ -409,7 +409,7 @@ namespace cage ScopeExit exit([&] { finish(); }); ProfilingScope profiling("asset decompression"); - profiling.set(asset->textName); + profiling.set(asset->textId); CAGE_ASSERT(!asset->failed); CAGE_ASSERT(!asset->assetHolder); CAGE_ASSERT(asset->decompress); @@ -436,7 +436,7 @@ namespace cage ScopeExit exit([&] { finish(); }); ProfilingScope profiling("asset loading"); - profiling.set(asset->textName); + profiling.set(asset->textId); CAGE_ASSERT(!asset->failed); CAGE_ASSERT(!asset->assetHolder); CAGE_ASSERT(asset->load); @@ -482,7 +482,7 @@ namespace cage ScopeExit exit([&] { finish(); }); ProfilingScope profiling("asset unloading"); - profiling.set(asset->textName); + profiling.set(asset->textId); asset->assetHolder.clear(); @@ -701,7 +701,7 @@ namespace cage impl->schemes[scheme] = value; } - void AssetManager::load_(uint32 scheme, uint32 realName, const String &textName, Holder &&value) + void AssetManager::load_(uint32 scheme, uint32 realName, const String &textId, Holder &&value) { AssetManagerImpl *impl = (AssetManagerImpl *)this; CAGE_ASSERT(scheme < impl->schemes.size()); @@ -709,12 +709,12 @@ namespace cage auto &c = impl->privateIndex[realName]; c.fabricated = true; c.references++; - Holder asset = systemMemory().createHolder(impl, scheme, realName, textName, std::move(value)); + Holder asset = systemMemory().createHolder(impl, scheme, realName, textId, std::move(value)); c.versions.insert(c.versions.begin(), asset.share()); impl->publish(std::move(asset)); } - void AssetManager::load_(uint32 scheme, uint32 realName, const String &textName, const AssetScheme &customScheme, Holder &&customData) + void AssetManager::load_(uint32 scheme, uint32 realName, const String &textId, const AssetScheme &customScheme, Holder &&customData) { AssetManagerImpl *impl = (AssetManagerImpl *)this; CAGE_ASSERT(scheme < impl->schemes.size()); @@ -724,7 +724,7 @@ namespace cage auto &c = impl->privateIndex[realName]; c.fabricated = true; c.references++; - Holder asset = systemMemory().createHolder(impl, scheme, realName, textName, customScheme, std::move(customData)); + Holder asset = systemMemory().createHolder(impl, scheme, realName, textId, customScheme, std::move(customData)); c.versions.insert(c.versions.begin(), asset.share()); lock.clear(); if (asset->fetch) @@ -756,7 +756,7 @@ namespace cage return {}; // different scheme if (a->failed) { - CAGE_LOG_THROW(Stringizer() + "asset real name: " + realName + ", text name: " + a->textName); + CAGE_LOG_THROW(Stringizer() + "asset real name: " + realName + ", text name: " + a->textId); CAGE_THROW_ERROR(Exception, "accessing asset that failed to load"); } CAGE_ASSERT(a->ref); @@ -809,9 +809,9 @@ namespace cage CAGE_THROW_ERROR(Exception, "file is not a cage asset"); if (h.version != CurrentAssetVersion) CAGE_THROW_ERROR(Exception, "cage asset version mismatch"); - if (h.textName[sizeof(h.textName) - 1] != 0) + if (h.textId[sizeof(h.textId) - 1] != 0) CAGE_THROW_ERROR(Exception, "cage asset text name not bounded"); - asset->textName = h.textName; + asset->textId = h.textId; if (h.scheme >= impl->schemes.size()) CAGE_THROW_ERROR(Exception, "cage asset scheme out of range"); asset->scheme = h.scheme; @@ -855,12 +855,12 @@ namespace cage { version = CurrentAssetVersion; String name = name_; - static constexpr uint32 MaxTexName = sizeof(textName) - 1; + static constexpr uint32 MaxTexName = sizeof(textId) - 1; if (name.length() > MaxTexName) name = String() + ".." + subString(name, name.length() - MaxTexName + 2, m); CAGE_ASSERT(name.length() <= MaxTexName); CAGE_ASSERT(name.length() > 0); - detail::memcpy(textName, name.c_str(), name.length()); + detail::memcpy(textId, name.c_str(), name.length()); scheme = schemeIndex; } } diff --git a/sources/libcore/assets/assetStructs.cpp b/sources/libcore/assets/assetStructs.cpp index 94c3d12f..c5c6539c 100644 --- a/sources/libcore/assets/assetStructs.cpp +++ b/sources/libcore/assets/assetStructs.cpp @@ -5,8 +5,6 @@ #include #include #include -#include -#include namespace cage { @@ -46,49 +44,6 @@ namespace cage return s; } - namespace - { - void processTextPackLoad(AssetContext *context) - { - Holder texts = newTextPack(); - Deserializer des(context->originalData); - uint32 cnt; - des >> cnt; - for (uint32 i = 0; i < cnt; i++) - { - uint32 name; - String val; - des >> name >> val; - texts->set(name, val); - } - context->assetHolder = std::move(texts).cast(); - } - } - - AssetScheme genAssetSchemeTextPack() - { - AssetScheme s; - s.load.bind<&processTextPackLoad>(); - s.typeHash = detail::typeHash(); - return s; - } - - String loadFormattedString(AssetManager *assets, uint32 asset, uint32 text, String params) - { - if (asset == 0 || text == 0) - return params; - auto a = assets->get(asset); - if (a) - { - std::vector ps; - while (!params.empty()) - ps.push_back(split(params, "|")); - return a->format(text, ps); - } - else - return ""; - } - namespace { void processColliderLoad(AssetContext *context) diff --git a/sources/libcore/assets/texts.cpp b/sources/libcore/assets/texts.cpp new file mode 100644 index 00000000..146cb2ba --- /dev/null +++ b/sources/libcore/assets/texts.cpp @@ -0,0 +1,290 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cage +{ + namespace + { + struct LanguageValues : private Noncopyable + { + LanguageCode lang; + ankerl::unordered_dense::map values; + }; + + Serializer &operator<<(Serializer &s, const LanguageValues &v) + { + s << v.lang << v.values; + return s; + } + + Deserializer &operator>>(Deserializer &s, LanguageValues &v) + { + s >> v.lang >> v.values; + return s; + } + + class TextsImpl : public Texts + { + public: + std::vector values; + FlatSet languages; + ankerl::unordered_dense::set names; + FlatSet ids; + + void clear() + { + values.clear(); + languages.clear(); + names.clear(); + ids.clear(); + } + + void importBuffer(PointerRange buffer) + { + clear(); + Deserializer des(buffer); + des >> values >> languages >> names >> ids; + CAGE_ASSERT(des.available() == 0); + } + + Holder> exportBuffer() const + { + MemoryBuffer buff; + Serializer ser(buff); + ser << values << languages << names << ids; + return std::move(buff); + } + + ankerl::unordered_dense::map &map(const LanguageCode &lang) + { + for (auto &it : values) + { + if (it.lang == lang) + return it.values; + } + values.push_back({}); + values.back().lang = lang; + languages.insert(lang); + return values.back().values; + } + }; + } + + void Texts::clear() + { + TextsImpl *impl = (TextsImpl *)this; + impl->clear(); + } + + Holder Texts::copy() const + { + Holder res = newTexts(); + res->importBuffer(exportBuffer()); // todo more efficient + return res; + } + + Holder> Texts::exportBuffer() const + { + const TextsImpl *impl = (const TextsImpl *)this; + return impl->exportBuffer(); + } + + void Texts::importBuffer(PointerRange buffer) + { + TextsImpl *impl = (TextsImpl *)this; + impl->importBuffer(buffer); + } + + void Texts::set(uint32 id, const String &text, const LanguageCode &language) + { + CAGE_ASSERT(id != 0); + TextsImpl *impl = (TextsImpl *)this; + impl->map(language)[id] = text; + impl->ids.insert(id); + } + + uint32 Texts::set(const String &name, const String &text, const LanguageCode &language) + { + CAGE_ASSERT(name != ""); + TextsImpl *impl = (TextsImpl *)this; + const uint32 id = HashString(name); + set(id, text, language); + impl->names.insert(name); + return id; + } + + Holder> Texts::allLanguages() const + { + const TextsImpl *impl = (const TextsImpl *)this; + return PointerRangeHolder(impl->languages.begin(), impl->languages.end()); + } + + Holder> Texts::allNames() const + { + const TextsImpl *impl = (const TextsImpl *)this; + return PointerRangeHolder(impl->names.begin(), impl->names.end()); + } + + Holder> Texts::allIds() const + { + const TextsImpl *impl = (const TextsImpl *)this; + return PointerRangeHolder(impl->ids.begin(), impl->ids.end()); + } + + String Texts::get(uint32 id, const LanguageCode &language) const + { + CAGE_ASSERT(id != 0); + const TextsImpl *impl = (const TextsImpl *)this; + for (const auto &it : impl->values) + { + if (it.lang == language) + { + auto it2 = it.values.find(id); + if (it2 != it.values.end()) + return it2->second; + break; + } + } + return ""; + } + + Holder newTexts() + { + return systemMemory().createImpl(); + } + + namespace + { + struct TextsAsset + { + Holder t; + + ~TextsAsset() + { + if (t) + textsRemove(+t); + } + }; + + void processTextsLoad(AssetContext *context) + { + Holder h = newTexts(); + h->importBuffer(context->originalData); + Holder ass = systemMemory().createHolder(); + ass->t = std::move(h); + textsAdd(+ass->t); + Holder t = Holder(+ass->t, std::move(ass)); + context->assetHolder = std::move(t).cast(); + } + } + + AssetScheme genAssetSchemeTexts() + { + AssetScheme s; + s.load.bind<&processTextsLoad>(); + s.typeHash = detail::typeHash(); + return s; + } + + namespace + { + Holder mut = newRwMutex(); + std::vector languages = { "en_US", "" }; + FlatSet sources; + } + + void textsSetLanguages(PointerRange langs) + { + ScopeLock lock(mut, WriteLockTag()); + languages = std::vector(langs.begin(), langs.end()); + } + + void textsSetLanguages(const String &languages) + { + std::vector v; + String l = languages; + while (!l.empty()) + v.push_back(split(l, ";")); + textsSetLanguages(v); + } + + void textsAdd(const Texts *txt) + { + ScopeLock lock(mut, WriteLockTag()); + sources.insert(txt); + } + + void textsRemove(const Texts *txt) + { + ScopeLock lock(mut, WriteLockTag()); + sources.erase(txt); + } + + String textsGet(uint32 id, String params) + { + if (id == 0) + return params; + ScopeLock lock(mut, ReadLockTag()); + for (const LanguageCode &lang : languages) + { + for (const auto src : sources) + { + const String s = src->get(id, lang); + if (!s.empty()) + return textFormat(s, params); + } + } + return params; + } + + String textFormat(String res, const String ¶ms) + { + uint32 pc = 1; + for (char c : params) + pc += c == '|'; + + const auto &find = [&](uint32 idx) -> String + { + if (idx > pc) + return ""; + uint32 i = 0; + uint32 k = 0; + while (k < idx) + k += params[i++] == '|'; + String res; + while (i < params.length() && params[i] != '|') + res += String(params[i++]); + return res; + }; + + while (true) + { + String prev = split(res, "{"); + if (res == "") + return prev + res; + String mid = split(res, "}"); + if (!mid.empty() && isDigitsOnly(mid)) + { + const uint32 idx = toUint32(mid); + mid = find(idx); + } + else + mid = ""; + res = prev + mid + res; + } + return res; + } +} diff --git a/sources/libcore/textPack.cpp b/sources/libcore/textPack.cpp deleted file mode 100644 index 02441173..00000000 --- a/sources/libcore/textPack.cpp +++ /dev/null @@ -1,119 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -namespace cage -{ - namespace - { - class TextPackImpl : public TextPack - { - public: - ankerl::unordered_dense::map texts; - }; - } - - String TextPack::format(const String &format, PointerRange params) - { - String res = format; - while (true) - { - String prev = split(res, "{"); - if (res == "") - return prev + res; - String mid = split(res, "}"); - if (isDigitsOnly(mid)) - { - uint32 idx = toUint32(mid); - if (idx < params.size()) - mid = params[idx]; - else - mid = ""; - } - else - mid = ""; - res = prev + mid + res; - } - } - - void TextPack::clear() - { - TextPackImpl *impl = (TextPackImpl *)this; - impl->texts.clear(); - } - - Holder TextPack::copy() const - { - Holder res = newTextPack(); - res->importBuffer(exportBuffer()); // todo more efficient - return res; - } - - Holder> TextPack::exportBuffer() const - { - TextPackImpl *impl = (TextPackImpl *)this; - MemoryBuffer buff; - Serializer ser(buff); - ser << numeric_cast(impl->texts.size()); - for (const auto &it : impl->texts) - ser << it.first << it.second; - return PointerRangeHolder(PointerRange(buff)); - } - - void TextPack::importBuffer(PointerRange buffer) - { - TextPackImpl *impl = (TextPackImpl *)this; - clear(); - Deserializer des(buffer); - uint32 cnt = 0; - des >> cnt; - impl->texts.reserve(cnt); - for (uint32 i = 0; i < cnt; i++) - { - uint32 n = 0; - String s; - des >> n >> s; - set(n, s); - } - CAGE_ASSERT(des.available() == 0); - } - - void TextPack::set(uint32 name, const String &text) - { - CAGE_ASSERT(name != 0); - TextPackImpl *impl = (TextPackImpl *)this; - impl->texts[name] = text; - } - - void TextPack::erase(uint32 name) - { - CAGE_ASSERT(name != 0); - TextPackImpl *impl = (TextPackImpl *)this; - impl->texts.erase(name); - } - - String TextPack::get(uint32 name) const - { - CAGE_ASSERT(name != 0); - const TextPackImpl *impl = (const TextPackImpl *)this; - auto it = impl->texts.find(name); - if (it == impl->texts.end()) - return ""; - return it->second; - } - - String TextPack::format(uint32 name, PointerRange params) const - { - CAGE_ASSERT(name != 0); - return format(get(name), params); - } - - Holder newTextPack() - { - return systemMemory().createImpl(); - } -} diff --git a/sources/libengine/assets/font.cpp b/sources/libengine/assets/font.cpp index 4cbdc20c..e6f2a3a8 100644 --- a/sources/libengine/assets/font.cpp +++ b/sources/libengine/assets/font.cpp @@ -12,7 +12,7 @@ namespace cage void processLoad(AssetContext *context) { Holder font = newFont(); - font->setDebugName(context->textName); + font->setDebugName(context->textId); Deserializer des(context->originalData); FontHeader data; diff --git a/sources/libengine/assets/model.cpp b/sources/libengine/assets/model.cpp index c73120e2..a7ea0704 100644 --- a/sources/libengine/assets/model.cpp +++ b/sources/libengine/assets/model.cpp @@ -36,7 +36,7 @@ namespace cage msh->layer = data.renderLayer; msh->bones = data.skeletonBones; - msh->setDebugName(context->textName); // last command to apply it to all subresources + msh->setDebugName(context->textId); // last command to apply it to all subresources context->assetHolder = std::move(msh).cast(); } diff --git a/sources/libengine/assets/renderObject.cpp b/sources/libengine/assets/renderObject.cpp index 7fca908d..16a08d39 100644 --- a/sources/libengine/assets/renderObject.cpp +++ b/sources/libengine/assets/renderObject.cpp @@ -12,7 +12,7 @@ namespace cage void processLoad(AssetContext *context) { Holder obj = newRenderObject(); - obj->setDebugName(context->textName); + obj->setDebugName(context->textId); Deserializer des(context->originalData); RenderObjectHeader h; diff --git a/sources/libengine/assets/shaderProgram.cpp b/sources/libengine/assets/shaderProgram.cpp index 0700e944..33c17665 100644 --- a/sources/libengine/assets/shaderProgram.cpp +++ b/sources/libengine/assets/shaderProgram.cpp @@ -14,7 +14,7 @@ namespace cage void processLoad(AssetContext *context) { Holder shr = newMultiShaderProgram(); - shr->setDebugName(context->textName); + shr->setDebugName(context->textId); Deserializer des(context->originalData); { diff --git a/sources/libengine/assets/sound.cpp b/sources/libengine/assets/sound.cpp index 7b049571..a9e86d6b 100644 --- a/sources/libengine/assets/sound.cpp +++ b/sources/libengine/assets/sound.cpp @@ -61,7 +61,7 @@ namespace cage CAGE_ASSERT(snd.frames == poly->frames()); CAGE_ASSERT(snd.sampleRate == poly->sampleRate()); Holder source = newSound(); - source->setDebugName(context->textName); + source->setDebugName(context->textId); source->referenceDistance = snd.referenceDistance; source->rolloffFactor = snd.rolloffFactor; source->gain = snd.gain; diff --git a/sources/libengine/assets/texture.cpp b/sources/libengine/assets/texture.cpp index 1d6d2b92..77fce460 100644 --- a/sources/libengine/assets/texture.cpp +++ b/sources/libengine/assets/texture.cpp @@ -68,7 +68,7 @@ namespace cage des >> data; Holder tex = newTexture(data.target); - tex->setDebugName(context->textName); + tex->setDebugName(context->textId); tex->filters(data.filterMin, data.filterMag, data.filterAniso); tex->wraps(data.wrapX, data.wrapY, data.wrapZ); diff --git a/sources/libengine/graphics/renderPipeline.cpp b/sources/libengine/graphics/renderPipeline.cpp index c96e99b5..d393c406 100644 --- a/sources/libengine/graphics/renderPipeline.cpp +++ b/sources/libengine/graphics/renderPipeline.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include // all the constants @@ -860,7 +860,7 @@ namespace cage prepare.font = assets->get(pt.font); if (!prepare.font) return; - const String str = loadFormattedString(assets, pt.assetName, pt.textName, pt.value); + const String str = textsGet(pt.textId, pt.value); const uint32 count = prepare.font->glyphsCount(str); if (count == 0) return; diff --git a/sources/libengine/gui/guiBuilder.cpp b/sources/libengine/gui/guiBuilder.cpp index a2554f69..69499626 100644 --- a/sources/libengine/gui/guiBuilder.cpp +++ b/sources/libengine/gui/guiBuilder.cpp @@ -109,9 +109,9 @@ namespace cage return *this; } - BuilderItem BuilderItem::text(uint32 assetName, uint32 textName, const String ¶meters) + BuilderItem BuilderItem::text(uint32 textId, const String ¶meters) { - return text(GuiTextComponent{ parameters, assetName, textName }); + return text(GuiTextComponent{ parameters, textId }); } BuilderItem BuilderItem::textFormat(const GuiTextFormatComponent &textFormat) diff --git a/sources/libengine/gui/items.cpp b/sources/libengine/gui/items.cpp index 90ce2b80..3dc9d0db 100644 --- a/sources/libengine/gui/items.cpp +++ b/sources/libengine/gui/items.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include namespace cage @@ -299,7 +299,7 @@ namespace cage void TextItem::transcript() { GUI_COMPONENT(Text, t, hierarchy->ent); - String value = loadFormattedString(hierarchy->impl->assetMgr, t.assetName, t.textName, t.value); + String value = textsGet(t.textId, t.value); transcript(value); } diff --git a/sources/libsimple/gameloop.cpp b/sources/libsimple/gameloop.cpp index dadb182d..d2174502 100644 --- a/sources/libsimple/gameloop.cpp +++ b/sources/libsimple/gameloop.cpp @@ -18,7 +18,7 @@ #include #include #include // for sizeof in defineScheme -#include // for sizeof in defineScheme +#include // for sizeof in defineScheme #include #include #include @@ -529,7 +529,7 @@ namespace cage // core assets assets->defineScheme(genAssetSchemePack()); assets->defineScheme>(genAssetSchemeRaw()); - assets->defineScheme(genAssetSchemeTextPack()); + assets->defineScheme(genAssetSchemeTexts()); assets->defineScheme(genAssetSchemeCollider()); assets->defineScheme(genAssetSchemeSkeletonRig()); assets->defineScheme(genAssetSchemeSkeletalAnimation()); diff --git a/sources/test-core/assetManager.cpp b/sources/test-core/assetManager.cpp index 018bd416..633aebea 100644 --- a/sources/test-core/assetManager.cpp +++ b/sources/test-core/assetManager.cpp @@ -539,7 +539,7 @@ void testAssetManager() CAGE_TEST(String(ass.cageName) == "cageAss"); CAGE_TEST(ass.version > 0); CAGE_TEST(ass.flags == 0); - CAGE_TEST(String(ass.textName) == "abcdefghijklmnopqrstuvwxyz"); + CAGE_TEST(String(ass.textId) == "abcdefghijklmnopqrstuvwxyz"); CAGE_TEST(ass.compressedSize == 0); CAGE_TEST(ass.originalSize == 0); CAGE_TEST(ass.scheme == 42); @@ -551,7 +551,7 @@ void testAssetManager() CAGE_TEST(String(ass.cageName) == "cageAss"); CAGE_TEST(ass.version > 0); CAGE_TEST(ass.flags == 0); - CAGE_TEST(String(ass.textName) == "..rstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"); + CAGE_TEST(String(ass.textId) == "..rstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"); CAGE_TEST(ass.compressedSize == 0); CAGE_TEST(ass.originalSize == 0); CAGE_TEST(ass.scheme == 13); diff --git a/sources/test-core/strings.cpp b/sources/test-core/strings.cpp index 488e3ad7..f9d3db13 100644 --- a/sources/test-core/strings.cpp +++ b/sources/test-core/strings.cpp @@ -9,6 +9,7 @@ #include #include #include +#include void test(Real a, Real b); @@ -913,6 +914,19 @@ namespace for (const auto &v : vec) CAGE_TEST(*o++ == v); } + + void testTextsFormat() + { + CAGE_TEST(textFormat("", "") == ""); + CAGE_TEST(textFormat("abc", "") == "abc"); + CAGE_TEST(textFormat("", "abc") == ""); + CAGE_TEST(textFormat("abc{0}ghi", "def") == "abcdefghi"); + CAGE_TEST(textFormat("{1}def{0}", "ghi|abc") == "abcdefghi"); + CAGE_TEST(textFormat("abc{}ghi", "def") == "abcghi"); + CAGE_TEST(textFormat("abc{def}ghi", "def") == "abcghi"); + CAGE_TEST(textFormat("{3}def{2}", "hola|ho|ghi|abc|jey") == "abcdefghi"); + CAGE_TEST(textFormat("abc{10}ghi", "def|juj") == "abcghi"); + } } void testStrings() @@ -932,4 +946,5 @@ void testStrings() testStringizer(); testNaturalSortBasics(); testNaturalSortRandom(); + testTextsFormat(); }