Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extract chrome strings #21134

Merged
merged 5 commits into from
Oct 21, 2023
Merged

Extract chrome strings #21134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f2c02ab
Remove model slider from common
PunkPun Oct 19, 2023
a82eb34
Fix static linting
PunkPun Oct 18, 2023
5fc41ff
Add automated chrome string extractor.
PunkPun Oct 18, 2023
d228860
Automated translation extraction
PunkPun Oct 21, 2023
90da175
Manual fixup
PunkPun Oct 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
323 changes: 323 additions & 0 deletions OpenRA.Game/UtilityCommands/ExtractChromeStrings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,323 @@
#region Copyright & License Information
/*
* Copyright (c) The OpenRA Developers and Contributors
* This file is part of OpenRA, which is free software. It is made
* available to you under the terms of the GNU General Public License
* as published by the Free Software Foundation, either version 3 of
* the License, or (at your option) any later version. For more
* information, see COPYING.
*/
#endregion

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using OpenRA.Widgets;

namespace OpenRA.UtilityCommands
{
sealed class ExtractChromeStringsCommand : IUtilityCommand
{
string IUtilityCommand.Name { get { return "--extract-chrome-strings"; } }

bool IUtilityCommand.ValidateArguments(string[] args)
{
return true;
}

[Desc("Extract translatable strings that are not yet localized and update chrome layout.")]
void IUtilityCommand.Run(Utility utility, string[] args)
{
// HACK: The engine code assumes that Game.modData is set.
var modData = Game.ModData = utility.ModData;

var translatableFields = modData.ObjectCreator.GetTypes()
.Where(t => t.Name.EndsWith("Widget", StringComparison.InvariantCulture) && t.IsSubclassOf(typeof(Widget)))
.ToDictionary(
t => t.Name[..^6],
t => t.GetFields().Where(f => f.HasAttribute<TranslationReferenceAttribute>()).Select(f => f.Name).ToArray())
.Where(t => t.Value.Any())
.ToDictionary(t => t.Key, t => t.Value);

var chromeLayouts = modData.Manifest.ChromeLayout.GroupBy(c => c.Split('/')[0].Split('|')[0], c => c);

foreach (var layout in chromeLayouts)
{
var fluentFolder = layout.Key + "|languages";
var fluentPackage = modData.ModFiles.OpenPackage(fluentFolder);
var fluentPath = Path.Combine(fluentPackage.Name, "chrome/en.ftl");

var unsortedCandidates = new List<TranslationCandidate>();
var groupedCandidates = new Dictionary<HashSet<string>, List<TranslationCandidate>>();
var chromeFiles = new List<(string Path, List<MiniYamlNodeBuilder> Nodes)>();

// Get all translations.
foreach (var chrome in layout)
{
modData.ModFiles.TryGetPackageContaining(chrome, out var chromePackage, out var chromeName);
var chromePath = Path.Combine(chromePackage.Name, chromeName);

var yaml = MiniYaml.FromFile(chromePath, false).Select(n => new MiniYamlNodeBuilder(n)).ToList();
chromeFiles.Add((chromePath, yaml));

var translationCandidates = new List<TranslationCandidate>();
foreach (var node in yaml)
{
if (node.Key != null)
{
var nodeSplit = node.Key.Split('@');
var nodeId = nodeSplit.Length > 1 ? ClearContainersAndToLower(nodeSplit[1]) : null;
FromChromeLayout(node, translatableFields, nodeId, ref translationCandidates);
}
}

if (translationCandidates.Count > 0)
{
var chromeFilename = chrome.Split('/').Last();
groupedCandidates[new HashSet<string>() { chromeFilename }] = new List<TranslationCandidate>();
for (var i = 0; i < translationCandidates.Count; i++)
{
var candidate = translationCandidates[i];
candidate.Chrome = chromeFilename;
unsortedCandidates.Add(candidate);
}
}
}

// Join matching translations.
foreach (var candidate in unsortedCandidates)
{
HashSet<string> foundHash = null;
TranslationCandidate found = default;
foreach (var (hash, translation) in groupedCandidates)
{
foreach (var c in translation)
{
if (c.Key == candidate.Key && c.Type == candidate.Type && c.Translation == candidate.Translation)
{
foundHash = hash;
found = c;
break;
}
}

if (foundHash != null)
break;
}

if (foundHash == null)
{
var hash = groupedCandidates.Keys.First(t => t.First() == candidate.Chrome);
groupedCandidates[hash].Add(candidate);
continue;
}

var newHash = foundHash.Append(candidate.Chrome).ToHashSet();
candidate.Nodes.AddRange(found.Nodes);
groupedCandidates[foundHash].Remove(found);

var nHash = groupedCandidates.FirstOrDefault(t => t.Key.SetEquals(newHash));
if (nHash.Key != null)
groupedCandidates[nHash.Key].Add(candidate);
else
groupedCandidates[newHash] = new List<TranslationCandidate>() { candidate };
}

// Write to translation and yaml files.
Directory.CreateDirectory(Path.GetDirectoryName(fluentPath));
using (var fluentWriter = new StreamWriter(fluentPath, append: true))
{
foreach (var (chromeFilename, candidates) in groupedCandidates.OrderBy(t => string.Join(',', t.Key)))
{
if (candidates.Count == 0)
continue;

fluentWriter.WriteLine("## " + string.Join(", ", chromeFilename));

// Pushing blocks of translations to string first allows for fancier formatting.
var build = "";
foreach (var grouping in candidates.GroupBy(t => t.Key))
{
if (grouping.Count() == 1)
{
var candidate = grouping.First();
var translationKey = candidate.Key;
if (candidate.Type == "text")
translationKey = $"{translationKey}";
else
translationKey = $"{translationKey}-" + candidate.Type.Replace("text", "");

build += $"{translationKey} = {candidate.Translation}\n";
foreach (var node in candidate.Nodes)
node.Value.Value = translationKey;
}
else
{
if (build.Length > 1 && build.Substring(build.Length - 2, 2) != "\n\n")
build += "\n";

var translationKey = grouping.Key;
build += $"{translationKey} =\n";
foreach (var candidate in grouping)
{
var type = candidate.Type;
if (candidate.Type != "label")
{
if (candidate.Type == "text")
type = "label";
else
type = type.Replace("text", "");
}

build += $" .{type} = {candidate.Translation}\n";
foreach (var node in candidate.Nodes)
node.Value.Value = $"{translationKey}.{type}";
}

build += "\n";
}
}

fluentWriter.WriteLine(build.Trim('\n') + '\n');
}
}

foreach (var chromeFile in chromeFiles)
{
using (var chromeLayoutWriter = new StreamWriter(chromeFile.Path))
chromeLayoutWriter.WriteLine(chromeFile.Nodes.WriteToString());
}
}
}

static bool IsAlreadyTranslated(string translation)
{
if (translation == translation.ToLowerInvariant() && translation.Any(c => c == '-'))
{
Console.WriteLine("Skipping " + translation + " because it is already translated.");
return true;
}

return false;
}

struct TranslationCandidate
{
public string Chrome;
public readonly string Key;
public readonly string Type;
public readonly string Translation;
public readonly List<MiniYamlNodeBuilder> Nodes;

public TranslationCandidate(string key, string type, string translation, MiniYamlNodeBuilder node)
{
Chrome = null;
Key = key;
Type = type;
Translation = translation;
Nodes = new List<MiniYamlNodeBuilder>() { node };
}
}

static string ClearContainersAndToLower(string node)
{
return node
.Replace("Background", "")
.Replace("Container", "")
.Replace("Panel", "")
.ToLowerInvariant()
.Replace("headers", "");
}

static string ClearTypesAndToLower(string node)
{
return node
.Replace("LabelForInput", "Label")
.Replace("LabelWithHighlight", "Label")
.Replace("DropdownButton", "Dropdown")
.Replace("CheckboxButton", "Checkbox")
.Replace("MenuButton", "Button")
.Replace("WorldButton", "Button")
.Replace("ProductionTypeButton", "Button")
.ToLowerInvariant();
}

static void FromChromeLayout(MiniYamlNodeBuilder node, Dictionary<string, string[]> translatables, string container, ref List<TranslationCandidate> translations)
{
var nodeSplit = node.Key.Split('@');
var nodeType = nodeSplit[0];
var nodeId = nodeSplit.Length > 1 ? ClearContainersAndToLower(nodeSplit[1]) : null;

if ((nodeType == "Background" || nodeType == "Container") && nodeId != null)
container = nodeId;

// Get translatable types.
var validChildTypes = new List<(MiniYamlNodeBuilder Node, string Type, string Value)>();
foreach (var childNode in node.Value.Nodes)
{
if (translatables.ContainsKey(nodeType))
{
var childType = childNode.Key.Split('@')[0];
if (translatables[nodeType].Contains(childType)
&& !string.IsNullOrEmpty(childNode.Value.Value)
&& !IsAlreadyTranslated(childNode.Value.Value)
&& childNode.Value.Value.Any(char.IsLetterOrDigit))
{
var translationValue = childNode.Value.Value
.Replace("\\n", "\n ")
.Replace("{", "<")
.Replace("}", ">")
.Trim().Trim('\n');

validChildTypes.Add((childNode, childType.ToLowerInvariant(), translationValue));
}
}
}

// Generate translation key.
if (validChildTypes.Count > 0)
{
nodeType = ClearTypesAndToLower(nodeType);

var translationKey = nodeType;
if (!string.IsNullOrEmpty(container))
{
var containerType = string.Join('-', container.Split('_').Exclude(nodeType).Where(s => !string.IsNullOrEmpty(s)));
if (!string.IsNullOrEmpty(containerType))
translationKey = $"{translationKey}-{containerType}";
}

if (!string.IsNullOrEmpty(nodeId))
{
nodeId = string.Join('-', nodeId.Split('_')
.Except(string.IsNullOrEmpty(container) ? new string[] { nodeType } : container.Split('_').Append(nodeType))
.Where(s => !string.IsNullOrEmpty(s)));

if (!string.IsNullOrEmpty(nodeId))
translationKey = $"{translationKey}-{nodeId}";
}

foreach (var (childNode, childType, translationValue) in validChildTypes)
translations.Add(new TranslationCandidate(translationKey, childType, translationValue.Trim().Trim('\n'), childNode));
}

// Recurse.
foreach (var childNode in node.Value.Nodes)
if (childNode.Key == "Children")
foreach (var n in childNode.Value.Nodes)
FromChromeLayout(n, translatables, container, ref translations);
}

/// <summary>This is a helper method to find untranslated strings in chrome layouts.</summary>
public static void FindUntranslatedStringFields(ModData modData)
{
var types = modData.ObjectCreator.GetTypes();
foreach (var (type, fields) in types.Where(t => t.Name.EndsWith("Widget", StringComparison.InvariantCulture) && t.IsSubclassOf(typeof(Widget))).ToDictionary(t => t.Name[..^6],
t => t.GetFields().Where(f => f.Name != "Id" && f.IsPublic && f.FieldType == typeof(string) && !f.HasAttribute<TranslationReferenceAttribute>()).Distinct().Select(f => f.Name).ToList()))
if (fields.Count > 0)
PunkPun marked this conversation as resolved.
Show resolved Hide resolved
Console.WriteLine($"{type}Widget:\n {string.Join("\n ", fields)}");
}
}
}