Improvements to support locale and units (AcademySoftwareFoundation#892)

- Add utilities UnitConverterRegistry::convertToUnit and MaterialX::joinStrings.
- Use C Locale for string stream operations (autodesk-forks#1183).
- Add tests for unit and locale content.

resources/Materials/TestSuite/locale/numericformat.mtlx

@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<materialx version="1.38">
+  <standard_surface name="SR_numberformats" type="surfaceshader">
+    <input name="base_color" type="color3" value="0.18,0.18, 0.18"/>
+    <input name="specular_color" type="color3" value="0.05, 0.05,0.05" />
+    <input name="normal" type="vector3" value="1,0.5,1" />
+    <input name="specular_roughness" type="float" value="0.25" />
+    <input name="coat" type="float" value="1" />
+  </standard_surface>
+  <surfacematerial name="Number_formats" type="material">
+    <input name="surfaceshader" type="surfaceshader" nodename="SR_numberformats" />
+  </surfacematerial>
+</materialx>

resources/Materials/TestSuite/locale/utf8.mtlx

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<materialx version="1.38" colorspace="lin_rec709">
+  <standard_surface name="SR_Localetest" type="surfaceshader">
+    <input name="base" type="float" value="1" />
+    <input name="base_color" type="color3" value="0.18,0.18, 0.18" uiname="びまん性" />
+    <input name="specular" type="float" value="0" uiname="spéculaire"/>
+    <input name="specular_roughness" type="float" value="0.25" uiname="表面粗さ"/>
+    <input name="metalness" type="float" value="1" />
+    <input name="coat" type="float" value="1" />
+    <input name="coat_color" type="color3" value="0.05, 0.05,0.05" />
+    <input name="coat_roughness" type="float" value="0.20000000298023224" />
+  </standard_surface>
+  <surfacematerial name="Locale" type="material">
+    <input name="surfaceshader" type="surfaceshader" nodename="SR_Localetest" />
+  </surfacematerial>
+</materialx>

source/MaterialXCore/Unit.cpp

@@ -241,4 +241,70 @@ void UnitConverterRegistry::write(DocumentPtr doc) const
     }
 }
 
+bool UnitConverterRegistry::convertToUnit(DocumentPtr doc, const string& unitType, const string& targetUnit)
+{
+    UnitTypeDefPtr unitTypeDef = doc->getUnitTypeDef(unitType);
+    UnitConverterPtr unitConverter = getUnitConverter(unitTypeDef);
+
+    if (!unitTypeDef || !unitConverter)
+    {
+        return false;
+    }
+
+    bool convertedUnits = false;
+    for (ElementPtr elem : doc->traverseTree())
+    {
+        NodePtr pNode = elem->asA<Node>();
+        if (!pNode || !pNode->getInputCount())
+        {
+            continue;
+        }
+        for (InputPtr input : pNode->getInputs())
+        {
+            const std::string type = input->getType();
+            const ValuePtr value = input->getValue();
+            if (value && input->hasUnit() && (input->getUnitType() == unitType) && value)
+            {
+                if (type == getTypeString<float>())
+                {
+                    float originalval = value->asA<float>();
+                    float convertedValue = unitConverter->convert(originalval, input->getUnit(), targetUnit);
+                    input->setValue<float>(convertedValue);
+                    input->removeAttribute(ValueElement::UNIT_ATTRIBUTE);
+                    input->removeAttribute(ValueElement::UNITTYPE_ATTRIBUTE);
+                    convertedUnits = true;
+                }
+                else if (type == getTypeString<Vector2>())
+                {
+                    Vector2 originalval = value->asA<Vector2>();
+                    Vector2 convertedValue = unitConverter->convert(originalval, input->getUnit(), targetUnit);
+                    input->setValue<Vector2>(convertedValue);
+                    input->removeAttribute(ValueElement::UNIT_ATTRIBUTE);
+                    input->removeAttribute(ValueElement::UNITTYPE_ATTRIBUTE);
+                    convertedUnits = true;
+                }
+                else if (type == getTypeString<Vector3>())
+                {
+                    Vector3 originalval = value->asA<Vector3>();
+                    Vector3 convertedValue = unitConverter->convert(originalval, input->getUnit(), targetUnit);
+                    input->setValue<Vector3>(convertedValue);
+                    input->removeAttribute(ValueElement::UNIT_ATTRIBUTE);
+                    input->removeAttribute(ValueElement::UNITTYPE_ATTRIBUTE);
+                    convertedUnits = true;
+                }
+                else if (type == getTypeString<Vector4>())
+                {
+                    Vector4 originalval = value->asA<Vector4>();
+                    Vector4 convertedValue = unitConverter->convert(originalval, input->getUnit(), targetUnit);
+                    input->setValue<Vector4>(convertedValue);
+                    input->removeAttribute(ValueElement::UNIT_ATTRIBUTE);
+                    input->removeAttribute(ValueElement::UNITTYPE_ATTRIBUTE);
+                    convertedUnits = true;
+                }
+            }
+        }
+    }
+    return convertedUnits;
+}
+
 MATERIALX_NAMESPACE_END

source/MaterialXCore/Unit.h

@@ -195,6 +195,10 @@ class MX_CORE_API UnitConverterRegistry
     /// Create unit definitions in a document based on registered converters
     void write(DocumentPtr doc) const;
 
+    /// Convert input values which have a source unit to a given target unit.
+    /// Returns if any unit conversion occured.
+    bool convertToUnit(DocumentPtr doc, const string& unitType, const string& targetUnit);
+
   private:
     UnitConverterRegistry(const UnitConverterRegistry&) = delete;
     UnitConverterRegistry() { }

source/MaterialXCore/Util.cpp

@@ -91,6 +91,16 @@ StringVec splitString(const string& str, const string& sep)
     return split;
 }
 
+string joinStrings(const StringVec& stringVec, const string& sep)
+{
+    string res;
+    for (const string& name : stringVec)
+    {
+        res = res.empty() ? name : res + sep + name;
+    }
+    return res;
+}
+
 string replaceSubstrings(string str, const StringMap& stringMap)
 {
     for (const auto& pair : stringMap)
@@ -145,11 +155,7 @@ StringVec splitNamePath(const string& namePath)
 
 string createNamePath(const StringVec& nameVec)
 {
-    string res;
-    for (const string& name : nameVec)
-    {
-        res = res.empty() ? name : res + NAME_PATH_SEPARATOR + name;
-    }
+    string res = joinStrings(nameVec, NAME_PATH_SEPARATOR);
     return res;
 }
 

source/MaterialXCore/Util.h

@@ -35,6 +35,10 @@ MX_CORE_API string incrementName(const string& name);
 /// separator characters.
 MX_CORE_API StringVec splitString(const string& str, const string& sep);
 
+/// Join a vector of substrings into a single string, placing the given
+/// separator between each substring.
+MX_CORE_API string joinStrings(const StringVec& strVec, const string& sep);
+
 /// Apply the given substring substitutions to the input string.
 MX_CORE_API string replaceSubstrings(string str, const StringMap& stringMap);
 

source/MaterialXCore/Value.cpp

@@ -31,6 +31,7 @@ template <class T> using enable_if_std_vector_t =
 template <class T> void stringToData(const string& str, T& data)
 {
     std::stringstream ss(str);
+    ss.imbue(std::locale::classic());
     if (!(ss >> data))
     {
         throw ExceptionTypeError("Type mismatch in generic stringToData: " + str);
@@ -94,6 +95,7 @@ template <class T> void stringToData(const string& str, enable_if_std_vector_t<T
 template <class T> void dataToString(const T& data, string& str)
 {
     std::stringstream ss;
+    ss.imbue(std::locale::classic());
 
     // Set float format and precision for the stream
     const Value::FloatFormat fmt = Value::getFloatFormat();

source/MaterialXTest/MaterialXFormat/XmlIo.cpp

@@ -227,3 +227,75 @@ TEST_CASE("Load content", "[xmlio]")
     mx::DocumentPtr nonExistentDoc = mx::createDocument();
     REQUIRE_THROWS_AS(mx::readFromXmlFile(nonExistentDoc, "NonExistent.mtlx", mx::FileSearchPath(), &readOptions), mx::ExceptionFileMissing&);
 }
+
+TEST_CASE("Locale region testing", "[xmlio]")
+{
+    /// In the United States, the thousands separator is a comma, while in Germany it is a period.
+    /// Thus, one thousand twenty five is displayed as 1,025 in the United States and 1.025 in Germany.
+    ///
+    /// In a MaterialX document, a vector3 value of "1,1.5,2.0" should be interpreted as (1.0f, 1.5f, 2.0f).
+
+    try
+    {
+        // Set locale to de
+        std::locale deLocale("de_DE");
+        std::locale::global(deLocale);
+    }
+    catch (const std::runtime_error& e)
+    {
+        WARN("Unable to change locale " << e.what());
+        return;
+    }
+
+    mx::FilePath libraryPath("libraries/stdlib");
+    mx::FilePath testPath("resources/Materials/TestSuite/locale");
+    mx::FileSearchPath searchPath = libraryPath.asString() +
+        mx::PATH_LIST_SEPARATOR +
+        testPath.asString();
+
+    // Read the standard library.
+    std::vector<mx::DocumentPtr> libs;
+    for (const mx::FilePath& filename : libraryPath.getFilesInDirectory(mx::MTLX_EXTENSION))
+    {
+        mx::DocumentPtr lib = mx::createDocument();
+        mx::readFromXmlFile(lib, filename, searchPath);
+        libs.push_back(lib);
+    }
+
+    // Read and validate each example document.
+    for (const mx::FilePath& filename : testPath.getFilesInDirectory(mx::MTLX_EXTENSION))
+    {
+        mx::DocumentPtr doc = mx::createDocument();
+        mx::readFromXmlFile(doc, filename, searchPath);
+        for (mx::DocumentPtr lib : libs)
+        {
+            doc->importLibrary(lib);
+        }
+        std::string message;
+
+        bool docValid = doc->validate(&message);
+        if (!docValid)
+        {
+            WARN("[" + filename.asString() + "] " + message);
+        }
+        REQUIRE(docValid);
+
+        // Traverse the document tree
+        int valueElementCount = 0;
+        int uiAttributeCount = 0;
+        for (mx::ElementPtr elem : doc->traverseTree())
+        {
+            if (elem->isA<mx::ValueElement>())
+            {
+                valueElementCount++;
+                if (elem->hasAttribute("uiname"))
+                {
+                    REQUIRE(!elem->getAttribute("uiname").empty());
+                    uiAttributeCount++;
+                }
+            }
+        }
+        REQUIRE(valueElementCount > 0);
+        REQUIRE(uiAttributeCount > 0);
+    }
+}

source/PyMaterialX/PyMaterialXCore/PyUtil.cpp

@@ -21,6 +21,7 @@ void bindPyUtil(py::module& mod)
     mod.def("isValidName", &mx::isValidName);
     mod.def("incrementName", &mx::incrementName);
     mod.def("splitString", &mx::splitString);
+    mod.def("joinStrings", &mx::joinStrings);
     mod.def("replaceSubstrings", &mx::replaceSubstrings);
     mod.def("stringEndsWith", &mx::stringEndsWith);
     mod.def("splitNamePath", &mx::splitNamePath);

source/PyMaterialX/PyMaterialXFormat/PyXmlIo.cpp

@@ -35,6 +35,12 @@ void bindPyXmlIo(py::module& mod)
         py::arg("doc"), py::arg("writeOptions") = nullptr);
     mod.def("prependXInclude", mx::prependXInclude);
 
+    mod.def("getEnvironmentPath", &mx::getEnvironmentPath,
+        py::arg("sep") = mx::PATH_LIST_SEPARATOR);
+
+    mod.attr("PATH_LIST_SEPARATOR") = mx::PATH_LIST_SEPARATOR;
+    mod.attr("MATERIALX_SEARCH_PATH_ENV_VAR") = mx::MATERIALX_SEARCH_PATH_ENV_VAR;
+
     py::register_exception<mx::ExceptionParseError>(mod, "ExceptionParseError");
     py::register_exception<mx::ExceptionFileMissing>(mod, "ExceptionFileMissing");
 }