Support for local code pages that are not UTF-8 on Windows

sample/SimpleController/SR1WalkGraspController.cpp

@@ -55,9 +55,8 @@ class SR1WalkGraspController : public cnoid::SimpleController
         ostream& os = io->os();
 
         if(!qseq){
-            string filename = getNativePathString(
-                cnoid::stdx::filesystem::path(shareDirectory())
-                / "motion" / "SR1" / "SR1WalkPattern3.seq");
+            auto path = shareDirPath() / "motion" / "SR1" / "SR1WalkPattern3.seq";
+            string filename = path.make_preferred().string();
 
             BodyMotion motion;
             if(!motion.loadStandardYAMLformat(filename)){

sample/SimpleController/SR1WalkPatternController.cpp

@@ -67,8 +67,8 @@ class SR1WalkPatternController : public cnoid::SimpleController
             io->enableIO(joint);
         }
 
-        string filename = getNativePathString(
-            cnoid::stdx::filesystem::path(shareDirectory()) / "motion" / "SR1" / patternFile);
+        auto path = shareDirPath() / "motion" / "SR1" / patternFile;
+        auto filename = path.make_preferred().string();
 
         BodyMotion motion;
         if(!motion.loadStandardYAMLformat(filename)){

src/AssimpSceneLoader/AssimpSceneLoader.cpp

@@ -8,6 +8,7 @@
 #include <cnoid/MeshFilter>
 #include <cnoid/ImageIO>
 #include <cnoid/FileUtil>
+#include <cnoid/UTF8>
 #include <cnoid/Exception>
 #include <cnoid/NullOut>
 #include <cnoid/stdx/optional>
@@ -154,7 +155,7 @@ SgNode* AssimpSceneLoaderImpl::load(const std::string& filename)
         return nullptr;
     }
 
-    filesystem::path path(filename);
+    filesystem::path path(fromUTF8(filename));
     directoryPath = path.remove_filename();
 
     T_local = stdx::nullopt;
@@ -495,11 +496,11 @@ SgTexture* AssimpSceneLoaderImpl::convertAiTexture(unsigned int index)
     if(srcMaterial->GetTextureCount(aiTextureType_DIFFUSE) > 0){
         aiString path;
         if(srcMaterial->GetTexture(aiTextureType_DIFFUSE, 0, &path, NULL, NULL, NULL, NULL, NULL) == AI_SUCCESS){
-            filesystem::path filepath(path.data);
+            filesystem::path filepath(fromUTF8(path.data));
             if(!checkAbsolute(filepath)){
                 filepath = filesystem::lexically_normal(directoryPath / filepath);
             }
-            string textureFile = getAbsolutePathString(filepath);
+            string textureFile = toUTF8(stdx::filesystem::absolute(filepath).string());
 
             SgImagePtr image;
             ImagePathToSgImageMap::iterator p = imagePathToSgImageMap.find(textureFile);

src/Base/Archive.cpp

@@ -6,6 +6,7 @@
 #include "Item.h"
 #include "MessageView.h"
 #include <cnoid/FilePathVariableProcessor>
+#include <cnoid/UTF8>
 #include <cnoid/stdx/filesystem>
 #include <map>
 #include "gettext.h"
@@ -90,7 +91,8 @@ void Archive::initSharedInfo(const std::string& projectFile)
 {
     initSharedInfo();
 
-    auto projectDir = filesystem::absolute(projectFile).parent_path().generic_string();
+    auto projectDir = toUTF8(
+        filesystem::absolute(fromUTF8(projectFile)).parent_path().generic_string());
     shared->pathVariableProcessor->setBaseDirectory(projectDir);
     shared->pathVariableProcessor->setProjectDirectory(projectDir);
 }

src/Base/ExtCommandItem.cpp

@@ -11,6 +11,7 @@
 #include "PutPropertyFunction.h"
 #include "Archive.h"
 #include <cnoid/Sleep>
+#include <cnoid/UTF8>
 #include <cnoid/stdx/filesystem>
 #include <fmt/format.h>
 #include "gettext.h"
@@ -107,7 +108,7 @@ bool ExtCommandItem::execute()
         terminate();
         string actualCommand(command_);
 #ifdef _WIN32
-        if(filesystem::path(actualCommand).extension() != ".exe"){
+        if(filesystem::path(fromUTF8(actualCommand)).extension() != ".exe"){
             actualCommand += ".exe";
         }
         // quote the command string to support a path including spaces
@@ -127,7 +128,7 @@ bool ExtCommandItem::execute()
 
         } else {
             mv->put(fmt::format(_("External command \"{}\" cannot be executed."), actualCommand));
-            if(!filesystem::exists(actualCommand)){
+            if(!filesystem::exists(fromUTF8(actualCommand))){
                 mv->putln(_(" The command does not exist."));
             } else {
                 mv->putln("");

src/Base/FileDialog.cpp

@@ -2,6 +2,7 @@
 #include "AppConfig.h"
 #include "ProjectManager.h"
 #include <cnoid/ExecutablePath>
+#include <cnoid/UTF8>
 #include <cnoid/stdx/filesystem>
 #include <QBoxLayout>
 #include <QStyle>
@@ -107,8 +108,9 @@ void FileDialog::Impl::updatePresetDirectories()
         urls << QUrl::fromLocalFile(QDir::homePath());
     }
 
-    urls << QUrl::fromLocalFile(shareDirectory().c_str());
+    urls << QUrl::fromLocalFile(shareDir().c_str());
     auto projectDir = ProjectManager::instance()->currentProjectDirectory();
+
     if(!projectDir.empty()){
         urls << QUrl::fromLocalFile(projectDir.c_str());
     }
@@ -152,12 +154,12 @@ bool FileDialog::selectFilePath(const std::string& filePath)
 {
     bool selected = false;
     if(!filePath.empty()){
-        filesystem::path path(filePath);
+        filesystem::path path(fromUTF8(filePath));
         filesystem::path dir(path.parent_path());
         if(filesystem::exists(dir)){
-            setDirectory(dir.string());
+            setDirectory(toUTF8(dir.string()));
             if(filesystem::exists(path)){
-                selectFile(path.filename().string());
+                selectFile(toUTF8(path.filename().string()));
                 selected = true;
             }
         }

src/Base/Item.cpp

@@ -11,6 +11,7 @@
 #include "ItemClassRegistry.h"
 #include "PutPropertyFunction.h"
 #include <cnoid/ValueTree>
+#include <cnoid/UTF8>
 #include <cnoid/stdx/filesystem>
 #include <typeinfo>
 #include <bitset>
@@ -1354,7 +1355,7 @@ void Item::suggestFileUpdate()
 
 void Item::updateFileInformation(const std::string& filename, const std::string& format, Mapping* options)
 {
-    filesystem::path fpath(filename);
+    filesystem::path fpath(fromUTF8(filename));
     if(filesystem::exists(fpath)){
         impl->fileModificationTime = filesystem::last_write_time_to_time_t(fpath);
         impl->isConsistentWithFile = true;

src/Base/ItemFileDialog.cpp

@@ -4,6 +4,7 @@
 #include "RootItem.h"
 #include "MainWindow.h"
 #include "MessageView.h"
+#include <cnoid/UTF8>
 #include <cnoid/stdx/filesystem>
 #include <QMessageBox>
 #include <fmt/format.h>
@@ -246,7 +247,7 @@ bool ItemFileDialog::Impl::saveItem(Item* item)
     self->updatePresetDirectories();
 
     bool selected = false;
-    auto path = filesystem::path(item->filePath());
+    auto path = filesystem::path(fromUTF8(item->filePath()));
     if(path.stem().string() == item->name()){
         selected = selectFilePath(item->filePath());
     }
@@ -285,11 +286,11 @@ bool ItemFileDialog::Impl::selectFilePath(const std::string& filePath)
 
     if(!filePath.empty()){
 
-        filesystem::path path(filePath);
+        filesystem::path path(fromUTF8(filePath));
         filesystem::path dir(path.parent_path());
 
         if(filesystem::exists(dir)){
-            self->setDirectory(dir.string());
+            self->setDirectory(toUTF8(dir.string()));
 
             bool doTrySelectFile = false;
             auto dotext = path.extension().string();
@@ -309,7 +310,7 @@ bool ItemFileDialog::Impl::selectFilePath(const std::string& filePath)
                 }
             }
             if(doTrySelectFile && filesystem::exists(path)){
-                self->selectFile(path.filename().string());
+                self->selectFile(toUTF8(path.filename().string()));
                 selected = true;
             }
         }
@@ -330,7 +331,8 @@ std::string ItemFileDialog::Impl::getSaveFilename()
         auto exts = targetFileIO->extensions();
         if(!exts.empty()){
             bool hasExtension = false;
-            string dotextension = filesystem::path(filename).extension().string();
+            string dotextension =
+                filesystem::path(fromUTF8(filename)).extension().string();
             if(!dotextension.empty()){
                 string extension = dotextension.substr(1); // remove the first dot
                 if(std::find(exts.begin(), exts.end(), extension) != exts.end()){
@@ -432,9 +434,10 @@ bool ItemFileDialog::Impl::onFileDialogFinished(int result)
 
     if(mode == Save && result == QDialog::Accepted){
         auto filename = getSaveFilename();
-        if(filesystem::exists(filename)){
+        if(filesystem::exists(fromUTF8(filename))){
             self->fileDialog()->show();
-            QString file(filesystem::path(filename).filename().string().c_str());
+            QString file(
+                toUTF8(filesystem::path(fromUTF8(filename)).filename().string()).c_str());
             QString message(QString(_("%1 already exists. Do you want to replace it? ")).arg(file));
             auto button =
                 QMessageBox::warning(self, self->windowTitle(), message, QMessageBox::Ok | QMessageBox::Cancel);

src/Base/ItemPropertyWidget.cpp

@@ -9,6 +9,7 @@
 #include "FileDialog.h"
 #include <cnoid/ConnectionSet>
 #include <cnoid/ExecutablePath>
+#include <cnoid/UTF8>
 #include <cnoid/stdx/variant>
 #include <cnoid/stdx/filesystem>
 #include <QTableWidget>
@@ -256,7 +257,8 @@ QVariant PropertyItem::data(int role) const
                 const FilePathProperty& f = stdx::get<FilePathProperty>(value);
                 string filename = f.filename();
                 if(!f.isFullpathDisplayMode()){
-                    filename = filesystem::path(filename).filename().string();
+                    filename = toUTF8(
+                        filesystem::path(fromUTF8(filename)).filename().string());
                 }
                 return filename.c_str();
             }
@@ -266,7 +268,8 @@ QVariant PropertyItem::data(int role) const
             const FilePathProperty& f = stdx::get<FilePathProperty>(value);
             if(!f.isFullpathDisplayMode()){
                 string fullpath = f.filename();
-                string filename = filesystem::path(fullpath).filename().string();
+                string filename = toUTF8(
+                    filesystem::path(fromUTF8(fullpath)).filename().string());
                 if(filename != fullpath){
                     return fullpath.c_str();
                 }
@@ -364,7 +367,9 @@ QString FilePathEditor::value() const
     if(currentValue.baseDirectory().empty()){
         return lineEdit->text();
     } else {
-        return (filesystem::path(currentValue.baseDirectory()) / lineEdit->text().toStdString()).string().c_str();
+        return toUTF8(
+            (filesystem::path(fromUTF8(currentValue.baseDirectory()))
+            / fromUTF8(lineEdit->text().toStdString())).string()).c_str();
     }
 }
 
@@ -557,15 +562,15 @@ void CustomizedItemDelegate::openFileDialog(FilePathProperty value, FilePathEdit
 
     filesystem::path directory;
     if(!value.baseDirectory().empty()){
-        directory = value.baseDirectory();
+        directory = fromUTF8(value.baseDirectory());
     } else {
-        filesystem::path filenamePath(value.filename());
+        filesystem::path filenamePath(fromUTF8(value.filename()));
         if(filenamePath.is_absolute()){
             directory = filenamePath.parent_path();
         }
     }
     if(!directory.empty()){
-        dialog.setDirectory(directory.string());
+        dialog.setDirectory(toUTF8(directory.string()));
     }
 
     QStringList filters;
@@ -578,13 +583,14 @@ void CustomizedItemDelegate::openFileDialog(FilePathProperty value, FilePathEdit
     if(dialog.exec()){
         QStringList filenames;
         filenames = dialog.selectedFiles();
-        filesystem::path newDirectory(dialog.directory().absolutePath().toStdString());
+        filesystem::path newDirectory(
+            fromUTF8(dialog.directory().absolutePath().toStdString()));
         if(newDirectory != directory){
             value.setBaseDirectory("");
         }
         string filename(filenames.at(0).toStdString());
         if(!value.baseDirectory().empty()){
-            filename = filesystem::path(filename).filename().string();
+            filename = toUTF8(filesystem::path(fromUTF8(filename)).filename().string());
         }
         if(value.isExtensionRemovalModeForFileDialogSelection()){
             regex pattern1(".+\\(\\*\\.(.+)\\)");

src/Base/MessageLogItem.cpp

@@ -10,6 +10,7 @@
 #include "PutPropertyFunction.h"
 #include "Archive.h"
 #include <cnoid/ExecutablePath>
+#include <cnoid/UTF8>
 #include <cnoid/stdx/filesystem>
 #include <fstream>
 #include <regex>
@@ -139,7 +140,7 @@ void MessageLogItemImpl::openFile()
     if(fileMode.selectedIndex()==MessageLogItem::APPEND){
         ofs.open(filename, ios::app);
     }else{
-        stdx::filesystem::path path(filename);
+        stdx::filesystem::path path(fromUTF8(filename));
         if(stdx::filesystem::exists(path)){
             bool ok = showConfirmDialog(
                 _("Confirm"),
@@ -149,7 +150,7 @@ void MessageLogItemImpl::openFile()
                 return;
             }
         }
-        ofs.open(filename, ios::out);
+        ofs.open(path.string(), ios::out);
     }
 
     if(!ofs){
@@ -201,7 +202,7 @@ void MessageLogItemImpl::setFileName(const string& filename_)
         return;
 
     filename = filename_;
-    stdx::filesystem::path path(filename);
+    stdx::filesystem::path path(fromUTF8(filename));
     string ext = path.extension().string();
     if(ext != ".log"){
         filename += ".log";

src/Base/MovieRecorder.cpp

@@ -27,6 +27,7 @@
 #include "LazyCaller.h"
 #include <cnoid/ConnectionSet>
 #include <cnoid/Selection>
+#include <cnoid/UTF8>
 #include <cnoid/stdx/variant>
 #include <cnoid/stdx/filesystem>
 #include <QPainter>
@@ -221,8 +222,8 @@ class MovieRecorderImpl
     std::thread imageOutputThread;
     std::mutex imageQueueMutex;
     std::condition_variable imageQueueCondition;
-   string filenameFormat;
-
+    string filenameFormat;
+    
     MovieRecorderImpl(ExtensionManager* ext);
     ~MovieRecorderImpl();
     void setTargetView(View* view);
@@ -710,8 +711,8 @@ bool MovieRecorderImpl::setupViewAndFilenameFormat()
         return false;
     }
 
-    filesystem::path directory(dialog->directoryEntry.string());
-    filesystem::path basename(dialog->basenameEntry.string() + "{:08d}.png");
+    filesystem::path directory(fromUTF8(dialog->directoryEntry.string()));
+    filesystem::path basename(fromUTF8(dialog->basenameEntry.string() + "{:08d}.png"));
 
     if(directory.empty()){
         showWarningDialog(_("Please set a directory to output image files."));
@@ -728,7 +729,7 @@ bool MovieRecorderImpl::setupViewAndFilenameFormat()
         }
     }
 
-    filenameFormat = (directory / basename).string();
+    filenameFormat = toUTF8((directory / basename).string());
 
     if(dialog->imageSizeCheck.isChecked()){
         int width = dialog->imageWidthSpin.value();