Fix file name encoding issues with VST on Windows

Fix plugin loading and setting loading/saving

Author: PhysSong

include/IoHelper.h

@@ -0,0 +1,72 @@
+/*
+ * IoHelper.h - helper functions for file I/O
+ *
+ * Copyright (c) 2018 Hyunjin Song <tteu.ingog/at/gmail.com>
+ *
+ * This file is part of LMMS - https://lmms.io
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program (see COPYING); if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301 USA.
+ *
+ */
+
+
+#include "lmmsconfig.h"
+
+#include <cstdio>
+
+
+#ifdef _WIN32
+#include <windows.h>
+
+std::wstring toWString(const std::string& s)
+{
+	std::wstring ret;
+	int len = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.data(),
+			s.length(), nullptr, 0);
+	if (len == 0)
+	{
+		return ret;
+	}
+	ret.resize(len);
+	MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.data(), s.length(), &ret[0], len);
+	return ret;
+}
+#endif
+
+#ifdef LMMS_BUILD_WIN32
+#include <io.h>
+#define F_OPEN_UTF8(a, b) _wfopen(toWString(a).data(), L##b)
+#else
+#ifdef LMMS_HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#define F_OPEN_UTF8(a, b) fopen((a).data(), b)
+#endif
+
+int fileToDescriptor(FILE* f, bool closeFile = true)
+{
+	int fh;
+	if (f == NULL) {return -1;}
+
+#ifdef LMMS_BUILD_WIN32
+	fh = _dup(_fileno(f));
+#else
+	fh = dup(fileno(f));
+#endif
+
+	if (closeFile) {fclose(f);}
+	return fh;
+}

plugins/vst_base/RemoteVstPlugin.cpp

@@ -88,6 +88,7 @@ struct ERect
 #include "lmms_basics.h"
 #include "Midi.h"
 #include "communication.h"
+#include "IoHelper.h"
 
 #include "VstSyncData.h"
 
@@ -678,9 +679,9 @@ void RemoteVstPlugin::init( const std::string & _plugin_file )
 
 
 
-static void close_check( int fd )
+static void close_check( FILE* fp )
 {
-	if( close( fd ) )
+	if( fclose( fp ) )
 	{
 		perror( "close" );
 	}
@@ -790,7 +791,7 @@ void RemoteVstPlugin::destroyEditor()
 
 bool RemoteVstPlugin::load( const std::string & _plugin_file )
 {
-	if( ( m_libInst = LoadLibrary( _plugin_file.c_str() ) ) == NULL )
+	if( ( m_libInst = LoadLibraryW( toWString(_plugin_file).c_str() ) ) == NULL )
 	{
 		// give VstPlugin class a chance to start 32 bit version of RemoteVstPlugin
 		if( GetLastError() == ERROR_BAD_EXE_FORMAT )
@@ -1072,13 +1073,13 @@ void RemoteVstPlugin::saveChunkToFile( const std::string & _file )
 		const int len = pluginDispatch( 23, 0, 0, &chunk );
 		if( len > 0 )
 		{
-			int fd = open( _file.c_str(), O_WRONLY | O_BINARY );
-			if ( ::write( fd, chunk, len ) != len )
+			FILE* fp = F_OPEN_UTF8( _file, "wb" );
+			if ( fwrite( chunk, len, 1, fp ) != len )
 			{
 				fprintf( stderr,
 					"Error saving chunk to file.\n" );
 			}
-			close_check( fd );
+			close_check( fp );
 		}
 	}
 }
@@ -1237,7 +1238,7 @@ void RemoteVstPlugin::savePreset( const std::string & _file )
 	if (!isPreset &&!chunky) uIntToFile = (unsigned int) m_plugin->numPrograms;
 	pBank->numPrograms = endian_swap( uIntToFile );
 
-	FILE * stream = fopen( _file.c_str(), "w" );
+	FILE * stream = F_OPEN_UTF8( _file, "w" );
 	fwrite ( pBank, 1, 28, stream );
 	fwrite ( progName, 1, isPreset ? 28 : 128, stream );
 	if ( chunky ) {
@@ -1289,7 +1290,7 @@ void RemoteVstPlugin::loadPresetFile( const std::string & _file )
 	unsigned int * pLen = new unsigned int[ 1 ];
 	unsigned int len = 0;
 	sBank * pBank = (sBank*) new char[ sizeof( sBank ) ];
-	FILE * stream = fopen( _file.c_str(), "r" );
+	FILE * stream = F_OPEN_UTF8( _file, "r" );
 	if ( fread ( pBank, 1, 56, stream ) != 56 )
 	{
 		fprintf( stderr, "Error loading preset file.\n" );
@@ -1390,12 +1391,12 @@ void RemoteVstPlugin::loadChunkFromFile( const std::string & _file, int _len )
 {
 	char * chunk = new char[_len];
 
-	const int fd = open( _file.c_str(), O_RDONLY | O_BINARY );
-	if ( ::read( fd, chunk, _len ) != _len )
+	FILE* fp = F_OPEN_UTF8( _file, "rb" );
+	if ( fread( chunk, 1, _len, fp ) != _len )
 	{
 		fprintf( stderr, "Error loading chunk from file.\n" );
 	}
-	close_check( fd );
+	close_check( fp );
 
 	pluginDispatch( effSetChunk, 0, _len, chunk );