Fix some VST deadlocks/hangs

Author: DomClark

include/MainWindow.h

@@ -29,6 +29,7 @@
 #include <QtCore/QTimer>
 #include <QtCore/QList>
 #include <QMainWindow>
+#include <QThread>
 
 #include "ConfigManager.h"
 #include "SubWindow.h"
@@ -248,4 +249,11 @@ private slots:
 
 } ;
 
+class AutoSaveThread : public QThread
+{
+	Q_OBJECT
+public:
+	void run();
+} ;
+
 #endif

include/RemotePlugin.h

@@ -1072,6 +1072,14 @@ RemotePluginBase::message RemotePluginBase::waitForMessage(
 							const message & _wm,
 							bool _busy_waiting )
 {
+#ifndef BUILD_REMOTE_PLUGIN_CLIENT
+	if( _busy_waiting )
+	{
+		// No point processing events outside of the main thread
+		_busy_waiting = QThread::currentThread() ==
+					QCoreApplication::instance()->thread();
+	}
+#endif
 	while( !isInvalid() )
 	{
 #ifndef BUILD_REMOTE_PLUGIN_CLIENT

plugins/vst_base/RemoteVstPlugin.cpp

@@ -44,6 +44,10 @@
 
 #ifdef LMMS_BUILD_LINUX
 
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+
 #ifndef O_BINARY
 #define O_BINARY 0
 #endif
@@ -67,6 +71,7 @@
 
 
 #include <vector>
+#include <queue>
 #include <string>
 
 
@@ -113,7 +118,7 @@ class RemoteVstPlugin;
 
 RemoteVstPlugin * __plugin = NULL;
 
-DWORD __GuiThreadID = 0;
+HWND __MessageHwnd = NULL;
 
 
 
@@ -244,8 +249,38 @@ class RemoteVstPlugin : public RemotePluginClient
 		pthread_mutex_unlock( &m_pluginLock );
 	}
 
+	inline bool isProcessing() const
+	{
+		return m_processing;
+	}
+
+	inline void setProcessing( bool processing )
+	{
+		m_processing = processing;
+	}
+
+	inline void queueMessage( const message & m ) {
+		m_messageList.push( m );
+	}
+
+	inline bool shouldGiveIdle() const
+	{
+		return m_shouldGiveIdle;
+	}
+
+	inline void setShouldGiveIdle( bool shouldGiveIdle )
+	{
+		m_shouldGiveIdle = shouldGiveIdle;
+	}
+
+	void idle();
+	void processUIThreadMessages();
+
 	static DWORD WINAPI processingThread( LPVOID _param );
-	static DWORD WINAPI guiEventLoop( LPVOID _param );
+	static bool setupMessageWindow();
+	static DWORD WINAPI guiEventLoop();
+	static LRESULT CALLBACK messageWndProc( HWND hwnd, UINT uMsg,
+						WPARAM wParam, LPARAM lParam );
 
 
 private:
@@ -303,6 +338,10 @@ class RemoteVstPlugin : public RemotePluginClient
 	bool m_initialized;
 
 	pthread_mutex_t m_pluginLock;
+	bool m_processing;
+
+	std::queue<message> m_messageList;
+	bool m_shouldGiveIdle;
 
 
 	float * * m_inputs;
@@ -348,6 +387,9 @@ RemoteVstPlugin::RemoteVstPlugin( const char * socketPath ) :
 	m_windowHeight( 0 ),
 	m_initialized( false ),
 	m_pluginLock(),
+	m_processing( false ),
+	m_messageList(),
+	m_shouldGiveIdle( false ),
 	m_inputs( NULL ),
 	m_outputs( NULL ),
 	m_midiEvents(),
@@ -1418,8 +1460,7 @@ intptr_t RemoteVstPlugin::hostCallback( AEffect * _effect, int32_t _opcode,
 			SHOW_CALLBACK ("amc: audioMasterIdle\n" );
 			// call application idle routine (this will
 			// call effEditIdle for all open editors too)
-			PostThreadMessage( __GuiThreadID,
-						WM_USER, GiveIdle, 0 );
+			PostMessage( __MessageHwnd, WM_USER, GiveIdle, 0 );
 			return 0;
 
 		case audioMasterPinConnected:
@@ -1720,8 +1761,7 @@ intptr_t RemoteVstPlugin::hostCallback( AEffect * _effect, int32_t _opcode,
 		case audioMasterUpdateDisplay:
 			SHOW_CALLBACK( "amc: audioMasterUpdateDisplay\n" );
 			// something has changed, update 'multi-fx' display
-			PostThreadMessage( __GuiThreadID,
-						WM_USER, GiveIdle, 0 );
+			PostMessage( __MessageHwnd, WM_USER, GiveIdle, 0 );
 			return 0;
 
 #if kVstVersion > 2
@@ -1754,6 +1794,43 @@ intptr_t RemoteVstPlugin::hostCallback( AEffect * _effect, int32_t _opcode,
 
 
 
+void RemoteVstPlugin::idle()
+{
+	if( isProcessing() )
+	{
+		setShouldGiveIdle( true );
+		return;
+	}
+	setProcessing( true );
+	pluginDispatch( effEditIdle );
+	setShouldGiveIdle( false );
+	setProcessing( false );
+	// We might have received a message whilst idling
+	processUIThreadMessages();
+}
+
+
+
+
+void RemoteVstPlugin::processUIThreadMessages()
+{
+	setProcessing( true );
+	while( m_messageList.size() )
+	{
+		processMessage( m_messageList.front() );
+		m_messageList.pop();
+		if( shouldGiveIdle() )
+		{
+			pluginDispatch( effEditIdle );
+			setShouldGiveIdle( false );
+		}
+	}
+	setProcessing( false );
+}
+
+
+
+
 DWORD WINAPI RemoteVstPlugin::processingThread( LPVOID _param )
 {
 	RemoteVstPlugin * _this = static_cast<RemoteVstPlugin *>( _param );
@@ -1767,80 +1844,99 @@ DWORD WINAPI RemoteVstPlugin::processingThread( LPVOID _param )
 		}
 		else
 		{
-			PostThreadMessage( __GuiThreadID,
+			PostMessage( __MessageHwnd,
 					WM_USER,
 					ProcessPluginMessage,
 					(LPARAM) new message( m ) );
 		}
 	}
 
 	// notify GUI thread about shutdown
-	PostThreadMessage( __GuiThreadID, WM_USER, ClosePlugin, 0 );
+	PostMessage( __MessageHwnd, WM_USER, ClosePlugin, 0 );
 
 	return 0;
 }
 
 
 
 
-DWORD WINAPI RemoteVstPlugin::guiEventLoop( LPVOID _param )
+bool RemoteVstPlugin::setupMessageWindow()
 {
-	RemoteVstPlugin * _this = static_cast<RemoteVstPlugin *>( _param );
-
 	HMODULE hInst = GetModuleHandle( NULL );
 	if( hInst == NULL )
 	{
-		_this->debugMessage( "guiEventLoop(): can't get "
+		__plugin->debugMessage( "setupMessageWindow(): can't get "
 							"module handle\n" );
-		return -1;
+		return false;
 	}
 
-	HWND timerWindow = CreateWindowEx( 0, "LVSL", "dummy",
+	__MessageHwnd = CreateWindowEx( 0, "LVSL", "dummy",
 						0, 0, 0, 0, 0, NULL, NULL,
 								hInst, NULL );
+	SetWindowLongPtr( __MessageHwnd, GWLP_WNDPROC,
+		reinterpret_cast<LONG_PTR>( RemoteVstPlugin::messageWndProc ) );
 	// install GUI update timer
-	SetTimer( timerWindow, 1000, 50, NULL );
+	SetTimer( __MessageHwnd, 1000, 50, NULL );
+
+	return true;
+}
+
 
-	MSG msg;
 
-	bool quit = false;
-	while( quit == false && GetMessage( &msg, NULL, 0, 0 ) )
+
+DWORD WINAPI RemoteVstPlugin::guiEventLoop()
+{
+	MSG msg;
+	while( GetMessage( &msg, NULL, 0, 0 ) > 0 )
 	{
 		TranslateMessage( &msg );
 		DispatchMessage( &msg );
+	}
 
-		if( msg.message == WM_TIMER && _this->isInitialized() )
-		{
-			// give plugin some idle-time for GUI-update
-			_this->pluginDispatch( effEditIdle );
-		}
-		else if( msg.message == WM_USER )
+	return 0;
+}
+
+
+
+
+LRESULT CALLBACK RemoteVstPlugin::messageWndProc( HWND hwnd, UINT uMsg,
+						WPARAM wParam, LPARAM lParam )
+{
+	if( uMsg == WM_TIMER && __plugin->isInitialized() )
+	{
+		// give plugin some idle-time for GUI-update
+		__plugin->idle();
+		return 0;
+	}
+	else if( uMsg == WM_USER )
+	{
+		switch( wParam )
 		{
-			switch( msg.wParam )
+			case ProcessPluginMessage:
 			{
-				case ProcessPluginMessage:
+				message * m = (message *) lParam;
+				__plugin->queueMessage( *m );
+				delete m;
+				if( !__plugin->isProcessing() )
 				{
-					message * m = (message *) msg.lParam;
-					_this->processMessage( *m );
-					delete m;
-					break;
+					__plugin->processUIThreadMessages();
 				}
+				return 0;
+			}
 
-				case GiveIdle:
-					_this->pluginDispatch( effEditIdle );
-					break;
+			case GiveIdle:
+				__plugin->idle();
+				return 0;
 
-				case ClosePlugin:
-					quit = true;
-					break;
+			case ClosePlugin:
+				PostQuitMessage(0);
+				return 0;
 
-				default:
-					break;
-			}
+			default:
+				break;
 		}
 	}
-
-	return 0;
+	return DefWindowProc( hwnd, uMsg, wParam, lParam );
 }
 
 
@@ -1893,15 +1989,18 @@ int main( int _argc, char * * _argv )
 
 	if( __plugin->isInitialized() )
 	{
-		__GuiThreadID = GetCurrentThreadId();
+		if( RemoteVstPlugin::setupMessageWindow() == false )
+		{
+			return -1;
+		}
 		if( CreateThread( NULL, 0, RemoteVstPlugin::processingThread,
 						__plugin, 0, NULL ) == NULL )
 		{
 			__plugin->debugMessage( "could not create "
 							"processingThread\n" );
 			return -1;
 		}
-		RemoteVstPlugin::guiEventLoop( __plugin );
+		RemoteVstPlugin::guiEventLoop();
 	}