Get PD minimally working under Wine (again)

With this commit, PhotoDemon should once again start correctly under Wine.  A lot of things may not work well (or at all!), but at least the app loads and provides minimal functionality again.

(Turns out the biggest problem was setting layered window bits on the main window, which previously improved compositing performance of PD's toolbars under Vista+ but breaks completely under Wine.  Because PD no longer provides semi-transparent panel support, this line could be removed entirely.)

To further improve Wine compatibility, PD now sets a bunch of internal flags to "XP compatibility mode" when Wine is detected as the "OS".  This fixes a number of rendering issues that have nothing to do with XP per se but which produce noticeably better results under Wine.

That said, some UI issues, like never receiving WM_MOUSELEAVE events so all UI bits look hovered, are Wine-specific issues that I can't easily address at present.  I'll keep poking around to see if I can find fixes for these, but no promises on a timeline...

Author: tannerhelland

Classes/pdDisplays.cls

@@ -448,8 +448,7 @@ Private Sub FillDetailedDisplayInfo(ByVal displayIndex As Long)
         ' to Vista or later.
 
         'If we have not yet retrieved an EDID value for this monitor, attempt to do so now
-        'And OS.IsVistaOrLater
-        If (Not listOfDisplays(displayIndex).HasEDID) Then
+        If (Not listOfDisplays(displayIndex).HasEDID) And OS.IsVistaOrLater() Then
 
             'Retrieve and cache the device ID
             Dim devID As String

Classes/pdWindowManager.cls

@@ -711,12 +711,10 @@ Friend Sub RegisterMainForm(ByRef parentForm As Form)
 
     If PDMain.IsProgramRunning() Then
 
-        'Apply any unique styles to the parent window
+        'Apply any unique styles to the parent window.  (PD uses complex nested windows for toolbars,
+        ' and setting the WS_EX_COMPOSITED bit reduces flickering on older versions of Windows.)
         Dim newWinStyle As Long
         newWinStyle = GetWindowLong(parentForm.hWnd, GWL_EXSTYLE) Or WS_EX_COMPOSITED
-        
-        'On Vista+, layered windows consume a bit more resources but they play very nicely with DWM
-        If OS.IsVistaOrLater Then newWinStyle = newWinStyle Or WS_EX_LAYERED
         SetWindowLong m_MainWindowHWnd, GWL_EXSTYLE, newWinStyle
 
         'Enforce minimum size handling.  (Note that this will subclass the main window, which allows us to handle

Forms/MainWindow.frm

@@ -2138,8 +2138,8 @@ Private Sub Form_Load()
         PDDebug.LogAction "Program initialization complete.  Second baseline memory measurement:"
         PDDebug.LogAction vbNullString, PDM_Mem_Report
 
-        'Before setting focus to the main form, active a focus tracker; it catches some cases that VB's
-        ' built-in focus events do not.
+        'Before setting focus to the main form, activate a focus tracker.
+        ' (PD uses this class to catch some focus cases that VB's built-in focus events do not.)
         Set m_FocusDetector = New pdFocusDetector
         m_FocusDetector.StartFocusTracking FormMain.hWnd
 

Modules/OS.bas

@@ -276,9 +276,11 @@ End Type
 Private Declare Function CloseHandle Lib "kernel32" (ByVal Handle As Long) As Long
 Private Declare Function CreateToolhelp32Snapshot Lib "kernel32" (ByVal lFlags As Long, ByVal lProcessID As Long) As Long
 Private Declare Function GetAsyncKeyState Lib "user32" (ByVal vKey As Long) As Integer
-Private Declare Function GetKeyState Lib "user32" (ByVal vKey As Long) As Integer
 Private Declare Function GetCommandLineW Lib "kernel32" () As Long
+Private Declare Function GetKeyState Lib "user32" (ByVal vKey As Long) As Integer
 Private Declare Function GetModuleFileNameW Lib "kernel32" (ByVal hModule As Long, ByVal ptrToFileNameBuffer As Long, ByVal nSize As Long) As Long
+Private Declare Function GetModuleHandleW Lib "kernel32" (ByVal lpModuleName As Long) As Long
+Private Declare Function GetProcAddress Lib "kernel32" (ByVal hModule As Long, ByVal lpProcName As String) As Long
 Private Declare Sub GetNativeSystemInfo Lib "kernel32" (ByRef lpSystemInfo As OS_SystemInfo)
 Private Declare Sub GetSystemTimeAsFileTime Lib "kernel32" (ByRef dstTime As Currency)
 Private Declare Function GetTempPathW Lib "kernel32" (ByVal nBufferLength As Long, ByVal lpStrBuffer As Long) As Long
@@ -347,6 +349,9 @@ End Enum
 
 Private m_ThemingAvailable As PD_ThemingAvailable
 
+'Used to detect if we're running under Wine or not.  (Added in 2024-05 to try and solve Wine-specific problems.)
+Private m_InsideWine As Boolean, m_LookedForWineAlready As Boolean
+
 'Function for returning this application's current memory usage.  Note that this function will return warped
 ' values inside the IDE (because the reported total is for *all* of VB6, including the IDE itself).
 Public Function AppMemoryUsage(Optional returnPeakValue As Boolean = False) As Long
@@ -635,7 +640,7 @@ End Function
 'PD will attempt to forcibly enable DEP on Win 7+ (when compiled, obviously).
 ' This may decrease issues with garbage 3rd-party virus and malware scanners.
 Public Function EnableProcessDEP() As Boolean
-    If (OS.IsWin7OrLater And OS.IsProgramCompiled) Then
+    If (OS.IsWin7OrLater And OS.IsProgramCompiled And (Not OS.IsWinActuallyWine)) Then
         Const PROCESS_DEP_ENABLE As Long = &H1&
         SetProcessDEPPolicy PROCESS_DEP_ENABLE
     End If
@@ -757,36 +762,88 @@ End Function
 'Check for a version >= the specified version.
 Public Function IsVistaOrLater() As Boolean
     If (Not m_VersionInfoCached) Then CacheOSVersion
-    IsVistaOrLater = (m_OSVI.dwMajorVersion >= 6)
+    If OS.IsWinActuallyWine Then
+        IsVistaOrLater = False
+    Else
+        IsVistaOrLater = (m_OSVI.dwMajorVersion >= 6)
+    End If
 End Function
 
 Public Function IsWin7OrLater() As Boolean
     If (Not m_VersionInfoCached) Then CacheOSVersion
-    IsWin7OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 1))
+    If OS.IsWinActuallyWine Then
+        IsWin7OrLater = False
+    Else
+        IsWin7OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 1))
+    End If
 End Function
 
 Public Function IsWin8OrLater() As Boolean
     If (Not m_VersionInfoCached) Then CacheOSVersion
-    IsWin8OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 2))
+    If OS.IsWinActuallyWine Then
+        IsWin8OrLater = False
+    Else
+        IsWin8OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 2))
+    End If
 End Function
 
 Public Function IsWin81OrLater() As Boolean
     If (Not m_VersionInfoCached) Then CacheOSVersion
-    IsWin81OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 3))
+    If OS.IsWinActuallyWine Then
+        IsWin81OrLater = False
+    Else
+        IsWin81OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 3))
+    End If
 End Function
 
 ' (NOTE: the Win-10 check requires a manifest, so don't rely on it in the IDE.  Also, MS doesn't guarantee that this
 ' check will remain valid forever, though it does work as of Windows 10-1703.)
 Public Function IsWin10OrLater() As Boolean
     If (Not m_VersionInfoCached) Then CacheOSVersion
-    IsWin10OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 4))
+    If OS.IsWinActuallyWine Then
+        IsWin10OrLater = False
+    Else
+        IsWin10OrLater = (m_OSVI.dwMajorVersion > 6) Or ((m_OSVI.dwMajorVersion = 6) And (m_OSVI.dwMinorVersion >= 4))
+    End If
 End Function
 
 Public Function GetWin10Build() As Long
     If (Not m_VersionInfoCached) Then CacheOSVersion
     GetWin10Build = m_OSVI.dwBuildNumber
 End Function
 
+'Are we running inside Wine?  Thanks to StackOverflow for the implementation idea:
+' https://stackoverflow.com/questions/7372388/determine-whether-a-program-is-running-under-wine-at-runtime
+'
+'PD doesn't officially support Wine, but I've found that lying to PD and claiming that Wine is actually XP
+' produces a reasonably usable version of the app.
+Public Function IsWinActuallyWine() As Boolean
+    
+    'Cache check once, on first call
+    If (Not m_LookedForWineAlready) Then
+        
+        m_LookedForWineAlready = True
+        
+        Dim modName As String
+        modName = "ntdll.dll"
+        
+        Dim hMod As Long
+        hMod = GetModuleHandleW(StrPtr(modName))
+        If (hMod <> 0) Then
+            Dim hProc As Long
+            hProc = GetProcAddress(hMod, "wine_get_version")
+            m_InsideWine = (hProc <> 0)
+        Else
+            PDDebug.LogAction "WARNING: couldn't find ntdll?"
+            m_InsideWine = False
+        End If
+        
+    End If
+    
+    IsWinActuallyWine = m_InsideWine
+    
+End Function
+
 'Return the number of logical cores on this system
 Public Function LogicalCoreCount() As Long
     Dim tmpSysInfo As OS_SystemInfo

PhotoDemon.vbp

@@ -527,7 +527,7 @@ Description="PhotoDemon Photo Editor"
 CompatibleMode="0"
 MajorVer=2024
 MinorVer=4
-RevisionVer=26
+RevisionVer=40
 AutoIncrementVer=1
 ServerSupportFiles=0
 VersionComments="Copyright 2000-2024 Tanner Helland - photodemon.org"