Skip to content
Permalink
Browse files

Move Windows overlay setup and injection to mumble_ol.exe.

This changes Mumble on Windows to run a helper process,
mumble_ol.exe, instead of loading mumble_ol.dll itself.

Prior to this change, Mumble would load mumble_ol.dll and
call PrepareD3D9() and PrepareDXGI() to set up the overlay.
Then, if the overlay was enabled, it would call InstallHooks()
to enable automatic injection of the overlay into new processes.
Similarly, it would call RemoveHooks() to disable automatic
overlay injection when the overlay was disabled in Mmuble.

With this change, Mumble instead runs a helper process called
mumble_ol.exe. This process sets up the overlay (equivalent
to calling PrepareD3D9(), PrepareDXGI() and calling InstallHooks()).
While running, it'll automatically inject the overlay into new
processes, just like Mumble itself did previously. On normal exit,
the helper process calls RemoveHooks() to ensure automatic overlay
injection is disabled.

To enable the overlay, Mumble starts the helper process.
To disable the overlay, Mumble terminates the helper process.
If the helper process dies when it is supposed to be running,
Mumble restarts it to ensure that overlay injection keeps
working as intended.

This change is the first part of enabling both an x86 and an x64
overlay to be active in Mumble at the same time. Since we cannot
load a 32-bit DLL into a 64-bit process (or vice versa), we need
a helper process for each architecture to reach our goal.
Note however that this commit in itself does not make it possible
for Mumble to run both an x86 and an x64 overlay at the same time.
This will come later.
  • Loading branch information...
mkrautz committed Jan 15, 2015
1 parent 3b507d5 commit 9f1e60507b06ab17e7dea22d4d5e83dc4bbc0e7b
@@ -35,6 +35,9 @@
<Component Id="mumble_ol.dll">
<File Source="$(var.SourceDir)\release\mumble_ol.dll" KeyPath="yes" />
</Component>
<Component Id="mumble_ol.exe">
<File Source="$(var.SourceDir)\release\mumble_ol.exe" KeyPath="yes" />
</Component>
<Component Id="celt0.0.7.0.dll">
<File Source="$(var.SourceDir)\release\celt0.0.7.0.dll" KeyPath="yes" />
</Component>
@@ -172,6 +172,7 @@
<?endif ?>
<ComponentRef Id="mumble.exe" />
<ComponentRef Id="mumble_ol.dll" />
<ComponentRef Id="mumble_ol.exe" />
<?ifdef G15 ?>
<ComponentRef Id="mumble_g15_helper.exe" />
<?endif ?>
@@ -47,6 +47,7 @@ CONFIG *= ordered debug_and_release
win32 {
SUBDIRS *= 3rdparty/fx11-build
SUBDIRS *= overlay
SUBDIRS *= overlay/overlay_exe
!CONFIG(no-g15) {
SUBDIRS *= g15helper
}
@@ -31,6 +31,7 @@
#include "lib.h"

#include "overlay_blacklist.h"
#include "overlay_exe/overlay_exe.h"

static HANDLE hMapObject = NULL;
static HANDLE hHookMutex = NULL;
@@ -442,6 +443,48 @@ extern "C" __declspec(dllexport) unsigned int __cdecl GetOverlayMagicVersion() {
return OVERLAY_MAGIC_NUMBER;
}

// Via d3d9.cpp
extern "C" __declspec(dllexport) void __cdecl PrepareD3D9();
// Via dxgi.cpp
extern "C" __declspec(dllexport) void __cdecl PrepareDXGI();

extern "C" __declspec(dllexport) int __cdecl OverlayHelperProcessMain(unsigned int magic) {
int retval = 0;

if (GetOverlayMagicVersion() != magic) {
return OVERLAY_HELPER_ERROR_DLL_MAGIC_MISMATCH;
}

PrepareD3D9();
PrepareDXGI();

InstallHooks();

while (1) {
MSG msg;
BOOL ret;

ret = GetMessage(&msg, NULL, 0, 0);

// The ret variable is set to 0 on WM_QUIT,
// and -1 on error.
if (ret == 0) {
retval = 0;
break;
} else if (ret == -1) {
retval = -1001;
break;
}

TranslateMessage(&msg);
DispatchMessage(&msg);
}

RemoveHooks();

return retval;
}

static bool dllmainProcAttachCheckProcessIsBlacklisted(char procname[], char *p);
static bool createSharedDataMap();

@@ -452,7 +495,7 @@ static void dllmainProcAttach(char *procname) {
if (!p) {
// No blacklisting if the file has no path
} else if (GetProcAddress(NULL, "mumbleSelfDetection") != NULL) {
ods("Lib: Attached to self (own process). Blacklisted - no overlay injection.");
ods("Lib: Attached to overlay helper or Mumble process. Blacklisted - no overlay injection.");
bBlackListed = TRUE;
bMumble = TRUE;
} else {
@@ -0,0 +1,176 @@
/* Copyright (C) 2011-2013, Benjamin Jemlich <pcgod@users.sourceforge.net>
Copyright (C) 2013-2015 Mikkel Krautz <mikkel@krautz.dk>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
- Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
- Neither the name of the Mumble Developers nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include <windows.h>
#include <shlwapi.h>
#include <stdio.h>

#include <string>

#include "../overlay.h"
#include "overlay_exe.h"

#define UNUSED(x) ((void)x)

typedef int (*OverlayHelperProcessMain)(unsigned int magic);

// Signal to the overlay DLL that it should not inject
// into this process.
extern "C" __declspec(dllexport) void mumbleSelfDetection() {};

// Alert shows a fatal error dialog and waits for the user to click OK.
static void Alert(LPCWSTR title, LPCWSTR msg) {
MessageBox(NULL, msg, title, MB_OK|MB_ICONERROR);
}

// GetExecutableDirPath returns the directory that
// mumble_ol.exe resides in.
static std::wstring GetExecutableDirPath() {
wchar_t path[MAX_PATH];

if (GetModuleFileNameW(NULL, path, MAX_PATH) == 0)
return std::wstring();

if (!PathRemoveFileSpecW(path))
return std::wstring();

std::wstring exePath(path);
return exePath.append(L"\\");
}

// ConfigureEnvironment prepares mumble_ol.exe's environment to
// run mumble_ol.dll's OverlayHelperProcessMain() function.
static bool ConfigureEnvironment() {
std::wstring exePath = GetExecutableDirPath();

// Remove the current directory from the DLL search path.
if (!SetDllDirectoryW(L""))
return false;

// Set mumble_ol.exe's directory as the current working directory.
if (!SetCurrentDirectoryW(exePath.c_str()))
return false;

return true;
}

// GetAbsoluteMumbleOverlayDllPath returns the absolute path to
// mumble_ol.dll - the DLL containing the Mumble overlay code.
static std::wstring GetAbsoluteMumbleOverlayDllPath() {
std::wstring exePath = GetExecutableDirPath();
if (exePath.empty())
return std::wstring();

std::wstring absDLLPath(exePath);
absDLLPath.append(L"\\");
absDLLPath.append(L"mumble_ol.dll");
return absDLLPath;
}

int main(int argc, char **argv) {
UNUSED(argc);
UNUSED(argv);

// Mumble passes the OVERLAY_MAGIC_NUMBER it was built
// with the first argument to this helper process.
//
// This is done for two reasons.
//
// The first reason is that we need to ensure
// compatibility between Mumble, the overlay helper,
// and the overlay itself.
//
// The second reason is that it is an easy way to catch
// users accidentally double-clicking on the overlay
// helper's EXE file. We can detect these non-Mumble
// initiated launches by checking if we were passed any
// arguments at all. If no parameters are passed, we
// display a nice alert dialog directing users to
// 'mumble.exe' instead.
unsigned int magic = 0;
{
std::wstring commandLine(GetCommandLine());

// The command line will contain two consecutive spaces
// if the program was passed any arguments. If we don't
// find them, it probably means that a user has double-clicked
// the executable. Tell them to run 'mumble.exe' instead.
size_t sep = commandLine.find(std::wstring(L" "));
if (sep == std::string::npos) {
Alert(L"Mumble Overlay", L"This program is not meant to be run by itself. Run 'mumble.exe' instead.");
return OVERLAY_HELPER_ERROR_EXE_MISSING_MAGIC_ARGUMENT;
}

// We expect that the Mumble process passes the overlay
// magic number that it is built with to us.
std::wstring magicNumberStr = commandLine.substr(sep);
try {
unsigned long passedInMagic = std::stoul(magicNumberStr);
magic = static_cast<unsigned int>(passedInMagic);
} catch (std::exception &) {
return OVERLAY_HELPER_ERROR_EXE_INVALID_MAGIC_ARGUMENT;
}
}

if (magic != OVERLAY_MAGIC_NUMBER) {
return OVERLAY_HELPER_ERROR_EXE_MAGIC_MISMATCH;
}

if (!ConfigureEnvironment()) {
return OVERLAY_HELPER_ERROR_EXE_CONFIGURE_ENVIRONMENT;
}

std::wstring absDLLPath = GetAbsoluteMumbleOverlayDllPath();
if (absDLLPath.empty()) {
return OVERLAY_HELPER_ERROR_EXE_GET_DLL_PATH;
}

HMODULE dll = LoadLibraryExW(absDLLPath.c_str(), NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
if (!dll) {
return OVERLAY_HELPER_ERROR_EXE_LOAD_DLL;
}

OverlayHelperProcessMain entryPoint = reinterpret_cast<OverlayHelperProcessMain>(GetProcAddress(dll, "OverlayHelperProcessMain"));
if (!entryPoint) {
return OVERLAY_HELPER_ERROR_EXE_LOOKUP_ENTRY_POINT;
}

return entryPoint(magic);
}

int APIENTRY wWinMain(HINSTANCE instance, HINSTANCE prevInstance, wchar_t *cmdArg, int cmdShow) {
UNUSED(instance);
UNUSED(prevInstance);
UNUSED(cmdArg);
UNUSED(cmdShow);

return main(0, NULL);
}
@@ -0,0 +1,89 @@
/* Copyright (C) 2015, Mikkel Krautz <mikkel@krautz.dk>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
- Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
- Neither the name of the Mumble Developers nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#ifndef MUMBLE_OVERLAY_EXE_H_
#define MUMBLE_OVERLAY_EXE_H_

/// OverlayHelperError represents exit codes returned by the
/// overlay helper process (mumble_ol.exe) on Windows.
enum OverlayHelperError {
/// The overlay helper process was not passed a magic
/// number as its first and only argument.
OVERLAY_HELPER_ERROR_EXE_MISSING_MAGIC_ARGUMENT = -1,
/// The magic number on the command line of the overlay
/// helper process could not be converted to an integer.
OVERLAY_HELPER_ERROR_EXE_INVALID_MAGIC_ARGUMENT = -2,
/// The magic number on the command line of the overlay
/// helper did not match the built-in magic number of
/// the helper process.
OVERLAY_HELPER_ERROR_EXE_MAGIC_MISMATCH = -3,
/// The overlay helper process was unable to configure
/// its environment in preparation of loading the
/// overlay DLL.
OVERLAY_HELPER_ERROR_EXE_CONFIGURE_ENVIRONMENT = -4,
/// The overlay helper process was unable to get the
/// path to the overlay DLL.
OVERLAY_HELPER_ERROR_EXE_GET_DLL_PATH = -5,
/// The overlay helper process was unable to load the
/// overlay DLL.
OVERLAY_HELPER_ERROR_EXE_LOAD_DLL = -6,
/// The overlay helper process was uanble to look up
/// the 'OverlayHelperProcessMain' entry point in the
/// overlay DLL.
OVERLAY_HELPER_ERROR_EXE_LOOKUP_ENTRY_POINT = -7,

/// The magic number passed to the overlay DLL's
/// OverlayHelperProcessMain function did not match
/// the overlay DLL's built-in magic number.
OVERLAY_HELPER_ERROR_DLL_MAGIC_MISMATCH = -1000,
/// The overlay helper process exited due to an error
/// in the Windows message loop.
OVERLAY_HELPER_ERROR_DLL_MESSAGE_LOOP = -1001,
};

/// OverlayHelperErrorToString converts an OverlayHelperError value
/// to a printable string representation.
static inline const char *OverlayHelperErrorToString(OverlayHelperError err) {
#define OHE(x) case x: return #x
switch (err) {
OHE(OVERLAY_HELPER_ERROR_EXE_MISSING_MAGIC_ARGUMENT);
OHE(OVERLAY_HELPER_ERROR_EXE_INVALID_MAGIC_ARGUMENT);
OHE(OVERLAY_HELPER_ERROR_EXE_MAGIC_MISMATCH);
OHE(OVERLAY_HELPER_ERROR_EXE_CONFIGURE_ENVIRONMENT);
OHE(OVERLAY_HELPER_ERROR_EXE_GET_DLL_PATH);
OHE(OVERLAY_HELPER_ERROR_EXE_LOAD_DLL);
OHE(OVERLAY_HELPER_ERROR_EXE_LOOKUP_ENTRY_POINT);
OHE(OVERLAY_HELPER_ERROR_DLL_MAGIC_MISMATCH);
OHE(OVERLAY_HELPER_ERROR_DLL_MESSAGE_LOOP);
}
return NULL;
}

#endif
@@ -0,0 +1,38 @@
include(../../compiler.pri)

TEMPLATE = app
CONFIG -= qt
CONFIG += debug_and_release
TARGET = mumble_ol

win32 {
DEFINES += WIN32 _WIN32
RC_FILE = overlay_exe.rc
LIBS *= -luser32 -lshlwapi

CONFIG(release, debug|release) {
QMAKE_CXXFLAGS_RELEASE -= -MD
QMAKE_CXXFLAGS += -MT
}
CONFIG(debug, debug|release) {
QMAKE_CXXFLAGS_DEBUG -= -MDd
QMAKE_CXXFLAGS += -MTd
}
}

SOURCES *= overlay_exe.cpp

QMAKE_POST_LINK = $$QMAKE_POST_LINK$$escape_expand(\\n\\t)$$quote(mt.exe -nologo -updateresource:$(DESTDIR_TARGET);1 -manifest ../../src/mumble/mumble.appcompat.manifest)

CONFIG(debug, debug|release) {
CONFIG += console
DEFINES *= DEBUG
DESTDIR = ../../debug
}

CONFIG(release, debug|release) {
DEFINES *= NDEBUG
DESTDIR = ../../release
}

include(../../symbols.pri)

0 comments on commit 9f1e605

Please sign in to comment.
You can’t perform that action at this time.