Skip to content

Commit

Permalink
Add a check-box to enable/disable expensive working set monitoring.
Browse files Browse the repository at this point in the history 
Windows makes it cheap and easy to retrieve the size of the full
working set of a process, but expensive and tricky to calculate the
private working set or PSS. In order to avoid causing problems UIforETW
now defaults to just grabbing the full working set (of specified processes)
and only calculates the private WS and PSS when a checkbox is checked.
  • Loading branch information
@randomascii
randomascii committed Jul 7, 2015
1 parent 04bfa15 commit 16c2171
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 49 deletions.
1 change: 1 addition & 0 deletions UIforETW/Resource.h
View file
Expand Up @@ -58,6 +58,7 @@
#define IDC_VIRTUALALLOCSTACKS 1029
#define IDC_CHROMEDLLPATH 1030
#define IDC_BUILDDATE 1031
#define IDC_EXPENSIVEWS 1031
#define IDC_CHROME_CATEGORIES 1033
#define ID_TRACES_OPENTRACEINWPA 32771
#define ID_TRACES_DELETETRACE 32772
Expand Down
13 changes: 13 additions & 0 deletions UIforETW/Settings.cpp
View file
Expand Up @@ -89,6 +89,7 @@ void CSettings::DoDataExchange(CDataExchange* pDX)
DDX_Control(pDX, IDC_HEAPSTACKS, btHeapStacks_);
DDX_Control(pDX, IDC_CHROMEDLLPATH, btChromeDllPath_);
DDX_Control(pDX, IDC_WSMONITOREDPROCESSES, btWSMonitoredProcesses_);
DDX_Control(pDX, IDC_EXPENSIVEWS, btExpensiveWSMonitoring_);
DDX_Control(pDX, IDC_VIRTUALALLOCSTACKS, btVirtualAllocStacks_);
DDX_Control(pDX, IDC_CHROME_CATEGORIES, btChromeCategories_);

Expand All @@ -103,6 +104,7 @@ BEGIN_MESSAGE_MAP(CSettings, CDialogEx)
ON_BN_CLICKED(IDC_AUTOVIEWTRACES, &CSettings::OnBnClickedAutoviewtraces)
ON_BN_CLICKED(IDC_HEAPSTACKS, &CSettings::OnBnClickedHeapstacks)
ON_BN_CLICKED(IDC_VIRTUALALLOCSTACKS, &CSettings::OnBnClickedVirtualallocstacks)
ON_BN_CLICKED(IDC_EXPENSIVEWS, &CSettings::OnBnClickedExpensivews)
END_MESSAGE_MAP()

BOOL CSettings::OnInitDialog()
Expand All @@ -126,6 +128,7 @@ BOOL CSettings::OnInitDialog()
btWSMonitoredProcesses_.EnableWindow(FALSE);
else
SetDlgItemText(IDC_WSMONITOREDPROCESSES, WSMonitoredProcesses_.c_str());
CheckDlgButton(IDC_EXPENSIVEWS, bExpensiveWSMonitoring_);

if (toolTip_.Create(this))
{
Expand Down Expand Up @@ -154,6 +157,10 @@ BOOL CSettings::OnInitDialog()
toolTip_.AddTool(&btWSMonitoredProcesses_, L"Names of processes whose working sets will be "
L"monitored, separated by semi-colons. An empty string means no monitoring. A '*' means "
L"that all processes will be monitored. For instance 'chrome.exe;notepad.exe'");
toolTip_.AddTool(&btExpensiveWSMonitoring_, L"Check this to have private working set and PSS "
L"(proportional set size) calculated for monitored processes. This may consume "
L"dozens or hundreds of ms each time. Without this checked only full working "
L"set is calculated, which is cheap.");
toolTip_.AddTool(&btVirtualAllocStacks_, L"Check this to record call stacks on VirtualAlloc on all "
L"traces instead of just heap traces.");
}
Expand Down Expand Up @@ -293,3 +300,9 @@ void CSettings::OnBnClickedVirtualallocstacks()
{
bVirtualAllocStacks_ = !bVirtualAllocStacks_;
}


void CSettings::OnBnClickedExpensivews()
{
bExpensiveWSMonitoring_ = !bExpensiveWSMonitoring_;
}
3 changes: 3 additions & 0 deletions UIforETW/Settings.h
View file
Expand Up @@ -36,6 +36,7 @@ class CSettings : public CDialogEx
std::wstring heapTracingExes_;
std::wstring chromeDllPath_;
std::wstring WSMonitoredProcesses_;
bool bExpensiveWSMonitoring_ = false;
bool bChromeDeveloper_ = false;
bool bAutoViewTraces_ = false;
bool bHeapStacks_ = false;
Expand All @@ -46,6 +47,7 @@ class CSettings : public CDialogEx
CEdit btHeapTracingExe_;
CMFCEditBrowseCtrl btChromeDllPath_;
CEdit btWSMonitoredProcesses_;
CButton btExpensiveWSMonitoring_;
CEdit btExtraProviders_;
CEdit btExtraStackwalks_;
CComboBox btBufferSizes_;
Expand Down Expand Up @@ -77,4 +79,5 @@ class CSettings : public CDialogEx
afx_msg void OnBnClickedAutoviewtraces();
afx_msg void OnBnClickedHeapstacks();
afx_msg void OnBnClickedVirtualallocstacks();
afx_msg void OnBnClickedExpensivews();
};
38 changes: 20 additions & 18 deletions UIforETW/UIforETW.rc
View file
Expand Up @@ -124,7 +124,7 @@ BEGIN
EDITTEXT IDC_TRACENAMEEDIT,75,144,160,14,ES_AUTOHSCROLL | ES_WANTRETURN | NOT WS_VISIBLE | NOT WS_TABSTOP
END

IDD_SETTINGS DIALOGEX 0, 0, 401, 178
IDD_SETTINGS DIALOGEX 0, 0, 401, 190
STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | WS_POPUP | WS_CAPTION | WS_SYSMENU
CAPTION "Settings"
FONT 8, "MS Shell Dlg", 400, 0, 0x1
Expand All @@ -133,25 +133,27 @@ BEGIN
EDITTEXT IDC_HEAPEXE,89,7,133,14,ES_AUTOHSCROLL
LTEXT "&WS monitored processes:",IDC_STATIC,7,25,82,8
EDITTEXT IDC_WSMONITOREDPROCESSES,89,22,133,14,ES_AUTOHSCROLL
LTEXT "Extra providers:",IDC_STATIC,7,40,53,8
EDITTEXT IDC_EXTRAPROVIDERS,89,38,133,14,ES_AUTOHSCROLL
LTEXT "Extra stackwalks:",IDC_STATIC,7,57,57,8
EDITTEXT IDC_EXTRASTACKWALKS,89,55,133,14,ES_AUTOHSCROLL
LTEXT "Buffer sizes:",IDC_STATIC,7,75,41,8
COMBOBOX IDC_BUFFERSIZES,89,73,81,103,CBS_DROPDOWN | CBS_SORT | WS_VSCROLL | WS_TABSTOP
LTEXT "Chrome.dll &path:",IDC_STATIC,8,94,67,8
CONTROL "",IDC_CHROMEDLLPATH,"MfcEditBrowse",WS_BORDER | WS_TABSTOP | 0x80,89,91,133,14
PUSHBUTTON "&Copy startup profile",IDC_COPYSTARTUPPROFILE,7,119,75,14
PUSHBUTTON "Copy symbol &DLLs",IDC_COPYSYMBOLDLLS,7,137,75,14
CONTROL "Ch&rome developer",IDC_CHROMEDEVELOPER,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,110,75,10
CONTROL "&Auto view traces",IDC_AUTOVIEWTRACES,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,125,70,10
CONTROL "&Stacks on heap tracing",IDC_HEAPSTACKS,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,140,89,10
CONTROL "E&xpensive working set monitoring",IDC_EXPENSIVEWS,
"Button",BS_AUTOCHECKBOX | WS_TABSTOP,7,39,123,10
LTEXT "Extra providers:",IDC_STATIC,7,58,53,8
EDITTEXT IDC_EXTRAPROVIDERS,89,56,133,14,ES_AUTOHSCROLL
LTEXT "Extra stackwalks:",IDC_STATIC,7,75,57,8
EDITTEXT IDC_EXTRASTACKWALKS,89,73,133,14,ES_AUTOHSCROLL
LTEXT "Buffer sizes:",IDC_STATIC,7,93,41,8
COMBOBOX IDC_BUFFERSIZES,89,91,81,103,CBS_DROPDOWN | CBS_SORT | WS_VSCROLL | WS_TABSTOP
LTEXT "Chrome.dll &path:",IDC_STATIC,8,112,67,8
CONTROL "",IDC_CHROMEDLLPATH,"MfcEditBrowse",WS_BORDER | WS_TABSTOP | 0x80,89,109,133,14
PUSHBUTTON "&Copy startup profile",IDC_COPYSTARTUPPROFILE,7,137,75,14
PUSHBUTTON "Copy symbol &DLLs",IDC_COPYSYMBOLDLLS,7,155,75,14
CONTROL "Ch&rome developer",IDC_CHROMEDEVELOPER,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,128,75,10
CONTROL "&Auto view traces",IDC_AUTOVIEWTRACES,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,143,70,10
CONTROL "&Stacks on heap tracing",IDC_HEAPSTACKS,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,158,89,10
CONTROL "&VirtualAlloc stacks always",IDC_VIRTUALALLOCSTACKS,
"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,155,97,10
"Button",BS_AUTOCHECKBOX | WS_TABSTOP,103,173,97,10
LTEXT "Chrome tracing cate&gories:",IDC_STATIC,245,7,131,8
LISTBOX IDC_CHROME_CATEGORIES,245,18,149,135,LBS_OWNERDRAWFIXED | LBS_HASSTRINGS | LBS_NOINTEGRALHEIGHT | LBS_DISABLENOSCROLL | WS_VSCROLL | WS_TABSTOP
DEFPUSHBUTTON "OK",IDOK,286,157,50,14
PUSHBUTTON "Cancel",IDCANCEL,344,157,50,14
DEFPUSHBUTTON "OK",IDOK,286,169,50,14
PUSHBUTTON "Cancel",IDCANCEL,344,169,50,14
END


Expand Down Expand Up @@ -223,7 +225,7 @@ BEGIN
LEFTMARGIN, 7
RIGHTMARGIN, 394
TOPMARGIN, 7
BOTTOMMARGIN, 171
BOTTOMMARGIN, 183
END
END
#endif // APSTUDIO_INVOKED
Expand Down
8 changes: 6 additions & 2 deletions UIforETW/UIforETWDlg.cpp
View file
Expand Up @@ -381,7 +381,7 @@ BOOL CUIforETWDlg::OnInitDialog()
monitorThread_.StartThread(&traceDir_);

// Configure the working set monitor.
workingSetThread_.SetProcessFilter(WSMonitoredProcesses_);
workingSetThread_.SetProcessFilter(WSMonitoredProcesses_, bExpensiveWSMonitoring_);

DisablePagingExecutive();

Expand Down Expand Up @@ -1348,6 +1348,7 @@ void CUIforETWDlg::OnBnClickedSettings()
dlgSettings.heapTracingExes_ = heapTracingExes_;
dlgSettings.chromeDllPath_ = chromeDllPath_;
dlgSettings.WSMonitoredProcesses_ = WSMonitoredProcesses_;
dlgSettings.bExpensiveWSMonitoring_ = bExpensiveWSMonitoring_;
dlgSettings.bChromeDeveloper_ = bChromeDeveloper_;
dlgSettings.bAutoViewTraces_ = bAutoViewTraces_;
dlgSettings.bHeapStacks_ = bHeapStacks_;
Expand Down Expand Up @@ -1378,8 +1379,11 @@ void CUIforETWDlg::OnBnClickedSettings()

// Copy over the remaining settings.
chromeDllPath_ = dlgSettings.chromeDllPath_;

WSMonitoredProcesses_ = dlgSettings.WSMonitoredProcesses_;
workingSetThread_.SetProcessFilter(WSMonitoredProcesses_);
bExpensiveWSMonitoring_ = dlgSettings.bExpensiveWSMonitoring_;
workingSetThread_.SetProcessFilter(WSMonitoredProcesses_, bExpensiveWSMonitoring_);

bAutoViewTraces_ = dlgSettings.bAutoViewTraces_;
bHeapStacks_ = dlgSettings.bHeapStacks_;
bVirtualAllocStacks_ = dlgSettings.bVirtualAllocStacks_;
Expand Down
2 changes: 2 additions & 0 deletions UIforETW/UIforETWDlg.h
View file
Expand Up @@ -117,6 +117,8 @@ class CUIforETWDlg : public CDialogEx
// This starts and stops a thread that monitors process working sets.
CWorkingSetMonitor workingSetThread_;
std::wstring WSMonitoredProcesses_;
bool bExpensiveWSMonitoring_ = false;

// This starts and stops a thread that monitors battery status.
CPowerStatusMonitor PowerMonitor_;

Expand Down
72 changes: 44 additions & 28 deletions UIforETW/WorkingSet.cpp
View file
Expand Up @@ -84,48 +84,63 @@ void CWorkingSetMonitor::SampleWorkingSets()
HANDLE hProcess =
OpenProcess(PROCESS_QUERY_INFORMATION |
PROCESS_VM_READ, FALSE, pid);
ULONG_PTR wsPages = 0;
uint64_t PSSPages = 0;
ULONG_PTR privateWSPages = 0;

if (NULL != hProcess)
{
bool success = true;
if (!QueryWorkingSet(hProcess, &buffer[0], static_cast<DWORD>(buffer.size())))
if (bExpensiveWSMonitoring_)
{
success = false;
// Increase the buffer size based on the NumberOfEntries returned,
// with some padding in case the working set is increasing.
if (GetLastError() == ERROR_BAD_LENGTH)
if (!QueryWorkingSet(hProcess, &buffer[0], static_cast<DWORD>(buffer.size())))
{
numEntries = pwsBuffer->NumberOfEntries + pwsBuffer->NumberOfEntries / 4;
buffer.resize(sizeof(PSAPI_WORKING_SET_INFORMATION) + numEntries * sizeof(PSAPI_WORKING_SET_BLOCK));
pwsBuffer = reinterpret_cast<PSAPI_WORKING_SET_INFORMATION*>(&buffer[0]);
if (QueryWorkingSet(hProcess, &buffer[0], static_cast<DWORD>(buffer.size())))
success = false;
// Increase the buffer size based on the NumberOfEntries returned,
// with some padding in case the working set is increasing.
if (GetLastError() == ERROR_BAD_LENGTH)
{
success = true;
numEntries = pwsBuffer->NumberOfEntries + pwsBuffer->NumberOfEntries / 4;
buffer.resize(sizeof(PSAPI_WORKING_SET_INFORMATION) + numEntries * sizeof(PSAPI_WORKING_SET_BLOCK));
pwsBuffer = reinterpret_cast<PSAPI_WORKING_SET_INFORMATION*>(&buffer[0]);
if (QueryWorkingSet(hProcess, &buffer[0], static_cast<DWORD>(buffer.size())))
{
success = true;
}
}
}
}

if (success)
{
ULONG_PTR wsPages = pwsBuffer->NumberOfEntries;
uint64_t PSSPages = 0;
ULONG_PTR privateWSPages = 0;
for (ULONG_PTR page = 0; page < wsPages; ++page)
if (success)
{
if (!pwsBuffer->WorkingSetInfo[page].Shared)
wsPages = pwsBuffer->NumberOfEntries;
for (ULONG_PTR page = 0; page < wsPages; ++page)
{
++privateWSPages;
PSSPages += PSSMultiplier;
}
else
{
UIETWASSERT(pwsBuffer->WorkingSetInfo[page].ShareCount <= 7);
PSSPages += PSSMultiplier / pwsBuffer->WorkingSetInfo[page].ShareCount;
if (!pwsBuffer->WorkingSetInfo[page].Shared)
{
++privateWSPages;
PSSPages += PSSMultiplier;
}
else
{
UIETWASSERT(pwsBuffer->WorkingSetInfo[page].ShareCount <= 7);
PSSPages += PSSMultiplier / pwsBuffer->WorkingSetInfo[page].ShareCount;
}
}
totalPSSPages += PSSPages;
totalPrivateWSPages += privateWSPages;
}
}
else
{
PROCESS_MEMORY_COUNTERS memoryCounters = {sizeof(memoryCounters)};
if (GetProcessMemoryInfo(hProcess, &memoryCounters, sizeof(memoryCounters)))
{
wsPages = memoryCounters.WorkingSetSize / 4096;
}
}
if (success)
{
totalWSPages += wsPages;
totalPSSPages += PSSPages;
totalPrivateWSPages += privateWSPages;

wchar_t process[MAX_PATH + 100];
swprintf_s(process, L"%s (%u)", peInfo.szExeFile, pid);
Expand Down Expand Up @@ -180,7 +195,7 @@ CWorkingSetMonitor::~CWorkingSetMonitor()
CloseHandle(hExitEvent_);
}

void CWorkingSetMonitor::SetProcessFilter(const std::wstring& processes)
void CWorkingSetMonitor::SetProcessFilter(const std::wstring& processes, bool bExpensiveWSMonitoring)
{
// A 32-bit process on 64-bit Windows will not be able to read the
// full working set of 64-bit processes, so don't even try.
Expand All @@ -196,4 +211,5 @@ void CWorkingSetMonitor::SetProcessFilter(const std::wstring& processes)
processAll_ = false;
processes_ = split(processes, ';');
}
bExpensiveWSMonitoring_ = bExpensiveWSMonitoring;
}
4 changes: 3 additions & 1 deletion UIforETW/WorkingSet.h
View file
Expand Up @@ -29,7 +29,7 @@ class CWorkingSetMonitor
// the working set display should monitor. If this is the
// empty string then no processes will be monitored.
// If this is '*' then all processes will be monitored.
void SetProcessFilter(const std::wstring& processes);
void SetProcessFilter(const std::wstring& processes, bool expensiveWSMonitoring);
private:
static DWORD __stdcall StaticWSMonitorThread(LPVOID);
void WSMonitorThread();
Expand All @@ -49,6 +49,8 @@ class CWorkingSetMonitor
std::vector<std::wstring> processes_;
// This variable is protected by processesLock_;
bool processAll_ = false;
// This variable is protected by processesLock_;
bool bExpensiveWSMonitoring_ = false;

// Incrementing counter that will be the same for all samples recorded
// at the same time.
Expand Down

4 comments on commit 16c2171

@MagicAndre1981
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Windows includes the provider Microsoft-Windows-Kernel-Memory, when used with Keyword 0x40 KERNEL_MEM_KEYWORD_MEMINFO_EX, Windows captures every 0.5s: Count, ProcessID, WorkingSetPageCount, CommitPageCount, VirtualSizeInPages, PrivateWorkingSetPageCount.

So there is no need to capture your own data.

@randomascii
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting. I just tried that on Windows 10 and it looks very promising. The data shows up in Memory-> Virtual Memory Snapshots. Do you know what operating system versions support this provider?

Also, I like that it shows things like the SystemCacheWS - that is very useful data.

@MagicAndre1981
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Windows 8 (Build 9200) also supports KERNEL_MEM_KEYWORD_WS_SWAP (0x80). Win7 only supports KERNEL_MEM_KEYWORD_MEMINFO (keyword 0x20) which logs

            <data inType="win:UInt8" name="PriorityLevels" />
            <data inType="win:Pointer" name="ZeroPageCount" />
            <data inType="win:Pointer" name="FreePageCount" />
            <data inType="win:Pointer" name="ModifiedPageCount" />
            <data inType="win:Pointer" name="ModifiedNoWritePageCount" />
            <data inType="win:Pointer" name="BadPageCount" />
            <data count="PriorityLevels" inType="win:Pointer" name="StandbyPageCounts" />
            <data count="PriorityLevels" inType="win:Pointer" name="RepurposedPageCounts" />

Windows 8.1 (6.3.9600) an Win10 also supports KERNEL_MEM_KEYWORD_ACG (0x100), but I have no idea what this is.

I found this provider during my research about the memory compression of Windows 10. I want to trace which data from which processes get compressed.

@randomascii
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved to #80.

Please sign in to comment.