From 3847821b7a9cf7a3b4889339c19ddd810ebc39e8 Mon Sep 17 00:00:00 2001 From: Alexey Nazarov Date: Mon, 18 Jun 2018 00:36:50 +0300 Subject: [PATCH 1/5] Added USS (unique set size) memory option --- README.rst | 12 ++++++++ psrecord/main.py | 58 ++++++++++++++++++++++++++++++------- psrecord/tests/test_main.py | 32 +++++++++++++++++++- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 913b25c..66d7380 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,18 @@ To include sub-processes in the CPU and memory stats, use: psrecord 1330 --log activity.txt --include-children +USS memory +--------- + +The USS (Unique Set Size) is the memory which is unique to a process and +which would be freed if the process was terminated right now. USS option +work only for Linux, Windows and OSX systems. To include USS in the memory +stats, use: + +:: + + psrecord 1330 --log activity.txt --uss + Running tests ============= diff --git a/psrecord/main.py b/psrecord/main.py index 8ed716b..a9f5217 100644 --- a/psrecord/main.py +++ b/psrecord/main.py @@ -37,9 +37,9 @@ def get_percent(process): return process.get_cpu_percent() -def get_memory(process): +def get_memory(process, is_uss=False): try: - return process.memory_info() + return process.memory_full_info() if is_uss else process.memory_info() except AttributeError: return process.get_memory_info() @@ -60,6 +60,14 @@ def all_children(pr): return processes +def is_uss_possible(process): + try: + process.memory_full_info() + return True + except AttributeError: + print ("Flag --uss is ignored. Not found method memory_full_info()") + return False + def main(): parser = argparse.ArgumentParser( @@ -89,6 +97,12 @@ def main(): 'in a slower maximum sampling rate).', action='store_true') + parser.add_argument('--uss', + help='in additional to real memory usage show unique ' + 'set size (USS) as more suitable to know how ' + 'much memory program were using.', + action='store_true') + args = parser.parse_args() # Attach to process @@ -105,38 +119,49 @@ def main(): pid = sprocess.pid monitor(pid, logfile=args.log, plot=args.plot, duration=args.duration, - interval=args.interval, include_children=args.include_children) + interval=args.interval, include_children=args.include_children, + uss=args.uss) if sprocess is not None: sprocess.kill() def monitor(pid, logfile=None, plot=None, duration=None, interval=None, - include_children=False): + include_children=False, uss=False): # We import psutil here so that the module can be imported even if psutil # is not present (for example if accessing the version) import psutil pr = psutil.Process(pid) + + if uss: + uss = is_uss_possible(pr) # Record start time start_time = time.time() if logfile: f = open(logfile, 'w') - f.write("# {0:12s} {1:12s} {2:12s} {3:12s}\n".format( + f.write("# {0:12s} {1:12s} {2:12s} {3:12s}".format( 'Elapsed time'.center(12), 'CPU (%)'.center(12), 'Real (MB)'.center(12), 'Virtual (MB)'.center(12)) ) + if (uss): + f.write(" {0:12s}".format( + 'USS (MB)'.center(12)) + ) + f.write("\n") log = {} log['times'] = [] log['cpu'] = [] log['mem_real'] = [] log['mem_virtual'] = [] + if (uss): + log['mem_uss'] = [] try: @@ -166,29 +191,36 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, # Get current CPU and memory try: current_cpu = get_percent(pr) - current_mem = get_memory(pr) + current_mem = get_memory(pr, uss) except Exception: break current_mem_real = current_mem.rss / 1024. ** 2 current_mem_virtual = current_mem.vms / 1024. ** 2 + if (uss): + current_mem_uss = current_mem.uss / 1024. ** 2 # Get information for children if include_children: for child in all_children(pr): try: current_cpu += get_percent(child) - current_mem = get_memory(child) + current_mem = get_memory(child, uss) except Exception: continue current_mem_real += current_mem.rss / 1024. ** 2 current_mem_virtual += current_mem.vms / 1024. ** 2 + if (uss): + current_mem_uss += current_mem.uss / 1024. ** 2 if logfile: - f.write("{0:12.3f} {1:12.3f} {2:12.3f} {3:12.3f}\n".format( + f.write("{0:12.3f} {1:12.3f} {2:12.3f} {3:12.3f}".format( current_time - start_time, current_cpu, current_mem_real, current_mem_virtual)) + if (uss): + f.write(" {0:12.3f}".format(current_mem_uss)) + f.write("\n") f.flush() if interval is not None: @@ -200,6 +232,8 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, log['cpu'].append(current_cpu) log['mem_real'].append(current_mem_real) log['mem_virtual'].append(current_mem_virtual) + if (uss): + log['mem_uss'].append(current_mem_uss) except KeyboardInterrupt: # pragma: no cover pass @@ -220,12 +254,14 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, ax.set_xlabel('time (s)') ax.set_ylim(0., max(log['cpu']) * 1.2) + mem_selector = 'mem_uss' if uss else 'mem_real' ax2 = ax.twinx() - ax2.plot(log['times'], log['mem_real'], '-', lw=1, color='b') - ax2.set_ylim(0., max(log['mem_real']) * 1.2) + ax2.plot(log['times'], log[mem_selector], '-', lw=1, color='b') + ax2.set_ylim(0., max(log[mem_selector]) * 1.2) - ax2.set_ylabel('Real Memory (MB)', color='b') + mem_label= 'Unique set size (MB)' if uss else 'Real Memory (MB)' + ax2.set_ylabel(mem_label, color='b') ax.grid() diff --git a/psrecord/tests/test_main.py b/psrecord/tests/test_main.py index db3f848..ee95c40 100644 --- a/psrecord/tests/test_main.py +++ b/psrecord/tests/test_main.py @@ -3,7 +3,7 @@ import subprocess import psutil -from ..main import main, monitor, all_children +from ..main import main, monitor, all_children, is_uss_possible TEST_CODE = """ import subprocess @@ -12,6 +12,21 @@ """ +def test_is_uss_possible(): + + p = subprocess.Popen('sleep 10', shell=True) + pr = psutil.Process(p.pid) + + if (sys.platform.startswith("linux") + or sys.platform.startswith("win") + or sys.platform.startswith("darwin") + and psutil.version_info >= (2, 6)): + assert is_uss_possible(pr) == True + else: + assert is_uss_possible(pr) == False + + p.kill() + def test_all_children(tmpdir): filename = tmpdir.join('test.py').strpath @@ -66,3 +81,18 @@ def test_main(self): def test_main_by_id(self): sys.argv = ['psrecord', '--duration=3', str(os.getpid())] main() + + def test_logfile_uss(self, tmpdir): + filename = tmpdir.join('test_logfile').strpath + monitor(self.p.pid, logfile=filename, duration=3, uss=True) + assert os.path.exists(filename) + is_uss_active = (open(filename, 'r').readlines()[0].find("USS") > 0) + assert is_uss_active == True + + def test_plot_uss(self, tmpdir): + filename = tmpdir.join('test_plot.png').strpath + monitor(self.p.pid, plot=filename, duration=3, uss=True) + assert os.path.exists(filename) + + def test_uss_with_children(self, tmpdir): + monitor(os.getpid(), duration=3, include_children=True, uss=True) From 7227c9da87f1de690771cb9bc30a6d1774d04f2e Mon Sep 17 00:00:00 2001 From: Alexey Nazarov Date: Mon, 18 Jun 2018 00:36:50 +0300 Subject: [PATCH 2/5] Added USS (unique set size) memory option --- README.rst | 12 ++++++++ psrecord/main.py | 56 +++++++++++++++++++++++++++++-------- psrecord/tests/test_main.py | 16 ++++++++++- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 913b25c..66d7380 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,18 @@ To include sub-processes in the CPU and memory stats, use: psrecord 1330 --log activity.txt --include-children +USS memory +--------- + +The USS (Unique Set Size) is the memory which is unique to a process and +which would be freed if the process was terminated right now. USS option +work only for Linux, Windows and OSX systems. To include USS in the memory +stats, use: + +:: + + psrecord 1330 --log activity.txt --uss + Running tests ============= diff --git a/psrecord/main.py b/psrecord/main.py index 8ed716b..5bcc041 100644 --- a/psrecord/main.py +++ b/psrecord/main.py @@ -37,9 +37,9 @@ def get_percent(process): return process.get_cpu_percent() -def get_memory(process): +def get_memory(process, is_uss=False): try: - return process.memory_info() + return process.memory_full_info() if is_uss else process.memory_info() except AttributeError: return process.get_memory_info() @@ -89,6 +89,12 @@ def main(): 'in a slower maximum sampling rate).', action='store_true') + parser.add_argument('--uss', + help='in additional to real memory usage show unique ' + 'set size (USS) as more suitable to know how ' + 'much memory program were using.', + action='store_true') + args = parser.parse_args() # Attach to process @@ -105,38 +111,55 @@ def main(): pid = sprocess.pid monitor(pid, logfile=args.log, plot=args.plot, duration=args.duration, - interval=args.interval, include_children=args.include_children) + interval=args.interval, include_children=args.include_children, + uss=args.uss) if sprocess is not None: sprocess.kill() def monitor(pid, logfile=None, plot=None, duration=None, interval=None, - include_children=False): + include_children=False, uss=False): # We import psutil here so that the module can be imported even if psutil # is not present (for example if accessing the version) import psutil pr = psutil.Process(pid) + + # Check is it possible to use --uss flag + if uss: + try: + pr.memory_full_info() + uss = True + except AttributeError: + print ("Flag --uss is ignored. Not found method memory_full_info()") + uss = False # Record start time start_time = time.time() if logfile: f = open(logfile, 'w') - f.write("# {0:12s} {1:12s} {2:12s} {3:12s}\n".format( + f.write("# {0:12s} {1:12s} {2:12s} {3:12s}".format( 'Elapsed time'.center(12), 'CPU (%)'.center(12), 'Real (MB)'.center(12), 'Virtual (MB)'.center(12)) ) + if (uss): + f.write(" {0:12s}".format( + 'USS (MB)'.center(12)) + ) + f.write("\n") log = {} log['times'] = [] log['cpu'] = [] log['mem_real'] = [] log['mem_virtual'] = [] + if (uss): + log['mem_uss'] = [] try: @@ -166,29 +189,36 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, # Get current CPU and memory try: current_cpu = get_percent(pr) - current_mem = get_memory(pr) + current_mem = get_memory(pr, uss) except Exception: break current_mem_real = current_mem.rss / 1024. ** 2 current_mem_virtual = current_mem.vms / 1024. ** 2 + if (uss): + current_mem_uss = current_mem.uss / 1024. ** 2 # Get information for children if include_children: for child in all_children(pr): try: current_cpu += get_percent(child) - current_mem = get_memory(child) + current_mem = get_memory(child, uss) except Exception: continue current_mem_real += current_mem.rss / 1024. ** 2 current_mem_virtual += current_mem.vms / 1024. ** 2 + if (uss): + current_mem_uss += current_mem.uss / 1024. ** 2 if logfile: - f.write("{0:12.3f} {1:12.3f} {2:12.3f} {3:12.3f}\n".format( + f.write("{0:12.3f} {1:12.3f} {2:12.3f} {3:12.3f}".format( current_time - start_time, current_cpu, current_mem_real, current_mem_virtual)) + if (uss): + f.write(" {0:12.3f}".format(current_mem_uss)) + f.write("\n") f.flush() if interval is not None: @@ -200,6 +230,8 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, log['cpu'].append(current_cpu) log['mem_real'].append(current_mem_real) log['mem_virtual'].append(current_mem_virtual) + if (uss): + log['mem_uss'].append(current_mem_uss) except KeyboardInterrupt: # pragma: no cover pass @@ -220,12 +252,14 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, ax.set_xlabel('time (s)') ax.set_ylim(0., max(log['cpu']) * 1.2) + mem_selector = 'mem_uss' if uss else 'mem_real' ax2 = ax.twinx() - ax2.plot(log['times'], log['mem_real'], '-', lw=1, color='b') - ax2.set_ylim(0., max(log['mem_real']) * 1.2) + ax2.plot(log['times'], log[mem_selector], '-', lw=1, color='b') + ax2.set_ylim(0., max(log[mem_selector]) * 1.2) - ax2.set_ylabel('Real Memory (MB)', color='b') + mem_label= 'Unique set size (MB)' if uss else 'Real Memory (MB)' + ax2.set_ylabel(mem_label, color='b') ax.grid() diff --git a/psrecord/tests/test_main.py b/psrecord/tests/test_main.py index db3f848..b0710bc 100644 --- a/psrecord/tests/test_main.py +++ b/psrecord/tests/test_main.py @@ -11,7 +11,6 @@ p.wait() """ - def test_all_children(tmpdir): filename = tmpdir.join('test.py').strpath @@ -66,3 +65,18 @@ def test_main(self): def test_main_by_id(self): sys.argv = ['psrecord', '--duration=3', str(os.getpid())] main() + + def test_logfile_uss(self, tmpdir): + filename = tmpdir.join('test_logfile').strpath + monitor(self.p.pid, logfile=filename, duration=3, uss=True) + assert os.path.exists(filename) + # is_uss_active = (open(filename, 'r').readlines()[0].find("USS") > 0) + # assert is_uss_active == True + + def test_plot_uss(self, tmpdir): + filename = tmpdir.join('test_plot.png').strpath + monitor(self.p.pid, plot=filename, duration=3, uss=True) + assert os.path.exists(filename) + + def test_uss_with_children(self, tmpdir): + monitor(os.getpid(), duration=3, include_children=True, uss=True) From 0018a11a800168d2c0e4adeead46498baa821584 Mon Sep 17 00:00:00 2001 From: Alexey Nazarov Date: Mon, 18 Jun 2018 02:08:53 +0300 Subject: [PATCH 3/5] Fixed code style --- psrecord/main.py | 10 +++++----- psrecord/tests/test_main.py | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/psrecord/main.py b/psrecord/main.py index 84eb684..948d4f8 100644 --- a/psrecord/main.py +++ b/psrecord/main.py @@ -68,6 +68,7 @@ def is_uss_possible(process): print ("Flag --uss is ignored. Not found method memory_full_info()") return False + def main(): parser = argparse.ArgumentParser( @@ -134,14 +135,13 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, import psutil pr = psutil.Process(pid) - # Check is it possible to use --uss flag if uss: try: pr.memory_full_info() uss = True except AttributeError: - print ("Flag --uss is ignored. Not found method memory_full_info()") + print ("Flag --uss is ignored. Not found memory_full_info()") uss = False # Record start time @@ -155,7 +155,7 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, 'Real (MB)'.center(12), 'Virtual (MB)'.center(12)) ) - if (uss): + if(uss): f.write(" {0:12s}".format( 'USS (MB)'.center(12)) ) @@ -224,7 +224,7 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, current_cpu, current_mem_real, current_mem_virtual)) - if (uss): + if (uss): f.write(" {0:12.3f}".format(current_mem_uss)) f.write("\n") f.flush() @@ -266,7 +266,7 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, ax2.plot(log['times'], log[mem_selector], '-', lw=1, color='b') ax2.set_ylim(0., max(log[mem_selector]) * 1.2) - mem_label= 'Unique set size (MB)' if uss else 'Real Memory (MB)' + mem_label = 'Unique set size (MB)' if uss else 'Real Memory (MB)' ax2.set_ylabel(mem_label, color='b') ax.grid() diff --git a/psrecord/tests/test_main.py b/psrecord/tests/test_main.py index ef3b98e..292eabb 100644 --- a/psrecord/tests/test_main.py +++ b/psrecord/tests/test_main.py @@ -3,7 +3,7 @@ import subprocess import psutil -from ..main import main, monitor, all_children, is_uss_possible +from ..main import main, monitor, all_children TEST_CODE = """ import subprocess @@ -11,6 +11,7 @@ p.wait() """ + def test_all_children(tmpdir): filename = tmpdir.join('test.py').strpath @@ -70,11 +71,11 @@ def test_logfile_uss(self, tmpdir): filename = tmpdir.join('test_logfile').strpath monitor(self.p.pid, logfile=filename, duration=3, uss=True) assert os.path.exists(filename) - + def test_plot_uss(self, tmpdir): filename = tmpdir.join('test_plot.png').strpath monitor(self.p.pid, plot=filename, duration=3, uss=True) assert os.path.exists(filename) - + def test_uss_with_children(self, tmpdir): monitor(os.getpid(), duration=3, include_children=True, uss=True) From 65463c76ac38a34d29cb2d55a0de1c0208b7ade8 Mon Sep 17 00:00:00 2001 From: Alexey Nazarov Date: Mon, 18 Jun 2018 02:16:47 +0300 Subject: [PATCH 4/5] Fix codestyle --- psrecord/main.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/psrecord/main.py b/psrecord/main.py index 948d4f8..c95b3a7 100644 --- a/psrecord/main.py +++ b/psrecord/main.py @@ -59,16 +59,6 @@ def all_children(pr): processes += all_children(child) return processes - -def is_uss_possible(process): - try: - process.memory_full_info() - return True - except AttributeError: - print ("Flag --uss is ignored. Not found method memory_full_info()") - return False - - def main(): parser = argparse.ArgumentParser( @@ -141,7 +131,7 @@ def monitor(pid, logfile=None, plot=None, duration=None, interval=None, pr.memory_full_info() uss = True except AttributeError: - print ("Flag --uss is ignored. Not found memory_full_info()") + print("Flag --uss is ignored. Not found memory_full_info()") uss = False # Record start time From fe8e3630c3a1efe81627399e326194ce55f9a53a Mon Sep 17 00:00:00 2001 From: Alexey Nazarov Date: Mon, 18 Jun 2018 02:26:31 +0300 Subject: [PATCH 5/5] Fixed flake8 style error --- psrecord/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psrecord/main.py b/psrecord/main.py index c95b3a7..fba026a 100644 --- a/psrecord/main.py +++ b/psrecord/main.py @@ -59,6 +59,7 @@ def all_children(pr): processes += all_children(child) return processes + def main(): parser = argparse.ArgumentParser(