Permalink
Browse files

Rework get-about-memory.py, and create get_gc_cc_log.py.

  • Loading branch information...
1 parent 715069c commit 9d877736fdf0a4e03deff85183665b4ef212f6d8 @jlebar jlebar committed Oct 17, 2012
Showing with 453 additions and 200 deletions.
  1. +1 −0 .gitignore
  2. +0 −200 get-about-memory.py
  3. +114 −0 tools/get_about_memory.py
  4. +99 −0 tools/get_gc_cc_log.py
  5. 0 tools/include/__init__.py
  6. +239 −0 tools/include/device_utils.py
View
1 .gitignore
@@ -3,6 +3,7 @@
.userconfig
.var.profile
*.swp
+*.pyc
Adreno*
Makefile
abi/
View
200 get-about-memory.py
@@ -1,200 +0,0 @@
-#!/usr/bin/env python
-
-'''Get a dump of about:memory from all the processes running on your device.
-
-You can then view these dumps using Firefox on your desktop.
-
-We also include the output of b2g-procrank and b2g-ps.
-
-'''
-
-from __future__ import print_function
-from __future__ import division
-
-import sys
-import re
-import os
-import subprocess
-import textwrap
-import argparse
-import json
-from gzip import GzipFile
-from time import sleep
-
-if sys.version_info < (2,7):
- print('This script requires Python 2.7.')
- sys.exit(1)
-
-def shell(cmd, cwd=None):
- proc = subprocess.Popen(cmd, shell=True, cwd=cwd,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- (out, err) = proc.communicate()
- if proc.returncode:
- print("Command %s failed with error code %d" % (cmd, proc.returncode), file=sys.stderr)
- if err:
- print(err, file=sys.stderr)
- raise subprocess.CalledProcessError(proc.returncode, cmd, err)
- return out
-
-def get_pids():
- """Get the pids of all gecko processes running on the device.
-
- Returns a tuple (master_pid, child_pids), where child_pids is a list.
-
- """
- procs = shell("adb shell ps").split('\n')
- master_pid = None
- child_pids = []
- for line in procs:
- if re.search(r'/b2g\s*$', line):
- if master_pid:
- raise Exception("Two copies of b2g process found?")
- master_pid = int(line.split()[1])
- if re.search(r'/plugin-container\s*$', line):
- child_pids.append(int(line.split()[1]))
-
- if not master_pid:
- raise Exception("b2g does not appear to be running on the device.")
-
- return (master_pid, child_pids)
-
-def list_files():
- return set(['/data/local/tmp/' + f.strip() for f in
- shell("adb shell ls '/data/local/tmp'").split('\n')
- if f.strip().startswith('memory-report-')])
-
-def send_signal(args, pid):
- # killer is a program we put on the device which is like kill(1), except it
- # accepts signals above 31. It also understands "SIGRTn" to mean
- # SIGRTMIN + n.
- #
- # SIGRT0 dumps memory reports, and SIGRT1 first minimizes memory usage and
- # then dumps the reports.
- signal = 'SIGRT0' if not args.minimize_memory_usage else 'SIGRT1'
- shell("adb shell killer %s %d" % (signal, pid))
-
-def choose_output_dir(args):
- if args.output_directory:
- return args.output_directory
-
- for i in range(0, 1024):
- try:
- dir = 'about-memory-%d' % i
- os.mkdir(dir)
- return dir
- except:
- pass
- raise Exception("Couldn't create about-memory output directory.")
-
-def wait_for_all_files(num_expected_files, old_files):
- wait_interval = .25
- max_wait = 30
-
- warn_time = 5
- warned = False
-
- for i in range(0, int(max_wait / wait_interval)):
- new_files = list_files() - old_files
-
- # For some reason, print() doesn't work with the \r hack.
- sys.stdout.write('\rGot %d/%d files.' % (len(new_files), num_expected_files))
- sys.stdout.flush()
-
- if not warned and len(new_files) == 0 and i * wait_interval >= warn_time:
- warned = True
- sys.stdout.write('\r')
- print(textwrap.fill(textwrap.dedent("""\
- The device may be asleep and not responding to our signal.
- Try pressing a button on the device to wake it up.\n\n""")))
-
- if len(new_files) == num_expected_files:
- print('')
- return
-
- sleep(wait_interval)
-
- print("We've waited %ds but the only about:memory dumps we see are" % max_wait)
- print('\n'.join([' ' + f for f in new_files]))
- print('We expected %d but see only %d files. Giving up...' %
- (num_expected_files, len(new_files)))
- raise Exception("Missing some about:memory dumps.")
-
-def get_files(args, master_pid, child_pids, old_files):
- """Get the memory reporter dumps from the device and return the directory
- we saved them to.
-
- """
- num_expected_files = 1 + len(child_pids)
-
- wait_for_all_files(num_expected_files, old_files)
- new_files = list_files() - old_files
- dir = choose_output_dir(args)
- for f in new_files:
- shell('adb pull %s' % f, cwd=dir)
- pass
- print("Pulled files into %s." % dir)
- merge_files(dir, [os.path.basename(f) for f in new_files])
- return dir
-
-def merge_files(dir, files):
- """Merge the given memory reporter dump files into one giant file."""
- dumps = [json.load(GzipFile(os.path.join(dir, f))) for f in files]
-
- merged_dump = dumps[0]
- for dump in dumps[1:]:
- # All of the properties other than 'reports' must be identical in all
- # dumps, otherwise we can't merge them.
- if set(dump.keys()) != set(merged_dump.keys()):
- print("Can't merge dumps because they don't have the "
- "same set of properties.")
- return
- for prop in merged_dump:
- if prop != 'reports' and dump[prop] != merged_dump[prop]:
- print("Can't merge dumps because they don't have the "
- "same value for property '%s'" % prop)
-
- merged_dump['reports'] += dump['reports']
-
- json.dump(merged_dump,
- GzipFile(os.path.join(dir, 'merged-reports.gz'), 'w'),
- indent=2)
-
-def remove_new_files(old_files):
- # Hopefully this command line won't get too long for ADB.
- shell('adb shell rm %s' % ' '.join(["'%s'" % f for f in list_files() - old_files]))
-
-def get_procrank_etc(dir):
- shell('adb shell procrank > procrank', cwd=dir)
- shell('adb shell b2g-ps > b2g-ps', cwd=dir)
- shell('adb shell b2g-procrank > b2g-procrank', cwd=dir)
-
-def get_dumps(args):
- (master_pid, child_pids) = get_pids()
- old_files = list_files()
- send_signal(args, master_pid)
- dir = get_files(args, master_pid, child_pids, old_files)
- if args.remove_from_device:
- remove_new_files(old_files)
- get_procrank_etc(dir)
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(description=textwrap.dedent('''\
- This script pulls about:memory reports from a device. You can then
- open these reports in desktop Firefox by visiting about:memory.'''))
-
- parser.add_argument('--minimize', '-m', dest='minimize_memory_usage',
- action='store_true', default=False,
- help='Minimize memory usage before collecting the memory reports.')
-
- parser.add_argument('--directory', '-d', dest='output_directory',
- action='store', metavar='DIR',
- help=textwrap.dedent('''\
- The directory to store the reports in. By default, we'll store the
- reports in the directory about-memory-N, for some N.'''))
-
- parser.add_argument('--remove', '-r', dest='remove_from_device',
- action='store_true', default=False,
- help='Delete the reports from the device after pulling them.')
-
- args = parser.parse_args()
- get_dumps(args)
View
114 tools/get_about_memory.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+'''Get a dump of about:memory from all the processes running on your device.
+
+You can then view these dumps using a recent Firefox nightly on your desktop by
+opening about:memory and using the button at the bottom of the page to load the
+memory-reports file that this script creates.
+
+This script also saves the output of b2g-procrank and a few other diagnostic
+programs.
+
+'''
+
+from __future__ import print_function
+
+import sys
+if sys.version_info < (2,7):
+ # We need Python 2.7 because we import argparse.
+ print('This script requires Python 2.7.')
+ sys.exit(1)
+
+import os
+import textwrap
+import argparse
+import json
+from gzip import GzipFile
+
+import include.device_utils as utils
+
+def merge_files(dir, files):
+ '''Merge the given memory reporter dump files into one giant file.'''
+ dumps = [json.load(GzipFile(os.path.join(dir, f))) for f in files]
+
+ merged_dump = dumps[0]
+ for dump in dumps[1:]:
+ # All of the properties other than 'reports' must be identical in all
+ # dumps, otherwise we can't merge them.
+ if set(dump.keys()) != set(merged_dump.keys()):
+ print("Can't merge dumps because they don't have the "
+ "same set of properties.")
+ return
+ for prop in merged_dump:
+ if prop != 'reports' and dump[prop] != merged_dump[prop]:
+ print("Can't merge dumps because they don't have the "
+ "same value for property '%s'" % prop)
+
+ merged_dump['reports'] += dump['reports']
+
+ merged_reports_path = os.path.join (dir, 'memory-reports')
+ json.dump(merged_dump,
+ open(merged_reports_path, 'w'),
+ indent=2)
+ return merged_reports_path
+
+def get_dumps(args):
+ if args.output_directory:
+ out_dir = utils.create_specific_output_dir(args.output_directory)
+ else:
+ out_dir = utils.create_new_output_dir('about-memory-')
+
+ # Do this function inside a try/catch which will delete out_dir if the
+ # function throws and out_dir is empty.
+ def do_work():
+ signal = 'SIGRT0' if not args.minimize_memory_usage else 'SIGRT1'
+ new_files = utils.send_signal_and_pull_files(
+ signal=signal,
+ outfiles_prefixes=['memory-report-'],
+ remove_outfiles_from_device=not args.leave_on_device,
+ out_dir=out_dir)
+
+ merged_reports_path = merge_files(out_dir, new_files)
+ utils.pull_procrank_etc(out_dir)
+
+ if not args.keep_individual_reports:
+ for f in new_files:
+ os.remove(os.path.join(out_dir, f))
+
+ print()
+ print(textwrap.fill(textwrap.dedent('''\
+ To view this report, open Firefox on your desktop, load
+ about:memory, click "read reports from a file" at the bottom, and
+ open %s''' %
+ os.path.abspath(merged_reports_path))))
+
+ utils.run_and_delete_dir_on_exception(do_work, out_dir)
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+
+ parser.add_argument('--minimize', '-m', dest='minimize_memory_usage',
+ action='store_true', default=False,
+ help='Minimize memory usage before collecting the memory reports.')
+
+ parser.add_argument('--directory', '-d', dest='output_directory',
+ action='store', metavar='DIR',
+ help=textwrap.dedent('''\
+ The directory to store the reports in. By default, we'll store the
+ reports in the directory about-memory-N, for some N.'''))
+
+ parser.add_argument('--leave-on-device', '-l', dest='leave_on_device',
+ action='store_true', default=False,
+ help='Leave the reports on the device after pulling them.')
+
+ parser.add_argument('--keep-individual-reports',
+ dest='keep_individual_reports',
+ action='store_true', default=False,
+ help=textwrap.dedent('''\
+ Don't delete the individual memory reports which we merge to create
+ the memory-reports file. You shouldn't need to pass this parameter
+ except for debugging.'''))
+
+ args = parser.parse_args()
+ get_dumps(args)
View
99 tools/get_gc_cc_log.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+'''This script pulls GC and CC logs from all B2G processes on a device. These
+logs are primarily used by leak-checking tools.
+
+This script also saves the output of b2g-procrank and a few other diagnostic
+programs.
+
+'''
+
+from __future__ import print_function
+
+import sys
+if sys.version_info < (2,7):
+ # We need Python 2.7 because we import argparse.
+ print('This script requires Python 2.7.')
+ sys.exit(1)
+
+import os
+import sys
+import re
+import argparse
+import textwrap
+import subprocess
+
+import include.device_utils as utils
+
+def compress_logs(log_filenames, out_dir):
+ # Compress with xz if we can; otherwise, use gzip.
+ try:
+ utils.shell('xz -V', show_errors=False)
+ compression_prog='xz'
+ except subprocess.CalledProcessError:
+ compression_prog='gzip'
+
+ # Compress in parallel. While we're at it, we also strip off the
+ # long identifier from the filenames, if we can. (The filename is
+ # something like gc-log.PID.IDENTIFIER.log, where the identifier is
+ # something like the number of seconds since the epoch when the log was
+ # triggered.)
+ compression_procs = []
+ for f in log_filenames:
+ # Rename the log file if we can.
+ match = re.match(r'^([a-zA-Z-]+\.[0-9]+)\.[0-9]+.log$', f)
+ if match:
+ if not os.path.exists(os.path.join(out_dir, match.group(1))):
+ new_name = match.group(1) + '.log'
+ os.rename(os.path.join(out_dir, f),
+ os.path.join(out_dir, new_name))
+ f = new_name
+
+ # Start compressing.
+ compression_procs.append((f, subprocess.Popen([compression_prog, f],
+ cwd=out_dir)))
+ # Wait for all the compression processes to finish.
+ for (filename, proc) in compression_procs:
+ proc.wait()
+ if proc.returncode:
+ print('Compression of %s failed!' % filename)
+ raise subprocess.CalledProcessError(proc.returncode,
+ [compression_prog, filename],
+ None)
+def get_logs(args):
+ if args.output_directory:
+ out_dir = utils.create_specific_output_dir(args.output_directory)
+ else:
+ out_dir = utils.create_new_output_dir('gc-cc-logs-')
+
+ def do_work():
+ log_filenames = utils.send_signal_and_pull_files(
+ signal='SIGRT2',
+ outfiles_prefixes=['cc-edges.', 'gc-edges.'],
+ remove_outfiles_from_device=not args.leave_on_device,
+ out_dir=out_dir)
+
+ compress_logs(log_filenames, out_dir)
+ utils.pull_procrank_etc(out_dir)
+
+ utils.run_and_delete_dir_on_exception(do_work, out_dir)
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+
+ parser.add_argument('--directory', '-d', dest='output_directory',
+ action='store', metavar='DIR',
+ help=textwrap.dedent('''\
+ The directory to store the logs in. By default, we'll store the
+ reports in the directory gc-cc-logs-N, for some N.'''))
+
+ parser.add_argument('--leave-on-device', '-l', dest='leave_on_device',
+ action='store_true', default=False,
+ help=textwrap.dedent('''\
+ Leave the logs on the device after pulling them. (Note: These logs
+ can take up tens of megabytes and are stored uncompressed on the
+ device!)'''))
+
+ args = parser.parse_args()
+ get_logs(args)
View
0 tools/include/__init__.py
No changes.
View
239 tools/include/device_utils.py
@@ -0,0 +1,239 @@
+'''Utilities for interacting with a remote device.'''
+
+from __future__ import print_function
+from __future__ import division
+
+import os
+import sys
+import re
+import subprocess
+from time import sleep
+
+def remote_shell(cmd):
+ '''Run the given command on on the device and return stdout.'''
+ return shell("adb shell '%s'" % cmd)
+
+def shell(cmd, cwd=None, show_errors=True):
+ '''Run the given command as a shell script on the host machine.
+
+ If cwd is specified, we run the command from that directory; otherwise, we
+ run the command from the current working directory.
+
+ '''
+ proc = subprocess.Popen(cmd, shell=True, cwd=cwd,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ (out, err) = proc.communicate()
+ if proc.returncode:
+ if show_errors:
+ print('Command %s failed with error code %d' %
+ (cmd, proc.returncode), file=sys.stderr)
+ if err:
+ print(err, file=sys.stderr)
+ raise subprocess.CalledProcessError(proc.returncode, cmd, err)
+ return out
+
+def create_specific_output_dir(out_dir):
+ '''Create the given directory if it doesn't exist.
+
+ Throw an exception if a non-directory file exists with the same name.
+
+ '''
+ if os.path.exists(out_dir):
+ if os.path.isdir(out_dir):
+ # Directory already exists; we're all good.
+ return
+ else:
+ raise Exception(textwrap.dedent('''\
+ Can't use %s as output directory; something that's not a
+ directory already exists with that name.''' % out_dir))
+ os.mkdir(out_dir)
+
+def create_new_output_dir(out_dir_prefix):
+ '''Create a new directory whose name begins with out_dir_prefix.'''
+ for i in range(0, 1024):
+ try:
+ dir = '%s%d' % (out_dir_prefix, i)
+ os.mkdir(dir)
+ return dir
+ except:
+ pass
+ raise Exception("Couldn't create output directory.")
+
+def get_remote_b2g_pids():
+ '''Get the pids of all gecko processes running on the device.
+
+ Returns a tuple (master_pid, child_pids), where child_pids is a list.
+
+ '''
+ procs = remote_shell('ps').split('\n')
+ master_pid = None
+ child_pids = []
+ for line in procs:
+ if re.search(r'/b2g\s*$', line):
+ if master_pid:
+ raise Exception('Two copies of b2g process found?')
+ master_pid = int(line.split()[1])
+ if re.search(r'/plugin-container\s*$', line):
+ child_pids.append(int(line.split()[1]))
+
+ if not master_pid:
+ raise Exception('b2g does not appear to be running on the device.')
+
+ return (master_pid, child_pids)
+
+def pull_procrank_etc(out_dir):
+ '''Get the output of procrank and a few other diagnostic programs and save
+ it into out_dir.
+
+ '''
+ shell('adb shell procrank > procrank', cwd=out_dir)
+ shell('adb shell b2g-ps > b2g-ps', cwd=out_dir)
+ shell('adb shell b2g-procrank > b2g-procrank', cwd=out_dir)
+
+def run_and_delete_dir_on_exception(fun, dir):
+ '''Run the given function and, if it throws an exception, delete the given
+ directory, if it's empty, before re-throwing the exception.
+
+ You might want to wrap your call to send_signal_and_pull_files in this
+ function.'''
+ try:
+ fun()
+ except:
+ # os.rmdir will throw if the directory is non-empty, and a simple
+ # 'raise' will re-throw the exception from os.rmdir (if that throws),
+ # so we need to explicitly save the exception info here. See
+ # http://nedbatchelder.com/blog/200711/rethrowing_exceptions_in_python.html
+ exception_info = sys.exc_info()
+
+ try:
+ # Throws if the directory is not empty.
+ os.rmdir(out_dir)
+ except OSError:
+ pass
+
+ # Raise the original exception.
+ raise exception_info[1], None, exception_info[2]
+
+def send_signal_and_pull_files(signal,
+ outfiles_prefixes,
+ remove_outfiles_from_device,
+ out_dir):
+ '''Send a signal to the main B2G process and pull files created as a
+ result.
+
+ We send the given signal (which may be either a number of a string of the
+ form 'SIGRTn', which we interpret as the signal SIGRTMIN + n) and pull the
+ files generated into out_dir on the host machine. We only pull files
+ which were created after the signal was sent.
+
+ When we're done, we remove the files from the device if
+ remote_outfiles_from_device is true.
+
+ outfiles_prefixes must be a list containing the beginnings of the files we
+ expect to be created as a result of the signal. For example, if we expect
+ to see files named 'foo-XXX' and 'bar-YYY', we'd set outfiles_prefixes to
+ ['foo-', 'bar-'].
+
+ We expect to pull len(outfiles_prefixes) * (# b2g processes) files from the
+ device.
+
+ '''
+ (master_pid, child_pids) = get_remote_b2g_pids()
+ old_files = _list_remote_temp_files(outfiles_prefixes)
+ _send_remote_signal(signal, master_pid)
+
+ num_expected_files = len(outfiles_prefixes) * (1 + len(child_pids))
+ _wait_for_remote_files(outfiles_prefixes, num_expected_files, old_files)
+ new_files = _pull_remote_files(outfiles_prefixes, old_files, out_dir)
+ if remove_outfiles_from_device:
+ _remove_files_from_device(outfiles_prefixes, old_files)
+ return [os.path.basename(f) for f in new_files]
+
+# You probably don't need to call the functions below from outside this module,
+# but hey, maybe you do.
+
+def _send_remote_signal(signal, pid):
+ '''Send a signal to a process on the device.
+
+ signal can be either an integer or a string of the form 'SIGRTn' where n is
+ an integer. We interpret SIGRTn to mean the signal SIGRTMIN + n.
+
+ '''
+ # killer is a program we put on the device which is like kill(1), except it
+ # accepts signals above 31. It also understands "SIGRTn" per above.
+ remote_shell("killer %s %d" % (signal, pid))
+
+def _list_remote_temp_files(prefixes):
+ '''Return a set of absolute filenames in the device's temp directory which
+ start with one of the given prefixes.'''
+ return set(['/data/local/tmp/' + f.strip() for f in
+ remote_shell('ls /data/local/tmp').split('\n')
+ if any([f.strip().startswith(prefix) for prefix in prefixes])])
+
+def _wait_for_remote_files(outfiles_prefixes, num_expected_files, old_files):
+ '''Wait for files to appear on the remote device.
+
+ We wait until we see num_expected_files whose names begin with one of the
+ elements of outfiles_prefixes and which aren't in old_files appear in the
+ device's temp directory. If we don't see these files after a timeout
+ expires, we throw an exception.
+
+ '''
+ wait_interval = .25
+ max_wait = 30
+
+ warn_time = 5
+ warned = False
+
+ for i in range(0, int(max_wait / wait_interval)):
+ new_files = _list_remote_temp_files(outfiles_prefixes) - old_files
+
+ # For some reason, print() doesn't work with the \r hack.
+ sys.stdout.write('\rGot %d/%d files.' %
+ (len(new_files), num_expected_files))
+ sys.stdout.flush()
+
+ if not warned and len(new_files) == 0 and i * wait_interval >= warn_time:
+ warned = True
+ sys.stdout.write('\r')
+ print(textwrap.fill(textwrap.dedent("""\
+ The device may be asleep and not responding to our signal.
+ Try pressing a button on the device to wake it up.\n\n""")))
+
+ if len(new_files) == num_expected_files:
+ print('')
+ return
+
+ sleep(wait_interval)
+
+ print("We've waited %ds but the only relevant files we see are" % max_wait)
+ print('\n'.join([' ' + f for f in new_files]))
+ print('We expected %d but see only %d files. Giving up...' %
+ (num_expected_files, len(new_files)))
+ raise Exception("Unable to pull some files.")
+
+def _pull_remote_files(outfiles_prefixes, old_files, out_dir):
+ '''Pull files from the remote device's temp directory into out_dir.
+
+ We pull each file in the temp directory whose name begins with one of the
+ elements of outfiles_prefixes and which isn't listed in old_files.
+
+ '''
+ new_files = _list_remote_temp_files(outfiles_prefixes) - old_files
+ for f in new_files:
+ shell('adb pull %s' % f, cwd=out_dir)
+ pass
+ print("Pulled files into %s." % out_dir)
+ return new_files
+
+def _remove_files_from_device(outfiles_prefixes, old_files):
+ '''Remove files from the remote device's temp directory.
+
+ We remove all files starting with one of the elements of outfiles_prefixes
+ which aren't listed in old_files.
+
+ '''
+ files_to_remove = _list_remote_temp_files(outfiles_prefixes) - old_files
+
+ # Hopefully this command line won't get too long for ADB.
+ remote_shell('rm %s' % ' '.join([str(f) for f in files_to_remove]))

0 comments on commit 9d87773

Please sign in to comment.