Skip to content

Commit

Permalink
Auto merge of #20251 - jdm:memchart, r=ajeffrey
Browse files Browse the repository at this point in the history 
Chart memory reports over time

This is a tool that can take the output of Servo when run with `-m N` and generate an HTML file that charts the behaviour of the various labels over time.

Run with `./mach run http://url >/tmp/log; python etc/memory_reports_over_time.py /tmp/log`.

---
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] There are tests for these changes

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/20251)
<!-- Reviewable:end -->
  • Loading branch information
bors-servo committed Mar 9, 2018
2 parents c9f60c5 + dd1b43b commit 324e22d
Show file tree
Hide file tree
Showing 3 changed files with 255 additions and 1 deletion.
8 changes: 7 additions & 1 deletion components/profile/mem.rs
View file
Expand Up @@ -12,6 +12,7 @@ use std::borrow::ToOwned;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::thread;
use std::time::Instant;
use time::duration_from_seconds;

pub struct Profiler {
Expand All @@ -20,6 +21,9 @@ pub struct Profiler {

/// Registered memory reporters.
reporters: HashMap<String, Reporter>,

/// Instant at which this profiler was created.
created: Instant,
}

const JEMALLOC_HEAP_ALLOCATED_STR: &'static str = "jemalloc-heap-allocated";
Expand Down Expand Up @@ -69,6 +73,7 @@ impl Profiler {
Profiler {
port: port,
reporters: HashMap::new(),
created: Instant::now(),
}
}

Expand Down Expand Up @@ -111,7 +116,8 @@ impl Profiler {
}

fn handle_print_msg(&self) {
println!("Begin memory reports");
let elapsed = self.created.elapsed();
println!("Begin memory reports {}", elapsed.as_secs());
println!("|");

// Collect reports from memory reporters.
Expand Down
112 changes: 112 additions & 0 deletions etc/memory_chart.html
View file
@@ -0,0 +1,112 @@
<!DOCTYPE html>
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.2/Chart.bundle.min.js"></script>
<select id="graphs"></select>
<script>
function transformData(data, path) {
console.log(path);
var pathParts = path.split('!');
var name = pathParts[pathParts.length - 1];
var transformed = [];
for (const report of data.map(d => d.report)) {
var subData = findData(report, path);
transformed.push(subData.amount);
}
console.log(transformed);
return [{
label: 'Usage in MiB',
data: transformed,
fill: false,
}];
}

function findData(data, parent) {
parent = parent.split('!');
parent.reverse();
while (parent.length) {
var next = parent.pop();
console.log(next);
if ('children' in data) {
data = data.children;
}
data = data[next];
}
return data;
}

function makeOptions(data, initial) {
var sel = document.querySelector('#graphs');
sel.innerHTML = '';
// TODO: add support for labels that are not present in initial report.
var rootData = data[0].report;
console.log(Object.keys(rootData));
var remaining = Object.keys(rootData).map(k => [k, rootData[k], k]);
remaining.reverse();
while (remaining.length) {
var next = remaining.pop();
var children = Object.keys(next[1].children).map(k => [k, next[1].children[k], next[2] + '!' + k]);
children.reverse();
remaining.push.apply(remaining, children);

var opt = sel.appendChild(document.createElement('option'));
opt.innerText = '-'.repeat((next[2].match(/!/g) || []).length) + next[0];
opt.fullPath = next[2];
}
sel.onchange = function() {
var title = sel.value;
while (title[0] == '-') {
title = title.slice(1);
}
makeChart({
'labels': data.map(d => d.seconds + 's'),
'datasets': transformData(data, sel.selectedOptions[0].fullPath),
}, title);
};
sel.value = initial;
sel.onchange();
}

function makeChart(data, title) {
var canvas = document.querySelector('#myChart');
if (canvas) {
canvas.remove();
}
canvas = document.body.appendChild(document.createElement('canvas'));
canvas.id = 'myChart';
canvas.width = 800;
canvas.height = 600;
var ctx = canvas.getContext('2d');
var myChart = new Chart(ctx, {
type: 'line',
data: {
labels: data['labels'],
datasets: data['datasets']
},
options: {
title: {
display: true,
text: title,
},
responsive: false,
animation: {
duration: 0
},
elements: {
line: {
tension: 0
}
},
scales: {
yAxes: [{
ticks: {
beginAtZero:true
}
}]
}
}
});
}

var data = [/* json data */];
var initialGraph = "explicit";
makeOptions(data, initialGraph);
</script>
136 changes: 136 additions & 0 deletions etc/memory_reports_over_time.py
View file
@@ -0,0 +1,136 @@
#!/usr/bin/env python

# Copyright 2018 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import json
import os
import sys
import tempfile
import webbrowser


def extract_memory_reports(lines):
in_report = False
report_lines = []
times = []
for line in lines:
if line.startswith('Begin memory reports'):
in_report = True
report_lines += [[]]
times += [line.strip().split()[-1]]
elif line == 'End memory reports\n':
in_report = False
elif in_report:
report_lines[-1].append(line.strip())
return (report_lines, times)


def parse_memory_report(lines):
reports = {}
parents = []
last_separator_index = None
for line in lines:
assert(line[0] == '|')
line = line[1:]
if not line:
continue
separator_index = line.index('--')
if last_separator_index and separator_index <= last_separator_index:
while parents and parents[-1][1] >= separator_index:
parents.pop()

amount, unit, _, name = line.split()

dest_report = reports
for (parent, index) in parents:
dest_report = dest_report[parent]['children']
dest_report[name] = {
'amount': amount,
'unit': unit,
'children': {}
}

parents += [(name, separator_index)]
last_separator_index = separator_index
return reports


def transform_report_for_test(report):
transformed = {}
remaining = list(report.items())
while remaining:
(name, value) = remaining.pop()
transformed[name] = '%s %s' % (value['amount'], value['unit'])
remaining += map(lambda (k, v): (name + '/' + k, v), list(value['children'].items()))
return transformed


def test():
input = '''|
| 23.89 MiB -- explicit
| 21.35 MiB -- jemalloc-heap-unclassified
| 2.54 MiB -- url(https://servo.org/)
| 2.16 MiB -- js
| 1.00 MiB -- gc-heap
| 0.77 MiB -- decommitted
| 1.00 MiB -- non-heap
| 0.27 MiB -- layout-thread
| 0.27 MiB -- stylist
| 0.12 MiB -- dom-tree
|
| 25.18 MiB -- jemalloc-heap-active'''

expected = {
'explicit': '23.89 MiB',
'explicit/jemalloc-heap-unclassified': '21.35 MiB',
'explicit/url(https://servo.org/)': '2.54 MiB',
'explicit/url(https://servo.org/)/js': '2.16 MiB',
'explicit/url(https://servo.org/)/js/gc-heap': '1.00 MiB',
'explicit/url(https://servo.org/)/js/gc-heap/decommitted': '0.77 MiB',
'explicit/url(https://servo.org/)/js/non-heap': '1.00 MiB',
'explicit/url(https://servo.org/)/layout-thread': '0.27 MiB',
'explicit/url(https://servo.org/)/layout-thread/stylist': '0.27 MiB',
'explicit/url(https://servo.org/)/dom-tree': '0.12 MiB',
'jemalloc-heap-active': '25.18 MiB',
}
report = parse_memory_report(input.split('\n'))
transformed = transform_report_for_test(report)
assert(sorted(transformed.keys()) == sorted(expected.keys()))
for k, v in transformed.items():
assert(v == expected[k])
return 0


def usage():
print('%s --test - run automated tests' % sys.argv[0])
print('%s file - extract all memory reports that are present in file' % sys.argv[0])
return 1


if __name__ == "__main__":
if len(sys.argv) == 1:
sys.exit(usage())

if sys.argv[1] == '--test':
sys.exit(test())

with open(sys.argv[1]) as f:
lines = f.readlines()
(reports, times) = extract_memory_reports(lines)
json_reports = []
for (report_lines, seconds) in zip(reports, times):
report = parse_memory_report(report_lines)
json_reports += [{'seconds': seconds, 'report': report}]
with tempfile.NamedTemporaryFile(delete=False) as output:
thisdir = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(thisdir, 'memory_chart.html')) as template:
content = template.read()
output.write(content.replace('[/* json data */]', json.dumps(json_reports)))
webbrowser.open_new_tab('file://' + output.name)

0 comments on commit 324e22d

Please sign in to comment.