Skip to content
Permalink
actix
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
executable file 264 lines (199 sloc) 8.46 KB
#!/usr/bin/env python3
"""
Visualize benchmark results produced by test-image.py.
Usage:
render-tests.py
Currently processes all *.checks.bench.json files in the current dir.
"""
import json
from pathlib import Path
from string import digits
from natsort import natsorted
try:
from docopt import docopt
from marko.ext.gfm import gfm
import pygal
from pygal.style import Style
except ImportError as e:
raise Exception('Some external dependencies not found, install them using: pip install -r requirements.txt') from e
def render():
suffix = '.checks.bench.json'
suites = {}
for filepath in Path('.').glob(f'*{suffix}'):
name = filepath.name[:-len(suffix)]
print(f'Loading {filepath} as {name}.')
with open(filepath) as fp:
suites[name] = json.load(fp)
names = natsorted(suites.keys())
figure_filenames = render_figures(names, suites)
out_filename = Path('bench-results.md')
with open(out_filename, 'w') as out:
print(f'# Benchmark of {", ".join(names)}', file=out)
notes_file = Path('notes.md')
if notes_file.exists():
print(f'Including {notes_file} in resulting Markdown.')
with notes_file.open() as fp:
out.write(fp.read())
else:
print(f'File {notes_file} does not exist, create it to include it in resulting Markdown.')
print('## General Info & Checks', file=out)
render_checks(names, suites, out)
print('## Graphs', file=out)
print('*The graphs are interactive, view the rendered HTML locally to enjoy it.*\n', file=out)
for filename in figure_filenames:
# Use HTML instead of Markdown image to specify the width
print(f'<img type="image/svg+xml" src="{filename}" alt="{filename}" width="49%"/>', file=out)
print(f'Markdown output written to {out_filename}.')
render_html(out_filename, Path('bench-results.html'))
def render_checks(names, suites, out):
print(f'|Check|{"|".join(names)}|', file=out)
print(f'|{"|".join(["---"] * (len(names) + 1))}|', file=out)
per_impl_checks = {name: suite['checks'] for name, suite in suites.items()}
check_names = sorted(set().union(*(checks.keys() for checks in per_impl_checks.values())))
def sanitize(value):
if type(value) is float:
value = float(f'{value:.3g}') # round to 3 significant figures
return str(int(value) if value >= 100 else value)
return str(value)
for check_name in check_names:
values = [sanitize(per_impl_checks[name].get(check_name)) for name in names]
if len(values) > 1 and len(set(values)) > 1:
values = [f'**{value}**' for value in values]
print(f'|{check_name}|{"|".join(values)}|', file=out)
FIGURE_FUNCS = []
def figure(func):
"""Simple decorator to mark a function as a figure generator."""
FIGURE_FUNCS.append(func)
return func
def render_figures(names, suites):
filenames = []
# draw "sibling" suites using the same color
basenames = [name.rstrip(digits) for name in names]
uniq_basenames = sorted(set(basenames))
style = Style(colors=[Style.colors[uniq_basenames.index(basename)] for basename in basenames])
config = pygal.Config(legend_at_bottom=True, style=style)
for figure_func in FIGURE_FUNCS:
chart = figure_func(names, suites, config)
filename = f'bench-results.{figure_func.__name__}.svg'
chart.render_to_file(filename)
filenames.append(filename)
return filenames
@figure
def startup_time_figure(names, suites, config):
chart = pygal.Bar(config)
chart.title = 'Startup Time (s)'
for name in names:
chart.add(name, suites[name]['checks']['Startup time (to start responding) secs'])
return chart
@figure
def errors_vs_connections_figure(names, suites, config):
chart = pygal.Line(config)
chart.title = 'Error Response Ratio vs. Connections (%)'
connections_x_labels(chart, suites, skip=2)
for name in names:
chart.add(name, [div_or_none(s['request_errors_new'], s['request_errors_new'] + s['requests_new'], scale=100)
for s in suites[name]['stats'][2:]])
return chart
@figure
def requests_vs_connections_figure(names, suites, config):
chart = pygal.Line(config)
chart.title = 'Success Requests per Second vs. Connections (Req/s)'
connections_x_labels(chart, suites, skip=2)
for name in names:
chart.add(name, [s['requests_per_s'] for s in suites[name]['stats'][2:]])
return chart
@figure
def latency_vs_connections_50_figure(names, suites, config):
return latency_vs_connections_figure(50, names, suites, config)
@figure
def latency_vs_connections_90_figure(names, suites, config):
return latency_vs_connections_figure(90, names, suites, config)
@figure
def latency_vs_connections_99_figure(names, suites, config):
return latency_vs_connections_figure(99, names, suites, config)
def latency_vs_connections_figure(percentile, names, suites, config):
chart = pygal.Line(config, logarithmic=True)
chart.title = f'{percentile}th Percentile Latency vs. Connections (ms)'
connections_x_labels(chart, suites, skip=2)
for name in names:
chart.add(name, [s[f'latency_{percentile}p_ms'] if s['requests_new'] else None
for s in suites[name]['stats'][2:]])
return chart
@figure
def max_mem_usage_figure(names, suites, config):
chart = pygal.Line(config)
chart.title = 'Max Memory Usage vs. Connections (MiB)'
connections_x_labels(chart, suites)
for name in names:
chart.add(name, [s['mem_max_mb'] for s in suites[name]['stats']])
return chart
@figure
def max_mem_usage_per_requests_figure(names, suites, config):
chart = pygal.Line(config)
chart.title = 'Max Memory Usage per Success Requests/s vs. Connections (MB-second/Req)'
connections_x_labels(chart, suites, skip=2)
for name in names:
chart.add(name, [div_or_none(s['mem_max_mb'], s['requests_per_s']) for s in suites[name]['stats'][2:]])
return chart
@figure
def cpu_figure(names, suites, config):
chart = pygal.Line(config)
chart.title = 'CPU Time per Round vs. Connections (CPU-seconds)'
connections_x_labels(chart, suites, skip=2)
for name in names:
chart.add(name, [s['cpu_new_s'] for s in suites[name]['stats'][2:]])
return chart
@figure
def cpu_per_request_figure(names, suites, config):
chart = pygal.Line(config)
chart.title = 'CPU Time per Success Request vs. Connections (CPU-milliseconds/Req)'
connections_x_labels(chart, suites, skip=2)
for name in names:
chart.add(name, [div_or_none(s['cpu_new_s'], s['requests_new'], scale=1000) for s in suites[name]['stats'][2:]])
return chart
@figure
def cpu_vs_requests_figure(names, suites, config):
chart = pygal.XY(config)
chart.title = 'Cumulative Success Requests vs. Cumulative CPU Seconds'
chart.x_title = 'CPU-seconds'
chart.y_title = 'cumulative success requests'
for name in names:
chart.add(name, [
{'value': (s['cpu_total_s'], s['requests_total']), 'label': f'After {s["connections"]} connections round.'}
for s in suites[name]['stats']
])
return chart
def connections_x_labels(chart, suites, skip=0):
chart.x_labels = [f"{s['connections']} conn's" if s['connections'] else s['message']
for s in next(iter(suites.values()))['stats']][skip:]
chart.x_label_rotation = -30
def div_or_none(numerator, denominator, scale=1):
if not denominator:
return None
return scale * numerator / denominator
HTML_PREFIX = '''<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Benchmark Report</title>
</head>
<body>
'''
HTML_SUFFIX = ''' </body>
</html>
'''
def render_html(md_file, html_file):
with open(md_file) as in_fp, open(html_file, 'w') as out_fp:
rs = in_fp.read()
html = gfm(rs)
# Replace <img> by <embed> for pygal interactivity, http://www.pygal.org/en/latest/documentation/web.html
html = html.replace('<img', '<embed')
# Replace link to md with link to .html for better browsability at HTML level.
html = html.replace('/README.md">full benchmark', '/README.html">full benchmark')
out_fp.write(HTML_PREFIX)
out_fp.write(html)
out_fp.write(HTML_SUFFIX)
print(f'HTML output written to {html_file.resolve().as_uri()}.')
if __name__ == '__main__':
args = docopt(__doc__)
render()