Skip to content

Commit

Permalink
Add option to html2 reporter to disable statistical warnings
Browse files Browse the repository at this point in the history 
Fix #121
  • Loading branch information
Johannes Bechberger committed Aug 13, 2020
1 parent d90c6a0 commit e6413e5
Showing 1 changed file with 24 additions and 19 deletions.
43 changes: 24 additions & 19 deletions temci/report/report.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,9 @@ def report(self):
"mean_in_comparison_tables": Bool() // Default(True)
// Description("Show the mean related values in the big comparison table"),
"force_override": Bool() // Default(False)
// Description("Override the contents of the output directory if it already exists?")
// Description("Override the contents of the output directory if it already exists?"),
"hide_stat_warnings": Bool() // Default(False)
// Description("Hide warnings and errors related to statistical properties")
}))
class HTMLReporter2(AbstractReporter):
"""
Expand Down Expand Up @@ -348,6 +350,7 @@ def report(self):
runs = self.stats_helper.valid_runs()
self._percent_format = self.misc["percent_format"]
self._float_format = self.misc["float_format"]
self._hide_stat_warnings = self.misc["hide_stat_warnings"]
self._app_html = ""
html = """<html lang="en">
<head>
Expand Down Expand Up @@ -668,8 +671,6 @@ def _short_summary_of_tested_pair_property(self, obj: TestedPairProperty, extend
&= \\frac{ %f }{ %f}
\\end{align}
gives a number that helps to talk about the practical significance of the mean difference.
A tiny difference might be cool, but irrelevant (as caching effects are probably higher, use the
<pre>temci build</pre> if your curious about this).
""" % (obj.first.parent.description(), obj.second.parent.description(), str(obj.first.parent),
float(obj.mean_diff()), float(obj.first.mean())))
}, {
Expand All @@ -687,15 +688,15 @@ def _short_summary_of_tested_pair_property(self, obj: TestedPairProperty, extend
- \\overline{{\\text{{{second}}}}}}}{{
\\text{{max}}(\\sigma_\\text{{{first}}}, \\sigma_\\text{{{second}}}) }} \\\\
= & \\frac{{{md}}}{{{std}}} \\end{{align}}
It's important because, as <a href='http://www.cse.unsw.edu.au/~cs9242/15/lectures/05-perfx4.pdf'>
{context}
""".format(first=obj.first.parent.description(), second=obj.second.parent.description(),
md=obj.mean_diff(), std=obj.max_std_dev(),
context="" if self._hide_stat_warnings else """It's important because, as <a href='http://www.cse.unsw.edu.au/~cs9242/15/lectures/05-perfx4.pdf'>
Gernot Heiser</a> points out:
<ul>
<li>Don't believe any effect that is less than a standard deviation</li>
<li>Be highly suspicious if it is less than two standard deviations</li>
</ul>
""".format(first=obj.first.parent.description(), second=obj.second.parent.description(),
md=obj.mean_diff(), std=obj.max_std_dev()), trigger="hover click")
</ul>"""), trigger="hover click",)
}, {
"title": "... ci (lower bound)",
"func": lambda x: fnumber(x.mean_diff_ci(self.misc["alpha"])[0]),
Expand Down Expand Up @@ -781,13 +782,13 @@ def _short_summary_of_tested_pair(self, obj: TestedPair, extended: bool = False,
"format": self._percent_format,
"popover": _Popover(self, "Explanation", """
The mean difference relative to the maximum standard deviation is important,
because as <a href='http://www.cse.unsw.edu.au/~cs9242/15/lectures/05-perfx4.pdf'>
because it puts the value info context{context}
""".format(context="." if self._hide_stat_warnings else """, or as <a href='http://www.cse.unsw.edu.au/~cs9242/15/lectures/05-perfx4.pdf'>
Gernot Heiser</a> points out:
<ul>
<li>Don't believe any effect that is less than a standard deviation</li>
<li>Be highly suspicious if it is less than two standard deviations</li>
</ul>
""", trigger="hover click")
</ul>"""), trigger="hover click")
}])

if self.misc["min_in_comparison_tables"]:
Expand Down Expand Up @@ -1318,7 +1319,7 @@ def _short_summary_modal(self, obj: BaseStatObject) -> str:
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
<button type="button" class="btn btn-primary" data-dismiss="modal"
onclick="window.location='#{html_id}'">More informations</button>
onclick="window.location='#{html_id}'">More information</button>
</div>
</div>
</div>
Expand Down Expand Up @@ -1399,10 +1400,11 @@ def collapsible(title: str, msgs: t.List[StatMessage]):
</div>
""".format(**locals())
html = ""
if obj.has_errors():
html += collapsible('Severe warnings <span class="badge">{}</span>'.format(len(obj.errors())), obj.errors())
if obj.has_warnings():
html += collapsible('Warnings <span class="badge">{}</span>'.format(len(obj.warnings())), obj.warnings())
if not self._hide_stat_warnings:
if obj.has_errors():
html += collapsible('Severe warnings <span class="badge">{}</span>'.format(len(obj.errors())), obj.errors())
if obj.has_warnings():
html += collapsible('Warnings <span class="badge">{}</span>'.format(len(obj.warnings())), obj.warnings())
return html

_time = time.time()
Expand Down Expand Up @@ -1507,11 +1509,14 @@ def __init__(self, parent: HTMLReporter2, content: str = "", cell_class: str = "
self.parent = parent
assert link is None or modal_id is None
if color_class_obj is not None:
warnings_text = "" if self.parent._hide_stat_warnings \
else _color_explanation(color_class_obj)
if self.popover is None:
self.popover = _Popover(parent, "Explanation", _color_explanation(color_class_obj))
self.popover = _Popover(parent, "Explanation", warnings_text)
else:
self.popover.content += _color_explanation(color_class_obj)
self.cell_class += " " + _color_class(color_class_obj)
self.popover.content += warnings_text
if not self.parent._hide_stat_warnings:
self.cell_class += " " + _color_class(color_class_obj)
if (modal_id is not None and show_click_on_info != False) or (show_click_on_info is True and not link):
msg = "<p>Click on the cell to get more information.</p>"
if self.popover is None:
Expand Down

0 comments on commit e6413e5

Please sign in to comment.