diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 70bbed4e2..ff14d8163 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -62,10 +62,10 @@ jobs:
with:
name: Python Benchmark
output-file-path: examples/pytest/output.json
+ render-json-path: .github/workflows/pytest-config.json
skip-fetch-gh-pages: true
fail-on-alert: true
commit-msg-append: append
- one-chart-groups: group1
- run: node ./scripts/ci_validate_modification.js before_data.js 'Python Benchmark'
only-alert-with-cache:
diff --git a/.github/workflows/pytest-config.json b/.github/workflows/pytest-config.json
new file mode 100644
index 000000000..a7c4cc0a4
--- /dev/null
+++ b/.github/workflows/pytest-config.json
@@ -0,0 +1,16 @@
+{
+ "suites": {
+ "Python Benchmark with pytest-benchmark": {
+ "header": "Test Suite Title",
+ "description": "Description of test suite."
+ }
+ },
+ "groups": {
+ "group 1": {
+ "header": "Group 1 Title",
+ "description": "Description of group 1.",
+ "single_chart": true,
+ "xAxis": "id"
+ }
+ }
+}
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 0812c0ce1..5caa8169e 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -22,10 +22,10 @@ jobs:
with:
name: Python Benchmark with pytest-benchmark
output-file-path: examples/pytest/output.json
+ render-json-path: .github/workflows/pytest-config.json
# Use personal access token instead of GITHUB_TOKEN due to https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
- one-chart-groups: group1
# Show alert with commit comment on detecting possible performance regression
alert-threshold: '200%'
comment-on-alert: true
diff --git a/.gitignore b/.gitignore
index 618097274..c0c7ed47c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@
/coverage
.vscode/
src/assets/data.js
+src/assets/config.js
diff --git a/README.md b/README.md
index bd687b8ad..5de346ee0 100644
--- a/README.md
+++ b/README.md
@@ -3,18 +3,51 @@
This is a fork of `github-action-benchmark`, optimised for `pytest-benchmark` only.
It is hoped that eventually some/all of the features will be fed upstream:
-1. Allow for tests to divided into groups (see [pytest-benchmark markers](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#markers)), with sub-headings.
-2. `one-chart-groups` allow for specific groups to be plotted on the same chart.
-3. Capture of CPU information (shown in tooltip), useful for cross-referencing any changes in results against.
-4. Add top-level `extra` key to captured data, which include the Python version.
-5. Round values in tooltip to 5 significant figures
-6. Add `chart-xaxis`, to allow chart x-axis to be commit date or ID
-7. Add `commit-msg-append` option , useful for adding e.g. `[ci skip]` to commit message, but not having it in results titles
-8. Removed capture of commit author/committer, since it can be obtained from the commit id/url
-9. Split original HTML index text into files in `src/assets`folder, this allows for,
-10. Add `npm run serve`, for local testing of results page
-11. Add `overwrite-assets` option, as to whether these assets should be overridden.
-12. Add `metadata` option, to capture additional information from the github action run (like dependency/docker versions).
+1. More metadata is saved about the benchmark run, for later comparison:
+ - CPU data (cores, processors, speed) is specifically saved, per commit, in the `cpu` key (uses [systeminformation](https://www.npmjs.com/package/systeminformation) npm package).
+ - The python version is extracted from the pytest-benchmark data in an `extra` key
+ - A `metadata` action option is available to save additional data
+ - This data is all shown in the data point's tooltip
+
+2. The group name is saved for each test (see [pytest-benchmark markers](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#markers)), or `null` if no group is given.
+ In the web-page rendering, tests are then arranged by the group they are in, which can be given a sub-heading, description, etc, and also a group can be "consolidated" into a single chart (with handling of differing data points).
+
+3. A new `render-json-path` allows for a JSON file to be copied, which is used to configure the rendered web-page, e.g. for certain test suites and groups:
+
+ ```json
+ {
+ "suites": {
+ "name": {
+ "header": "Test Suite Title",
+ "description": "Description of test suite."
+ }
+ },
+ "groups": {
+ "group1": {
+ "header": "Group 1 Title",
+ "description": "Description of group 1.",
+ "single_chart": true,
+ "xAxis": "date",
+ "backgroundFill": false
+ }
+ }
+ }
+ ```
+
+4. Split original `index.html` into multiple HTML/JS files in the `src/assets` folder. This allows for easier testing and development of the output web-page.
+ This has additionally allowed for:
+ - Adding `npm run serve`, for local development of output web-page (using [light-server](https://www.npmjs.com/package/light-server))
+ - Adding `overwrite-assets` option, to specify whether any existing assets should be overridden during a commit to `gh-pages`.
+
+5. Improve formatting of charts:
+ - Color cycling for consolidated charts
+ - For legend, extract common test name prefix as title
+ - Data point tooltips: rounding values to 5 significant figures and better formatting of dates etc.
+ - Addition of the `chartjs-plugin-zoom`.
+
+6. Add `commit-msg-append` option , useful for adding e.g. `[ci skip]` to commit message, but not having it as part of the test suite key in the data JSON.
+7. Removed capture of commit author/committer, since it can be obtained from the commit id/url, and just bloats the data JSON.
+8. Renamed `max-items-in-chart` -> `max-data-items`, to better describe its function of truncating the saved data during a commit.
## Development Notes
@@ -29,12 +62,13 @@ You can also get the test coverage: `npm run coverage`
To fix linting issues: `npm run fix`
-For developing the webpage and associated JS/CSS, you can serve the assets: `npm serve`.
-Note, that this requires you place a `data.js` object in `src/assets`.
+For developing the webpage and associated JS/CSS, you can serve the assets: `npm run serve`.
+Note, that this requires you place a `data.js` and `config.js` object in `src/assets`.
To "release" a GH actions version, you need to create a branch containing all the compiled JS + node_modules.
To do this (or to update a release branch), first make sure the branch has been created,
-then you can run the script: `./scripts/prepare-release.sh branch-name`.
+then you can run the script: `./scripts/prepare-release.sh branch-name`, usually with a branch named e.g. `v2`.
+After this the branch should be tagged.
---
diff --git a/action.yml b/action.yml
index b446b19a1..6eba7a20b 100644
--- a/action.yml
+++ b/action.yml
@@ -66,17 +66,12 @@ inputs:
external-data-json-path:
description: 'JSON data file for storing benchmark results. When this input is set, github-action-benchmark no longer uses Git branch to store data. Instead, it reads and appends benchmark data from/to the file. User must store the file anywhere'
required: false
- max-items-in-chart:
- description: 'Max data points in a benchmark chart to avoid making the chart too busy. Value must be unsigned integer. No limit by default'
+ max-data-items:
+ description: 'Max data points to store per test suite. No limit by default, otherwise data will be truncated.'
required: false
- chart-xaxis:
- description: 'X-axis labels to use, commit "id" or "date"'
+ render-json-path:
+ description: 'A path to a file which contains the configuration JSON for rendering results.'
required: false
- default: id
- one-chart-groups:
- description: 'Benchmark groups (comma-delimited) whose tests should be plotted together on a single chart'
- required: false
- default: ''
overwrite-assets:
description: 'Whether to overwrite existing HTML/JS/CSS files on GitHub Pages'
required: false
diff --git a/src/assets/benchmark.css b/src/assets/benchmark.css
index 712608b25..98df2b4f3 100644
--- a/src/assets/benchmark.css
+++ b/src/assets/benchmark.css
@@ -1,84 +1,113 @@
html {
- font-family: BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif;
- -webkit-font-smoothing: antialiased;
- background-color: #fff;
- font-size: 16px;
- }
- body {
- color: #4a4a4a;
- margin: 8px;
- font-size: 1em;
- font-weight: 400;
- }
- header {
- margin-bottom: 8px;
- display: flex;
- flex-direction: column;
- }
- main {
- width: 100%;
- display: flex;
- flex-direction: column;
- }
- a {
- color: #3273dc;
- cursor: pointer;
- text-decoration: none;
- }
- a:hover {
- color: #000;
- }
- button {
- color: #fff;
- background-color: #3298dc;
- border-color: transparent;
- cursor: pointer;
- text-align: center;
- }
- button:hover {
- background-color: #2793da;
- flex: none;
- }
- .spacer {
- flex: auto;
- }
- .small {
- font-size: 0.75rem;
- }
- footer {
- margin-top: 16px;
- display: flex;
- align-items: center;
- }
- .header-label {
- margin-right: 4px;
- }
- .benchmark-set {
- margin: 8px 0;
- width: 100%;
- display: flex;
- flex-direction: column;
- }
- .benchmark-title {
- font-size: 3rem;
- font-weight: 600;
- word-break: break-word;
- text-align: center;
- }
- .benchmark-group {
- font-size: 2rem;
- font-weight: 400;
- word-break: break-word;
- text-align: center;
- }
- .benchmark-graphs {
- display: flex;
- flex-direction: row;
- justify-content: space-around;
- align-items: center;
- flex-wrap: wrap;
- width: 100%;
- }
- .benchmark-chart {
- max-width: 1000px;
- }
+ font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Fira Sans", "Droid Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
+ -webkit-font-smoothing: antialiased;
+ background-color: #fff;
+ font-size: 16px;
+}
+
+body {
+ color: #4a4a4a;
+ margin: 8px;
+ font-size: 1em;
+ font-weight: 400;
+}
+
+header {
+ margin-bottom: 8px;
+ display: flex;
+ flex-direction: column;
+}
+
+main {
+ width: 100%;
+ display: flex;
+ flex-direction: column;
+}
+
+a {
+ color: #3273dc;
+ cursor: pointer;
+ text-decoration: none;
+}
+
+a:hover {
+ color: #000;
+}
+
+button {
+ color: #fff;
+ background-color: #3298dc;
+ border-color: transparent;
+ cursor: pointer;
+ text-align: center;
+}
+
+button:hover {
+ background-color: #2793da;
+ flex: none;
+}
+
+.spacer {
+ flex: auto;
+}
+
+.small {
+ font-size: 0.75rem;
+}
+
+footer {
+ margin-top: 16px;
+ display: flex;
+ align-items: center;
+}
+
+.header-label {
+ margin-right: 4px;
+}
+
+.benchmark-set {
+ margin: 8px 0;
+ width: 100%;
+ display: flex;
+ flex-direction: column;
+}
+
+.benchmark-title {
+ font-size: 3rem;
+ font-weight: 600;
+ word-break: break-word;
+ text-align: center;
+}
+
+.benchmark-description {
+ font-size: 1rem;
+ text-align: center;
+}
+
+.benchmark-group-title {
+ font-size: 2rem;
+ font-weight: 400;
+ word-break: break-word;
+ text-align: left;
+ margin-left: 2%;
+}
+
+.benchmark-group-description {
+ font-size: 1rem;
+ text-align: left;
+ margin-left: 2%;
+ margin-bottom: 2rem;
+}
+
+.benchmark-graphs {
+ display: flex;
+ flex-direction: row;
+ justify-content: space-around;
+ align-items: center;
+ flex-wrap: wrap;
+ width: 100%;
+}
+
+.benchmark-chart {
+ max-width: 1000px;
+}
diff --git a/src/assets/funcs.js b/src/assets/funcs.js
index 9b82b9892..c918ec2e3 100644
--- a/src/assets/funcs.js
+++ b/src/assets/funcs.js
@@ -1,5 +1,8 @@
+// DOM independent functions module
+
'use strict';
+// convert the entries to to a map: {group-key: {test-name: data, ...}, ...}
export function collectBenchesPerTestCasePerGroup(entries) {
const map = new Map();
for (const entry of entries) {
@@ -30,23 +33,45 @@ function* cycle(iterable) {
}
}
-export function renderGraph(canvas, dataset, labels, xAxis, alpha = 60, labelString = 'iter/sec') {
+
+var longestCommonPrefix = function(strs) {
+ let prefix = ""
+ if(strs === null || strs.length === 0) return prefix
+
+ for (let i=0; i < strs[0].length; i++){
+ const char = strs[0][i] // loop through all characters of the very first string.
+
+ for (let j = 1; j < strs.length; j++){
+ // loop through all other strings in the array
+ if(strs[j][i] !== char) return prefix
+ }
+ prefix = prefix + char
+ }
+
+ return prefix
+}
+
+
+export function renderGraph(canvas, dataset, labels, xAxis, alpha = 60, fill = true, labelString = 'iter/sec') {
const colorCycle = cycle(['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']);
+ // if all test names start with a common prefix, then show this as a title
+ const tests_prefix = dataset.length > 1 ? longestCommonPrefix(dataset.map(s => s.name)): ''
+
const data = {
labels,
datasets: dataset.map(s => {
const color = colorCycle.next().value;
return {
- label: s.name,
+ label: s.name.slice(tests_prefix.length),
data: s.data.map(d => d ? d.bench.value : null),
borderColor: color,
- // fill: false,
+ fill: fill,
backgroundColor: color + `${alpha}`, // Add alpha for #rrggbbaa
spanGaps: true,
}
- })
+ }).sort((a, b) => { return a.label > b.label; })
};
const xAxes = {
scaleLabel: {
@@ -75,6 +100,18 @@ export function renderGraph(canvas, dataset, labels, xAxis, alpha = 60, labelStr
xAxes: [xAxes],
yAxes: [yAxes],
},
+ title: {
+ position: 'top',
+ text: tests_prefix,
+ display: tests_prefix.length > 0,
+ },
+ legend: {
+ position: 'top',
+ align: 'center',
+ // TODO legend titles only available in chartjs v3
+ // rather then chart title
+ // title: {text: 'hallo', display: true},
+ },
tooltips: {
callbacks: {
afterTitle: items => {
@@ -122,6 +159,17 @@ export function renderGraph(canvas, dataset, labels, xAxis, alpha = 60, labelStr
const url = data.commit.url;
window.open(url, '_blank');
},
+ plugins: {
+ zoom: {
+ // pan: {
+ // enabled: true
+ // },
+ zoom: {
+ enabled: true,
+ drag: true
+ }
+ }
+ }
};
new Chart(canvas, {
type: 'line',
diff --git a/src/assets/index.html b/src/assets/index.html
index 3aaccace2..53b6e42be 100644
--- a/src/assets/index.html
+++ b/src/assets/index.html
@@ -28,8 +28,11 @@
-
+
+
+
+