Skip to content

Commit 6685203

Browse files
alxndrsnalxndrsn
authored
Add script: repeat-perf-test.sh (#8760)
This script allows running the performance test suite against one or more different versions of the codebase.
1 parent 1f58173 commit 6685203

File tree

8 files changed

+191
-30
lines changed

8 files changed

+191
-30
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,4 @@ yarn.lock
2626
/.eslintcache
2727
release-todo.txt
2828
/perf-test-results/
29+
/dist-bundles/

‎TESTING.md‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,15 @@ environment. For example:
120120

121121
This overrides the path used to load PouchDB in the browser. We use this in CI
122122
to select different builds of the PouchDB library, for example to test the
123-
minified version, the Webpack version, etc.
123+
Webpack version, etc.
124+
125+
This is an alternative to `SRC_ROOT` and `USE_MINIFIED`.
126+
127+
#### `SRC_ROOT`
128+
129+
This overrides the path used to load all PouchDB files in the browser. We use
130+
this in performance tests to allow easily comparing two different versions of
131+
PouchDB, including plugin and adapter implementations.
124132

125133
#### `USE_MINIFIED`
126134

@@ -251,6 +259,7 @@ command-line options and their query string equivalents are:
251259
| `GREP` | `grep` |
252260
| `ITERATIONS` | `iterations` |
253261
| `PLUGINS` | `plugins` |
262+
| `SRC_ROOT` | `srcRoot` |
254263
| `POUCHDB_SRC` | `src` |
255264
| `USE_MINIFIED` | `useMinified` |
256265
| `VIEW_ADAPTERS` | `viewAdapters` |

‎bin/dev-server.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
var watch = require('watch-glob');
66
var http_server = require('http-server');
77
var debounce = require('lodash.debounce');
8-
var buildPouchDB = require('./build-pouchdb');
98
var browserify = require('browserify');
109
var fs = require('fs');
1110

@@ -32,13 +31,17 @@ if (process.env.COUCH_HOST) {
3231
if (process.env.ITERATIONS) {
3332
queryParams.iterations = process.env.ITERATIONS;
3433
}
34+
if (process.env.SRC_ROOT) {
35+
queryParams.srcRoot = process.env.SRC_ROOT;
36+
}
3537
if (process.env.USE_MINIFIED) {
3638
queryParams.useMinified = process.env.USE_MINIFIED;
3739
}
3840

3941
var rebuildPromise = Promise.resolve();
4042

4143
function rebuildPouch() {
44+
const buildPouchDB = require('./build-pouchdb');
4245
rebuildPromise = rebuildPromise.then(buildPouchDB).then(function () {
4346
console.log('Rebuilt packages/node_modules/pouchdb');
4447
}).catch(console.error);
@@ -85,7 +88,7 @@ function watchAll() {
8588
var filesWritten = false;
8689

8790
Promise.resolve().then(function () {
88-
if (process.env.CI) {
91+
if (process.env.CI || process.env.NO_REBUILD) {
8992
return; // don't bother rebuilding in CI; we already built
9093
}
9194
return Promise.all([

‎bin/repeat-perf-test.sh‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/bin/bash -eu
2+
3+
scriptName="$(basename "$0")"
4+
log() { echo "[$scriptName] $*"; }
5+
6+
npm_install() {
7+
# Don't use npm ci, as it requires package-lock.json to be in sync. This is
8+
# probably not the case when switching branches, especially as it's ignored by
9+
# git.
10+
npm install --no-fund --ignore-scripts --no-audit --prefer-offline --progress=false
11+
}
12+
13+
mkdir -p ./perf-test-results
14+
flagFileDevServerRunning=./perf-test-results/.dev-server-started
15+
cleanup() {
16+
if [[ -n ${SERVER_PID-} ]] && ps --pid "$SERVER_PID" >/dev/null; then
17+
log "Shutting down dev server..."
18+
kill "$SERVER_PID"
19+
log "Shutdown complete."
20+
fi
21+
! [[ -f "$flagFileDevServerRunning" ]] || rm "$flagFileDevServerRunning"
22+
}
23+
trap cleanup EXIT
24+
25+
if [[ -f "$flagFileDevServerRunning" ]]; then
26+
log "!!!"
27+
log "!!! Cannot start tests - flag file already exists at $flagFileDevServerRunning"
28+
log "!!! Are tests running in another process?"
29+
log "!!!"
30+
exit 1
31+
fi
32+
33+
if [[ "$#" -lt 1 ]]; then
34+
cat <<EOF
35+
36+
DESCRIPTION
37+
Repeatedly run the performance test suite against one or more versions of the codebase.
38+
39+
USAGE
40+
[PERF_REPEATS=<N>] $0 ...tree-ish
41+
42+
EOF
43+
exit 1
44+
fi
45+
46+
echo
47+
if [[ -z "${PERF_REPEATS-}" ]]; then
48+
log "Running perf tests ENDLESSLY on:"
49+
else
50+
log "Running perf tests $PERF_REPEATS times on:"
51+
fi
52+
log
53+
declare -a commits
54+
i=0
55+
for treeish in "$@"; do
56+
commits[i]="$(git rev-parse "$treeish")"
57+
description="$(git show --oneline --no-patch "$treeish")"
58+
log " $((i=i+1)). $description ($treeish)"
59+
done
60+
log
61+
log "!!! This may cause strange issues if you have uncomitted changes. !!!"
62+
log
63+
log "Press <enter> to continue."
64+
echo
65+
read -r
66+
67+
./bin/wait-for-couch.sh 20
68+
69+
mkdir -p dist-bundles
70+
71+
log "Building bundles..."
72+
for commit in "${commits[@]}"; do
73+
targetDir="dist-bundles/$commit"
74+
if [[ -d "$targetDir" ]]; then
75+
log "Skipping build for $commit - dist files already found at $targetDir."
76+
else
77+
log "Building commit $commit..."
78+
git checkout "$commit"
79+
npm_install # in case of different deps on different branches
80+
npm run build
81+
82+
mkdir -p "$targetDir"
83+
cp -r packages/node_modules/pouchdb/dist/. "$targetDir/"
84+
85+
git checkout -
86+
fi
87+
done
88+
89+
log "Building tests..."
90+
npm_install # in case of different deps on different branches
91+
npm run build-test
92+
93+
iterate_tests() {
94+
for commit in "${commits[@]}"; do
95+
log "Running perf tests on $commit..."
96+
SRC_ROOT="../../dist-bundles/$commit" \
97+
JSON_REPORTER=1 \
98+
PERF=1 \
99+
USE_MINIFIED=1 \
100+
MANUAL_DEV_SERVER=1 \
101+
node ./bin/test-browser.js
102+
103+
sleep 1
104+
done
105+
}
106+
107+
log "Installing playwright brower..."
108+
npx playwright install "${CLIENT:-firefox}"
109+
110+
log "Starting dev server..."
111+
NO_REBUILD=1 node -e "
112+
const { start } = require('./bin/dev-server.js');
113+
start(() => {
114+
console.log('[$scriptName] Dev server ready.');
115+
require('fs').writeFileSync('$flagFileDevServerRunning', '');
116+
});
117+
" &
118+
SERVER_PID=$!
119+
120+
until [[ -f "$flagFileDevServerRunning" ]]; do sleep 1; done
121+
log "Dev server started OK!"
122+
123+
log "Running tests..."
124+
if [[ -z "${PERF_REPEATS-}" ]]; then
125+
while true; do
126+
iterate_tests
127+
done
128+
else
129+
while ((PERF_REPEATS-- > 0)); do
130+
iterate_tests
131+
log "Iterations remaining: $PERF_REPEATS"
132+
done
133+
fi
134+
135+
log "All tests complete."

‎bin/run-test.sh‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -116,28 +116,13 @@ if [[ -n $SERVER ]]; then
116116
fi
117117
fi
118118

119-
if [ "$SERVER" == "couchdb-master" ]; then
120-
while [ '200' != "$(curl -s -o /dev/null -w '%{http_code}' ${COUCH_HOST})" ]; do
121-
echo waiting for couch to load... ;
122-
sleep 1;
123-
done
119+
./bin/wait-for-couch.sh 20
124120

121+
if [ "$SERVER" == "couchdb-master" ]; then
122+
printf '\nEnabling CORS...'
125123
./node_modules/.bin/add-cors-to-couchdb $COUCH_HOST
126124
fi
127125

128-
printf "Waiting for host to start on %s..." "$COUCH_HOST"
129-
WAITING=0
130-
until curl --output /dev/null --silent --head --fail --max-time 2 $COUCH_HOST; do
131-
if [ $WAITING -eq 4 ]; then
132-
printf '\nHost failed to start\n'
133-
exit 1
134-
fi
135-
((WAITING=WAITING+1))
136-
printf '.'
137-
sleep 5
138-
done
139-
printf '\nHost started :)'
140-
141126
if [ "$CLIENT" == "unit" ]; then
142127
npm run test-unit
143128
elif [ "$CLIENT" == "node" ]; then

‎bin/test-browser.js‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ const stacktraceParser = require('stacktrace-parser');
1010
var MochaSpecReporter = require('mocha').reporters.Spec;
1111
const createMochaStatsCollector = require('mocha/lib/stats-collector');
1212

13-
var devserver = require('./dev-server.js');
14-
1513
// BAIL=0 to disable bailing
1614
var bail = process.env.BAIL !== '0';
1715

@@ -53,6 +51,7 @@ const qs = {
5351
autoCompaction: process.AUTO_COMPACTION,
5452
SERVER: process.env.SERVER,
5553
SKIP_MIGRATION: process.env.SKIP_MIGRATION,
54+
srcRoot: process.env.SRC_ROOT,
5655
src: process.env.POUCHDB_SRC,
5756
useMinified: process.env.USE_MINIFIED,
5857
plugins: process.env.PLUGINS,
@@ -185,6 +184,8 @@ function BenchmarkJsonReporter(runner) {
185184
if (runner.failed) {
186185
console.log('Runner failed; JSON will not be writted.');
187186
} else {
187+
results.srcRoot = process.env.SRC_ROOT;
188+
188189
const resultsDir = 'perf-test-results';
189190
fs.mkdirSync(resultsDir, { recursive: true });
190191

@@ -276,6 +277,10 @@ async function startTest() {
276277
}
277278
}
278279

279-
devserver.start(function () {
280+
if (process.env.MANUAL_DEV_SERVER) {
280281
startTest();
281-
});
282+
} else {
283+
// dev-server.js rebuilds bundles when required
284+
const devserver = require('./dev-server.js');
285+
devserver.start(startTest);
286+
}

‎bin/wait-for-couch.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash -eu
2+
3+
maxWait="${1:-0}"
4+
5+
printf "Waiting for host to start on %s..." "$COUCH_HOST"
6+
7+
WAITING=0
8+
until [[ '200' = "$(curl -s -o /dev/null -w '%{http_code}' "$COUCH_HOST")" ]]; do
9+
((WAITING=WAITING+1))
10+
if [ $WAITING -eq "$maxWait" ]; then
11+
printf '\nHost failed to start\n'
12+
exit 1
13+
fi
14+
printf '.'
15+
sleep 1
16+
done
17+
18+
printf '\nHost started :)'

‎tests/common-utils.js‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -108,21 +108,26 @@ const srcExtension = () => {
108108
return params.useMinified ? 'min.js' : 'js';
109109
};
110110

111+
const srcRoot = () => {
112+
const params = commonUtils.params();
113+
return params.srcRoot || '../../packages/node_modules/pouchdb/dist';
114+
};
115+
111116
commonUtils.pouchdbSrc = function () {
112-
const scriptPath = '../../packages/node_modules/pouchdb/dist';
113117
const params = commonUtils.params();
118+
if (params.src && params.srcRoot) {
119+
throw new Error('Cannot use POUCHDB_SRC and SRC_ROOT options together.');
120+
}
114121
if (params.src && params.useMinified) {
115122
throw new Error('Cannot use POUCHDB_SRC and USE_MINIFIED options together.');
116123
}
117-
return params.src || `${scriptPath}/pouchdb.${srcExtension()}`;
124+
return params.src || `${srcRoot()}/pouchdb.${srcExtension()}`;
118125
};
119126

120127
commonUtils.loadPouchDBForBrowser = function (plugins) {
121-
var scriptPath = '../../packages/node_modules/pouchdb/dist';
122-
123128
plugins = plugins.map((plugin) => {
124129
plugin = plugin.replace(/^pouchdb-(adapter-)?/, '');
125-
return `${scriptPath}/pouchdb.${plugin}.${srcExtension()}`;
130+
return `${srcRoot()}/pouchdb.${plugin}.${srcExtension()}`;
126131
});
127132

128133
var scripts = [commonUtils.pouchdbSrc()].concat(plugins);

0 commit comments

Comments
 (0)