Skip to content

Commit

Permalink
🔨 More customizable benchmark (#518)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dubzzz
dubzzz authored Jan 12, 2023
1 parent 7587984 commit 0281cfd
Showing 1 changed file with 162 additions and 83 deletions.
245 changes: 162 additions & 83 deletions perf/benchmark.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,108 +7,181 @@
//
// Or against another generator:
// $: yarn bench mersenne
//
// Or only benchmark generators against each others on new:
// $: yarn bench self
//
// Or only benchmarks on new against default generator:
// $: yarn bench alone

const { Bench } = require('tinybench');
const libReference = require('../lib-reference/pure-rand-default');
const libTest = require('../lib-test/pure-rand-default');

const SEED = Date.now() | 0;
const PROF_GEN = process.argv[2] || 'xoroshiro128plus';
console.info(`Generator: ${PROF_GEN}`);
console.info(`Seed : ${SEED}\n`);
const NUM_RUNS = process.argv[3] || '';
const NUM_INTS = process.argv[4] || '';

const numIterations = Number.isNaN(+NUM_RUNS) ? 1_000 : +NUM_RUNS;
const numInts = Number.isNaN(+NUM_INTS) ? 100 : +NUM_RUNS;

const numInts = 100_000;
const numIterations = 1_000;
console.info(`Generator : ${PROF_GEN}`);
console.info(`Iterations : ${numIterations}`);
console.info(`Numbers per run: ${numInts}`);

function builtInNoDistribution(from, to) {
const out = Math.random();
return from + ~~(out * (to - from + 1)); // ~~ is Math.floor
}

function noDistribution(from, to, g) {
const out = g.unsafeNext();
const out = g.unsafeNext() >>> 0;
return from + (out % (to - from + 1));
}

function fillBench(bench) {
bench.add('built-in (no distrib)', () => {
const g = libReference[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
builtInNoDistribution(0, i, g);
}
});
bench.add('reference (no distrib)', () => {
const g = libReference[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
noDistribution(0, i, g);
}
});
bench.add('reference (uniform small)', () => {
const g = libReference[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformIntDistribution(0, i, g);
}
});
bench.add('reference (uniform large)', () => {
const g = libReference[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformIntDistribution(-0x1_0000_0000, i, g);
}
});
bench.add('reference (uniform small array)', () => {
const g = libReference[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformArrayIntDistribution({ sign: 1, data: [0] }, { sign: 1, data: [i] }, g);
}
});
bench.add('reference (uniform large array)', () => {
const g = libReference[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformIntDistribution({ sign: -1, data: [1, 0] }, { sign: 1, data: [0, i] }, g);
}
});
bench.add('reference (jump)', () => {
const g = libReference[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
g.unsafeJump();
}
});
bench.add('test (no distrib)', () => {
const g = libTest[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
noDistribution(0, i, g);
}
});
bench.add('test (uniform small)', () => {
const g = libTest[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
libTest.unsafeUniformIntDistribution(0, i, g);
}
});
bench.add('test (uniform large)', () => {
const g = libTest[PROF_GEN](SEED);
let seed = Date.now() | 0;
const nextSeed = () => {
seed = (seed + 1) | 0;
return seed;
};
const compareMode = PROF_GEN === 'self';
const configurations = compareMode
? [
{ libName: 'test', generator: 'congruential32' },
{ libName: 'test', generator: 'mersenne' },
{ libName: 'test', generator: 'xorshift128plus' },
{ libName: 'test', generator: 'xoroshiro128plus' },
]
: PROF_GEN === 'alone'
? [{ libName: 'test', generator: 'xoroshiro128plus' }]
: [
{ libName: 'reference', generator: PROF_GEN },
{ libName: 'test', generator: PROF_GEN },
];

const safeBench = (details, fn) => {
const name = JSON.stringify(details);
fn();
bench.add(name, fn);
console.info(`✔️ ${name}`);
};

safeBench({ range: 'S' }, () => {
for (let i = 0; i !== numInts; ++i) {
libTest.unsafeUniformIntDistribution(-0x1_0000_0000, i, g);
builtInNoDistribution(0, i);
}
});
bench.add('test (uniform small array)', () => {
const g = libTest[PROF_GEN](SEED);
safeBench({ range: 'M' }, () => {
for (let i = 0; i !== numInts; ++i) {
libTest.unsafeUniformArrayIntDistribution({ sign: 1, data: [0] }, { sign: 1, data: [i] }, g);
builtInNoDistribution(i, 0x7fff_ffff);
}
});
bench.add('test (uniform large array)', () => {
const g = libTest[PROF_GEN](SEED);
safeBench({ range: 'L' }, () => {
for (let i = 0; i !== numInts; ++i) {
libTest.unsafeUniformIntDistribution({ sign: -1, data: [1, 0] }, { sign: 1, data: [0, i] }, g);
builtInNoDistribution(-0x8000_0000, i);
}
});
bench.add('test (jump)', () => {
const g = libTest[PROF_GEN](SEED);
for (let i = 0; i !== numInts; ++i) {
g.unsafeJump();
for (const { libName, generator } of configurations) {
const libReference = require('../lib-' + libName + '/pure-rand-default');
safeBench({ libName, generator, range: 'S' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
noDistribution(0, i, g);
}
});
safeBench({ libName, generator, range: 'M' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
noDistribution(i, 0x7fff_ffff, g);
}
});
safeBench({ libName, generator, range: 'L' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
noDistribution(-0x8000_0000, i, g);
}
});
if (!compareMode) {
safeBench({ libName, generator, algorithm: 'uniform', range: 'S' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformIntDistribution(0, i, g);
}
});
safeBench({ libName, generator, algorithm: 'uniform', range: 'M' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformIntDistribution(i, 0x7fff_ffff, g);
}
});
safeBench({ libName, generator, algorithm: 'uniform', range: 'L' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformIntDistribution(-0x8000_0000, i, g);
}
});
safeBench({ libName, generator, algorithm: 'uniform', range: 'XL' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformIntDistribution(-0x1_0000_0000, i, g);
}
});
safeBench({ libName, generator, algorithm: 'array uniform', range: 'S' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformArrayIntDistribution({ sign: 1, data: [0] }, { sign: 1, data: [i] }, g);
}
});
safeBench({ libName, generator, algorithm: 'array uniform', range: 'M' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformArrayIntDistribution({ sign: 1, data: [i] }, { sign: 1, data: [0x7fff_ffff] }, g);
}
});
safeBench({ libName, generator, algorithm: 'array uniform', range: 'L' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformArrayIntDistribution({ sign: -1, data: [0x8000_0000] }, { sign: 1, data: [i] }, g);
}
});
safeBench({ libName, generator, algorithm: 'array uniform', range: 'XL' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformArrayIntDistribution({ sign: -1, data: [1, 0] }, { sign: 1, data: [0, i] }, g);
}
});
safeBench({ libName, generator, algorithm: 'bigint uniform', range: 'S' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformBigIntDistribution(BigInt(0), BigInt(i), g);
}
});
safeBench({ libName, generator, algorithm: 'bigint uniform', range: 'M' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformBigIntDistribution(BigInt(i), BigInt(0x7fff_ffff), g);
}
});
safeBench({ libName, generator, algorithm: 'bigint uniform', range: 'L' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformBigIntDistribution(BigInt(-0x8000_0000), BigInt(i), g);
}
});
safeBench({ libName, generator, algorithm: 'bigint uniform', range: 'XL' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
libReference.unsafeUniformBigIntDistribution(BigInt(-0x1_0000_0000), BigInt(i), g);
}
});
safeBench({ libName, generator, algorithm: 'jump' }, () => {
const g = libReference[generator](nextSeed());
for (let i = 0; i !== numInts; ++i) {
g.unsafeJump();
}
});
}
});
}
console.info('');
}

async function runner() {
Expand All @@ -118,13 +191,19 @@ async function runner() {
await bench.run();

console.table(
bench.tasks.map(({ name, result }) => ({
'Task Name': name,
Mean: result?.mean,
P75: result?.p75,
P99: result?.p99,
RME: result?.rme,
}))
bench.tasks.map(({ name, result }) => {
const { libName, generator, algorithm, range } = JSON.parse(name);
return {
Library: libName ?? '—',
RNG: generator ?? '—',
Algorithm: algorithm ?? '—',
Range: range ?? '—',
Mean: result?.mean,
P75: result?.p75,
P99: result?.p99,
RME: result?.rme,
};
})
);
}
runner();

0 comments on commit 0281cfd

Please sign in to comment.