Skip to content

Commit

Permalink
Added BENCH/TEST_PRNG, replacing other ad-hoc sources of randomness
Browse files Browse the repository at this point in the history 
When you add a function to every benchmark suite, you know if should
probably be provided by the benchmark runner itself. That being said,
randomness in tests/benchmarks is a bit tricky because it needs to be
strictly controlled and reproducible.

No global state is used, allowing tests/benches to maintain multiple
randomness stream which can be useful for checking results during a run.

There's an argument for having global prng state in that the prng could
be preserved across power-loss, but I have yet to see a use for this,
and it would add a significant requirement to any future test/bench runner.
  • Loading branch information
@geky
geky committed Dec 7, 2022
1 parent d8e7ffb commit b0382fa
Show file tree
Hide file tree
Showing 10 changed files with 134 additions and 124 deletions.
34 changes: 10 additions & 24 deletions benches/bench_dir.toml
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,3 @@


# deterministic prng
code = '''
static uint32_t xorshift32(uint32_t *state) {
uint32_t x = *state;
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*state = x;
return x;
}
'''

[cases.bench_dir_open]
# 0 = in-order
# 1 = reversed-order
Expand All @@ -37,7 +23,7 @@ code = '''
uint32_t file_prng = i;
for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
buffer[k] = xorshift32(&file_prng);
buffer[k] = BENCH_PRNG(&file_prng);
}
lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
}
Expand All @@ -52,7 +38,7 @@ code = '''
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: xorshift32(&prng) % N;
: BENCH_PRNG(&prng) % N;
sprintf(name, "file%08x", i_);
lfs_file_t file;
lfs_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0;
Expand All @@ -61,7 +47,7 @@ code = '''
for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
assert(buffer[k] == xorshift32(&file_prng));
assert(buffer[k] == BENCH_PRNG(&file_prng));
}
}
Expand Down Expand Up @@ -93,7 +79,7 @@ code = '''
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: xorshift32(&prng) % N;
: BENCH_PRNG(&prng) % N;
sprintf(name, "file%08x", i_);
lfs_file_t file;
lfs_file_open(&lfs, &file, name,
Expand All @@ -102,7 +88,7 @@ code = '''
uint32_t file_prng = i_;
for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
buffer[k] = xorshift32(&file_prng);
buffer[k] = BENCH_PRNG(&file_prng);
}
lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
}
Expand Down Expand Up @@ -139,7 +125,7 @@ code = '''
uint32_t file_prng = i;
for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
buffer[k] = xorshift32(&file_prng);
buffer[k] = BENCH_PRNG(&file_prng);
}
lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
}
Expand All @@ -154,7 +140,7 @@ code = '''
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: xorshift32(&prng) % N;
: BENCH_PRNG(&prng) % N;
sprintf(name, "file%08x", i_);
int err = lfs_remove(&lfs, name);
assert(!err || err == LFS_ERR_NOENT);
Expand Down Expand Up @@ -185,7 +171,7 @@ code = '''
uint32_t file_prng = i;
for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
buffer[k] = xorshift32(&file_prng);
buffer[k] = BENCH_PRNG(&file_prng);
}
lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
}
Expand Down Expand Up @@ -235,7 +221,7 @@ code = '''
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: xorshift32(&prng) % N;
: BENCH_PRNG(&prng) % N;
printf("hm %d\n", i);
sprintf(name, "dir%08x", i_);
int err = lfs_mkdir(&lfs, name);
Expand Down Expand Up @@ -271,7 +257,7 @@ code = '''
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: xorshift32(&prng) % N;
: BENCH_PRNG(&prng) % N;
sprintf(name, "dir%08x", i_);
int err = lfs_remove(&lfs, name);
assert(!err || err == LFS_ERR_NOENT);
Expand Down
24 changes: 5 additions & 19 deletions benches/bench_file.toml
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,3 @@


# deterministic prng
code = '''
static uint32_t xorshift32(uint32_t *state) {
uint32_t x = *state;
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*state = x;
return x;
}
'''

[cases.bench_file_read]
# 0 = in-order
# 1 = reversed-order
Expand All @@ -33,7 +19,7 @@ code = '''
for (lfs_size_t i = 0; i < chunks; i++) {
uint32_t chunk_prng = i;
for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
buffer[j] = xorshift32(&chunk_prng);
buffer[j] = BENCH_PRNG(&chunk_prng);
}
lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
Expand All @@ -50,14 +36,14 @@ code = '''
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (chunks-1-i)
: xorshift32(&prng) % chunks;
: BENCH_PRNG(&prng) % chunks;
lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET)
=> i_*CHUNK_SIZE;
lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
uint32_t chunk_prng = i_;
for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
assert(buffer[j] == xorshift32(&chunk_prng));
assert(buffer[j] == BENCH_PRNG(&chunk_prng));
}
}
Expand Down Expand Up @@ -91,10 +77,10 @@ code = '''
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (chunks-1-i)
: xorshift32(&prng) % chunks;
: BENCH_PRNG(&prng) % chunks;
uint32_t chunk_prng = i_;
for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
buffer[j] = xorshift32(&chunk_prng);
buffer[j] = BENCH_PRNG(&chunk_prng);
}
lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET)
Expand Down
13 changes: 13 additions & 0 deletions runners/bench_runner.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,19 @@ void bench_trace(const char *fmt, ...) {
}


// bench prng
uint32_t bench_prng(uint32_t *state) {
// A simple xorshift32 generator, easily reproducible. Keep in mind
// determinism is much more important than actual randomness here.
uint32_t x = *state;
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*state = x;
return x;
}


// bench recording state
static struct lfs_config *bench_cfg = NULL;
static lfs_emubd_io_t bench_last_readed = 0;
Expand Down
6 changes: 6 additions & 0 deletions runners/bench_runner.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ struct bench_suite {
};


// deterministic prng for pseudo-randomness in benches
uint32_t bench_prng(uint32_t *state);

#define BENCH_PRNG(state) bench_prng(state)


// access generated bench defines
intmax_t bench_define(size_t define);

Expand Down
13 changes: 13 additions & 0 deletions runners/test_runner.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -545,6 +545,19 @@ void test_trace(const char *fmt, ...) {
}


// test prng
uint32_t test_prng(uint32_t *state) {
// A simple xorshift32 generator, easily reproducible. Keep in mind
// determinism is much more important than actual randomness here.
uint32_t x = *state;
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*state = x;
return x;
}


// encode our permutation into a reusable id
static void perm_printid(
const struct test_suite *suite,
Expand Down
6 changes: 6 additions & 0 deletions runners/test_runner.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ struct test_suite {
};


// deterministic prng for pseudo-randomness in testes
uint32_t test_prng(uint32_t *state);

#define TEST_PRNG(state) test_prng(state)


// access generated test defines
intmax_t test_define(size_t define);

Expand Down

0 comments on commit b0382fa

Please sign in to comment.