Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fuzz: ensure all all minor fuzzers compile #1341

Merged
merged 1 commit into from
Dec 5, 2023
Merged

fuzz: ensure all all minor fuzzers compile #1341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/workflows/linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,13 @@ jobs:
- run: zig/zig build fuzz_vsr_superblock -- --seed 123 --events-max 3
- run: zig/zig build fuzz_vsr_superblock_quorums -- --seed 123

# Disabled for now, since they're slow.
# Build the fuzzers, but don't run them, since they're slow.
- run: zig/zig build build_fuzz_lsm_manifest_log
- run: zig/zig build build_fuzz_vsr_free_set
- run: zig/zig build build_fuzz_lsm_forest

# # This both checks that the hash_log builds and acts as a regression test for
# # https://github.com/tigerbeetle/tigerbeetle/issues/404
# - run: zig/zig build fuzz_lsm_manifest_log -- --seed 123 --events-max 400
# - run: zig/zig build fuzz_vsr_free_set -- --seed 123
# - run: zig/zig build fuzz_lsm_forest -Dhash-log-mode=create -Doptimize=ReleaseSafe -- --seed 16319736705930193193 --events-max 10000 && zig/zig build fuzz_lsm_forest -Dhash-log-mode=check -- --seed 16319736705930193193 --events-max 10000

# Check some build steps that would otherwise not get checked.
Expand Down
198 changes: 130 additions & 68 deletions src/lsm/manifest_log_fuzz.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,43 +75,9 @@ fn run_fuzz(
.write_latency_mean = 1 + random.uintLessThan(u64, 40),
};

var storage = try Storage.init(allocator, constants.storage_size_max, storage_options);
defer storage.deinit(allocator);

var storage_verify = try Storage.init(allocator, constants.storage_size_max, storage_options);
defer storage_verify.deinit(allocator);

var superblock = try SuperBlock.init(allocator, .{
.storage = &storage,
.storage_size_limit = constants.storage_size_max,
});
defer superblock.deinit(allocator);

var superblock_verify = try SuperBlock.init(allocator, .{
.storage = &storage_verify,
.storage_size_limit = constants.storage_size_max,
});
defer superblock_verify.deinit(allocator);

var grid = try Grid.init(allocator, .{
.superblock = &superblock,
.missing_blocks_max = 0,
.missing_tables_max = 0,
});
defer grid.deinit(allocator);

var grid_verify = try Grid.init(allocator, .{
.superblock = &superblock_verify,
.missing_blocks_max = 0,
.missing_tables_max = 0,
});
defer grid_verify.deinit(allocator);

var env = try Environment.init(allocator, .{
.grid = &grid,
.grid_verify = &grid_verify,
});
defer env.deinit(allocator);
var env: Environment = undefined;
try env.init(allocator, storage_options);
defer env.deinit();

{
env.format_superblock();
Expand All @@ -120,6 +86,9 @@ fn run_fuzz(
env.open_superblock();
env.wait(&env.manifest_log);

env.open_free_set();
env.wait(&env.manifest_log);

env.open();
env.wait(&env.manifest_log);
}
Expand Down Expand Up @@ -288,44 +257,110 @@ fn generate_events(
}

const Environment = struct {
const FreeSetEncoded = vsr.FreeSetEncodedType(Storage);

allocator: std.mem.Allocator,
superblock_context: SuperBlock.Context = undefined,
storage: Storage,
storage_verify: Storage,
superblock: SuperBlock,
superblock_verify: SuperBlock,
superblock_context: SuperBlock.Context,

grid: Grid,
grid_verify: Grid,

manifest_log: ManifestLog,
manifest_log_verify: ManifestLog,
manifest_log_model: ManifestLogModel,
manifest_log_opening: ?ManifestLogModel.TableMap = null,
pending: usize = 0,
manifest_log_opening: ?ManifestLogModel.TableMap,
pending: u32,

fn init(
env: *Environment, // In-place construction for stable addresses.
allocator: std.mem.Allocator,
options: struct {
grid: *Grid,
grid_verify: *Grid,
},
) !Environment {
var manifest_log_model = try ManifestLogModel.init(allocator);
errdefer manifest_log_model.deinit();

var manifest_log = try ManifestLog.init(allocator, options.grid, manifest_log_options);
errdefer manifest_log.deinit(allocator);

var manifest_log_verify =
try ManifestLog.init(allocator, options.grid_verify, manifest_log_options);
errdefer manifest_log_verify.deinit(allocator);

return Environment{
.allocator = allocator,
.manifest_log = manifest_log,
.manifest_log_verify = manifest_log_verify,
.manifest_log_model = manifest_log_model,
};
storage_options: Storage.Options,
) !void {
comptime var fields_initialized = 0;

fields_initialized += 1;
env.allocator = allocator;

fields_initialized += 1;
env.storage =
try Storage.init(allocator, constants.storage_size_max, storage_options);
errdefer env.storage.deinit(allocator);

fields_initialized += 1;
env.storage_verify =
try Storage.init(allocator, constants.storage_size_max, storage_options);
errdefer env.storage_verify.deinit(allocator);

fields_initialized += 1;
env.superblock = try SuperBlock.init(allocator, .{
.storage = &env.storage,
.storage_size_limit = constants.storage_size_max,
});
errdefer env.superblock.deinit(allocator);

fields_initialized += 1;
env.superblock_verify = try SuperBlock.init(allocator, .{
.storage = &env.storage_verify,
.storage_size_limit = constants.storage_size_max,
});
errdefer env.superblock_verify.deinit(allocator);

fields_initialized += 1;
env.superblock_context = undefined;

fields_initialized += 1;
env.grid = try Grid.init(allocator, .{
.superblock = &env.superblock,
.missing_blocks_max = 0,
.missing_tables_max = 0,
});
errdefer env.grid.deinit(allocator);

fields_initialized += 1;
env.grid_verify = try Grid.init(allocator, .{
.superblock = &env.superblock_verify,
.missing_blocks_max = 0,
.missing_tables_max = 0,
});
errdefer env.grid_verify.deinit(allocator);

fields_initialized += 1;
env.manifest_log = try ManifestLog.init(allocator, &env.grid, manifest_log_options);
errdefer env.manifest_log.deinit(allocator);

fields_initialized += 1;
env.manifest_log_verify =
try ManifestLog.init(allocator, &env.grid_verify, manifest_log_options);
errdefer env.manifest_log_verify.deinit(allocator);

fields_initialized += 1;
env.manifest_log_model = try ManifestLogModel.init(allocator);
errdefer env.manifest_log_model.deinit();

fields_initialized += 1;
env.manifest_log_opening = null;
fields_initialized += 1;
env.pending = 0;

comptime assert(fields_initialized == std.meta.fields(@This()).len);
}

fn deinit(env: *Environment, allocator: std.mem.Allocator) void {
env.manifest_log.deinit(allocator);
env.manifest_log_verify.deinit(env.allocator);
env.manifest_log_model.deinit();
fn deinit(env: *Environment) void {
assert(env.manifest_log_opening == null);
env.manifest_log_model.deinit();
env.manifest_log_verify.deinit(env.allocator);
env.manifest_log.deinit(env.allocator);
env.grid_verify.deinit(env.allocator);
env.grid.deinit(env.allocator);
env.superblock_verify.deinit(env.allocator);
env.superblock.deinit(env.allocator);
env.storage_verify.deinit(env.allocator);
env.storage.deinit(env.allocator);
env.* = undefined;
}

fn wait(env: *Environment, manifest_log: *ManifestLog) void {
Expand Down Expand Up @@ -360,6 +395,22 @@ const Environment = struct {
env.pending -= 1;
}

fn open_free_set(env: *Environment) void {
assert(env.pending == 0);
env.pending += 1;
env.grid.free_set_encoded.open(
&env.grid,
env.superblock.working.free_set_reference(),
open_free_set_callback,
);
}

fn open_free_set_callback(free_set_encoded: *FreeSetEncoded) void {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This particular callback is what motivated the overall init refactor. For this to work, I need to make Grid (not *Grid) a part of the environment, and I figured that I'd rather make everything a part of environment anyway.

const grid = @fieldParentPtr(Grid, "free_set_encoded", free_set_encoded);
const env = @fieldParentPtr(Environment, "grid", grid);
env.pending -= 1;
}

fn open(env: *Environment) void {
assert(env.pending == 0);

Expand Down Expand Up @@ -411,7 +462,11 @@ const Environment = struct {
try env.manifest_log_model.checkpoint();

env.pending += 1;
env.manifest_log.checkpoint(checkpoint_callback);
env.manifest_log.checkpoint(checkpoint_manifest_log_callback);
env.wait(&env.manifest_log);

env.pending += 1;
env.grid.checkpoint(checkpoint_grid_callback);
env.wait(&env.manifest_log);

const vsr_state = &env.manifest_log.superblock.working.vsr_state;
Expand All @@ -420,7 +475,7 @@ const Environment = struct {
// VSRState.monotonic() asserts that the previous_checkpoint id changes.
// In a normal replica this is guaranteed – even if the LSM is idle and no blocks
// are acquired or released, the client sessions are necessarily mutated.
var reply = std.mem.zeroInit(vsr.Header, .{
var reply = std.mem.zeroInit(vsr.Header.Reply, .{
.cluster = 0,
.command = .reply,
.op = vsr_state.checkpoint.commit_min + 1,
Expand All @@ -438,6 +493,7 @@ const Environment = struct {
&env.superblock_context,
.{
.manifest_references = env.manifest_log.checkpoint_references(),
.free_set_reference = env.grid.free_set_encoded.checkpoint_reference(),
.commit_min_checksum = vsr_state.checkpoint.commit_min_checksum + 1,
.commit_min = vsr.Checkpoint.checkpoint_after(vsr_state.checkpoint.commit_min),
.commit_max = vsr.Checkpoint.checkpoint_after(vsr_state.commit_max),
Expand All @@ -450,11 +506,16 @@ const Environment = struct {
try env.verify();
}

fn checkpoint_callback(manifest_log: *ManifestLog) void {
fn checkpoint_manifest_log_callback(manifest_log: *ManifestLog) void {
const env = @fieldParentPtr(Environment, "manifest_log", manifest_log);
env.pending -= 1;
}

fn checkpoint_grid_callback(grid: *Grid) void {
const env = @fieldParentPtr(Environment, "grid", grid);
env.pending -= 1;
}

fn checkpoint_superblock_callback(context: *SuperBlock.Context) void {
const env = @fieldParentPtr(Environment, "superblock_context", context);
env.pending -= 1;
Expand Down Expand Up @@ -590,6 +651,7 @@ const ManifestLogModel = struct {
const removed = model.tables.fetchRemove(table_info.address).?;
assert(std.meta.eql(removed.value, table_info));
},
.reserved => unreachable,
}
}
model.appends.clearRetainingCapacity();
Expand Down