Skip to content

Commit

Permalink
storage: Test more runtime crash scenarios
Browse files Browse the repository at this point in the history
  • Loading branch information
@ohsayan
ohsayan committed Mar 28, 2024
1 parent 267c9a6 commit 8c21f12
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 55 deletions.
4 changes: 2 additions & 2 deletions server/src/engine/storage/v2/raw/journal/raw/tests/journal_ops.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,8 @@ fn journal_open_close() {
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::AttemptingEvent(1),
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(1),
JournalReaderTraceEvent::DriverEventExpectingReopenGotReopen,
JournalReaderTraceEvent::ReopenSuccess,
// now read close event
Expand Down Expand Up @@ -263,8 +263,8 @@ fn journal_with_server_single_event() {
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
// now read reopen event
JournalReaderTraceEvent::AttemptingEvent(2),
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(2),
JournalReaderTraceEvent::DriverEventExpectingReopenGotReopen,
JournalReaderTraceEvent::ReopenSuccess,
// now read close event
Expand Down
161 changes: 108 additions & 53 deletions server/src/engine/storage/v2/raw/journal/raw/tests/recovery.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,20 @@ fn emulate_corrupted_final_event(
}
}

fn apply_event_mix(jrnl: &mut RawJournalWriter<SimpleDBJournal>) -> RuntimeResult<u64> {
let mut op_count = 0;
let mut sdb = SimpleDB::new();
for num in 1..=100 {
op_count += 1;
sdb.push(jrnl, format!("key-{num}"))?;
if num % 10 == 0 {
op_count += 1;
sdb.pop(jrnl)?;
}
}
Ok(op_count)
}

#[test]
fn corruption_before_close() {
let initializers: Vec<Initializer> = vec![
Expand All @@ -133,17 +147,8 @@ fn corruption_before_close() {
}),
// open, apply mix of events, close
("close_event_corruption.db", |jrnl_id| {
let mut operation_count = 0;
let mut sdb = SimpleDB::new();
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
for num in 1..=100 {
operation_count += 1;
sdb.push(&mut jrnl, format!("key-{num}"))?;
if num % 10 == 0 {
operation_count += 1;
sdb.pop(&mut jrnl)?;
}
}
let operation_count = apply_event_mix(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?;
Ok(operation_count)
}),
Expand Down Expand Up @@ -241,14 +246,25 @@ fn corruption_before_close() {

#[test]
fn corruption_after_reopen() {
let initializers: Vec<Initializer> = vec![("corruption_after_reopen.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
RawJournalWriter::close_driver(&mut jrnl)?;
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(1)
})];
let initializers: Vec<Initializer> = vec![
("corruption_after_reopen.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
RawJournalWriter::close_driver(&mut jrnl)?;
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(1)
}),
("corruption_after_ropen_multi_before_close.db", |jrnl_id| {
let mut jrnl = create_journal::<SimpleDBJournal>(jrnl_id)?;
let operation_count = apply_event_mix(&mut jrnl)?;
RawJournalWriter::close_driver(&mut jrnl)?;
drop(jrnl);
// reopen, but don't close
open_journal::<SimpleDBJournal>(jrnl_id, &SimpleDB::new(), JournalSettings::default())?;
Ok(operation_count + 1) // + 1 since we have the reopen event which is the next event that'll vanish
}),
];
emulate_corrupted_final_event(
initializers,
|journal_id, repaired_last_event_id, trim_size, open_result| {
Expand All @@ -265,28 +281,53 @@ fn corruption_after_reopen() {
*/
let mut jrnl =
open_result.expect(&format!("failed at {trim_size} for journal {journal_id}"));
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::ClosedAndReachedEof,
JournalReaderTraceEvent::Completed,
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
prev_id: 0
},
JournalWriterTraceEvent::DriverEventCompleted,
JournalWriterTraceEvent::ReinitializeComplete,
],
"failed at {trim_size} for journal {journal_id}"
);
if repaired_last_event_id == 1 {
// empty log, only the reopen
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::ClosedAndReachedEof,
JournalReaderTraceEvent::Completed,
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
prev_id: 0
},
JournalWriterTraceEvent::DriverEventCompleted,
JournalWriterTraceEvent::ReinitializeComplete,
],
"failed at {trim_size} for journal {journal_id}"
);
} else {
assert_eq!(
&trace[trace.len() - 12..],
intovec![
JournalReaderTraceEvent::ServerEventAppliedSuccess,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id - 1), // close event
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::ClosedAndReachedEof,
JournalReaderTraceEvent::Completed,
JournalWriterTraceEvent::ReinitializeAttempt,
JournalWriterTraceEvent::DriverEventAttemptCommit {
event: DriverEventKind::Reopened,
event_id: repaired_last_event_id,
prev_id: repaired_last_event_id - 1 // close event
},
JournalWriterTraceEvent::DriverEventCompleted,
JournalWriterTraceEvent::ReinitializeComplete
]
)
}
// now close this so that this works with the post repair handler
RawJournalWriter::close_driver(&mut jrnl).unwrap();
let _ = obtain_offsets();
Expand All @@ -296,19 +337,33 @@ fn corruption_after_reopen() {
open_result.unwrap_err().kind(),
&ErrorKind::IoError(IoErrorKind::UnexpectedEof.into())
);
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
);
if repaired_last_event_id == 1 {
// empty log, only the reopen
assert_eq!(
trace,
intovec![
JournalReaderTraceEvent::Initialized,
JournalReaderTraceEvent::LookingForEvent,
JournalReaderTraceEvent::AttemptingEvent(0),
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
);
} else {
assert_eq!(
&trace[trace.len() - 5..],
intovec![
JournalReaderTraceEvent::DriverEventExpectingClose,
JournalReaderTraceEvent::DriverEventCompletedBlockRead,
JournalReaderTraceEvent::DriverEventExpectedCloseGotClose,
JournalReaderTraceEvent::DriverEventExpectingReopenBlock,
JournalReaderTraceEvent::AttemptingEvent(repaired_last_event_id)
]
);
}
}
},
|journal_id, trim_size, repair_result, reopen_result| {
Expand Down

0 comments on commit 8c21f12

Please sign in to comment.