Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Normalize import format #339

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 6 additions & 5 deletions ctl/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@

//! # Raft Engine Control

use std::path::Path;
use std::sync::Arc;
use std::{path::Path, sync::Arc};

use clap::{crate_authors, crate_version, Parser};
use raft_engine::env::{DefaultFileSystem, FileSystem};
use raft_engine::internals::LogQueue;
use raft_engine::{Engine, Error, Result as EngineResult};
use raft_engine::{
env::{DefaultFileSystem, FileSystem},
internals::LogQueue,
Engine, Error, Result as EngineResult,
};

#[derive(Debug, clap::Parser)]
#[clap(
Expand Down
7 changes: 2 additions & 5 deletions examples/fork.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
use std::path::Path;
use std::sync::Arc;
use std::{path::Path, sync::Arc};

use raft_engine::env::DefaultFileSystem;
use raft_engine::Config;
use raft_engine::Engine;
use raft_engine::{env::DefaultFileSystem, Config, Engine};

fn main() {
let mut args = std::env::args();
Expand Down
15 changes: 15 additions & 0 deletions rustfmt.toml
Original file line number Diff line number Diff line change
@@ -1 +1,16 @@
version = "Two"
unstable_features = true

comment_width = 80
wrap_comments = true
format_code_in_doc_comments = true
format_macro_bodies = true
format_macro_matchers = true
normalize_comments = true
normalize_doc_attributes = true
condense_wildcard_suffixes = true
newline_style = "Unix"
use_field_init_shorthand = true
use_try_shorthand = true
imports_granularity = "Crate"
group_imports = "StdExternalCrate"
13 changes: 8 additions & 5 deletions src/codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

#![allow(dead_code)]

use std::io::{self, ErrorKind, Write};
use std::mem;
use std::{
io::{self, ErrorKind, Write},
mem,
};

use byteorder::{BigEndian, ByteOrder, LittleEndian, WriteBytesExt};
use thiserror::Error;
Expand Down Expand Up @@ -357,10 +359,11 @@ pub fn read_u8(data: &mut BytesSlice<'_>) -> Result<u8> {

#[cfg(test)]
mod tests {
use super::*;
use std::{f32, f64, i16, i32, i64, io::ErrorKind, u16, u32, u64};

use protobuf::CodedOutputStream;
use std::io::ErrorKind;
use std::{f32, f64, i16, i32, i64, u16, u32, u64};

use super::*;

const U16_TESTS: &[u16] = &[
i16::MIN as u16,
Expand Down
11 changes: 6 additions & 5 deletions src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@
use log::{info, warn};
use serde::{Deserialize, Serialize};

use crate::pipe_log::Version;
use crate::{util::ReadableSize, Result};
use crate::{pipe_log::Version, util::ReadableSize, Result};

const MIN_RECOVERY_READ_BLOCK_SIZE: usize = 512;
const MIN_RECOVERY_THREADS: usize = 1;
Expand Down Expand Up @@ -343,9 +342,11 @@ mod tests {
let mut load: Config = toml::from_str(old).unwrap();
load.sanitize().unwrap();
// Downgrade to older version.
assert!(toml::to_string(&load)
.unwrap()
.contains("tolerate-corrupted-tail-records"));
assert!(
toml::to_string(&load)
.unwrap()
.contains("tolerate-corrupted-tail-records")
);
}

#[test]
Expand Down
10 changes: 6 additions & 4 deletions src/consistency.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

use hashbrown::HashMap;

use crate::file_pipe_log::ReplayMachine;
use crate::log_batch::{LogItemBatch, LogItemContent};
use crate::pipe_log::{FileId, LogQueue};
use crate::Result;
use crate::{
file_pipe_log::ReplayMachine,
log_batch::{LogItemBatch, LogItemContent},
pipe_log::{FileId, LogQueue},
Result,
};

/// A `ConsistencyChecker` scans for log entry holes in a log queue. It will
/// return a list of corrupted raft groups along with their last valid log
Expand Down
134 changes: 78 additions & 56 deletions src/engine.rs
Original file line number Diff line number Diff line change
@@ -1,28 +1,32 @@
// Copyright (c) 2017-present, PingCAP, Inc. Licensed under Apache-2.0.

use std::cell::{Cell, RefCell};
use std::marker::PhantomData;
use std::path::Path;
use std::sync::{mpsc, Arc, Mutex};
use std::thread::{Builder as ThreadBuilder, JoinHandle};
use std::time::{Duration, Instant};
use std::{
cell::{Cell, RefCell},
marker::PhantomData,
path::Path,
sync::{mpsc, Arc, Mutex},
thread::{Builder as ThreadBuilder, JoinHandle},
time::{Duration, Instant},
};

use log::{error, info};
use protobuf::{parse_from_bytes, Message};

use crate::config::{Config, RecoveryMode};
use crate::consistency::ConsistencyChecker;
use crate::env::{DefaultFileSystem, FileSystem};
use crate::event_listener::EventListener;
use crate::file_pipe_log::debug::LogItemReader;
use crate::file_pipe_log::{DefaultMachineFactory, FilePipeLog, FilePipeLogBuilder};
use crate::log_batch::{Command, LogBatch, MessageExt};
use crate::memtable::{EntryIndex, MemTableRecoverContextFactory, MemTables};
use crate::metrics::*;
use crate::pipe_log::{FileBlockHandle, FileId, LogQueue, PipeLog};
use crate::purge::{PurgeHook, PurgeManager};
use crate::write_barrier::{WriteBarrier, Writer};
use crate::{perf_context, Error, GlobalStats, Result};
use crate::{
config::{Config, RecoveryMode},
consistency::ConsistencyChecker,
env::{DefaultFileSystem, FileSystem},
event_listener::EventListener,
file_pipe_log::{debug::LogItemReader, DefaultMachineFactory, FilePipeLog, FilePipeLogBuilder},
log_batch::{Command, LogBatch, MessageExt},
memtable::{EntryIndex, MemTableRecoverContextFactory, MemTables},
metrics::*,
perf_context,
pipe_log::{FileBlockHandle, FileId, LogQueue, PipeLog},
purge::{PurgeHook, PurgeManager},
write_barrier::{WriteBarrier, Writer},
Error, GlobalStats, Result,
};

const METRICS_FLUSH_INTERVAL: Duration = Duration::from_secs(30);
/// Max times for `write`.
Expand Down Expand Up @@ -106,11 +110,13 @@
let memtables_clone = memtables.clone();
let metrics_flusher = ThreadBuilder::new()
.name("re-metrics".into())
.spawn(move || loop {
stats_clone.flush_metrics();
memtables_clone.flush_metrics();
if rx.recv_timeout(METRICS_FLUSH_INTERVAL).is_ok() {
break;
.spawn(move || {
loop {
stats_clone.flush_metrics();
memtables_clone.flush_metrics();
if rx.recv_timeout(METRICS_FLUSH_INTERVAL).is_ok() {
break;
}

Check warning on line 119 in src/engine.rs

View check run for this annotation

Codecov / codecov/patch

src/engine.rs#L119

Added line #L119 was not covered by tests
}
})?;

Expand Down Expand Up @@ -625,18 +631,24 @@

#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::env::{ObfuscatedFileSystem, Permission};
use crate::file_pipe_log::{parse_reserved_file_name, FileNameExt};
use crate::log_batch::AtomicGroupBuilder;
use crate::pipe_log::Version;
use crate::test_util::{generate_entries, PanicGuard};
use crate::util::ReadableSize;
use std::{
collections::{BTreeSet, HashSet},
fs::OpenOptions,
path::PathBuf,
};

use kvproto::raft_serverpb::RaftLocalState;
use raft::eraftpb::Entry;
use std::collections::{BTreeSet, HashSet};
use std::fs::OpenOptions;
use std::path::PathBuf;

use super::*;
use crate::{
env::{ObfuscatedFileSystem, Permission},
file_pipe_log::{parse_reserved_file_name, FileNameExt},
log_batch::AtomicGroupBuilder,
pipe_log::Version,
test_util::{generate_entries, PanicGuard},
util::ReadableSize,
};

pub(crate) type RaftLogEngine<F = DefaultFileSystem> = Engine<F>;
impl<F: FileSystem> RaftLogEngine<F> {
Expand Down Expand Up @@ -1208,9 +1220,11 @@
// GC all log entries. Won't trigger purge because total size is not enough.
let count = engine.compact_to(1, 100);
assert_eq!(count, 100);
assert!(!engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append));
assert!(
!engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append)
);

// Append more logs to make total size greater than `purge_threshold`.
for index in 100..250 {
Expand All @@ -1220,9 +1234,11 @@
// GC first 101 log entries.
assert_eq!(engine.compact_to(1, 101), 1);
// Needs to purge because the total size is greater than `purge_threshold`.
assert!(engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append));
assert!(
engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append)
);

let old_min_file_seq = engine.file_span(LogQueue::Append).0;
let will_force_compact = engine.purge_expired_files().unwrap();
Expand All @@ -1236,9 +1252,11 @@

assert_eq!(engine.compact_to(1, 102), 1);
// Needs to purge because the total size is greater than `purge_threshold`.
assert!(engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append));
assert!(
engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append)
);
let will_force_compact = engine.purge_expired_files().unwrap();
// The region needs to be force compacted because the threshold is reached.
assert!(!will_force_compact.is_empty());
Expand Down Expand Up @@ -1327,9 +1345,11 @@
engine.append(11, 1, 11, Some(&data));

// The engine needs purge, and all old entries should be rewritten.
assert!(engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append));
assert!(
engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append)
);
assert!(engine.purge_expired_files().unwrap().is_empty());
assert!(engine.file_span(LogQueue::Append).0 > 1);

Expand Down Expand Up @@ -1363,9 +1383,11 @@
}
}

assert!(engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append));
assert!(
engine
.purge_manager
.needs_rewrite_log_files(LogQueue::Append)
);
assert!(engine.purge_expired_files().unwrap().is_empty());
}

Expand Down Expand Up @@ -1670,7 +1692,7 @@
}

drop(engine);
//dump dir with raft groups. 8 element in raft groups 7 and 2 elements in raft
// dump dir with raft groups. 8 element in raft groups 7 and 2 elements in raft
// groups 8
let dump_it = Engine::dump_with_file_system(dir.path(), fs.clone()).unwrap();
let total = dump_it
Expand All @@ -1680,7 +1702,7 @@
.count();
assert!(total == 10);

//dump file
// dump file
let file_id = FileId {
queue: LogQueue::Rewrite,
seq: 1,
Expand All @@ -1697,10 +1719,10 @@
.count();
assert!(0 == total);

//dump dir that does not exists
// dump dir that does not exists
assert!(Engine::dump_with_file_system(Path::new("/not_exists_dir"), fs.clone()).is_err());

//dump file that does not exists
// dump file that does not exists
let mut not_exists_file = PathBuf::from(dir.as_ref());
not_exists_file.push("not_exists_file");
assert!(Engine::dump_with_file_system(not_exists_file.as_path(), fs).is_err());
Expand Down Expand Up @@ -1733,7 +1755,7 @@
let script1 = "".to_owned();
RaftLogEngine::unsafe_repair_with_file_system(
dir.path(),
None, /* queue */
None, // queue
script1,
fs.clone(),
)
Expand All @@ -1752,7 +1774,7 @@
.to_owned();
RaftLogEngine::unsafe_repair_with_file_system(
dir.path(),
None, /* queue */
None, // queue
script2,
fs.clone(),
)
Expand Down Expand Up @@ -1814,7 +1836,7 @@
.to_owned();
RaftLogEngine::unsafe_repair_with_file_system(
dir.path(),
None, /* queue */
None, // queue
script,
fs.clone(),
)
Expand Down
11 changes: 6 additions & 5 deletions src/env/default.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@

#[cfg(feature = "failpoints")]
use std::io::{Error, ErrorKind};
use std::io::{Read, Result as IoResult, Seek, SeekFrom, Write};
use std::path::Path;
use std::sync::Arc;
use std::{
io::{Read, Result as IoResult, Seek, SeekFrom, Write},
path::Path,
sync::Arc,
};

use fail::fail_point;

use crate::env::log_fd::LogFd;
use crate::env::{FileSystem, Handle, Permission, WriteExt};
use crate::env::{log_fd::LogFd, FileSystem, Handle, Permission, WriteExt};

/// A low-level file adapted for standard interfaces including [`Seek`],
/// [`Write`] and [`Read`].
Expand Down