Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Batch prewrite and commit requests to same region across batch commands #5756

Merged
merged 65 commits into from Nov 27, 2019
Merged
Changes from 4 commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
79a8d48
batch commit and prewrite to the same region
tabokie Oct 29, 2019
80a4971
fix commit callback and panics
tabokie Oct 29, 2019
e77b294
remove batch txn
tabokie Oct 30, 2019
8040a15
fix loose end
tabokie Oct 30, 2019
b968a74
major overhaul
tabokie Oct 31, 2019
af9526d
address comment
tabokie Oct 31, 2019
7aa4405
Merge branch 'master' into batch-write
tabokie Oct 31, 2019
f9d94b0
fix batching
tabokie Oct 31, 2019
ac74884
Merge branch 'master' into batch-write
tabokie Nov 1, 2019
2cb1506
polishing
tabokie Nov 1, 2019
2bd58fd
Merge branch 'master' into batch-write
tabokie Nov 4, 2019
6cfdb71
minor fix
tabokie Nov 4, 2019
60381c4
remove legacy code
tabokie Nov 4, 2019
85c1b60
bump
tabokie Nov 5, 2019
4da17d3
use seperate txn
tabokie Nov 5, 2019
35a9b9a
fmt
tabokie Nov 5, 2019
dbf92ff
Merge branch 'master' into batch-write
tabokie Nov 5, 2019
1f33f80
remove debug log and reuse mvcc txn
tabokie Nov 5, 2019
d672fe7
Merge branch 'master' into batch-write
tabokie Nov 6, 2019
762a0f5
add test and fix pessimistic commit
tabokie Nov 6, 2019
adfa751
remove maybe_clone
tabokie Nov 6, 2019
2428860
use latency for batch limiter
tabokie Nov 6, 2019
6af9a79
Merge branch 'master' into batch-write
tabokie Nov 6, 2019
c05778e
turn up latency threshold
tabokie Nov 6, 2019
d603cb5
turn up latency threshold
tabokie Nov 6, 2019
f1dec94
fix limiter
tabokie Nov 6, 2019
47d94e0
tune up latency threshold
tabokie Nov 7, 2019
1814f84
use writing bytes to limit write batch
tabokie Nov 7, 2019
37e5065
refine limiter
tabokie Nov 7, 2019
6ccdb94
Merge branch 'master' into batch-write
tabokie Nov 8, 2019
6b737db
use pending commands to limit write batch
tabokie Nov 8, 2019
e12c108
tune up threshold
tabokie Nov 8, 2019
02759fd
Merge branch 'master' of https://github.com/tikv/tikv into batch-write
tabokie Nov 11, 2019
e3be9cc
tune up threshold
tabokie Nov 11, 2019
c2e13f0
fix error from merge
tabokie Nov 11, 2019
320dc80
limit batch size below 2MB
tabokie Nov 11, 2019
7dbb08a
use one reader for multiple txn
tabokie Nov 11, 2019
a125498
tune up threshold
tabokie Nov 11, 2019
3eafd78
tune down threshold
tabokie Nov 11, 2019
9b80be0
revert reader reuse
tabokie Nov 13, 2019
e44dac1
refine limiter
tabokie Nov 13, 2019
e5aa74e
Merge branch 'master' into batch-write
tabokie Nov 13, 2019
af3a532
refine limiter
tabokie Nov 13, 2019
8f1dc42
Merge branch 'master' into batch-write
tabokie Nov 13, 2019
32fb615
major tidy up
tabokie Nov 13, 2019
ef9a264
Merge branch 'master' into batch-write
tabokie Nov 13, 2019
fc6751d
Merge branch 'master' into batch-write
tabokie Nov 13, 2019
5e54d88
refine parameter
tabokie Nov 14, 2019
aa2cde8
Merge branch 'master' into batch-write
tabokie Nov 14, 2019
a26406d
Merge branch 'master' into batch-write
tabokie Nov 14, 2019
7d77623
Merge branch 'master' into batch-write
tabokie Nov 18, 2019
19e513b
limiter fights against jittering
tabokie Nov 18, 2019
04685d3
refinement
tabokie Nov 19, 2019
6f07d84
partial revert
tabokie Nov 19, 2019
9f19d03
Merge branch 'master' into batch-write
tabokie Nov 19, 2019
39c5511
refine limiter
tabokie Nov 19, 2019
66b766e
tune thread load limit
tabokie Nov 20, 2019
28d3a05
refinement
tabokie Nov 20, 2019
e819807
Merge branch 'master' of https://github.com/tikv/tikv into batch-write
tabokie Nov 21, 2019
5ce31cc
address comment
tabokie Nov 21, 2019
f29f965
Merge branch 'master' of https://github.com/tikv/tikv into batch-write
tabokie Nov 21, 2019
f67c7dd
fix comment
tabokie Nov 21, 2019
5fd730e
tune up stat interval
tabokie Nov 21, 2019
02c6cd4
Merge branch 'master' into batch-write
tabokie Nov 21, 2019
b4e227c
Merge branch 'master' of https://github.com/tikv/tikv into batch-write
tabokie Nov 21, 2019
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -34,6 +34,7 @@ const LOAD_STATISTICS_SLOTS: usize = 4;
const LOAD_STATISTICS_INTERVAL: Duration = Duration::from_millis(100);
pub const GRPC_THREAD_PREFIX: &str = "grpc-server";
pub const READPOOL_NORMAL_THREAD_PREFIX: &str = "store-read-norm";
pub const SCHED_POOL_THREAD_PREFIX: &str = "sched-worker";
pub const STATS_THREAD_PREFIX: &str = "transport-stats";

/// The TiKV server
@@ -59,6 +60,8 @@ pub struct Server<T: RaftStoreRouter + 'static, S: StoreAddrResolver + 'static>
grpc_thread_load: Arc<ThreadLoad>,
readpool_normal_concurrency: usize,
readpool_normal_thread_load: Arc<ThreadLoad>,
sched_pool_concurrency: usize,
sched_pool_thread_load: Arc<ThreadLoad>,
timer: Handle,
}

@@ -83,6 +86,8 @@ impl<T: RaftStoreRouter, S: StoreAddrResolver + 'static> Server<T, S> {
let readpool_normal_concurrency = storage.readpool_normal_concurrency();
let readpool_normal_thread_load =
Arc::new(ThreadLoad::with_threshold(cfg.heavy_load_threshold));
let sched_pool_concurrency = storage.sched_pool_concurrency();
let sched_pool_thread_load = Arc::new(ThreadLoad::with_threshold(cfg.heavy_load_threshold));

let env = Arc::new(
EnvBuilder::new()
@@ -100,6 +105,7 @@ impl<T: RaftStoreRouter, S: StoreAddrResolver + 'static> Server<T, S> {
snap_worker.scheduler(),
Arc::clone(&grpc_thread_load),
Arc::clone(&readpool_normal_thread_load),
Arc::clone(&sched_pool_thread_load),
cfg.enable_request_batch,
if cfg.enable_request_batch && cfg.request_batch_enable_cross_command {
Some(Duration::from(cfg.request_batch_wait_duration))
@@ -153,6 +159,8 @@ impl<T: RaftStoreRouter, S: StoreAddrResolver + 'static> Server<T, S> {
grpc_thread_load,
readpool_normal_concurrency,
readpool_normal_thread_load,
sched_pool_concurrency,
sched_pool_thread_load,
timer: GLOBAL_TIMER_HANDLE.clone(),
};

@@ -219,13 +227,21 @@ impl<T: RaftStoreRouter, S: StoreAddrResolver + 'static> Server<T, S> {
stats.set_thread_target(self.readpool_normal_concurrency);
stats
};
let mut sched_pool_load_stats = {
let tl = Arc::clone(&self.sched_pool_thread_load);
let mut stats =
ThreadLoadStatistics::new(LOAD_STATISTICS_SLOTS, SCHED_POOL_THREAD_PREFIX, tl);
stats.set_thread_target(self.sched_pool_concurrency);
stats
};
self.stats_pool.as_ref().unwrap().spawn(
self.timer
.interval(Instant::now(), LOAD_STATISTICS_INTERVAL)
.map_err(|_| ())
.for_each(move |i| {
grpc_load_stats.record(i);
readpool_normal_load_stats.record(i);
sched_pool_load_stats.record(i);
Ok(())
}),
);
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.