Latency spike when inserting edge with batch size of 1024

[kikimo]

Please check the FAQ documentation before raising an issue

We continue observed latency spike when inserting edge with batch size of 1024, latency spike 1086815 vs normal 48874 :

insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47589 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46583 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47045 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:1028017 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45934 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46012 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46351 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47799 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46490 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46793 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:48733 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47053 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:48381 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47029 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46681 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:48063 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47839 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45776 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47882 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46381 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44486 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46547 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:48080 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:49942 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44523 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45387 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:47456 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:54132 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:51732 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44010 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44960 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45173 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45391 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45070 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45224 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44493 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45363 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44965 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45112 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:43971 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46546 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45925 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46701 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45703 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45014 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44799 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45026 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44761 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44723 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45161 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:1086815 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:48874 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46853 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46113 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45388 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:46788 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:45542 LatencyDetailUs:map[]})}), err: <nil>
insert resp: ExecResponse({Result_:ResponseCommon({FailedParts:[] LatencyInUs:44694 LatencyDetailUs:map[]})}), err: <nil>

Your Environments (required)

• OS: uname -a
• Compiler: g++ --version or clang++ --version
• CPU: lscpu
• Commit id (e.g. a3ffc7d8)

How To Reproduce(required)

Steps to reproduce the behavior:

1. Step 1
2. Step 2
3. Step 3

Expected behavior

Additional context

[liuyu85cn]

may be raft buffer related.