Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Proactive flush of delta merge layer #7756

Open
wants to merge 34 commits into
base: master
Choose a base branch
from
Open

Proactive flush of delta merge layer #7756

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
dbfec4a
background delta flush
hehechen Mar 1, 2023
2f873bf
fix lock
hehechen Mar 1, 2023
ffd10d1
add lock held
CalvinNeo May 29, 2023
c6391ab
Merge remote-tracking branch 'upstream/master' into calvin_delta_flus…
CalvinNeo May 29, 2023
08a7c45
add tests
CalvinNeo May 29, 2023
6c5986f
add findByRangeChecked
CalvinNeo May 30, 2023
a7723f4
f
CalvinNeo May 30, 2023
03a3da3
fix
CalvinNeo May 30, 2023
9c0e733
lock
CalvinNeo May 31, 2023
2105a5a
fix lock
CalvinNeo Jun 15, 2023
253061a
try recursive
CalvinNeo Jun 16, 2023
df454d5
dont lock kvstore
CalvinNeo Jun 16, 2023
5686b5c
tto be revert: remove all locks
CalvinNeo Jun 16, 2023
94c9265
Revert "tto be revert: remove all locks"
CalvinNeo Jun 19, 2023
7db53fe
to revert really remove lock
CalvinNeo Jun 19, 2023
ba3596c
revert
CalvinNeo Jun 20, 2023
2dfaab1
fix 2 fg lock
CalvinNeo Jun 20, 2023
6d6cc37
add some tests
CalvinNeo Jun 25, 2023
9cb9ec1
support test of notify
CalvinNeo Jun 27, 2023
6aa3dc6
settings,tests,proxy updates
CalvinNeo Jun 28, 2023
6b7b256
fmt
CalvinNeo Jul 3, 2023
c4b3173
a
CalvinNeo Jul 6, 2023
e75bd6d
Merge remote-tracking branch 'upstream/master' into calvin_delta_flus…
CalvinNeo Jul 6, 2023
c080671
remove get flush state when exec_compact_log
CalvinNeo Jul 12, 2023
3b93e12
metrics: part1
CalvinNeo Jul 20, 2023
2ea06c2
z
CalvinNeo Jul 20, 2023
a0ecfe5
update jul26
CalvinNeo Jul 25, 2023
eb698cc
fix proxy
CalvinNeo Jul 25, 2023
ac8e88f
get flushed state
CalvinNeo Jul 25, 2023
d4f0961
mew log
CalvinNeo Aug 1, 2023
4de6a63
aaaa
CalvinNeo Aug 3, 2023
e9e7fce
f
CalvinNeo Aug 4, 2023
2d08004
a
CalvinNeo Aug 4, 2023
4366d87
rrrr
CalvinNeo Aug 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion contrib/tiflash-proxy
Open in desktop
Submodule tiflash-proxy updated 28 files
+41 −5 components/raftstore/src/coprocessor/dispatcher.rs
+22 −1 components/raftstore/src/coprocessor/mod.rs
+192 −55 components/raftstore/src/store/fsm/apply.rs
+6 −2 components/raftstore/src/store/fsm/peer.rs
+3 −0 proxy_components/engine_store_ffi/src/core/forward_raft/command.rs
+30 −0 proxy_components/engine_store_ffi/src/core/forward_raft/region.rs
+1 −0 proxy_components/engine_store_ffi/src/ffi/mod.rs
+25 −1 proxy_components/engine_store_ffi/src/observer.rs
+1 −0 proxy_components/engine_tiflash/src/engine.rs
+13 −1 proxy_components/engine_tiflash/src/proxy_utils/proxy_ext.rs
+1 −0 proxy_components/mock-engine-store/src/mock_cluster/cluster_ext.rs
+5 −2 proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs
+3 −1 proxy_components/mock-engine-store/src/mock_cluster/v1/cluster_ext_v1.rs
+14 −1 proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs
+47 −0 proxy_components/proxy_ffi/src/apply_router_helper.rs
+4 −0 proxy_components/proxy_ffi/src/engine_store_helper_impls.rs
+22 −5 proxy_components/proxy_ffi/src/interfaces.rs
+1 −0 proxy_components/proxy_ffi/src/lib.rs
+20 −0 proxy_components/proxy_ffi/src/raftstore_proxy.rs
+29 −1 proxy_components/proxy_ffi/src/raftstore_proxy_helper_impls.rs
+5 −0 proxy_components/proxy_server/src/run.rs
+1 −0 proxy_scripts/ci_check.sh
+0 −2 proxy_tests/proxy/shared/ingest.rs
+1 −0 proxy_tests/proxy/shared/mod.rs
+81 −0 proxy_tests/proxy/shared/proactive_flush.rs
+56 −0 proxy_tests/proxy/shared/write.rs
+1 −1 raftstore-proxy/ffi/src/RaftStoreProxyFFI/@version
+11 −2 raftstore-proxy/ffi/src/RaftStoreProxyFFI/ProxyFFI.h
20 changes: 12 additions & 8 deletions dbms/src/Common/FailPoint.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ namespace DB
M(exception_before_dmfile_remove_encryption) \
M(exception_before_dmfile_remove_from_disk) \
M(force_triggle_background_merge_delta) \
M(force_triggle_foreground_flush) \
M(exception_before_mpp_make_non_root_mpp_task_active) \
M(exception_before_mpp_register_non_root_mpp_task) \
M(exception_before_mpp_register_tunnel_for_non_root_mpp_task) \
Expand Down Expand Up @@ -100,6 +99,9 @@ namespace DB
M(force_stop_background_checkpoint_upload) \
M(skip_seek_before_read_dmfile) \
M(exception_after_large_write_exceed) \
M(proactive_flush_force_set_type) \
M(proactive_flush_between_persist_cache_and_region) \
M(proactive_flush_between_persist_regions) \
M(exception_when_fetch_disagg_pages)

#define APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) \
Expand All @@ -114,13 +116,15 @@ namespace DB
M(pause_before_register_non_root_mpp_task) \
M(pause_before_make_non_root_mpp_task_active)

#define APPLY_FOR_PAUSEABLE_FAILPOINTS(M) \
M(pause_when_reading_from_dt_stream) \
M(pause_when_writing_to_dt_store) \
M(pause_when_ingesting_to_dt_store) \
M(pause_when_altering_dt_store) \
M(pause_after_copr_streams_acquired) \
M(pause_query_init)
#define APPLY_FOR_PAUSEABLE_FAILPOINTS(M) \
M(pause_when_reading_from_dt_stream) \
M(pause_when_writing_to_dt_store) \
M(pause_when_ingesting_to_dt_store) \
M(pause_when_altering_dt_store) \
M(pause_after_copr_streams_acquired) \
M(pause_query_init) \
M(pause_proactive_flush_before_persist_region) \
M(pause_passive_flush_before_persist_region)

#define APPLY_FOR_RANDOM_FAILPOINTS(M) \
M(random_tunnel_wait_timeout_failpoint) \
Expand Down
52 changes: 40 additions & 12 deletions dbms/src/Common/TiFlashMetrics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,12 +125,6 @@ namespace DB
M(tiflash_schema_apply_duration_seconds, "Bucketed histogram of ddl apply duration", Histogram, \
F(type_sync_schema_apply_duration, {{"type", "sync_schema_duration"}}, ExpBuckets{0.001, 2, 20}), \
F(type_sync_table_schema_apply_duration, {{"type", "sync_table_schema_duration"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_raft_read_index_count, "Total number of raft read index", Counter) \
M(tiflash_stale_read_count, "Total number of stale read", Counter) \
M(tiflash_raft_read_index_duration_seconds, "Bucketed histogram of raft read index duration", Histogram, \
F(type_raft_read_index_duration, {{"type", "tmt_raft_read_index_duration"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_raft_wait_index_duration_seconds, "Bucketed histogram of raft wait index duration", Histogram, \
F(type_raft_wait_index_duration, {{"type", "tmt_raft_wait_index_duration"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_syncing_data_freshness, "The freshness of tiflash data with tikv data", Histogram, \
F(type_syncing_data_freshness, {{"type", "data_freshness"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_storage_read_tasks_count, "Total number of storage engine read tasks", Counter) \
Expand All @@ -148,7 +142,11 @@ namespace DB
F(type_seg_split_fg, {"type", "seg_split_fg"}), \
F(type_seg_split_ingest, {"type", "seg_split_ingest"}), \
F(type_seg_merge_bg_gc, {"type", "seg_merge_bg_gc"}), \
F(type_place_index_update, {"type", "place_index_update"})) \
F(type_place_index_update, {"type", "place_index_update"}), \
F(type_compact_log_segment_bg, {"type", "compact_log_segment_bg"}), \
F(type_compact_log_segment_fg, {"type", "compact_log_segment_fg"}), \
F(type_compact_log_region_bg, {"type", "compact_log_region_bg"}), \
F(type_compact_log_region_fg, {"type", "compact_log_region_fg"})) \
M(tiflash_storage_subtask_duration_seconds, "Bucketed histogram of storage's sub task duration", Histogram, \
F(type_delta_merge_bg, {{"type", "delta_merge_bg"}}, ExpBuckets{0.001, 2, 20}), \
F(type_delta_merge_bg_gc, {{"type", "delta_merge_bg_gc"}}, ExpBuckets{0.001, 2, 20}), \
Expand All @@ -160,7 +158,11 @@ namespace DB
F(type_seg_split_fg, {{"type", "seg_split_fg"}}, ExpBuckets{0.001, 2, 20}), \
F(type_seg_split_ingest, {{"type", "seg_split_ingest"}}, ExpBuckets{0.001, 2, 20}), \
F(type_seg_merge_bg_gc, {{"type", "seg_merge_bg_gc"}}, ExpBuckets{0.001, 2, 20}), \
F(type_place_index_update, {{"type", "place_index_update"}}, ExpBuckets{0.001, 2, 20})) \
F(type_place_index_update, {{"type", "place_index_update"}}, ExpBuckets{0.001, 2, 20}), \
F(type_compact_log_bg, {{"type", "compact_log_bg"}}, ExpBuckets{0.001, 2, 20}), \
F(type_compact_log_fg, {{"type", "compact_log_fg"}}, ExpBuckets{0.001, 2, 20}), \
F(type_compact_log_fg_dm, {{"type", "compact_log_fg_dm"}}, ExpBuckets{0.001, 2, 20}), \
F(type_compact_log_bg_dm, {{"type", "compact_log_bg_dm"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_storage_throughput_bytes, "Calculate the throughput of tasks of storage in bytes", Gauge, /**/ \
F(type_write, {"type", "write"}), /**/ \
F(type_ingest, {"type", "ingest"}), /**/ \
Expand Down Expand Up @@ -253,16 +255,42 @@ namespace DB
F(type_apply_snapshot_predecode_sst2dt, {{"type", "snapshot_predecode_sst2dt"}}, ExpBuckets{0.05, 2, 10}), \
F(type_apply_snapshot_predecode_upload, {{"type", "snapshot_predecode_upload"}}, ExpBuckets{0.05, 2, 10}), \
F(type_apply_snapshot_flush, {{"type", "snapshot_flush"}}, ExpBuckets{0.05, 2, 10})) \
M(tiflash_raft_process_keys, "Total number of keys processed in some types of Raft commands", Counter, \
F(type_apply_snapshot, {"type", "apply_snapshot"}), F(type_ingest_sst, {"type", "ingest_sst"})) \
M(tiflash_raft_apply_write_command_duration_seconds, "Bucketed histogram of applying write command Raft logs", Histogram, \
M(tiflash_raft_apply_write_command_duration_seconds, "Bucketed histogram of applying write command Raft logs", \
Histogram, /* like tiflash_raft_command_duration_seconds but are smaller tasks */ \
F(type_write, {{"type", "write"}}, ExpBuckets{0.0005, 2, 20}), \
F(type_admin, {{"type", "admin"}}, ExpBuckets{0.0005, 2, 20}), \
F(type_flush_region, {{"type", "flush_region"}}, ExpBuckets{0.0005, 2, 20})) \
M(tiflash_raft_process_keys, "Total number of keys processed in some types of Raft commands", Counter, \
F(type_apply_snapshot, {"type", "apply_snapshot"}), F(type_ingest_sst, {"type", "ingest_sst"})) \
M(tiflash_raft_upstream_latency, "The latency that tikv sends raft log to tiflash.", Histogram, \
F(type_write, {{"type", "write"}}, ExpBuckets{0.001, 2, 30})) \
M(tiflash_raft_write_data_to_storage_duration_seconds, "Bucketed histogram of writting region into storage layer", Histogram, \
F(type_decode, {{"type", "decode"}}, ExpBuckets{0.0005, 2, 20}), F(type_write, {{"type", "write"}}, ExpBuckets{0.0005, 2, 20})) \
F(type_decode, {{"type", "decode"}}, ExpBuckets{0.0005, 2, 20}), \
F(type_write, {{"type", "write"}}, ExpBuckets{0.0005, 2, 20})) \
M(tiflash_raft_raft_log_lag_count, "Bucketed histogram raft index lag", Histogram, \
F(type_compact_index, {{"type", "compact_index"}}, EqualWidthBuckets{0, 200, 5}), \
F(type_applied_index, {{"type", "applied_index"}}, EqualWidthBuckets{0, 200, 5})) \
M(tiflash_raft_raft_events_count, "Raft event counter", Counter, \
F(type_pre_exec_compact, {{"type", "pre_exec_compact"}}), \
F(type_flush_apply_snapshot, {{"type", "flush_apply_snapshot"}}), \
F(type_flush_ingest_sst, {{"type", "flush_ingest_sst"}}), \
F(type_flush_useless_admin, {{"type", "flush_useless_admin"}}), \
F(type_flush_useful_admin, {{"type", "flush_useful_admin"}}), \
F(type_flush_passive, {{"type", "flush_passive"}}), \
F(type_flush_proactive, {{"type", "flush_proactive"}}), \
F(type_flush_log_gap, {{"type", "flush_log_gap"}}), \
F(type_flush_size, {{"type", "flush_size"}}), \
F(type_flush_rowcount, {{"type", "flush_rowcount"}}), \
F(type_exec_compact, {{"type", "exec_compact"}})) \
M(tiflash_raft_region_flush_size, "Bucketed histogram of region flushed size", Histogram, \
F(type_flushed, {{"type", "flushed"}}, ExpBuckets{32, 2, 16}), \
F(type_unflushed, {{"type", "unflushed"}}, ExpBuckets{32, 2, 16})) \
M(tiflash_raft_read_index_count, "Total number of raft read index", Counter) \
M(tiflash_stale_read_count, "Total number of stale read", Counter) \
M(tiflash_raft_read_index_duration_seconds, "Bucketed histogram of raft read index duration", Histogram, \
F(type_raft_read_index_duration, {{"type", "tmt_raft_read_index_duration"}}, ExpBuckets{0.001, 2, 20})) \
M(tiflash_raft_wait_index_duration_seconds, "Bucketed histogram of raft wait index duration", Histogram, \
F(type_raft_wait_index_duration, {{"type", "tmt_raft_wait_index_duration"}}, ExpBuckets{0.001, 2, 20})) \
/* required by DBaaS */ \
M(tiflash_server_info, "Indicate the tiflash server info, and the value is the start timestamp (s).", Gauge, \
F(start_time, {"version", TiFlashBuildInfo::getReleaseVersion()}, {"hash", TiFlashBuildInfo::getGitHash()})) \
Expand Down