cdc: Exponentialize resolved ts scan backoff (#15119) (#16483)

close #15112

Make the scan task retry wait time exponential, so that less logs will be produced.

Signed-off-by: Yang Zhang <yang.zhang@pingcap.com>

Co-authored-by: Yang Zhang <yang.zhang@pingcap.com>
Co-authored-by: tonyxuqqi <tonyxuqi@outlook.com>
Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>

components/resolved_ts/src/scanner.rs

@@ -31,7 +31,7 @@ use crate::{
 
 const DEFAULT_SCAN_BATCH_SIZE: usize = 128;
 const GET_SNAPSHOT_RETRY_TIME: u32 = 3;
-const GET_SNAPSHOT_RETRY_BACKOFF_STEP: Duration = Duration::from_millis(25);
+const GET_SNAPSHOT_RETRY_BACKOFF_STEP: Duration = Duration::from_millis(100);
 
 pub struct ScanTask {
     pub handle: ObserveHandle,
@@ -178,7 +178,9 @@ impl<T: 'static + CdcHandle<E>, E: KvEngine> ScannerPool<T, E> {
             if retry_times != 0 {
                 let mut backoff = GLOBAL_TIMER_HANDLE
                     .delay(
-                        std::time::Instant::now() + retry_times * GET_SNAPSHOT_RETRY_BACKOFF_STEP,
+                        std::time::Instant::now()
+                            + GET_SNAPSHOT_RETRY_BACKOFF_STEP
+                                .mul_f64(10_f64.powi(retry_times as i32 - 1)),
                     )
                     .compat()
                     .fuse();