feat: adjust block sync timeouts (#6342)

Description
---
Adjusted block sync timeouts to enable block sync over tor when we have
many full blocks. Previous values were unrealistic and unfeasible. This
PR provides feasible default values for full blocks sync over tor for a
fresh base node.

Motivation and Context
---
A recent stress test on `esemeralda` provided a blockchain with many
full blocks. Performing a fresh base node block sync over tor in a
controlled environment proved impossible due to inherent network
latencies. Although there may be a fix to improve the tor latencies with
full blocks sync, it is not yet available.

```rust
2024-05-14 17:16:00.091735600 [c::bn::block_sync] TRACE [block sync timings] 1.1 #1501 Latency in 134.48s
2024-05-14 17:18:56.454123100 [c::bn::block_sync] TRACE [block sync timings] 1.1 #1502 Latency in 173.69s
2024-05-14 17:21:15.535060700 [c::bn::block_sync] TRACE [block sync timings] 1.1 #1503 Latency in 136.89s
2024-05-14 17:24:15.852873000 [c::bn::block_sync] TRACE [block sync timings] 1.1 #1504 Latency in 178.57s
2024-05-14 17:26:53.273228300 [c::bn::block_sync] TRACE [block sync timings] 1.1 #1505 Latency in 155.62s
```

How Has This Been Tested?
---
Many system-level tests.

What process can a PR reviewer use to test or verify this change?
---
Code review and system-level tests.

<!-- Checklist -->
<!-- 1. Is the title of your PR in the form that would make nice release
notes? The title, excluding the conventional commit
tag, will be included exactly as is in the CHANGELOG, so please think
about it carefully. -->


Breaking Changes
---

- [x] None
- [ ] Requires data directory on base node to be deleted
- [ ] Requires hard fork
- [ ] Other - Please specify

<!-- Does this include a breaking change? If so, include this line as a
footer -->
<!-- BREAKING CHANGE: Description what the user should do, e.g. delete a
database, resync the chain -->

base_layer/core/src/base_node/sync/config.rs

@@ -50,19 +50,20 @@ pub struct BlockchainSyncConfig {
     pub validation_concurrency: usize,
     /// The RPC deadline to set on sync clients. If this deadline is reached, a new sync peer will be selected for
     /// sync.
+    #[serde(with = "serializers::seconds")]
     pub rpc_deadline: Duration,
 }
 
 impl Default for BlockchainSyncConfig {
     fn default() -> Self {
         Self {
-            initial_max_sync_latency: Duration::from_secs(15),
-            max_latency_increase: Duration::from_secs(2),
-            ban_period: Duration::from_secs(60 * 60 * 2), // 2 hours
-            short_ban_period: Duration::from_secs(240),   // 4 mins
+            initial_max_sync_latency: Duration::from_secs(240), // Syncing many full blocks over tor require this
+            max_latency_increase: Duration::from_secs(10),      // Syncing many full blocks over tor require this
+            ban_period: Duration::from_secs(60 * 60 * 2),       // 2 hours
+            short_ban_period: Duration::from_secs(240),         // 4 mins
             forced_sync_peers: Default::default(),
             validation_concurrency: 6,
-            rpc_deadline: Duration::from_secs(15),
+            rpc_deadline: Duration::from_secs(240), // Syncing many full blocks over tor require this
         }
     }
 }

base_layer/core/src/base_node/sync/horizon_state_sync/synchronizer.rs

@@ -251,7 +251,7 @@ impl<'a, B: BlockchainBackend + 'static> HorizonStateSynchronization<'a, B> {
 
         let config = RpcClient::builder()
             .with_deadline(self.config.rpc_deadline)
-            .with_deadline_grace_period(Duration::from_secs(3));
+            .with_deadline_grace_period(Duration::from_secs(5));
 
         let mut client = conn
             .connect_rpc_using_builder::<rpc::BaseNodeSyncRpcClient>(config)

common/config/presets/c_base_node_c.toml

@@ -83,20 +83,25 @@ track_reorgs = true
 #service.block_sync_trigger = 5
 
 [base_node.state_machine]
-# The initial max sync latency. If a peer fails to stream a header/block within this deadline another sync peer will be
-# selected. If there are no further peers the sync will be restarted with an increased by `max_latency_increase`.
-#blockchain_sync_config.initial_max_sync_latency = 15
-# If all sync peers exceed latency increase allowed latency by this value
-#blockchain_sync_config.max_latency_increase =2
-# Longer ban period for potentially malicious infractions (protocol violations etc.)
+# The initial max sync latency (seconds). If a peer fails to stream a header/block within this deadline another sync
+# peer will be selected. If there are no further peers the sync will be restarted with an increased by
+# `max_latency_increase`. [default = 240]
+blockchain_sync_config.initial_max_sync_latency = 240
+# If all sync peers exceed latency increase allowed latency by this value (seconds) [default = 10]
+blockchain_sync_config.max_latency_increase = 10
+# Longer ban period (seconds) for potentially malicious infractions (protocol violations etc.) [default = 2 hours]
 #blockchain_sync_config.ban_period = 7_200 # 2 * 60 * 60
-# Short ban period for infractions that are likely not malicious (slow to respond spotty connections etc)
+# Short ban period (seconds) for infractions that are likely not malicious (slow to respond spotty connections etc)
+# [default = 4 minutes]
 #blockchain_sync_config.short_ban_period = 240
 # An allowlist of sync peers from which to sync. No other peers will be selected for sync. If empty sync peers
-# are chosen based on their advertised chain metadata.
+# are chosen based on their advertised chain metadata. [default = []]
 #blockchain_sync_config.forced_sync_peers = []
-# Number of threads to use for validation
+# Number of threads to use for validation [default = 6]
 #blockchain_sync_config.validation_concurrency = 6
+# The RPC deadline to set on sync clients. If this deadline is reached, a new sync peer will be selected for sync.
+# [default = 240]
+blockchain_sync_config.rpc_deadline = 240
 
 # The maximum amount of VMs that RandomX will be use (default = 0)
 #max_randomx_vms = 0