feat(sandbox): fast forward timestamp and epoch height

chain/chain/src/chain.rs

@@ -4334,8 +4334,11 @@ impl<'a> ChainUpdate<'a> {
         provenance: &Provenance,
         on_challenge: &mut dyn FnMut(ChallengeBody),
     ) -> Result<(), Error> {
-        // Refuse blocks from the too distant future.
-        if header.timestamp() > Clock::utc() + Duration::seconds(ACCEPTABLE_TIME_DIFFERENCE) {
+        // Refuse blocks from the too distant future. Only exception to this is while we're within
+        // sandbox, we're allowed to jump to the future via fast forwarding.
+        if !cfg!(feature = "sandbox")
+            && header.timestamp() > Clock::utc() + Duration::seconds(ACCEPTABLE_TIME_DIFFERENCE)
+        {
             return Err(ErrorKind::InvalidBlockFutureTime(header.timestamp()).into());
         }
 

chain/client/src/client.rs

@@ -67,6 +67,10 @@ pub struct Client {
     #[cfg(feature = "test_features")]
     pub adv_produce_blocks_only_valid: bool,
 
+    /// Fast Forward accrued delta height used to calculate fast forwarded timestamps for each block.
+    #[cfg(feature = "sandbox")]
+    pub(crate) accrued_fastforward_delta: near_primitives::types::BlockHeightDelta,
+
     pub config: ClientConfig,
     pub sync_status: SyncStatus,
     pub chain: Chain,
@@ -172,6 +176,8 @@ impl Client {
             adv_produce_blocks: false,
             #[cfg(feature = "test_features")]
             adv_produce_blocks_only_valid: false,
+            #[cfg(feature = "sandbox")]
+            accrued_fastforward_delta: 0,
             config,
             sync_status,
             chain,
@@ -494,7 +500,8 @@ impl Client {
         let next_epoch_protocol_version =
             self.runtime_adapter.get_epoch_protocol_version(&next_epoch_id)?;
 
-        let block = Block::produce(
+        #[allow(unused_mut)] // unused if not in sandbox
+        let mut block = Block::produce(
             this_epoch_protocol_version,
             next_epoch_protocol_version,
             prev_header,
@@ -516,6 +523,24 @@ impl Client {
             block_merkle_root,
         );
 
+        // If sandbox is enabled, set the fast-forwarded block timestamp to the produced block.
+        // This is current time + total amount of time forwarded from fast-forwarding.
+        #[cfg(feature = "sandbox")]
+        {
+            let header = match block {
+                Block::BlockV1(ref mut block) => &mut block.header,
+                Block::BlockV2(ref mut block) => &mut block.header,
+            };
+            let mut inner_lite = match header {
+                BlockHeader::BlockHeaderV1(ref mut header) => &mut header.inner_lite,
+                BlockHeader::BlockHeaderV2(ref mut header) => &mut header.inner_lite,
+                BlockHeader::BlockHeaderV3(ref mut header) => &mut header.inner_lite,
+            };
+
+            // Set block timestamp to the actual fast forwarded timestamp:
+            inner_lite.timestamp = to_timestamp(Clock::utc()) + self.sandbox_delta_time();
+        }
+
         // Update latest known even before returning block out, to prevent race conditions.
         self.chain.mut_store().save_latest_known(LatestKnown {
             height: next_height,
@@ -960,6 +985,18 @@ impl Client {
         Ok(())
     }
 
+    /// Gets the advanced timestamp delta in nanoseconds for sandbox once it has been fast-forwarded
+    #[cfg(feature = "sandbox")]
+    pub fn sandbox_delta_time(&self) -> u64 {
+        let avg_block_prod_time = (self.config.min_block_production_delay.as_nanos()
+            + self.config.max_block_production_delay.as_nanos())
+            / 2;
+        (self.accrued_fastforward_delta as u128 * avg_block_prod_time).try_into().expect(&format!(
+            "Too high of a delta_height {} to convert into u64",
+            self.accrued_fastforward_delta
+        ))
+    }
+
     pub fn send_approval(
         &mut self,
         parent_hash: &CryptoHash,

chain/client/src/client_actor.rs

@@ -775,9 +775,11 @@ impl ClientActor {
 
         #[cfg(feature = "sandbox")]
         let latest_known = if let Some(delta_height) = self.fastforward_delta.take() {
+            self.client.accrued_fastforward_delta += delta_height;
+            let timestamp_delta_ns = self.client.sandbox_delta_time();
             let new_latest_known = near_chain::types::LatestKnown {
                 height: latest_known.height + delta_height,
-                seen: near_primitives::utils::to_timestamp(Clock::utc()),
+                seen: near_primitives::utils::to_timestamp(Clock::utc()) + timestamp_delta_ns,
             };
 
             self.client.chain.mut_store().save_latest_known(new_latest_known.clone())?;

chain/client/src/test_utils.rs

@@ -66,6 +66,11 @@ use near_primitives::utils::MaybeValidated;
 
 pub type PeerManagerMock = Mocker<PeerManagerActor>;
 
+/// min block production time in milliseconds
+pub const MIN_BLOCK_PROD_TIME: u64 = 100;
+/// max block production time in milliseconds
+pub const MAX_BLOCK_PROD_TIME: u64 = 200;
+
 const TEST_SEED: RngSeed = [3; 32];
 /// Sets up ClientActor and ViewClientActor viewing the same store/runtime.
 pub fn setup(
@@ -284,8 +289,8 @@ pub fn setup_mock_with_validity_period_and_no_epoch_sync(
             5,
             account_id,
             skip_sync_wait,
-            100,
-            200,
+            MIN_BLOCK_PROD_TIME,
+            MAX_BLOCK_PROD_TIME,
             enable_doomslug,
             false,
             false,

chain/epoch_manager/Cargo.toml

@@ -28,6 +28,7 @@ near-store = { path = "../../core/store" }
 near-chain-configs = { path = "../../core/chain-configs" }
 
 [features]
+sandbox = []
 expensive_tests = []
 protocol_feature_chunk_only_producers = [
   "near-primitives/protocol_feature_chunk_only_producers",

chain/epoch_manager/src/lib.rs

@@ -382,7 +382,8 @@ impl EpochManager {
         let validator_stake =
             epoch_info.validators_iter().map(|r| r.account_and_stake()).collect::<HashMap<_, _>>();
         let next_epoch_id = self.get_next_epoch_id_from_info(block_info)?;
-        let next_epoch_info = self.get_epoch_info(&next_epoch_id)?.clone();
+        #[allow(unused_mut)] // unused mut when not in sandbox
+        let mut next_epoch_info = self.get_epoch_info(&next_epoch_id)?.clone();
         self.save_epoch_validator_info(store_update, block_info.epoch_id(), &epoch_summary)?;
 
         let EpochSummary {
@@ -410,6 +411,19 @@ impl EpochManager {
             )
         };
         let next_next_epoch_config = self.config.for_protocol_version(next_version);
+        #[cfg(feature = "sandbox")]
+        {
+            // explaination on calculation:
+            // (block_height / length) gives us epoch_height, but we index from 1, so +1 at the end.
+            // Note, this is a retroactive update for sandbox since we can receive fast-forward
+            // requests in the middle of an epoch. block_height is different based on when it gets
+            // called. When fast-forwarded, we need to retroactively adjust the epoch height, but
+            // with a normal `finalize_epoch`, it's the start of the epoch's next block height, so
+            // (height - 1) is needed after adding +1 at the end.
+            *next_epoch_info.epoch_height_mut() =
+                ((block_info.height() - 1) / next_next_epoch_config.epoch_length) + 1;
+        }
+
         let next_next_epoch_info = match proposals_to_epoch_info(
             next_next_epoch_config,
             rng_seed,
@@ -445,6 +459,13 @@ impl EpochManager {
         // This epoch info is computed for the epoch after next (T+2),
         // where epoch_id of it is the hash of last block in this epoch (T).
         self.save_epoch_info(store_update, &next_next_epoch_id, next_next_epoch_info)?;
+
+        // Store next_epoch info again since sandbox changes it. Need to retroactively update
+        // since a fast-forward request can happen in the middle of an epoch.
+        if cfg!(feature = "sandbox") {
+            self.save_epoch_info(store_update, &next_epoch_id, next_epoch_info)?;
+        }
+
         Ok(())
     }
 
@@ -4006,4 +4027,55 @@ mod tests2 {
             epoch_manager.epoch_validators_ordered_unique.get(&epoch_id).unwrap().clone();
         assert_eq!(epoch_validators_unique, epoch_validators_unique_in_cache);
     }
+
+    #[test]
+    #[cfg(feature = "sandbox")]
+    fn test_sandbox_epoch_height_change() {
+        use near_primitives::types::BlockHeightDelta;
+        const EPOCH_LENGTH: BlockHeightDelta = 5;
+        const BLOCKS_TO_FAST_FORWARD: BlockHeightDelta = 1000;
+
+        // setup epoch manager
+        let amount_staked = 1_000_000;
+        let validators = vec![
+            ("test1".parse().unwrap(), amount_staked),
+            ("test2".parse().unwrap(), amount_staked),
+        ];
+        let mut epoch_manager =
+            setup_default_epoch_manager(validators, EPOCH_LENGTH, 1, 10, 0, 90, 60);
+        let h = hash_range(11);
+        record_block(&mut epoch_manager, CryptoHash::default(), h[0], 0, vec![]);
+
+        // Initially record 7 blocks without any fast-forwarding:
+        for i in 1..8 {
+            record_block(&mut epoch_manager, h[i - 1], h[i], i as u64, vec![]);
+        }
+
+        // First two epoch heights should be 1 and 2
+        let epoch_info = epoch_manager.get_epoch_info(&EpochId(h[0])).unwrap().clone();
+        assert_eq!(epoch_info.epoch_height(), 1);
+        let epoch_info = epoch_manager.get_epoch_info(&EpochId(h[5])).unwrap().clone();
+        assert_eq!(epoch_info.epoch_height(), 2);
+
+        // Then record some fast-forwarded blocks, starting in the middle of an epoch:
+        for i in 8..11 {
+            record_block(
+                &mut epoch_manager,
+                h[i - 1],
+                h[i],
+                i as u64 + BLOCKS_TO_FAST_FORWARD,
+                vec![],
+            );
+        }
+
+        // Fast-forward in the middle of an epoch should reflect in that the epoch height got updated:
+        let epoch_info = epoch_manager.get_epoch_info(&EpochId(h[5])).unwrap().clone();
+        let expected_epoch_height = (5 + BLOCKS_TO_FAST_FORWARD) / EPOCH_LENGTH + 1;
+        assert_eq!(epoch_info.epoch_height(), expected_epoch_height);
+        // Subsequent one should just be additonal prev epoch height + 1
+        // NOTE: that the requests happens on the 8th recorded block because that's when the sandbox
+        // request to fast-forward would have been sent.
+        let epoch_info = epoch_manager.get_epoch_info(&EpochId(h[8])).unwrap().clone();
+        assert_eq!(epoch_info.epoch_height(), expected_epoch_height + 1);
+    }
 }

integration-tests/src/tests/client/sandbox.rs

@@ -1,13 +1,13 @@
 use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::sync::Arc;
 
 use actix::System;
 
 use near_actix_test_utils::run_actix;
 use near_chain::{ChainGenesis, Provenance, RuntimeAdapter};
 use near_chain_configs::Genesis;
-use near_client::test_utils::{setup_mock, TestEnv};
+use near_client::test_utils::{setup_mock, TestEnv, MAX_BLOCK_PROD_TIME, MIN_BLOCK_PROD_TIME};
 use near_crypto::{InMemorySigner, KeyType};
 use near_logger_utils::init_test_logger;
 use near_network::types::{
@@ -119,36 +119,68 @@ fn test_patch_account() {
 
 #[test]
 fn test_fast_forward() {
+    const BLOCKS_TO_FASTFORWARD: BlockHeight = 10_000;
+    const BLOCKS_TO_PRODUCE: u64 = 20;
+
+    // the 1_000_000 is to convert milliseconds to nanoseconds, where MIN/MAX_BLOCK_PROD_TIME are in milliseconds:
+    const FAST_FORWARD_DELTA: u64 = (BLOCKS_TO_FASTFORWARD + BLOCKS_TO_PRODUCE) * 1_000_000;
+
     init_test_logger();
     run_actix(async {
         let count = Arc::new(AtomicUsize::new(0));
+        let first_block_timestamp = Arc::new(AtomicU64::new(0));
+
         // Produce 20 blocks
-        let (client, _view_client) = setup_mock(
+        let (_client, _view_client) = setup_mock(
             vec!["test".parse().unwrap()],
             "test".parse().unwrap(),
             true,
             false,
-            Box::new(move |msg, _ctx, _| {
+            Box::new(move |msg, _ctx, client| {
                 if let NetworkRequests::Block { block } = msg.as_network_requests_ref() {
                     let height = block.header().height();
+                    let timestamp = block.header().raw_timestamp();
+
                     count.fetch_add(1, Ordering::Relaxed);
-                    if count.load(Ordering::Relaxed) >= 20 {
+                    let at = count.load(Ordering::Relaxed);
+
+                    // Fast forward by 10,000 blocks after the first block produced:
+                    if at == 1 {
+                        first_block_timestamp.store(timestamp, Ordering::Relaxed);
+                        client.do_send(NetworkClientMessages::Sandbox(
+                            NetworkSandboxMessage::SandboxFastForward(BLOCKS_TO_FASTFORWARD),
+                        ));
+                    }
+
+                    // Check at final block if timestamp and block height matches up:
+                    if at >= BLOCKS_TO_PRODUCE as usize {
+                        let before_forward = first_block_timestamp.load(Ordering::Relaxed);
+                        let min_forwarded_time =
+                            before_forward + FAST_FORWARD_DELTA * MIN_BLOCK_PROD_TIME;
+                        let max_forwarded_time =
+                            before_forward + FAST_FORWARD_DELTA * MAX_BLOCK_PROD_TIME;
                         assert!(
-                            height >= 10000,
+                            height >= BLOCKS_TO_FASTFORWARD,
                             "Was not able to fast forward. Current height: {}",
                             height
                         );
+
+                        // Check if final block timestamp is within fast forwarded time min and max:
+                        assert!(
+                            (min_forwarded_time..max_forwarded_time).contains(&timestamp),
+                            "Forwarded timestamp {} is not within expected range ({},{})",
+                            timestamp,
+                            min_forwarded_time,
+                            max_forwarded_time
+                        );
+
                         System::current().stop();
                     }
                 }
                 PeerManagerMessageResponse::NetworkResponses(NetworkResponses::NoResponse)
             }),
         );
 
-        // Fast forward by 10,000 blocks:
-        client.do_send(NetworkClientMessages::Sandbox(NetworkSandboxMessage::SandboxFastForward(
-            10000,
-        )));
         near_network::test_utils::wait_or_panic(5000);
     });
 }

nearcore/Cargo.toml

@@ -147,4 +147,5 @@ sandbox = [
   "near-client/sandbox",
   "node-runtime/sandbox",
   "near-jsonrpc/sandbox",
+  "near-epoch-manager/sandbox",
 ]