feat(sandbox): fast forward timestamp and epoch height

chain/chain/src/chain.rs

@@ -4193,8 +4193,11 @@ impl<'a> ChainUpdate<'a> {
         provenance: &Provenance,
         on_challenge: &mut dyn FnMut(ChallengeBody),
     ) -> Result<(), Error> {
-        // Refuse blocks from the too distant future.
-        if header.timestamp() > Clock::utc() + Duration::seconds(ACCEPTABLE_TIME_DIFFERENCE) {
+        // Refuse blocks from the too distant future. Only exception to this is while we're within
+        // sandbox, we're allowed to jump to the future via fast forwarding.
+        if cfg!(feature = "sandbox")
+            && header.timestamp() > Clock::utc() + Duration::seconds(ACCEPTABLE_TIME_DIFFERENCE)
+        {
             return Err(ErrorKind::InvalidBlockFutureTime(header.timestamp()).into());
         }
 

chain/client/src/client.rs

@@ -67,6 +67,10 @@ pub struct Client {
     #[cfg(feature = "test_features")]
     pub adv_produce_blocks_only_valid: bool,
 
+    /// Fast Forward accrued delta height used to calculate fast forwarded timestamps for each block.
+    #[cfg(feature = "sandbox")]
+    pub(crate) accrued_fastforward_delta: near_primitives::types::BlockHeightDelta,
+
     pub config: ClientConfig,
     pub sync_status: SyncStatus,
     pub chain: Chain,
@@ -172,6 +176,8 @@ impl Client {
             adv_produce_blocks: false,
             #[cfg(feature = "test_features")]
             adv_produce_blocks_only_valid: false,
+            #[cfg(feature = "sandbox")]
+            accrued_fastforward_delta: 0,
             config,
             sync_status,
             chain,
@@ -494,7 +500,8 @@ impl Client {
         let next_epoch_protocol_version =
             self.runtime_adapter.get_epoch_protocol_version(&next_epoch_id)?;
 
-        let block = Block::produce(
+        #[allow(unused_mut)] // unused if not in sandbox
+        let mut block = Block::produce(
             this_epoch_protocol_version,
             next_epoch_protocol_version,
             prev_header,
@@ -516,6 +523,24 @@ impl Client {
             block_merkle_root,
         );
 
+        // If sandbox is enabled, set the fast-forwarded block timestamp to the produced block.
+        // This is current time + total amount of time forwarded from fast-forwarding.
+        #[cfg(feature = "sandbox")]
+        {
+            let header = match block {
+                Block::BlockV1(ref mut block) => &mut block.header,
+                Block::BlockV2(ref mut block) => &mut block.header,
+            };
+            let mut inner_lite = match header {
+                BlockHeader::BlockHeaderV1(ref mut header) => &mut header.inner_lite,
+                BlockHeader::BlockHeaderV2(ref mut header) => &mut header.inner_lite,
+                BlockHeader::BlockHeaderV3(ref mut header) => &mut header.inner_lite,
+            };
+
+            // Set block timestamp to the actual fast forwarded timestamp:
+            inner_lite.timestamp = to_timestamp(Clock::utc()) + self.sandbox_delta_time();
+        }
+
         // Update latest known even before returning block out, to prevent race conditions.
         self.chain.mut_store().save_latest_known(LatestKnown {
             height: next_height,
@@ -960,6 +985,18 @@ impl Client {
         Ok(())
     }
 
+    /// Gets the advanced timestamp delta in nanoseconds for sandbox once it has been fast-forwarded
+    #[cfg(feature = "sandbox")]
+    pub fn sandbox_delta_time(&self) -> u64 {
+        let avg_block_prod_time = (self.config.min_block_production_delay.as_nanos()
+            + self.config.max_block_production_delay.as_nanos())
+            / 2;
+        (self.accrued_fastforward_delta as u128 * avg_block_prod_time).try_into().expect(&format!(
+            "Too high of a delta_height {} to convert into u64",
+            self.accrued_fastforward_delta
+        ))
+    }
+
     pub fn send_approval(
         &mut self,
         parent_hash: &CryptoHash,

chain/client/src/client_actor.rs

@@ -777,9 +777,11 @@ impl ClientActor {
 
         #[cfg(feature = "sandbox")]
         let latest_known = if let Some(delta_height) = self.fastforward_delta.take() {
+            self.client.accrued_fastforward_delta += delta_height;
+            let timestamp_delta_ns = self.client.sandbox_delta_time();
             let new_latest_known = near_chain::types::LatestKnown {
                 height: latest_known.height + delta_height,
-                seen: near_primitives::utils::to_timestamp(Clock::utc()),
+                seen: near_primitives::utils::to_timestamp(Clock::utc()) + timestamp_delta_ns,
             };
 
             self.client.chain.mut_store().save_latest_known(new_latest_known.clone())?;

chain/client/src/test_utils.rs

@@ -66,6 +66,11 @@ use near_primitives::utils::MaybeValidated;
 
 pub type PeerManagerMock = Mocker<PeerManagerActor>;
 
+/// min block production time in milliseconds
+pub const MIN_BLOCK_PROD_TIME: u64 = 100;
+/// max block production time in milliseconds
+pub const MAX_BLOCK_PROD_TIME: u64 = 200;
+
 const TEST_SEED: RngSeed = [3; 32];
 /// Sets up ClientActor and ViewClientActor viewing the same store/runtime.
 pub fn setup(
@@ -284,8 +289,8 @@ pub fn setup_mock_with_validity_period_and_no_epoch_sync(
             5,
             account_id,
             skip_sync_wait,
-            100,
-            200,
+            MIN_BLOCK_PROD_TIME,
+            MAX_BLOCK_PROD_TIME,
             enable_doomslug,
             false,
             false,

chain/epoch_manager/Cargo.toml

@@ -28,6 +28,7 @@ near-store = { path = "../../core/store" }
 near-chain-configs = { path = "../../core/chain-configs" }
 
 [features]
+sandbox = []
 expensive_tests = []
 protocol_feature_chunk_only_producers = ["near-primitives/protocol_feature_chunk_only_producers", "near-chain-configs/protocol_feature_chunk_only_producers", "near-chain/protocol_feature_chunk_only_producers"]
 protocol_feature_fix_staking_threshold = ["near-primitives/protocol_feature_fix_staking_threshold"]

chain/epoch_manager/src/lib.rs

@@ -382,7 +382,8 @@ impl EpochManager {
         let validator_stake =
             epoch_info.validators_iter().map(|r| r.account_and_stake()).collect::<HashMap<_, _>>();
         let next_epoch_id = self.get_next_epoch_id_from_info(block_info)?;
-        let next_epoch_info = self.get_epoch_info(&next_epoch_id)?.clone();
+        #[allow(unused_mut)] // unused mut when not in sandbox
+        let mut next_epoch_info = self.get_epoch_info(&next_epoch_id)?.clone();
         self.save_epoch_validator_info(store_update, block_info.epoch_id(), &epoch_summary)?;
 
         let EpochSummary {
@@ -410,6 +411,12 @@ impl EpochManager {
             )
         };
         let next_next_epoch_config = self.config.for_protocol_version(next_version);
+        #[cfg(feature = "sandbox")]
+        {
+            *next_epoch_info.epoch_height_mut() =
+                block_info.height() / next_next_epoch_config.epoch_length;
+        }
+
         let next_next_epoch_info = match proposals_to_epoch_info(
             next_next_epoch_config,
             rng_seed,
@@ -445,6 +452,12 @@ impl EpochManager {
         // This epoch info is computed for the epoch after next (T+2),
         // where epoch_id of it is the hash of last block in this epoch (T).
         self.save_epoch_info(store_update, &next_next_epoch_id, next_next_epoch_info)?;
+
+        // Store next_epoch info again since sandbox changes it:
+        if cfg!(feature = "sandbox") {
+            self.save_epoch_info(store_update, &next_epoch_id, next_epoch_info)?;
+        }
+
         Ok(())
     }
 

integration-tests/src/tests/client/sandbox.rs

@@ -1,13 +1,13 @@
 use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::sync::Arc;
 
 use actix::System;
 
 use near_actix_test_utils::run_actix;
 use near_chain::{ChainGenesis, Provenance, RuntimeAdapter};
 use near_chain_configs::Genesis;
-use near_client::test_utils::{setup_mock, TestEnv};
+use near_client::test_utils::{setup_mock, TestEnv, MAX_BLOCK_PROD_TIME, MIN_BLOCK_PROD_TIME};
 use near_crypto::{InMemorySigner, KeyType};
 use near_logger_utils::init_test_logger;
 use near_network::types::{
@@ -119,36 +119,68 @@ fn test_patch_account() {
 
 #[test]
 fn test_fast_forward() {
+    const BLOCKS_TO_FASTFORWARD: BlockHeight = 10_000;
+    const BLOCKS_TO_PRODUCE: u64 = 20;
+
+    // the 1_000_000 is to convert milliseconds to nanoseconds, where MIN/MAX_BLOCK_PROD_TIME are in milliseconds:
+    const FAST_FORWARD_DELTA: u64 = (BLOCKS_TO_FASTFORWARD + BLOCKS_TO_PRODUCE) * 1_000_000;
+
     init_test_logger();
     run_actix(async {
         let count = Arc::new(AtomicUsize::new(0));
+        let first_block_timestamp = Arc::new(AtomicU64::new(0));
+
         // Produce 20 blocks
-        let (client, _view_client) = setup_mock(
+        let (_client, _view_client) = setup_mock(
             vec!["test".parse().unwrap()],
             "test".parse().unwrap(),
             true,
             false,
-            Box::new(move |msg, _ctx, _| {
+            Box::new(move |msg, _ctx, client| {
                 if let NetworkRequests::Block { block } = msg.as_network_requests_ref() {
                     let height = block.header().height();
+                    let timestamp = block.header().raw_timestamp();
+
                     count.fetch_add(1, Ordering::Relaxed);
-                    if count.load(Ordering::Relaxed) >= 20 {
+                    let at = count.load(Ordering::Relaxed);
+
+                    // Fast forward by 10,000 blocks after the first block produced:
+                    if at == 1 {
+                        first_block_timestamp.store(timestamp, Ordering::Relaxed);
+                        client.do_send(NetworkClientMessages::Sandbox(
+                            NetworkSandboxMessage::SandboxFastForward(BLOCKS_TO_FASTFORWARD),
+                        ));
+                    }
+
+                    // Check at final block if timestamp and block height matches up:
+                    if at >= BLOCKS_TO_PRODUCE as usize {
+                        let before_forward = first_block_timestamp.load(Ordering::Relaxed);
+                        let min_forwarded_time =
+                            before_forward + FAST_FORWARD_DELTA * MIN_BLOCK_PROD_TIME;
+                        let max_forwarded_time =
+                            before_forward + FAST_FORWARD_DELTA * MAX_BLOCK_PROD_TIME;
                         assert!(
-                            height >= 10000,
+                            height >= BLOCKS_TO_FASTFORWARD,
                             "Was not able to fast forward. Current height: {}",
                             height
                         );
+
+                        // Check if final block timestamp is within fast forwarded time min and max:
+                        assert!(
+                            (min_forwarded_time..max_forwarded_time).contains(&timestamp),
+                            "Forwarded timestamp {} is not within expected range ({},{})",
+                            timestamp,
+                            min_forwarded_time,
+                            max_forwarded_time
+                        );
+
                         System::current().stop();
                     }
                 }
                 PeerManagerMessageResponse::NetworkResponses(NetworkResponses::NoResponse)
             }),
         );
 
-        // Fast forward by 10,000 blocks:
-        client.do_send(NetworkClientMessages::Sandbox(NetworkSandboxMessage::SandboxFastForward(
-            10000,
-        )));
         near_network::test_utils::wait_or_panic(5000);
     });
 }

nearcore/Cargo.toml

@@ -89,4 +89,4 @@ force_wasmer2 = ["near-vm-runner/force_wasmer2"]
 force_wasmer0 = ["near-vm-runner/force_wasmer0"]
 force_wasmtime = ["near-vm-runner/force_wasmtime"]
 
-sandbox = ["near-client/sandbox", "node-runtime/sandbox", "near-jsonrpc/sandbox"]
+sandbox = ["near-client/sandbox", "node-runtime/sandbox", "near-jsonrpc/sandbox", "near-epoch-manager/sandbox"]

pytest/tests/sandbox/fast_forward.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 # test fast fowarding by a specific block height within a sandbox node. This will
-# fail if the block height is not past the forwarded height.
+# fail if the block height is not past the forwarded height. Also we will test
+# for the timestamps and epoch height being adjusted correctly after the block
+# height is changed.
 
+import datetime
 import sys, time
 import pathlib
 
@@ -11,17 +14,53 @@
 from utils import figure_out_sandbox_binary
 
 # startup a RPC node
-BLOCKS_TO_FASTFORARD = 10000
+MIN_BLOCK_PROD_TIME = 1  # seconds
+MAX_BLOCK_PROD_TIME = 2  # seconds
+EPOCH_LENGTH = 10
+BLOCKS_TO_FASTFORWARD = 10000
 CONFIG = figure_out_sandbox_binary()
-nodes = start_cluster(1, 0, 1, CONFIG, [["epoch_length", 10]], {})
+CONFIG.update({
+    "consensus": {
+        "min_block_production_delay": {
+            "secs": MIN_BLOCK_PROD_TIME,
+            "nanos": 0,
+        },
+        "max_block_production_delay": {
+            "secs": MAX_BLOCK_PROD_TIME,
+            "nanos": 0,
+        },
+    }
+})
+
+nodes = start_cluster(1, 0, 1, CONFIG, [["epoch_length", EPOCH_LENGTH]], {})
 
 # request to fast forward
 nodes[0].json_rpc('sandbox_fast_forward', {
-    "delta_height": BLOCKS_TO_FASTFORARD,
+    "delta_height": BLOCKS_TO_FASTFORWARD,
 })
 
 # wait a little for it to fast forward
 time.sleep(3)
 
 # Assert at the end that the node is past the amounts of blocks we specified
-assert nodes[0].get_latest_block().height > BLOCKS_TO_FASTFORARD
+assert nodes[0].get_latest_block().height > BLOCKS_TO_FASTFORWARD
+
+# Assert that we're within the bounds of fast forward timestamp between range of min and max:
+sync_info = nodes[0].get_status()['sync_info']
+earliest = datetime.datetime.strptime(sync_info['earliest_block_time'][:-4],
+                                      '%Y-%m-%dT%H:%M:%S.%f')
+latest = datetime.datetime.strptime(sync_info['latest_block_time'][:-4],
+                                    '%Y-%m-%dT%H:%M:%S.%f')
+
+min_forwarded_secs = datetime.timedelta(
+    0, BLOCKS_TO_FASTFORWARD * MIN_BLOCK_PROD_TIME)
+max_forwarded_secs = datetime.timedelta(
+    0, BLOCKS_TO_FASTFORWARD * MAX_BLOCK_PROD_TIME)
+min_forwarded_time = earliest + min_forwarded_secs
+max_forwarded_time = earliest + max_forwarded_secs
+
+assert min_forwarded_time < latest < max_forwarded_time
+
+# Check to see that the epoch height has been updated correctly:
+epoch_height = nodes[0].get_validators()['result']['epoch_height']
+assert epoch_height >= BLOCKS_TO_FASTFORWARD / EPOCH_LENGTH

test-utils/state-viewer/Cargo.toml

@@ -36,6 +36,6 @@ near-client = { path = "../../chain/client" }
 testlib = { path = "../../test-utils/testlib" }
 
 [features]
-sandbox = ["node-runtime/sandbox", "near-chain/sandbox", "near-network/sandbox", "near-client/sandbox"]
+sandbox = ["node-runtime/sandbox", "near-chain/sandbox", "near-network/sandbox", "near-client/sandbox", "near-epoch-manager/sandbox"]
 nightly_protocol_features = ["nearcore/nightly_protocol_features"]
 nightly_protocol = ["nearcore/nightly_protocol"]