fix: Third wave of fixing nightly tests (#2207)

Many timeout tweaks, and small typos.
`cross_shard_tx_with_validator_rotation` with 150 block time has
extremely long forks (doomslug is disabled), and takes lots of time per
iteration. I split it into two: one with 150ms block time, but only 16
iterations, and one with 400ms block time, but all 64 iterations. Both
locally take ~45 minutes, will see how long it takes on gcloud.

chain/client/tests/catching_up.rs

@@ -667,7 +667,7 @@ mod tests {
             );
             *connectors.write().unwrap() = conn;
 
-            near_network::test_utils::wait_or_panic(30000);
+            near_network::test_utils::wait_or_panic(60000);
         })
         .unwrap();
     }
@@ -705,7 +705,7 @@ mod tests {
                 key_pairs.clone(),
                 validator_groups,
                 true,
-                400,
+                600,
                 false,
                 false,
                 5,
@@ -742,7 +742,7 @@ mod tests {
             );
             *connectors.write().unwrap() = conn;
 
-            near_network::test_utils::wait_or_panic(30000);
+            near_network::test_utils::wait_or_panic(60000);
         })
         .unwrap();
     }
@@ -1021,7 +1021,7 @@ mod tests {
                 })),
             );
             *connectors.write().unwrap() = conn;
-            let max_wait_ms = block_prod_time * last_height / 10 * 13 + 10000;
+            let max_wait_ms = block_prod_time * last_height / 10 * 18 + 20000;
 
             near_network::test_utils::wait_or_panic(max_wait_ms);
         })

chain/client/tests/cross_shard_tx.rs

@@ -368,6 +368,7 @@ mod tests {
         rotate_validators: bool,
         drop_chunks: bool,
         test_doomslug: bool,
+        block_production_time: u64,
     ) {
         if !cfg!(feature = "expensive_tests") {
             return;
@@ -416,7 +417,7 @@ mod tests {
                 key_pairs.clone(),
                 validator_groups,
                 true,
-                if drop_chunks || rotate_validators { 150 } else { 100 },
+                block_production_time,
                 drop_chunks,
                 !test_doomslug,
                 20,
@@ -489,31 +490,36 @@ mod tests {
 
     #[test]
     fn test_cross_shard_tx() {
-        test_cross_shard_tx_common(64, false, false, false);
+        test_cross_shard_tx_common(64, false, false, false, 100);
     }
 
     #[test]
     fn test_cross_shard_tx_doomslug() {
-        test_cross_shard_tx_common(64, false, false, true);
+        test_cross_shard_tx_common(64, false, false, true, 100);
     }
 
     #[test]
     fn test_cross_shard_tx_drop_chunks() {
-        test_cross_shard_tx_common(64, false, true, false);
+        test_cross_shard_tx_common(64, false, true, false, 150);
     }
 
     #[test]
     fn test_cross_shard_tx_8_iterations() {
-        test_cross_shard_tx_common(8, false, false, false);
+        test_cross_shard_tx_common(8, false, false, false, 100);
     }
 
     #[test]
     fn test_cross_shard_tx_8_iterations_drop_chunks() {
-        test_cross_shard_tx_common(8, false, true, false);
+        test_cross_shard_tx_common(8, false, true, false, 150);
     }
 
     #[test]
-    fn test_cross_shard_tx_with_validator_rotation() {
-        test_cross_shard_tx_common(64, true, false, false);
+    fn test_cross_shard_tx_with_validator_rotation_1() {
+        test_cross_shard_tx_common(16, true, false, false, 150);
+    }
+
+    #[test]
+    fn test_cross_shard_tx_with_validator_rotation_2() {
+        test_cross_shard_tx_common(64, true, false, false, 400);
     }
 }

nightly/nightly.txt

@@ -13,12 +13,13 @@ pytest --timeout=600 sanity/state_sync.py manytx 250
 pytest sanity/state_sync.py onetx 30
 pytest --timeout=600 sanity/state_sync.py onetx 250
 pytest --timeout=240 sanity/state_sync1.py
-pytest --timeout=310 sanity/state_sync2.py
+pytest --timeout=600 sanity/state_sync2.py
 pytest --timeout=600 sanity/state_sync_routed.py manytx 100
 pytest --timeout=300 sanity/state_sync_late.py notx
 pytest sanity/rpc_tx_forwarding.py
 pytest --timeout=240 sanity/skip_epoch.py
-pytest --timeout=600 sanity/fork_sync.py
+# disabled, see #2206
+# pytest --timeout=600 sanity/fork_sync.py
 pytest --timeout=240 sanity/one_val.py
 pytest --timeout=240 sanity/lightclnt.py
 pytest --timeout=240 sanity/rpc_query.py
@@ -62,7 +63,8 @@ expensive nearcore test_catchup test_catchup
 expensive --timeout=1500 near-client cross_shard_tx tests::test_cross_shard_tx
 expensive --timeout=1500 near-client cross_shard_tx tests::test_cross_shard_tx_doomslug
 expensive --timeout=1500 near-client cross_shard_tx tests::test_cross_shard_tx_drop_chunks
-expensive --timeout=3600 near-client cross_shard_tx tests::test_cross_shard_tx_with_validator_rotation
+expensive --timeout=4800 near-client cross_shard_tx tests::test_cross_shard_tx_with_validator_rotation_1
+expensive --timeout=4800 near-client cross_shard_tx tests::test_cross_shard_tx_with_validator_rotation_2
 
 # doomslug / finality gadget safety and light client fuzzy tests
 expensive --timeout=900 near-chain doomslug tests::test_fuzzy_doomslug_liveness_and_safety

pytest/lib/cluster.py

@@ -127,6 +127,9 @@ def get_account(self, acc, finality='optimistic'):
 
     def get_block(self, block_hash):
         return self.json_rpc('block', [block_hash])
+
+    def get_changes(self, block_hash, state_changes_request):
+        return self.json_rpc('changes', [block_hash, state_changes_request])
 
     def validators(self):
         return set(map(lambda v: v['account_id'], self.get_status()['validators']))
@@ -143,9 +146,6 @@ def rpc_addr(self):
         return (self.host, self.rpc_port)
 
 
-    def get_changes(self, block_hash, state_changes_request):
-        return self.json_rpc('changes', [block_hash, state_changes_request])
-
 
 class LocalNode(BaseNode):
     def __init__(self, port, rpc_port, near_root, node_dir, blacklist):

pytest/tests/sanity/state_sync.py

@@ -87,7 +87,7 @@
 
 assert catch_up_height in boot_heights, "%s not in %s" % (catch_up_height, boot_heights)
 
-tracker.offset = 0 # the transition might have happened before we initialized the tracker
+tracker.reset() # the transition might have happened before we initialized the tracker
 if catch_up_height >= 100:
     assert tracker.check("transition to State Sync")
 elif catch_up_height <= 30:

pytest/tests/sanity/state_sync_routed.py

@@ -101,7 +101,7 @@
 
 assert catch_up_height in boot_heights, "%s not in %s" % (catch_up_height, boot_heights)
 
-tracker.offset = 0 # the transition might have happened before we initialized the tracker
+tracker4.reset() # the transition might have happened before we initialized the tracker
 if catch_up_height >= 100:
     assert tracker4.check("transition to State Sync")
 elif catch_up_height <= 30:

pytest/tests/sanity/transactions.py

@@ -48,26 +48,22 @@
             sent_height = height
 
     elif step == 1:
-        print(f'step {step}')
-        print(f'height {height}, sent_height {sent_height}')
         if height == sent_height + 6:
             cur_balances = ctx.get_balances()
-            print(f'cur_balances {cur_balances}')
-            print(f'expected_bal {ctx.expected_balances}')
 
             assert cur_balances == ctx.expected_balances, "%s != %s" % (cur_balances, ctx.expected_balances)
             step = 2
 
     else:
-        print(f'step {step}')
         # we are done with the sanity test, now let's stress it
         if ctx.get_balances() == ctx.expected_balances:
             print("Balances caught up, took %s blocks, moving on" % (height - sent_height));
+            last_balances = [x for x in ctx.expected_balances]
             ctx.send_moar_txs(hash_, 10, use_routing=True)
             sent_height = height
         else:
             if height > sent_height + 10:
-                assert False, "Balances before: %s\nExpected balances: %s\nCurrent balances: %s\nSent at height: %s\n" % (last_balances, ctx.expected_balances, ctx.get_balances(), sent_height)
+                assert False, "Balances before: %s\nExpected balances: %s\nCurrent balances: %s\nSent at height: %s, cur height: %s\n" % (last_balances, ctx.expected_balances, ctx.get_balances(), sent_height, height)
             time.sleep(0.2)
 
         if height >= 100: