Merge pull request #1266 from input-output-hk/kderme/restart-rollback

Avoid Rollbacks during restart

cardano-chain-gen/src/Cardano/Mock/ChainSync/Server.hs

@@ -12,6 +12,7 @@ module Cardano.Mock.ChainSync.Server
     forkServerThread
   , withServerHandle
   , stopServer
+  , restartServer
 
     -- * ServerHandle api
   , ServerHandle (..)
@@ -84,9 +85,12 @@ import           Cardano.Mock.Chain hiding (rollback)
 import           Cardano.Mock.ChainDB
 import           Cardano.Mock.ChainSync.State
 
+{- HLINT ignore "Use readTVarIO" -}
+
 data ServerHandle m blk = ServerHandle
   { chainProducerState :: StrictTVar m (ChainProducerState blk)
-  , threadHandle :: Async ()
+  , threadHandle :: StrictTVar m (Async ())
+  , forkAgain :: m (Async ())
   }
 
 replaceGenesis :: MonadSTM m => ServerHandle m blk -> State blk -> STM m ()
@@ -110,8 +114,16 @@ rollback handle point =
       Nothing -> error $ "point " <> show point <> " not in chain"
       Just st' -> st'
 
-stopServer :: ServerHandle m blk -> IO ()
-stopServer = cancel . threadHandle
+restartServer :: ServerHandle IO blk -> IO ()
+restartServer sh = do
+    stopServer sh
+    thread <- forkAgain sh
+    atomically $ writeTVar (threadHandle sh) thread
+
+stopServer :: ServerHandle IO blk -> IO ()
+stopServer sh = do
+    srvThread <- atomically $ readTVar $ threadHandle sh
+    cancel srvThread
 
 type MockServerConstraint blk =
     ( SerialiseNodeToClientConstraints blk
@@ -137,8 +149,10 @@ forkServerThread
     -> IO (ServerHandle IO blk)
 forkServerThread iom config initSt netMagic path = do
     chainSt <- newTVarIO $ initChainProducerState config initSt
-    thread <- async $ runLocalServer iom (configCodec config) netMagic path chainSt
-    pure $ ServerHandle chainSt thread
+    let runThread = async $ runLocalServer iom (configCodec config) netMagic path chainSt
+    thread <- runThread
+    threadVar <- newTVarIO thread
+    pure $ ServerHandle chainSt threadVar runThread
 
 withServerHandle
     :: forall blk a. MockServerConstraint blk

cardano-chain-gen/src/Cardano/Mock/Forging/Interpreter.hs

@@ -19,6 +19,7 @@ module Cardano.Mock.Forging.Interpreter
   , forgeNextFindLeader
   , forgeNext
   , forgeNextAfter
+  , rollbackInterpreter
   , getCurrentInterpreterState
   , getCurrentLedgerState
   , getCurrentEpoch
@@ -62,9 +63,8 @@ import           Cardano.Mock.Forging.Types
 
 import           NoThunks.Class (OnlyCheckWhnfNamed (..))
 
-import           Ouroboros.Consensus.Block (BlockForging, BlockNo (..), BlockProtocol, EpochNo,
-                   ForgeStateInfo, ShouldForge (..), SlotNo (..), blockNo, blockSlot,
-                   checkShouldForge)
+import           Ouroboros.Consensus.Block (BlockForging, BlockProtocol, EpochNo, ForgeStateInfo,
+                   ShouldForge (..), checkShouldForge)
 import qualified Ouroboros.Consensus.Block as Block
 import           Ouroboros.Consensus.Cardano.Block (LedgerState (..), StandardAlonzo,
                    StandardBabbage, StandardShelley)
@@ -95,6 +95,7 @@ import qualified Ouroboros.Consensus.TypeFamilyWrappers as Consensus
 import           Ouroboros.Consensus.Util.IOLike (Exception, NoThunks, StrictMVar, modifyMVar,
                    newMVar, readMVar, swapMVar)
 import           Ouroboros.Consensus.Util.Orphans ()
+import           Ouroboros.Network.Block
 
 import           System.Directory (doesPathExist)
 
@@ -366,6 +367,30 @@ tryAllForging interpreter interState currentSlot xs = do
         NotLeader -> do
           pure Nothing
 
+rollbackInterpreter :: Interpreter -> CardanoPoint -> IO ()
+rollbackInterpreter interpreter pnt = do
+    interState <- getCurrentInterpreterState interpreter
+    !chain' <- case rollbackChainDB (istChain interState) pnt of
+          Just c -> pure c
+          Nothing -> throwIO RollbackFailed
+    let newSt = currentState chain'
+    let tip = headTip chain'
+    let (nextSlot, nextBlock) = case tip of
+          TipGenesis -> (SlotNo 0, BlockNo 1)
+          Tip slt _ blkNo -> (slt + 1, blkNo + 1)
+    let !newInterState =
+          InterpreterState
+            { istChain = chain'
+            , istForecast = mkForecast cfg newSt
+            , istSlot = nextSlot
+            , istNextBlockNo = nextBlock
+            , istFingerprint = istFingerprint interState
+            }
+    void $ swapMVar (interpState interpreter) newInterState
+  where
+    cfg :: TopLevelConfig CardanoBlock
+    cfg = interpTopLeverConfig interpreter
+
 getCurrentInterpreterState :: Interpreter -> IO  InterpreterState
 getCurrentInterpreterState = readMVar . interpState
 

cardano-chain-gen/src/Cardano/Mock/Forging/Types.hs

@@ -3,6 +3,7 @@
 
 module Cardano.Mock.Forging.Types
   ( CardanoBlock
+  , CardanoPoint
   , TPraosStandard
   , PraosStandard
   , ForgingError (..)
@@ -22,6 +23,7 @@ import           Ouroboros.Consensus.Protocol.Praos (Praos)
 import           Ouroboros.Consensus.Protocol.TPraos (TPraos)
 import           Ouroboros.Consensus.Shelley.Eras (StandardAlonzo, StandardBabbage, StandardCrypto,
                    StandardShelley)
+import           Ouroboros.Network.Block (Point)
 
 import           Cardano.Ledger.Address
 import qualified Cardano.Ledger.Core as Core
@@ -36,6 +38,8 @@ type PraosStandard = Praos StandardCrypto
 
 type CardanoBlock = Consensus.CardanoBlock StandardCrypto
 
+type CardanoPoint = Point CardanoBlock
+
 data MockBlock = MockBlock
   { txs :: ![TxEra]
   , node :: !NodeId
@@ -63,6 +67,7 @@ data ForgingError
   | FailedToValidateSlot !SlotNo !Int !FilePath
   | NotExpectedSlotNo !SlotNo !SlotNo !Int
   | FingerprintDecodeError !String
+  | RollbackFailed
   deriving (Show, Exception)
 
 data UTxOIndex era

cardano-chain-gen/test/Test/Cardano/Db/Mock/UnifiedApi.hs

@@ -15,6 +15,7 @@ module Test.Cardano.Db.Mock.UnifiedApi
   , fillUntilNextEpoch
   , fillEpochs
   , fillEpochPercentage
+  , rollbackTo
   , registerAllStakeCreds
   ) where
 
@@ -157,6 +158,11 @@ fillEpochPercentage interpreter mockServer percentage = do
   let blocksToCreate = div (percentage * blocksPerEpoch) 100
   replicateM blocksToCreate $forgeNextFindLeaderAndSubmit interpreter mockServer []
 
+rollbackTo :: Interpreter -> ServerHandle IO CardanoBlock -> CardanoPoint -> IO ()
+rollbackTo interpreter mockServer point = do
+    rollbackInterpreter interpreter point
+    atomically $ rollback mockServer point
+
 registerAllStakeCreds :: Interpreter -> ServerHandle IO CardanoBlock -> IO CardanoBlock
 registerAllStakeCreds interpreter mockServer = do
   blk <- forgeWithStakeCreds interpreter

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo.hs

@@ -671,11 +671,11 @@ mirRewardRollback =
       assertRewardCounts dbSync st True Nothing [(StakeIndexNew 1, (0,0,0,1,0))]
 
       atomically $ rollback mockServer (blockPoint $ last c)
-      assertBlockNoBackoff dbSync (fromIntegral $ 4 + length (a <> b <> c))
+      assertBlockNoBackoff dbSync (fromIntegral $ 4 + length (a <> b <> c <> d))
       assertRewardCounts dbSync st True Nothing [(StakeIndexNew 1, (0,0,0,1,0))]
       stopDBSync dbSync
       startDBSync dbSync
-      assertBlockNoBackoff dbSync (fromIntegral $ 4 + length (a <> b <> c))
+      assertBlockNoBackoff dbSync (fromIntegral $ 4 + length (a <> b <> c <> d))
       assertRewardCounts dbSync st True Nothing [(StakeIndexNew 1, (0,0,0,1,0))]
 
       forM_ d $ atomically . addBlock mockServer
@@ -790,10 +790,12 @@ rollbackBoundary =
 
       assertRewardCount dbSync 3
       atomically $ rollback mockServer (blockPoint $ last blks)
-      assertBlockNoBackoff dbSync (2 + length a + length blks)
+      assertBlockNoBackoff dbSync (2 + length a + length blks + length blks')
       forM_ blks' $ atomically . addBlock mockServer
       assertBlockNoBackoff dbSync (2 + length a + length blks + length blks')
       assertRewardCount dbSync 3
+      blks'' <- fillUntilNextEpoch interpreter mockServer
+      assertBlockNoBackoff dbSync (2 + length a + length blks + length blks' + length blks'')
   where
     testLabel = "rollbackBoundary-alonzo"
 

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage.hs

@@ -71,12 +71,17 @@ unitTests iom knownMigrations =
           , test "sync one block" addSimple
           , test "restart db-sync" restartDBSync
           , test "sync small chain" addSimpleChain
+          , test "node restart" nodeRestart
+          , test "node restart boundary" nodeRestartBoundary
           ]
       , testGroup "rollbacks"
           [ test "simple rollback" simpleRollback
           , test "sync bigger chain" bigChain
           , test "rollback while db-sync is off" restartAndRollback
           , test "rollback further" rollbackFurther
+          , test "big rollbacks executed lazily" lazyRollback
+          , test "lazy rollback on restart" lazyRollbackRestart
+          , test "rollback while rollbacking" doubleRollback
           , test "rollback stake address cache" stakeAddressRollback
           ]
       , testGroup "different configs"
@@ -218,6 +223,35 @@ addSimpleChain =
   where
     testLabel = "addSimpleChain"
 
+nodeRestart :: IOManager -> [(Text, Text)] -> Assertion
+nodeRestart =
+    withFullConfig babbageConfig testLabel $ \interpreter mockServer dbSync -> do
+      startDBSync dbSync
+      void $ forgeAndSubmitBlocks interpreter mockServer 5
+      assertBlockNoBackoff dbSync 5
+
+      restartServer mockServer
+
+      void $ forgeAndSubmitBlocks interpreter mockServer 5
+      assertBlockNoBackoff dbSync 10
+  where
+    testLabel = "nodeRestart"
+
+nodeRestartBoundary :: IOManager -> [(Text, Text)] -> Assertion
+nodeRestartBoundary =
+    withFullConfig babbageConfig testLabel $ \interpreter mockServer dbSync -> do
+      startDBSync dbSync
+      blks <- fillUntilNextEpoch interpreter mockServer
+      assertBlockNoBackoff dbSync $ length blks
+
+      restartServer mockServer
+
+      void $ forgeAndSubmitBlocks interpreter mockServer 5
+      assertBlockNoBackoff dbSync $ 5 + length blks
+
+  where
+    testLabel = "nodeRestartBoundary"
+
 restartDBSync :: IOManager -> [(Text, Text)] -> Assertion
 restartDBSync =
     withFullConfig babbageConfig testLabel $ \interpreter mockServer dbSync -> do
@@ -324,6 +358,66 @@ rollbackFurther =
   where
     testLabel = "rollbackFurther"
 
+lazyRollback :: IOManager -> [(Text, Text)] -> Assertion
+lazyRollback =
+  withFullConfig babbageConfig testLabel $ \interpreter mockServer dbSync -> do
+    startDBSync  dbSync
+    lastBlk <- last <$> forgeAndSubmitBlocks interpreter mockServer 200
+    void $ forgeAndSubmitBlocks interpreter mockServer 70
+    assertBlockNoBackoff dbSync 270
+    rollbackTo interpreter mockServer (blockPoint lastBlk)
+    -- Here we create the fork.
+    void $ withBabbageFindLeaderAndSubmitTx interpreter mockServer $
+        Babbage.mkSimpleDCertTx [(StakeIndexNew 1, DCertDeleg . RegKey)]
+    void $ forgeAndSubmitBlocks interpreter mockServer 40
+    assertBlockNoBackoff dbSync 241
+  where
+    testLabel = "lazyRollback"
+
+lazyRollbackRestart :: IOManager -> [(Text, Text)] -> Assertion
+lazyRollbackRestart =
+  withFullConfig babbageConfig testLabel $ \interpreter mockServer dbSync -> do
+    startDBSync  dbSync
+    lastBlk <- last <$> forgeAndSubmitBlocks interpreter mockServer 220
+    void $ forgeAndSubmitBlocks interpreter mockServer 60
+    assertBlockNoBackoff dbSync 280
+
+    stopDBSync dbSync
+    rollbackTo interpreter mockServer (blockPoint lastBlk)
+
+    startDBSync  dbSync
+    -- Here we create the fork.
+    void $ withBabbageFindLeaderAndSubmitTx interpreter mockServer $
+        Babbage.mkSimpleDCertTx [(StakeIndexNew 1, DCertDeleg . RegKey)]
+    void $ forgeAndSubmitBlocks interpreter mockServer 30
+    assertBlockNoBackoff dbSync 251
+  where
+    testLabel = "lazyRollbackRestart"
+
+doubleRollback :: IOManager -> [(Text, Text)] -> Assertion
+doubleRollback =
+  withFullConfig babbageConfig testLabel $ \interpreter mockServer dbSync -> do
+    startDBSync  dbSync
+    lastBlk1 <- last <$> forgeAndSubmitBlocks interpreter mockServer 150
+    lastBlk2 <- last <$> forgeAndSubmitBlocks interpreter mockServer 100
+    void $ forgeAndSubmitBlocks interpreter mockServer 100
+    assertBlockNoBackoff dbSync 350
+
+    rollbackTo interpreter mockServer (blockPoint lastBlk2)
+    -- Here we create the fork.
+    void $ withBabbageFindLeaderAndSubmitTx interpreter mockServer $
+        Babbage.mkSimpleDCertTx [(StakeIndexNew 1, DCertDeleg . RegKey)]
+    void $ forgeAndSubmitBlocks interpreter mockServer 50
+
+    rollbackTo interpreter mockServer (blockPoint lastBlk1)
+    void $ withBabbageFindLeaderAndSubmitTx interpreter mockServer $
+        Babbage.mkSimpleDCertTx [(StakeIndexNew 0, DCertDeleg . RegKey)]
+    void $ forgeAndSubmitBlocks interpreter mockServer 50
+
+    assertBlockNoBackoff dbSync 201
+  where
+    testLabel = "doubleRollback"
+
 stakeAddressRollback :: IOManager -> [(Text, Text)] -> Assertion
 stakeAddressRollback =
   withFullConfig babbageConfig testLabel $ \interpreter mockServer dbSync -> do
@@ -976,10 +1070,12 @@ rollbackBoundary =
 
       assertRewardCount dbSync 3
       atomically $ rollback mockServer (blockPoint $ last blks)
-      assertBlockNoBackoff dbSync (2 + length a + length blks)
+      assertBlockNoBackoff dbSync (2 + length a + length blks + length blks')
       forM_ blks' $ atomically . addBlock mockServer
       assertBlockNoBackoff dbSync (2 + length a + length blks + length blks')
       assertRewardCount dbSync 3
+      blks'' <- fillUntilNextEpoch interpreter mockServer
+      assertBlockNoBackoff dbSync (2 + length a + length blks + length blks' + length blks'')
   where
     testLabel = "rollbackBoundary"
 

cardano-chain-gen/test/testfiles/fingerprint/doubleRollback

@@ -0,0 +1 @@
+[1,3,4,7,8,9,10,11,14,30,31,33,34,35,40,42,44,61,62,74,76,82,85,88,89,93,95,102,104,106,111,114,120,123,141,143,146,149,158,170,171,178,186,190,198,201,207,209,210,221,223,227,245,249,250,259,260,265,272,276,283,284,294,296,301,305,306,311,312,317,318,321,322,325,331,333,337,339,345,347,348,350,365,375,377,379,382,391,394,401,407,411,413,415,424,428,445,453,467,469,471,474,487,489,490,492,497,498,515,519,520,524,528,529,530,531,554,557,567,568,575,576,597,603,615,616,631,637,640,641,643,650,654,656,657,662,668,676,678,688,694,695,698,700,701,702,703,716,725,727,729,730,732,733,738,747,750,771,774,777,778,780,785,796,798,799,805,808,819,821,822,827,831,833,835,838,841,844,847,848,858,859,860,863,864,872,875,889,906,912,913,914,917,942,943,945,948,950,952,954,955,959,967,971,980,981,983,993,997,1004,1006,1009,1010,1013,1015,1019,1025,1028,1031,1039,1043,1047,1050,1054,1056,1057,1062,1065,1089,1090,1092,1094,1098,1101,1103,1105,1107,1115,1116,1122,1125,1126,1130,1139,1146,1150,1156,1158,1159,1165,1166,1168,1172,1176,1183,1185,1187,1188,1214,1216,1222,1223,1225,1227,1236,1242,1245,1264,1267,1270,1279,1284,1294,1304,1311,1313,1314,1318,1322,1323,1327,1333,1339,1345,1349,1356,1358,1367,1373,1384,1391,1399,1402,1405,1407,1409,1410,1432,1437,1443,1446,1450,1451,1456,1466,1481,1489,1491,1496,1498,1500,1503,1515,1526,1529,1534,1550,1551,1556,1565,1567,1574,1583,1584,1596,1600,1606,1610,1613,1616,1622,1632,1634,1644,1646,1647,1650,1651,1654,1659,1661,1664,1670,1674,1675,1679,1680,1684,1685,1686,1166,1168,1172,1176,1183,1185,1187,1188,1214,1216,1222,1223,1225,1227,1236,1242,1245,1264,1267,1270,1279,1284,1294,1304,1311,1313,1314,1318,1322,1323,1327,1333,1339,1345,1349,1356,1358,1367,1373,1384,1391,1399,1402,1405,1407,1409,1410,1432,1437,1443,1446,729,730,732,733,738,747,750,771,774,777,778,780,785,796,798,799,805,808,819,821,822,827,831,833,835,838,841,844,847,848,858,859,860,863,864,872,875,889,906,912,913,914,917,942,943,945,948,950,952,954,955]

cardano-chain-gen/test/testfiles/fingerprint/lazyRollback

@@ -0,0 +1 @@
+[1,3,4,7,8,9,10,11,14,30,31,33,34,35,40,42,44,61,62,74,76,82,85,88,89,93,95,102,104,106,111,114,120,123,141,143,146,149,158,170,171,178,186,190,198,201,207,209,210,221,223,227,245,249,250,259,260,265,272,276,283,284,294,296,301,305,306,311,312,317,318,321,322,325,331,333,337,339,345,347,348,350,365,375,377,379,382,391,394,401,407,411,413,415,424,428,445,453,467,469,471,474,487,489,490,492,497,498,515,519,520,524,528,529,530,531,554,557,567,568,575,576,597,603,615,616,631,637,640,641,643,650,654,656,657,662,668,676,678,688,694,695,698,700,701,702,703,716,725,727,729,730,732,733,738,747,750,771,774,777,778,780,785,796,798,799,805,808,819,821,822,827,831,833,835,838,841,844,847,848,858,859,860,863,864,872,875,889,906,912,913,914,917,942,943,945,948,950,952,954,955,959,967,971,980,981,983,993,997,1004,1006,1009,1010,1013,1015,1019,1025,1028,1031,1039,1043,1047,1050,1054,1056,1057,1062,1065,1089,1090,1092,1094,1098,1101,1103,1105,1107,1115,1116,1122,1125,1126,1130,1139,1146,1150,1156,1158,1159,1165,1166,1168,1172,1176,1183,1185,1187,1188,1214,1216,1222,1223,1225,1227,1236,1242,1245,1264,1267,1270,955,959,967,971,980,981,983,993,997,1004,1006,1009,1010,1013,1015,1019,1025,1028,1031,1039,1043,1047,1050,1054,1056,1057,1062,1065,1089,1090,1092,1094,1098,1101,1103,1105,1107,1115,1116,1122,1125,1126,1130,1139,1146,1150,1156,1158,1159,1165,1166,1168,1172,1176,1183,1185,1187,1188,1214,1216,1222,1223,1225,1227,1236,1242,1245,1264,1267,1270,1279]

cardano-chain-gen/test/testfiles/fingerprint/lazyRollbackRestart

@@ -0,0 +1 @@
+[1,3,4,7,8,9,10,11,14,30,31,33,34,35,40,42,44,61,62,74,76,82,85,88,89,93,95,102,104,106,111,114,120,123,141,143,146,149,158,170,171,178,186,190,198,201,207,209,210,221,223,227,245,249,250,259,260,265,272,276,283,284,294,296,301,305,306,311,312,317,318,321,322,325,331,333,337,339,345,347,348,350,365,375,377,379,382,391,394,401,407,411,413,415,424,428,445,453,467,469,471,474,487,489,490,492,497,498,515,519,520,524,528,529,530,531,554,557,567,568,575,576,597,603,615,616,631,637,640,641,643,650,654,656,657,662,668,676,678,688,694,695,698,700,701,702,703,716,725,727,729,730,732,733,738,747,750,771,774,777,778,780,785,796,798,799,805,808,819,821,822,827,831,833,835,838,841,844,847,848,858,859,860,863,864,872,875,889,906,912,913,914,917,942,943,945,948,950,952,954,955,959,967,971,980,981,983,993,997,1004,1006,1009,1010,1013,1015,1019,1025,1028,1031,1039,1043,1047,1050,1054,1056,1057,1062,1065,1089,1090,1092,1094,1098,1101,1103,1105,1107,1115,1116,1122,1125,1126,1130,1139,1146,1150,1156,1158,1159,1165,1166,1168,1172,1176,1183,1185,1187,1188,1214,1216,1222,1223,1225,1227,1236,1242,1245,1264,1267,1270,1279,1284,1294,1304,1311,1313,1314,1318,1322,1323,1043,1047,1050,1054,1056,1057,1062,1065,1089,1090,1092,1094,1098,1101,1103,1105,1107,1115,1116,1122,1125,1126,1130,1139,1146,1150,1156,1158,1159,1165,1166]

cardano-chain-gen/test/testfiles/fingerprint/nodeRestart

@@ -0,0 +1 @@
+[1,3,4,7,8,9,10,11,14,30]

cardano-chain-gen/test/testfiles/fingerprint/nodeRestartBoundary

@@ -0,0 +1 @@
+[1,3,4,7,8,9,10,11,14,30,31,33,34,35,40,42,44,61,62,74,76,82,85,88,89,93,95,102,104,106,111,114,120,123,141,143,146,149,158,170,171,178,186,190,198,201,207,209,210,221,223,227,245,249,250,259,260,265,272,276,283,284,294,296,301,305,306,311,312,317,318,321,322,325,331,333,337,339,345,347,348,350,365,375,377,379,382,391,394,401,407,411,413,415,424,428,445,453,467,469,471,474,487,489,490,492,497,498,515,519,520,524,528,529]