From 0db9898a12542b4e9ba41d4266840acd55b012e3 Mon Sep 17 00:00:00 2001
From: Erik de Castro Lopo <erikd@mega-nerd.com>
Date: Fri, 15 Jan 2021 17:59:24 +1100
Subject: [PATCH] db-sync: Handle case where latest ledger state file cannot be
 parsed

If the latest ledger state file is corrupt (ie maybe truncated) just
delete it and try the previous one.

Closes: https://github.com/input-output-hk/cardano-db-sync/issues/483
---
 cardano-db-sync/src/Cardano/DbSync.hs         |  6 ++--
 .../src/Cardano/DbSync/Database.hs            |  2 +-
 .../src/Cardano/DbSync/LedgerState.hs         | 30 ++++++++++++++-----
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/cardano-db-sync/src/Cardano/DbSync.hs b/cardano-db-sync/src/Cardano/DbSync.hs
index 19dd55350..1b38af96f 100644
--- a/cardano-db-sync/src/Cardano/DbSync.hs
+++ b/cardano-db-sync/src/Cardano/DbSync.hs
@@ -144,10 +144,10 @@ runDbSyncNode plugin enp =
       liftIO $ do
         -- Must run plugin startup after the genesis distribution has been inserted/validate.
         runDbStartup trce plugin
-
         case genCfg of
           GenesisCardano _ bCfg _sCfg -> do
             ledgerVar <- initLedgerStateVar genCfg
+            loadLatestLedgerState (envLedgerStateDir genesisEnv) ledgerVar
             runDbSyncNodeNodeClient genesisEnv ledgerVar
                 iomgr trce plugin (cardanoCodecConfig bCfg) (enpSocketPath enp)
   where
@@ -286,13 +286,13 @@ logDbState trce = do
 
 getLatestPoints :: LedgerStateDir -> IO [Point CardanoBlock]
 getLatestPoints ledgerStateDir = do
-    xs <- listLedgerStateSlotNos ledgerStateDir
+    xs <- lsfSlotNo <<$>> listLedgerStateFilesOrdered ledgerStateDir
     ys <- catMaybes <$> DB.runDbNoLogging (mapM DB.querySlotHash xs)
     pure $ mapMaybe convert ys
   where
     convert :: (SlotNo, ByteString) -> Maybe (Point CardanoBlock)
     convert (slot, hashBlob) =
-      fmap (Point . Point.block slot) (convertHashBlob hashBlob)
+      Point . Point.block slot <$> convertHashBlob hashBlob
 
     convertHashBlob :: ByteString -> Maybe (HeaderHash CardanoBlock)
     convertHashBlob = Just . fromRawHash (Proxy @CardanoBlock)
diff --git a/cardano-db-sync/src/Cardano/DbSync/Database.hs b/cardano-db-sync/src/Cardano/DbSync/Database.hs
index fc04a778c..2661aed60 100644
--- a/cardano-db-sync/src/Cardano/DbSync/Database.hs
+++ b/cardano-db-sync/src/Cardano/DbSync/Database.hs
@@ -95,7 +95,7 @@ runActions trce env plugin ledgerState actions = do
             pure Done
         ([], DbRollBackToPoint sn:ys) -> do
             runRollbacks trce plugin sn
-            liftIO $ loadLedgerState (envLedgerStateDir env) ledgerState sn
+            liftIO $ loadLedgerStateAtSlot (envLedgerStateDir env) ledgerState sn
             dbAction Continue ys
         (ys, zs) -> do
           insertBlockList trce env ledgerState plugin ys
diff --git a/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs b/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs
index 579fece31..80ccb8c2f 100644
--- a/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs
+++ b/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs
@@ -12,8 +12,9 @@ module Cardano.DbSync.LedgerState
   , LedgerStateFile (..)
   , applyBlock
   , initLedgerStateVar
-  , listLedgerStateSlotNos
-  , loadLedgerState
+  , ledgerStateTipSlot
+  , loadLatestLedgerState
+  , loadLedgerStateAtSlot
   , readLedgerState
   , saveLedgerState
   , listLedgerStateFilesOrdered
@@ -143,6 +144,11 @@ applyBlock env (LedgerStateVar stateVar) blk =
         Left err -> panic err
         Right result -> result
 
+ledgerStateTipSlot :: LedgerStateVar -> IO SlotNo
+ledgerStateTipSlot (LedgerStateVar stateVar) = do
+  lstate <- readTVarIO stateVar
+  pure $ fromWithOrigin (SlotNo 0) . ledgerTipSlot $ ledgerState (clsState lstate)
+
 saveLedgerState :: LedgerStateDir -> LedgerStateVar -> LedgerStateSnapshot -> SyncState -> IO ()
 saveLedgerState lsd@(LedgerStateDir stateDir) (LedgerStateVar stateVar) snapshot synced = do
   atomically $ writeTVar stateVar ledger
@@ -181,8 +187,17 @@ saveLedgerState lsd@(LedgerStateDir stateDir) (LedgerStateVar stateVar) snapshot
     codecConfig :: CodecConfig (CardanoBlock StandardCrypto)
     codecConfig = configCodec (clsConfig ledger)
 
-loadLedgerState :: LedgerStateDir -> LedgerStateVar -> SlotNo -> IO ()
-loadLedgerState stateDir (LedgerStateVar stateVar) slotNo = do
+loadLatestLedgerState :: LedgerStateDir -> LedgerStateVar -> IO ()
+loadLatestLedgerState stateDir ledgerVar = do
+  files <- listLedgerStateFilesOrdered stateDir
+  ledger <- readLedgerState ledgerVar
+  mcs <- firstJustM (loadFile ledger) files
+  case mcs of
+    Just cs -> atomically $ writeTVar (unLedgerStateVar ledgerVar) cs
+    Nothing -> pure ()
+
+loadLedgerStateAtSlot :: LedgerStateDir -> LedgerStateVar -> SlotNo -> IO ()
+loadLedgerStateAtSlot stateDir (LedgerStateVar stateVar) slotNo = do
   -- Read current state to get the LedgerConfig and CodecConfig.
   lstate <- readLedgerState (LedgerStateVar stateVar)
   -- Load the state
@@ -260,7 +275,9 @@ loadFile ledger lsf = do
         Left (_ :: IOException) -> pure Nothing
         Right bs ->
           case decode bs of
-            Left err -> panic (textShow err)
+            Left _err -> do
+              safeRemoveFile fp
+              pure Nothing
             Right ls -> pure $ Just ls
 
 
@@ -295,9 +312,6 @@ listLedgerStateFilesOrdered (LedgerStateDir stateDir) = do
     revSlotNoOrder :: LedgerStateFile -> LedgerStateFile -> Ordering
     revSlotNoOrder a b = compare (lsfSlotNo b) (lsfSlotNo a)
 
-listLedgerStateSlotNos :: LedgerStateDir -> IO [SlotNo]
-listLedgerStateSlotNos = fmap3 lsfSlotNo listLedgerStateFilesOrdered
-
 readLedgerState :: LedgerStateVar -> IO CardanoLedgerState
 readLedgerState (LedgerStateVar stateVar) = readTVarIO stateVar