db-sync: Handle case where latest ledger state file cannot be parsed

If the latest ledger state file is corrupt (ie maybe truncated) just
delete it and try the previous one.

Closes: #483

cardano-db-sync/src/Cardano/DbSync.hs

@@ -144,10 +144,10 @@ runDbSyncNode plugin enp =
       liftIO $ do
         -- Must run plugin startup after the genesis distribution has been inserted/validate.
         runDbStartup trce plugin
-
         case genCfg of
           GenesisCardano _ bCfg _sCfg -> do
             ledgerVar <- initLedgerStateVar genCfg
+            loadLatestLedgerState (envLedgerStateDir genesisEnv) ledgerVar
             runDbSyncNodeNodeClient genesisEnv ledgerVar
                 iomgr trce plugin (cardanoCodecConfig bCfg) (enpSocketPath enp)
   where
@@ -286,13 +286,13 @@ logDbState trce = do
 
 getLatestPoints :: LedgerStateDir -> IO [Point CardanoBlock]
 getLatestPoints ledgerStateDir = do
-    xs <- listLedgerStateSlotNos ledgerStateDir
+    xs <- lsfSlotNo <<$>> listLedgerStateFilesOrdered ledgerStateDir
     ys <- catMaybes <$> DB.runDbNoLogging (mapM DB.querySlotHash xs)
     pure $ mapMaybe convert ys
   where
     convert :: (SlotNo, ByteString) -> Maybe (Point CardanoBlock)
     convert (slot, hashBlob) =
-      fmap (Point . Point.block slot) (convertHashBlob hashBlob)
+      Point . Point.block slot <$> convertHashBlob hashBlob
 
     convertHashBlob :: ByteString -> Maybe (HeaderHash CardanoBlock)
     convertHashBlob = Just . fromRawHash (Proxy @CardanoBlock)

cardano-db-sync/src/Cardano/DbSync/Database.hs

@@ -95,7 +95,7 @@ runActions trce env plugin ledgerState actions = do
             pure Done
         ([], DbRollBackToPoint sn:ys) -> do
             runRollbacks trce plugin sn
-            liftIO $ loadLedgerState (envLedgerStateDir env) ledgerState sn
+            liftIO $ loadLedgerStateAtSlot (envLedgerStateDir env) ledgerState sn
             dbAction Continue ys
         (ys, zs) -> do
           insertBlockList trce env ledgerState plugin ys

cardano-db-sync/src/Cardano/DbSync/LedgerState.hs

@@ -12,8 +12,9 @@ module Cardano.DbSync.LedgerState
   , LedgerStateFile (..)
   , applyBlock
   , initLedgerStateVar
-  , listLedgerStateSlotNos
-  , loadLedgerState
+  , ledgerStateTipSlot
+  , loadLatestLedgerState
+  , loadLedgerStateAtSlot
   , readLedgerState
   , saveLedgerState
   , listLedgerStateFilesOrdered
@@ -143,6 +144,11 @@ applyBlock env (LedgerStateVar stateVar) blk =
         Left err -> panic err
         Right result -> result
 
+ledgerStateTipSlot :: LedgerStateVar -> IO SlotNo
+ledgerStateTipSlot (LedgerStateVar stateVar) = do
+  lstate <- readTVarIO stateVar
+  pure $ fromWithOrigin (SlotNo 0) . ledgerTipSlot $ ledgerState (clsState lstate)
+
 saveLedgerState :: LedgerStateDir -> LedgerStateVar -> LedgerStateSnapshot -> SyncState -> IO ()
 saveLedgerState lsd@(LedgerStateDir stateDir) (LedgerStateVar stateVar) snapshot synced = do
   atomically $ writeTVar stateVar ledger
@@ -181,8 +187,17 @@ saveLedgerState lsd@(LedgerStateDir stateDir) (LedgerStateVar stateVar) snapshot
     codecConfig :: CodecConfig (CardanoBlock StandardCrypto)
     codecConfig = configCodec (clsConfig ledger)
 
-loadLedgerState :: LedgerStateDir -> LedgerStateVar -> SlotNo -> IO ()
-loadLedgerState stateDir (LedgerStateVar stateVar) slotNo = do
+loadLatestLedgerState :: LedgerStateDir -> LedgerStateVar -> IO ()
+loadLatestLedgerState stateDir ledgerVar = do
+  files <- listLedgerStateFilesOrdered stateDir
+  ledger <- readLedgerState ledgerVar
+  mcs <- firstJustM (loadFile ledger) files
+  case mcs of
+    Just cs -> atomically $ writeTVar (unLedgerStateVar ledgerVar) cs
+    Nothing -> pure ()
+
+loadLedgerStateAtSlot :: LedgerStateDir -> LedgerStateVar -> SlotNo -> IO ()
+loadLedgerStateAtSlot stateDir (LedgerStateVar stateVar) slotNo = do
   -- Read current state to get the LedgerConfig and CodecConfig.
   lstate <- readLedgerState (LedgerStateVar stateVar)
   -- Load the state
@@ -260,7 +275,9 @@ loadFile ledger lsf = do
         Left (_ :: IOException) -> pure Nothing
         Right bs ->
           case decode bs of
-            Left err -> panic (textShow err)
+            Left _err -> do
+              safeRemoveFile fp
+              pure Nothing
             Right ls -> pure $ Just ls
 
 
@@ -295,9 +312,6 @@ listLedgerStateFilesOrdered (LedgerStateDir stateDir) = do
     revSlotNoOrder :: LedgerStateFile -> LedgerStateFile -> Ordering
     revSlotNoOrder a b = compare (lsfSlotNo b) (lsfSlotNo a)
 
-listLedgerStateSlotNos :: LedgerStateDir -> IO [SlotNo]
-listLedgerStateSlotNos = fmap3 lsfSlotNo listLedgerStateFilesOrdered
-
 readLedgerState :: LedgerStateVar -> IO CardanoLedgerState
 readLedgerState (LedgerStateVar stateVar) = readTVarIO stateVar