diff --git a/cardano-chain-gen/cardano-chain-gen.cabal b/cardano-chain-gen/cardano-chain-gen.cabal index 286fe6468..34b04451a 100644 --- a/cardano-chain-gen/cardano-chain-gen.cabal +++ b/cardano-chain-gen/cardano-chain-gen.cabal @@ -44,6 +44,7 @@ library Cardano.Mock.ChainDB Cardano.Mock.ChainSync.Server Cardano.Mock.ChainSync.State + Cardano.Mock.Forging.Examples Cardano.Mock.Forging.Interpreter Cardano.Mock.Forging.Tx.Alonzo Cardano.Mock.Forging.Tx.Alonzo.ScriptsExamples diff --git a/cardano-chain-gen/src/Cardano/Mock/Forging/Examples.hs b/cardano-chain-gen/src/Cardano/Mock/Forging/Examples.hs new file mode 100644 index 000000000..f20a6e11e --- /dev/null +++ b/cardano-chain-gen/src/Cardano/Mock/Forging/Examples.hs @@ -0,0 +1,42 @@ +module Cardano.Mock.Forging.Examples where + +import Cardano.Prelude hiding (length, (.)) + +import Data.List.Extra + +import Cardano.Ledger.Mary.Value +import Cardano.Ledger.Shelley.API +import Cardano.Mock.Forging.Interpreter +import qualified Cardano.Mock.Forging.Tx.Alonzo as Alonzo +import Cardano.Mock.Forging.Tx.Generic +import Cardano.Mock.Forging.Types + +delegateAndSendBlocks :: Int -> Interpreter -> IO [CardanoBlock] +delegateAndSendBlocks n interpreter = do + addrFrom <- withAlonzoLedgerState interpreter $ resolveAddress (UTxOIndex 0) + registerBlocks <- forM (chunksOf 500 creds) $ \blockCreds -> do + blockTxs <- withAlonzoLedgerState interpreter $ \_st -> + forM (chunksOf 10 blockCreds) $ \txCreds -> -- 10 per tx + Alonzo.mkDCertTx (fmap (DCertDeleg . RegKey) txCreds) (Wdrl mempty) + forgeNextFindLeader interpreter (TxAlonzo <$> blockTxs) + + delegateBlocks <- forM (chunksOf 500 creds) $ \blockCreds -> do + blockTxs <- withAlonzoLedgerState interpreter $ \st -> + forM (chunksOf 10 blockCreds) $ \txCreds -> --do -- 10 per tx + Alonzo.mkDCertTx + (fmap (\ (poolIx, cred) -> DCertDeleg $ Delegate $ Delegation cred (resolvePool (PoolIndex poolIx) st)) + (zip (cycle [0,1,2]) txCreds)) + (Wdrl mempty) + forgeNextFindLeader interpreter (TxAlonzo <$> blockTxs) + + let utxoIndex = UTxOAddress addrFrom + sendBlocks <- forM (chunksOf 500 addresses) $ \blockAddresses -> do + blockTxs <- withAlonzoLedgerState interpreter $ \st -> + forM (chunksOf 10 blockAddresses) $ \txAddresses -> + Alonzo.mkPaymentTx' utxoIndex (fmap (\addr -> (UTxOAddress addr, Value 1 mempty)) txAddresses) st + forgeNextFindLeader interpreter (TxAlonzo <$> blockTxs) + pure $ registerBlocks <> delegateBlocks <> sendBlocks + where + creds = createStakeCredentials n + pcreds = createPaymentCredentials n + addresses = fmap (\(pcred, cred) -> Addr Testnet pcred (StakeRefBase cred)) (zip pcreds creds) diff --git a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo.hs b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo.hs index 55d7e4afb..665687452 100644 --- a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo.hs +++ b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo.hs @@ -8,7 +8,7 @@ module Cardano.Mock.Forging.Tx.Alonzo where -import Cardano.Prelude hiding ((.)) +import Cardano.Prelude hiding (sum, (.)) import qualified Data.Map.Strict as Map import Data.Maybe (fromJust) @@ -109,6 +109,24 @@ mkPaymentTx inputIndex outputIndex amount fees sta = do change = TxOut addr' (valueFromList (fromIntegral $ fromIntegral inputValue - amount - fees) []) Strict.SNothing Right $ mkSimpleTx True $ consPaymentTxBody input mempty (StrictSeq.fromList [output, change]) (Coin fees) mempty +mkPaymentTx' :: AlonzoUTxOIndex + -> [(AlonzoUTxOIndex, Value StandardCrypto)] + -> AlonzoLedgerState + -> Either ForgingError (ValidatedTx (AlonzoEra StandardCrypto)) +mkPaymentTx' inputIndex outputIndex sta = do + inputPair <- fst <$> resolveUTxOIndex inputIndex sta + outps <- mapM mkOuts outputIndex + + let inps = Set.singleton $ fst inputPair + TxOut addr' (Value inputValue _) _ = snd inputPair + outValue = sum ((\ (Value vl _) -> vl) . snd <$> outputIndex) + change = TxOut addr' (valueFromList (fromIntegral $ fromIntegral inputValue - outValue) []) Strict.SNothing + Right $ mkSimpleTx True $ consPaymentTxBody inps mempty (StrictSeq.fromList $ outps ++ [change]) (Coin 0) mempty + where + mkOuts (outIx, vl) = do + addr <- resolveAddress outIx sta + Right $ TxOut addr vl Strict.SNothing + mkLockByScriptTx :: AlonzoUTxOIndex -> [Bool] -> Integer -> Integer -> AlonzoLedgerState -> Either ForgingError (ValidatedTx (AlonzoEra StandardCrypto)) diff --git a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Generic.hs b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Generic.hs index fcc1d5148..36192f7f8 100644 --- a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Generic.hs +++ b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Generic.hs @@ -11,6 +11,9 @@ module Cardano.Mock.Forging.Tx.Generic , resolveUTxOIndex , resolveStakeCreds , resolvePool + , createStakeCredentials + , createPaymentCredentials + , mkDummyScriptHash ) where import Cardano.Prelude hiding (length, (.)) @@ -24,7 +27,9 @@ import Cardano.Ledger.Address import Cardano.Ledger.BaseTypes import qualified Cardano.Ledger.Core as Core import Cardano.Ledger.Credential +import Cardano.Ledger.Crypto (ADDRHASH) import Cardano.Ledger.Era (Crypto) +import Cardano.Ledger.Hashes (ScriptHash (ScriptHash)) import Cardano.Ledger.Keys import Cardano.Ledger.Shelley.LedgerState hiding (LedgerState) import Cardano.Ledger.Shelley.TxBody @@ -39,6 +44,8 @@ import qualified Ouroboros.Consensus.Shelley.Ledger.Ledger as Consensus import Cardano.Mock.Forging.Tx.Alonzo.ScriptsExamples import Cardano.Mock.Forging.Types +import Test.Cardano.Ledger.Shelley.Examples.Consensus (mkDummyHash) + resolveAddress :: forall era. (Crypto era ~ StandardCrypto, HasField "address" (Core.TxOut era) (Addr (Crypto era))) => UTxOIndex era -> LedgerState (ShelleyBlock era) -> Either ForgingError (Addr (Crypto era)) @@ -161,3 +168,14 @@ unregisteredPools = , KeyHash "222462543264795t3298745680239746523897456238974563298348" , KeyHash "33323876542397465497834256329487563428975634827956348975" ] + +createStakeCredentials :: Int -> [StakeCredential StandardCrypto] +createStakeCredentials n = + fmap (KeyHashObj . KeyHash . mkDummyHash (Proxy @(ADDRHASH StandardCrypto))) [1..n] + +createPaymentCredentials :: Int -> [PaymentCredential StandardCrypto] +createPaymentCredentials n = + fmap (KeyHashObj . KeyHash . mkDummyHash (Proxy @(ADDRHASH StandardCrypto))) [1..n] + +mkDummyScriptHash :: Int -> ScriptHash StandardCrypto +mkDummyScriptHash n = ScriptHash $ mkDummyHash (Proxy @(ADDRHASH StandardCrypto)) n diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/UnifiedApi.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/UnifiedApi.hs index afaf817fb..9d55b506a 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/UnifiedApi.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/UnifiedApi.hs @@ -1,6 +1,7 @@ module Test.Cardano.Db.Mock.UnifiedApi where import Control.Monad.Class.MonadSTM.Strict +import Data.Word (Word64) import Cardano.Slotting.Slot (SlotNo (..)) @@ -28,6 +29,12 @@ forgeNextFindLeaderAndSubmit interpreter mockServer txs' = do atomically $ addBlock mockServer blk pure blk +forgeNextSkipSlotsFindLeaderAndSubmit :: Interpreter -> ServerHandle IO CardanoBlock -> Word64 -> [TxEra] -> IO CardanoBlock +forgeNextSkipSlotsFindLeaderAndSubmit interpreter mockServer skipSlots txs' = do + blk <- forgeNextAfter interpreter skipSlots txs' + atomically $ addBlock mockServer blk + pure blk + forgeAndSubmitBlocks :: Interpreter -> ServerHandle IO CardanoBlock -> Int -> IO [CardanoBlock] forgeAndSubmitBlocks interpreter mockServer blocksToCreate = do forM [1..blocksToCreate] $ \_ -> forgeNextFindLeaderAndSubmit interpreter mockServer [] diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit.hs index 6dfbc3cfe..512c36499 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit.hs @@ -36,6 +36,7 @@ import Cardano.SMASH.Server.PoolDataLayer import Cardano.SMASH.Server.Types import Cardano.Mock.ChainSync.Server +import Cardano.Mock.Forging.Examples import Cardano.Mock.Forging.Interpreter import qualified Cardano.Mock.Forging.Tx.Alonzo as Alonzo import Cardano.Mock.Forging.Tx.Alonzo.ScriptsExamples @@ -95,6 +96,14 @@ unitTests iom knownMigrations = , test "rollback on epoch boundary" rollbackBoundary , test "single MIR Cert multiple outputs" singleMIRCertMultiOut ] + , testGroup "stake distribution" + [ test "stake distribution from genesis" stakeDistGenesis + , test "2000 delegations" delegations2000 + , test "2001 delegations" delegations2001 + , test "8000 delegations" delegations8000 + , test "many delegations" delegationsMany + , test "many delegations, not dense chain" delegationsManyNotDense + ] , testGroup "plutus spend scripts" [ test "simple script lock" simpleScript , test "unlock script in same block" unlockScriptSameBlock @@ -842,6 +851,129 @@ singleMIRCertMultiOut = where testLabel = "singleMIRCertMultiOut" +stakeDistGenesis :: IOManager -> [(Text, Text)] -> Assertion +stakeDistGenesis = + withFullConfig "config" testLabel $ \interpreter mockServer dbSync -> do + startDBSync dbSync + a <- fillUntilNextEpoch interpreter mockServer + assertBlockNoBackoff dbSync (fromIntegral $ length a - 1) + -- There are 5 delegations in genesis + assertEpochStake dbSync 5 + where + testLabel = "stakeDistGenesis" + +delegations2000 :: IOManager -> [(Text, Text)] -> Assertion +delegations2000 = + withFullConfig "config" testLabel $ \interpreter mockServer dbSync -> do + startDBSync dbSync + a <- delegateAndSendBlocks 1995 interpreter + forM_ a $ atomically . addBlock mockServer + b <- fillUntilNextEpoch interpreter mockServer + c <- fillUntilNextEpoch interpreter mockServer + + assertBlockNoBackoff dbSync (fromIntegral $ length a + length b + length c - 1) + -- There are exactly 2000 entries on the second epoch, 5 from genesis and 1995 manually added + assertEpochStakeEpoch dbSync 2 2000 + + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertBlockNoBackoff dbSync (fromIntegral $ length a + length b + length c) + assertEpochStakeEpoch dbSync 2 2000 + where + testLabel = "delegations2000" + +delegations2001 :: IOManager -> [(Text, Text)] -> Assertion +delegations2001 = + withFullConfig "config" testLabel $ \interpreter mockServer dbSync -> do + startDBSync dbSync + a <- delegateAndSendBlocks 1996 interpreter + forM_ a $ atomically . addBlock mockServer + b <- fillUntilNextEpoch interpreter mockServer + c <- fillUntilNextEpoch interpreter mockServer + + assertBlockNoBackoff dbSync (fromIntegral $ length a + length b + length c - 1) + -- The first block of epoch inserts 2000 out of 2001 epoch distribution. + assertEpochStakeEpoch dbSync 2 2000 + -- The remaining entry is inserted on the next block. + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertBlockNoBackoff dbSync (fromIntegral $ length a + length b + length c) + assertEpochStakeEpoch dbSync 2 2001 + where + testLabel = "delegations2001" + +delegations8000 :: IOManager -> [(Text, Text)] -> Assertion +delegations8000 = + withFullConfig "config" testLabel $ \interpreter mockServer dbSync -> do + startDBSync dbSync + a <- delegateAndSendBlocks 7995 interpreter + forM_ a $ atomically . addBlock mockServer + b <- fillEpochs interpreter mockServer 3 + + assertBlockNoBackoff dbSync (fromIntegral $ length a + length b - 1) + assertEpochStakeEpoch dbSync 3 2000 + + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertEpochStakeEpoch dbSync 3 4000 + + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertEpochStakeEpoch dbSync 3 6000 + + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertEpochStakeEpoch dbSync 3 8000 + + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertEpochStakeEpoch dbSync 3 8000 + where + testLabel = "delegations8000" + +delegationsMany :: IOManager -> [(Text, Text)] -> Assertion +delegationsMany = + withFullConfig "config" testLabel $ \interpreter mockServer dbSync -> do + startDBSync dbSync + a <- delegateAndSendBlocks 40000 interpreter + forM_ a $ atomically . addBlock mockServer + b <- fillEpochs interpreter mockServer 5 + + -- too long. We cannot use default delays + assertBlockNoBackoffTimes (repeat 10) dbSync (fromIntegral $ length a + length b - 1) + -- The slice size here is + -- 1 + div (delegationsLen * 5) expectedBlocks = 2001 + -- instead of 2000, because there are many delegations + assertEpochStakeEpoch dbSync 7 2001 + + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertEpochStakeEpoch dbSync 7 4002 + + _ <- forgeNextFindLeaderAndSubmit interpreter mockServer [] + assertEpochStakeEpoch dbSync 7 6003 + where + testLabel = "delegationsMany" + +delegationsManyNotDense :: IOManager -> [(Text, Text)] -> Assertion +delegationsManyNotDense = + withFullConfig "config" testLabel $ \interpreter mockServer dbSync -> do + startDBSync dbSync + a <- delegateAndSendBlocks 40000 interpreter + forM_ a $ atomically . addBlock mockServer + b <- fillEpochs interpreter mockServer 5 + + -- too long. We cannot use default delays + assertBlockNoBackoffTimes (repeat 10) dbSync (fromIntegral $ length a + length b - 1) + -- The slice size here is + -- 1 + div (delegationsLen * 5) expectedBlocks = 2001 + -- instead of 2000, because there are many delegations + assertEpochStakeEpoch dbSync 7 2001 + + -- Blocks come on average every 5 slots. If we skip 15 slots before each block, + -- we are expected to get only 1/4 of the expected blocks. The adjusted slices + -- should still be long enough to cover everything. + replicateM_ 40 $ + forgeNextSkipSlotsFindLeaderAndSubmit interpreter mockServer 15 [] + + -- Even if the chain is not dense, all distributions are inserted. + assertEpochStakeEpoch dbSync 7 40005 + where + testLabel = "delegationsManyNotDense" + simpleScript :: IOManager -> [(Text, Text)] -> Assertion simpleScript = withFullConfig "config" testLabel $ \interpreter mockServer dbSync -> do diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs index 5ba17b016..6dbf8f109 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs @@ -25,7 +25,7 @@ import Data.Word (Word64) import GHC.Records (HasField (..)) import Database.Esqueleto.Legacy (InnerJoin (..), SqlExpr, countRows, from, on, select, - unValue, (==.), (^.)) + unValue, val, where_, (==.), (^.)) import Database.Persist.Sql (Entity, SqlBackend, entityVal) import Database.PostgreSQL.Simple (SqlError (..)) @@ -66,8 +66,11 @@ assertRewardCount env n = assertEqBackoff env queryRewardCount n defaultDelays "Unexpected rewards count" assertBlockNoBackoff :: DBSyncEnv -> Int -> IO () -assertBlockNoBackoff env blockNo = - assertEqBackoff env queryBlockHeight (Just $ fromIntegral blockNo) defaultDelays "Unexpected BlockNo" +assertBlockNoBackoff = assertBlockNoBackoffTimes defaultDelays + +assertBlockNoBackoffTimes :: [Int] -> DBSyncEnv -> Int -> IO () +assertBlockNoBackoffTimes times env blockNo = + assertEqBackoff env queryBlockHeight (Just $ fromIntegral blockNo) times "Unexpected BlockNo" defaultDelays :: [Int] defaultDelays = [1,2,4,8,16,32,64,128] @@ -201,6 +204,24 @@ assertRewardCounts env st filterAddr expected = do pure (reward, stake_addr ^. StakeAddressHashRaw) pure $ fmap (bimap entityVal unValue) res +assertEpochStake :: DBSyncEnv -> Word64 -> IO () +assertEpochStake env expected = + assertEqBackoff env q expected defaultDelays "Unexpected epoch stake counts" + where + q = + maybe 0 unValue . listToMaybe <$> + (select . from $ \(_a :: SqlExpr (Entity EpochStake)) -> pure countRows) + +assertEpochStakeEpoch :: DBSyncEnv -> Word64 -> Word64 -> IO () +assertEpochStakeEpoch env e expected = + assertEqBackoff env q expected defaultDelays "Unexpected epoch stake counts" + where + q = + maybe 0 unValue . listToMaybe <$> + (select . from $ \(a :: SqlExpr (Entity EpochStake)) -> do + where_ (a ^. EpochStakeEpochNo ==. val e) + pure countRows + ) assertAlonzoCounts :: DBSyncEnv -> (Word64, Word64, Word64, Word64, Word64, Word64, Word64, Word64) -> IO () assertAlonzoCounts env expected = diff --git a/cardano-chain-gen/test/testfiles/config/genesis.byron.json b/cardano-chain-gen/test/testfiles/config/genesis.byron.json index cf088f937..25ced6be5 100644 --- a/cardano-chain-gen/test/testfiles/config/genesis.byron.json +++ b/cardano-chain-gen/test/testfiles/config/genesis.byron.json @@ -26,6 +26,6 @@ { "summand": "155381000000000" , "multiplier": "43946000000" } , "unlockStakeEpoch": "18446744073709551615" } -, "protocolConsts": { "k": 216 , "protocolMagic": 42 } +, "protocolConsts": { "k": 10 , "protocolMagic": 42 } , "avvmDistr": {} } diff --git a/cardano-chain-gen/test/testfiles/config/test-config.json b/cardano-chain-gen/test/testfiles/config/test-config.json index 60a3306d4..64e59bf85 100644 --- a/cardano-chain-gen/test/testfiles/config/test-config.json +++ b/cardano-chain-gen/test/testfiles/config/test-config.json @@ -4,7 +4,7 @@ "ApplicationName": "cardano-sl", "ApplicationVersion": 0, "ByronGenesisFile": "genesis.byron.json", - "ByronGenesisHash": "462bb9869a5a6e4325cc294ca659d68607e8a6f37b5be96ea663fdedfe2b5949", + "ByronGenesisHash": "865a9da19944fc00cefe5a0cdcb6dc9d3964a74681d9488ab79967f13a6400b3", "LastKnownBlockVersion-Alt": 0, "LastKnownBlockVersion-Major": 5, "LastKnownBlockVersion-Minor": 1, diff --git a/cardano-chain-gen/test/testfiles/fingerprint/delegations2000 b/cardano-chain-gen/test/testfiles/fingerprint/delegations2000 new file mode 100644 index 000000000..969f76da4 --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/delegations2000 @@ -0,0 +1 @@ +[3,4,6,13,22,31,43,46,52,54,56,64,69,73,76,78,79,90,98,99,104,105,106,109,111,113,114,120,122,124,132,142,143,144,152,158,163,167,199,201,209,210,216,219,235,237,254,261,268,273,275,279,280,285,303,304,306,309,310,313,322,325,327,329,330,331,335,340,346,348,353,361,366,372,373,376,377,378,383,388,391,392,393,397,406,408,409,413,420,422,430,438,440,444,452,454,456,460,461,468,469,471,478,483,488,503,507,511,513,514,529,530,536,547,550,554,556,558,561,562,565,566,577,591,594,595,596,599,605,606,611,620,625,627,629,643,644,655,658,667,676,684,690,695,701,704,711,715,720,723,730,732,737,739,741,748,752,755,758,765,772,776,779,780,781,785,790,806,810,816,818,820,822,835,843,846,852,854,855,858,864,865,866,875,878,880,883,888,891,900,904,905,906,907,911,914,917,920,925,934,939,945,951,953,959,962,968,971,975,979,983,988,992,993,996,1002,1003] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/delegations2001 b/cardano-chain-gen/test/testfiles/fingerprint/delegations2001 new file mode 100644 index 000000000..51e5c28f8 --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/delegations2001 @@ -0,0 +1 @@ +[3,4,6,13,22,31,43,46,52,54,56,64,69,73,76,78,79,90,98,99,104,105,106,109,111,113,114,120,122,124,132,142,143,144,152,158,163,167,199,201,209,210,216,219,235,237,254,261,268,273,275,279,280,285,303,304,306,309,310,313,322,325,327,329,330,331,335,340,346,348,353,361,366,372,373,376,377,378,383,388,391,392,393,397,406,408,409,413,420,422,430,438,440,444,452,454,456,460,461,468,469,471,478,483,488,503,507,511,513,514,529,530,536,547,550,554,556,558,561,562,565,566,577,591,594,595,596,599,605,606,611,620,625,627,629,643,644,655,658,667,676,684,690,695,701,704,711,715,720,723,730,732,737,739,741,748,752,755,758,765,772,776,779,780,781,785,790,806,810,816,818,820,822,835,843,846,852,854,855,858,864,865,866,875,878,880,883,888,891,900,904,905,906,907,911,914,917,920,925,934,939,945,951,953,959,962,968,971,975,979,983,988,992,993,996,1006,1011] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/delegations8000 b/cardano-chain-gen/test/testfiles/fingerprint/delegations8000 new file mode 100644 index 000000000..f0a18e6c0 --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/delegations8000 @@ -0,0 +1 @@ +[3,4,6,13,22,31,43,46,52,54,56,64,69,73,76,78,79,90,98,99,104,105,106,109,111,113,114,120,122,124,132,142,143,144,152,158,163,167,199,201,209,210,216,219,235,237,254,261,268,273,275,279,280,285,303,304,306,309,310,313,322,325,327,329,330,331,335,340,346,348,353,361,366,372,373,376,377,378,383,388,391,392,393,397,406,408,409,413,420,422,430,438,440,444,452,454,456,460,461,468,469,471,478,483,488,503,507,511,513,514,529,530,536,547,550,554,556,558,561,562,565,566,577,591,594,595,596,599,605,606,611,620,625,627,629,643,644,655,658,667,676,684,690,695,701,704,711,715,720,723,730,732,737,739,741,748,752,755,758,765,772,776,779,780,781,785,790,806,810,816,818,820,822,835,843,846,852,854,855,858,864,865,866,875,878,880,883,888,891,900,904,905,906,907,911,914,917,920,925,934,939,945,951,953,959,962,968,971,975,979,983,988,992,993,996,1002,1007,1012,1014,1021,1022,1025,1026,1028,1038,1041,1058,1059,1060,1062,1063,1064,1069,1070,1079,1084,1092,1093,1109,1110,1111,1119,1120,1132,1134,1135,1148,1149,1167,1173,1178,1189,1192,1194,1195,1196,1202,1203,1209,1221,1223,1232,1233,1240,1246,1257,1263,1267,1268,1271,1273,1277,1280,1282,1285,1297,1299,1306,1313,1318,1319,1323,1325,1328,1333,1336,1341,1344,1348,1350,1352,1359,1362,1371,1372,1378,1379,1383,1393,1397,1399,1414,1417,1420,1421,1433,1434,1435,1460,1466,1467,1472,1474,1476,1479,1481,1482,1485,1497,1499,1503,1506,1515,1517,1521] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/delegationsMany b/cardano-chain-gen/test/testfiles/fingerprint/delegationsMany new file mode 100644 index 000000000..9de734b61 --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/delegationsMany @@ -0,0 +1 @@ +[3,4,6,13,22,31,43,46,52,54,56,64,69,73,76,78,79,90,98,99,104,105,106,109,111,113,114,120,122,124,132,142,143,144,152,158,163,167,199,201,209,210,216,219,235,237,254,261,268,273,275,279,280,285,303,304,306,309,310,313,322,325,327,329,330,331,335,340,346,348,353,361,366,372,373,376,377,378,383,388,391,392,393,397,406,408,409,413,420,422,430,438,440,444,452,454,456,460,461,468,469,471,478,483,488,503,507,511,513,514,529,530,536,547,550,554,556,558,561,562,565,566,577,591,594,595,596,599,605,606,611,620,625,627,629,643,644,655,658,667,676,684,690,695,701,704,711,715,720,723,730,732,737,739,741,748,752,755,758,765,772,776,779,780,781,785,790,806,810,816,818,820,822,835,843,846,852,854,855,858,864,865,866,875,878,880,883,888,891,900,904,905,906,907,911,914,917,920,925,934,939,945,951,953,959,962,968,971,975,979,983,988,992,993,996,1004,1010,1013,1019,1031,1034,1038,1041,1052,1060,1061,1069,1076,1082,1090,1105,1107,1113,1116,1126,1128,1130,1134,1136,1149,1154,1156,1157,1158,1160,1169,1176,1177,1185,1186,1190,1200,1202,1205,1207,1208,1209,1214,1215,1222,1227,1230,1234,1235,1239,1257,1259,1264,1272,1275,1278,1283,1285,1288,1290,1293,1304,1308,1321,1325,1326,1331,1334,1337,1340,1344,1348,1349,1350,1362,1364,1367,1369,1372,1373,1376,1384,1385,1388,1391,1397,1403,1406,1408,1415,1417,1427,1431,1432,1435,1436,1437,1438,1465,1466,1470,1472,1474,1491,1499,1500,1518,1529,1545,1546,1559,1560,1562,1573,1574,1576,1579,1586,1606,1610,1613,1614,1620,1621,1629,1634,1642,1646,1648,1653,1660,1665,1668,1674,1683,1686,1699,1703,1715,1716,1718,1719,1731,1745,1752,1761,1764,1766,1769,1777,1784,1786,1791,1792,1799,1804,1814,1821,1836,1839,1841,1851,1852,1869,1879,1880,1882,1888,1891,1905,1906,1907,1908,1915,1925,1926,1929,1933,1939,1946,1955,1956,1959,1967,1974,1978,1981,1985,1988,1991,1997,2001,2002,2011,2015,2016,2017,2022,2030,2031,2033,2037,2042,2050,2056,2060,2064,2070,2077,2079,2088,2090,2095,2096,2104,2105,2111,2112,2116,2123,2130,2132,2135,2137,2138,2140,2147,2149,2154,2167,2174,2177,2182,2185,2189,2192,2194,2206,2218,2224,2228,2232,2234,2257,2268,2269,2270,2271,2273,2278,2279,2287,2289,2290,2292,2306,2311,2313,2315,2319,2327,2330,2351,2352,2356,2362,2364,2369,2377,2383,2386,2395,2400,2404,2405,2406,2413,2415,2423,2424,2431,2432,2437,2438,2449,2464,2468,2469,2477,2486,2503,2504,2511,2517,2519,2522,2526,2527,2529,2544,2549,2551,2553,2559,2567,2569,2570,2571,2579,2581,2587,2588,2592,2594,2595,2607,2609,2610,2614,2617,2620,2622,2632,2636,2638,2644,2645,2646,2647,2649,2652,2655,2656,2658,2662,2666,2668,2674,2675,2676,2692,2698,2701,2713,2717,2727,2728,2730,2735,2738,2740,2742,2743,2763,2774,2785,2786,2788,2791,2802,2805,2809,2814,2816,2819,2833,2836,2839,2844,2845,2846,2858,2863,2867,2869,2871,2872,2874,2880,2882,2883,2893,2894,2895,2900,2909,2925,2929,2958,2960,2962,2965,2966,2973,2974,2979,2985,2987,3000,3001,3005,3008,3020,3028,3030,3031,3033,3036,3044,3046,3050,3061,3064,3066,3071,3076,3082,3088,3091,3106,3108,3117,3122,3123,3129,3130,3132,3135,3136,3144,3172,3174,3177,3178,3182,3190,3193,3200,3204,3210,3212,3214,3215,3216,3218,3219,3229,3230,3234,3241,3245,3248,3254,3263,3264,3268,3278,3280,3286,3289,3292,3295,3305,3306,3309,3317,3319,3320,3328,3333,3334,3336,3346,3356,3359,3360,3363,3364,3365,3368,3370,3373,3379,3384,3387,3389,3390,3395,3399,3409,3422,3425,3435,3438,3447,3450,3452,3461,3463,3467,3472,3477,3481,3485,3488,3491,3493,3495,3496,3498,3500,3502,3504] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/delegationsManyNotDense b/cardano-chain-gen/test/testfiles/fingerprint/delegationsManyNotDense new file mode 100644 index 000000000..c51ffeda4 --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/delegationsManyNotDense @@ -0,0 +1 @@ +[3,4,6,13,22,31,43,46,52,54,56,64,69,73,76,78,79,90,98,99,104,105,106,109,111,113,114,120,122,124,132,142,143,144,152,158,163,167,199,201,209,210,216,219,235,237,254,261,268,273,275,279,280,285,303,304,306,309,310,313,322,325,327,329,330,331,335,340,346,348,353,361,366,372,373,376,377,378,383,388,391,392,393,397,406,408,409,413,420,422,430,438,440,444,452,454,456,460,461,468,469,471,478,483,488,503,507,511,513,514,529,530,536,547,550,554,556,558,561,562,565,566,577,591,594,595,596,599,605,606,611,620,625,627,629,643,644,655,658,667,676,684,690,695,701,704,711,715,720,723,730,732,737,739,741,748,752,755,758,765,772,776,779,780,781,785,790,806,810,816,818,820,822,835,843,846,852,854,855,858,864,865,866,875,878,880,883,888,891,900,904,905,906,907,911,914,917,920,925,934,939,945,951,953,959,962,968,971,975,979,983,988,992,993,996,1004,1010,1013,1019,1031,1034,1038,1041,1052,1060,1061,1069,1076,1082,1090,1105,1107,1113,1116,1126,1128,1130,1134,1136,1149,1154,1156,1157,1158,1160,1169,1176,1177,1185,1186,1190,1200,1202,1205,1207,1208,1209,1214,1215,1222,1227,1230,1234,1235,1239,1257,1259,1264,1272,1275,1278,1283,1285,1288,1290,1293,1304,1308,1321,1325,1326,1331,1334,1337,1340,1344,1348,1349,1350,1362,1364,1367,1369,1372,1373,1376,1384,1385,1388,1391,1397,1403,1406,1408,1415,1417,1427,1431,1432,1435,1436,1437,1438,1465,1466,1470,1472,1474,1491,1499,1500,1518,1529,1545,1546,1559,1560,1562,1573,1574,1576,1579,1586,1606,1610,1613,1614,1620,1621,1629,1634,1642,1646,1648,1653,1660,1665,1668,1674,1683,1686,1699,1703,1715,1716,1718,1719,1731,1745,1752,1761,1764,1766,1769,1777,1784,1786,1791,1792,1799,1804,1814,1821,1836,1839,1841,1851,1852,1869,1879,1880,1882,1888,1891,1905,1906,1907,1908,1915,1925,1926,1929,1933,1939,1946,1955,1956,1959,1967,1974,1978,1981,1985,1988,1991,1997,2001,2002,2011,2015,2016,2017,2022,2030,2031,2033,2037,2042,2050,2056,2060,2064,2070,2077,2079,2088,2090,2095,2096,2104,2105,2111,2112,2116,2123,2130,2132,2135,2137,2138,2140,2147,2149,2154,2167,2174,2177,2182,2185,2189,2192,2194,2206,2218,2224,2228,2232,2234,2257,2268,2269,2270,2271,2273,2278,2279,2287,2289,2290,2292,2306,2311,2313,2315,2319,2327,2330,2351,2352,2356,2362,2364,2369,2377,2383,2386,2395,2400,2404,2405,2406,2413,2415,2423,2424,2431,2432,2437,2438,2449,2464,2468,2469,2477,2486,2503,2504,2511,2517,2519,2522,2526,2527,2529,2544,2549,2551,2553,2559,2567,2569,2570,2571,2579,2581,2587,2588,2592,2594,2595,2607,2609,2610,2614,2617,2620,2622,2632,2636,2638,2644,2645,2646,2647,2649,2652,2655,2656,2658,2662,2666,2668,2674,2675,2676,2692,2698,2701,2713,2717,2727,2728,2730,2735,2738,2740,2742,2743,2763,2774,2785,2786,2788,2791,2802,2805,2809,2814,2816,2819,2833,2836,2839,2844,2845,2846,2858,2863,2867,2869,2871,2872,2874,2880,2882,2883,2893,2894,2895,2900,2909,2925,2929,2958,2960,2962,2965,2966,2973,2974,2979,2985,2987,3000,3001,3005,3008,3020,3028,3030,3031,3033,3036,3044,3046,3050,3061,3064,3066,3071,3076,3082,3088,3091,3106,3108,3117,3122,3123,3129,3130,3132,3135,3136,3144,3172,3174,3177,3178,3182,3190,3193,3200,3204,3210,3212,3214,3215,3216,3218,3219,3229,3230,3234,3241,3245,3248,3254,3263,3264,3268,3278,3280,3286,3289,3292,3295,3305,3306,3309,3317,3319,3320,3328,3333,3334,3336,3346,3356,3359,3360,3363,3364,3365,3368,3370,3373,3379,3384,3387,3389,3390,3395,3399,3409,3422,3425,3435,3438,3447,3450,3452,3461,3463,3467,3472,3477,3481,3485,3488,3491,3493,3495,3496,3498,3500,3521,3543,3560,3576,3603,3624,3641,3657,3676,3694,3716,3732,3759,3787,3807,3826,3845,3862,3883,3903,3924,3948,3966,3983,4003,4019,4046,4062,4082,4101,4118,4136,4157,4173,4192,4210,4226,4246,4262,4284] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/stakeDistGenesis b/cardano-chain-gen/test/testfiles/fingerprint/stakeDistGenesis new file mode 100644 index 000000000..42445c6bb --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/stakeDistGenesis @@ -0,0 +1 @@ +[3,4,6,13,22,31,43,46,52,54,56,64,69,73,76,78,79,90,98,99,104,105,106,109,111,113,114,120,122,124,132,142,143,144,152,158,163,167,199,201,209,210,216,219,235,237,254,261,268,273,275,279,280,285,303,304,306,309,310,313,322,325,327,329,330,331,335,340,346,348,353,361,366,372,373,376,377,378,383,388,391,392,393,397,406,408,409,413,420,422,430,438,440,444,452,454,456,460,461,468,469,471,478,483,488,503] \ No newline at end of file diff --git a/cardano-db-sync/src/Cardano/DbSync/Default.hs b/cardano-db-sync/src/Cardano/DbSync/Default.hs index 6405d8308..4416d908d 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Default.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Default.hs @@ -23,7 +23,7 @@ import Cardano.DbSync.Era.Shelley.Adjust (adjustEpochRewards) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Shelley.Insert (insertShelleyBlock) import Cardano.DbSync.Era.Shelley.Insert.Epoch (finalizeEpochBulkOps, forceInsertRewards, - insertPoolDepositRefunds, isEmptyEpochBulkOps, postEpochRewards, postEpochStake) + insertPoolDepositRefunds, isEmptyEpochBulkOps, postEpochRewards) import Cardano.DbSync.Era.Shelley.Validate (validateEpochRewards) import Cardano.DbSync.Error import Cardano.DbSync.LedgerState (LedgerEvent (..), LedgerStateSnapshot (..), applyBlock, @@ -153,12 +153,6 @@ handleLedgerEvents tracer lenv point = , show (unEpochNo $ Generic.rwdEpoch rwds), " ", renderPoint point ] postEpochRewards lenv rwds point - LedgerStakeDist sdist -> do - liftIO . logInfo tracer $ mconcat - [ "Handling ", show (Map.size (Generic.sdistStakeMap sdist)), " stakes for epoch " - , show (unEpochNo $ Generic.sdistEpochNo sdist), " ", renderPoint point - ] - postEpochStake lenv sdist point LedgerRewardDist rwd -> lift $ stashPoolRewards tracer lenv rwd LedgerMirDist md -> diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs index 972d847ef..bd659a89c 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs @@ -78,10 +78,7 @@ insertABOBBoundary -> ExceptT SyncNodeError (ReaderT SqlBackend m) () insertABOBBoundary tracer blk details = do -- Will not get called in the OBFT part of the Byron era. - let prevHash = case Byron.boundaryPrevHash (Byron.boundaryHeader blk) of - Left gh -> Byron.genesisToHeaderHash gh - Right hh -> Byron.unHeaderHash hh - pbid <- liftLookupFail "insertABOBBoundary" $ DB.queryBlockId prevHash + pbid <- liftLookupFail "insertABOBBoundary" $ DB.queryBlockId (Byron.ebbPrevHash blk) slid <- lift . DB.insertSlotLeader $ DB.SlotLeader { DB.slotLeaderHash = BS.replicate 28 '\0' @@ -123,7 +120,7 @@ insertABlock => Trace IO Text -> Bool -> Byron.ABlock ByteString -> SlotDetails -> ExceptT SyncNodeError (ReaderT SqlBackend m) () insertABlock tracer firstBlockOfEpoch blk details = do - pbid <- liftLookupFail "insertABlock" $ DB.queryBlockId (Byron.unHeaderHash $ Byron.blockPreviousHash blk) + pbid <- liftLookupFail "insertABlock" $ DB.queryBlockId (Byron.blockPreviousHash blk) slid <- lift . DB.insertSlotLeader $ Byron.mkSlotLeader blk blkId <- lift . DB.insertBlock $ DB.Block diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Util.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Util.hs index 40919702f..336609000 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Util.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Util.hs @@ -15,6 +15,8 @@ module Cardano.DbSync.Era.Byron.Util , blockNumber , blockPayload , blockPreviousHash + , ebbPrevHash + , prevHash , epochNumber , genesisToHeaderHash , protocolVersion @@ -40,6 +42,7 @@ import qualified Cardano.Chain.Slotting as Byron import qualified Cardano.Chain.UTxO as Byron import qualified Cardano.Chain.Update as Byron +import qualified Ouroboros.Consensus.Byron.Ledger.Block as Byron import qualified Cardano.Db as DB @@ -91,8 +94,19 @@ blockPayload :: Byron.ABlock a -> [Byron.TxAux] blockPayload = Byron.unTxPayload . Byron.bodyTxPayload . Byron.blockBody -blockPreviousHash :: Byron.ABlock a -> Byron.HeaderHash -blockPreviousHash = Byron.headerPrevHash . Byron.blockHeader +blockPreviousHash :: Byron.ABlock a -> ByteString +blockPreviousHash = unHeaderHash . Byron.headerPrevHash . Byron.blockHeader + +ebbPrevHash :: Byron.ABoundaryBlock a -> ByteString +ebbPrevHash bblock = + case Byron.boundaryPrevHash (Byron.boundaryHeader bblock) of + Left gh -> genesisToHeaderHash gh + Right hh -> unHeaderHash hh + +prevHash :: Byron.ByronBlock -> ByteString +prevHash blk = case Byron.byronBlockRaw blk of + Byron.ABOBBlock ablk -> blockPreviousHash ablk + Byron.ABOBBoundary abblk -> ebbPrevHash abblk epochNumber :: Byron.ABlock ByteString -> Word64 -> Word64 epochNumber blk slotsPerEpoch = diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs index d1ecd7517..5dc31454a 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs @@ -14,6 +14,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Block , blockHash , slotLeaderHash + , blockPrevHash ) where import qualified Cardano.Api.Shelley as Api diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/StakeDist.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/StakeDist.hs index a2e30bff8..0b3f0ce0a 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/StakeDist.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/StakeDist.hs @@ -1,14 +1,21 @@ {-# LANGUAGE DataKinds #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE GADTs #-} {-# LANGUAGE NoImplicitPrelude #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TupleSections #-} + module Cardano.DbSync.Era.Shelley.Generic.StakeDist - ( StakeDist (..) - , epochStakeDist + ( StakeSliceRes (..) + , StakeSlice (..) , stakeDistPoolHashKeys , stakeDistStakeCreds + , getSecurityParameter + , getStakeSlice ) where import Cardano.Prelude +import Prelude (id) import Cardano.Crypto.Hash (hashToBytes) @@ -21,82 +28,135 @@ import Cardano.Ledger.Keys (KeyHash (..), KeyRole (..)) import qualified Cardano.Ledger.Shelley.EpochBoundary as Shelley import qualified Cardano.Ledger.Shelley.LedgerState as Shelley hiding (_delegations) -import Cardano.Slotting.Slot (EpochNo (..)) - import Cardano.DbSync.Era.Shelley.Generic.StakeCred import Cardano.DbSync.Era.Shelley.Generic.StakePoolKeyHash import Cardano.DbSync.Types +import Data.Compact.VMap (VB, VMap (..), VP) import qualified Data.Compact.VMap as VMap import qualified Data.Map.Strict as Map import qualified Data.Set as Set +import qualified Data.Vector.Generic as VG +import Ouroboros.Consensus.Block import Ouroboros.Consensus.Cardano.Block (LedgerState (..)) - +import Ouroboros.Consensus.Config +import Ouroboros.Consensus.HardFork.Combinator import Ouroboros.Consensus.Ledger.Extended (ExtLedgerState (..)) -import Ouroboros.Consensus.Shelley.Ledger.Block (ShelleyBlock) +import Ouroboros.Consensus.Node.ProtocolInfo +import Ouroboros.Consensus.Protocol.Abstract +import Ouroboros.Consensus.Shelley.Ledger import qualified Ouroboros.Consensus.Shelley.Ledger.Ledger as Consensus +data StakeSliceRes = + Slice StakeSlice Bool -- True if this is the final slice for this epoch. Can be used for logging. + | NoSlices -data StakeDist = StakeDist - { sdistEpochNo :: !EpochNo - , sdistStakeMap :: !(Map StakeCred (Coin, StakePoolKeyHash)) +data StakeSlice = StakeSlice + { sliceEpochNo :: !EpochNo + , sliceDistr :: !(Map StakeCred (Coin, StakePoolKeyHash)) } deriving Eq -epochStakeDist :: Ledger.Network -> EpochNo -> ExtLedgerState CardanoBlock -> Maybe StakeDist -epochStakeDist network epoch els = +emptySlice :: EpochNo -> StakeSlice +emptySlice epoch = StakeSlice epoch Map.empty + +getSecurityParameter :: ConsensusProtocol (BlockProtocol blk) + => ProtocolInfo IO blk -> Word64 +getSecurityParameter = maxRollbacks . configSecurityParam . pInfoConfig + +-- 'sliceIndex' can match the epochBlockNo for every block. +-- +-- 'minSliceSize' has to be constant or it could cause missing data. +-- If this value is too small it will be adjusted to a 'defaultEpochSliceSize' +-- which is big enough to cover all delegations. +-- On mainnet, for a value minSliceSize = 2000, it will be used as the actual size of slices +-- until the size of delegations grows up to 8.6M, in which case, the size of slices +-- will be adjusted. +getStakeSlice :: ConsensusProtocol (BlockProtocol blk) + => ProtocolInfo IO blk -> Ledger.Network + -> EpochNo -> Word64 -> Word64 -> ExtLedgerState CardanoBlock -> StakeSliceRes +getStakeSlice pInfo network epoch sliceIndex minSliceSize els = case ledgerState els of - LedgerStateByron _ -> Nothing - LedgerStateShelley sls -> Just $ genericStakeDist network epoch sls - LedgerStateAllegra als -> Just $ genericStakeDist network epoch als - LedgerStateMary mls -> Just $ genericStakeDist network epoch mls - LedgerStateAlonzo als -> Just $ genericStakeDist network epoch als - --- Use Set because they guarantee unique elements. -stakeDistPoolHashKeys :: StakeDist -> Set StakePoolKeyHash -stakeDistPoolHashKeys = Set.fromList . map snd . Map.elems . sdistStakeMap - -stakeDistStakeCreds :: StakeDist -> Set StakeCred -stakeDistStakeCreds = Map.keysSet . sdistStakeMap - --- ------------------------------------------------------------------------------------------------- - -genericStakeDist :: forall era. Ledger.Network -> EpochNo -> LedgerState (ShelleyBlock era) -> StakeDist -genericStakeDist network epoch lstate = - StakeDist - { sdistEpochNo = epoch - , sdistStakeMap = stakeMap - } + LedgerStateByron _ -> NoSlices + LedgerStateShelley sls -> genericStakeSlice pInfo network epoch sliceIndex minSliceSize sls + LedgerStateAllegra als -> genericStakeSlice pInfo network epoch sliceIndex minSliceSize als + LedgerStateMary mls -> genericStakeSlice pInfo network epoch sliceIndex minSliceSize mls + LedgerStateAlonzo als -> genericStakeSlice pInfo network epoch sliceIndex minSliceSize als + +genericStakeSlice :: forall era c blk. (c ~ Crypto era, ConsensusProtocol (BlockProtocol blk)) + => ProtocolInfo IO blk -> Ledger.Network -> EpochNo -> Word64 -> Word64 + -> LedgerState (ShelleyBlock era) -> StakeSliceRes +genericStakeSlice pInfo network epoch sliceIndex minSliceSize lstate + | index > delegationsLen = NoSlices + | index == delegationsLen = Slice (emptySlice epoch) True + | index + epochSliceSize > delegationsLen = Slice (mkSlice (delegationsLen - index)) True + | otherwise = Slice (mkSlice epochSliceSize) False where - stakeMap :: Map StakeCred (Coin, StakePoolKeyHash) - stakeMap = Map.intersectionWith (,) stakeCoinMap stakePoolMap - - stakeCoinMap :: Map StakeCred Coin - stakeCoinMap = mapBimap (toStakeCred network) Ledger.fromCompact stMap - - stMap :: Map (Credential 'Staking (Crypto era)) (Ledger.CompactForm Coin) - stMap = VMap.toMap . Shelley.unStake $ Shelley._stake stakeSet - - stakePoolMap :: Map StakeCred StakePoolKeyHash - stakePoolMap = mapBimap (toStakeCred network) convertStakePoolkeyHash delMap - - delMap :: Map (Credential 'Staking (Crypto era)) (KeyHash 'StakePool (Crypto era)) - delMap = VMap.toMap $ Shelley._delegations stakeSet - -- We use '_pstakeSet' here instead of '_pstateMark' because the stake addresses for the -- later may not have been added to the database yet. That means that when these values -- are added to the database, the epoch number where they become active is the current -- epoch plus one. - stakeSet :: Shelley.SnapShot (Crypto era) - stakeSet = Shelley._pstakeSet . Shelley.esSnapshots . Shelley.nesEs + stakeSnapshot :: Shelley.SnapShot c + stakeSnapshot = Shelley._pstakeSet . Shelley.esSnapshots . Shelley.nesEs $ Consensus.shelleyLedgerState lstate - convertStakePoolkeyHash :: KeyHash 'StakePool (Crypto era) -> StakePoolKeyHash + delegations :: VMap.KVVector VB VB (Credential 'Staking c, KeyHash 'StakePool c) + delegations = VMap.unVMap $ Shelley._delegations stakeSnapshot + + delegationsLen :: Word64 + delegationsLen = fromIntegral $ VG.length delegations + + stakes :: VMap VB VP (Credential 'Staking c) (Ledger.CompactForm Coin) + stakes = Shelley.unStake $ Shelley._stake stakeSnapshot + + lookupStake :: Credential 'Staking c -> Maybe Coin + lookupStake cred = Ledger.fromCompact <$> VMap.lookup cred stakes + + -- This is deterministic for the whole epoch and is the constant size of slices + -- until the data are over. This means the last slice could be of smaller size and slices + -- after that will be empty. + epochSliceSize :: Word64 + epochSliceSize = + max minSliceSize defaultEpochSliceSize + where + -- On mainnet this is 2160 + k :: Word64 + k = getSecurityParameter pInfo + + -- On mainnet this is 21600 + expectedBlocks :: Word64 + expectedBlocks = 10 * k + + -- This size of slices is enough to cover the whole list, even if only + -- the 20% of the expected blocks appear in an epoch. + defaultEpochSliceSize :: Word64 + defaultEpochSliceSize = 1 + div (delegationsLen * 5) expectedBlocks + + -- The starting index of the data in the delegation vector. + index :: Word64 + index = sliceIndex * epochSliceSize + + mkSlice :: Word64 -> StakeSlice + mkSlice size = + StakeSlice + { sliceEpochNo = epoch + , sliceDistr = distribution + } + where + delegationsSliced :: VMap VB VB (Credential 'Staking c) (KeyHash 'StakePool c) + delegationsSliced = VMap $ VG.slice (fromIntegral index) (fromIntegral size) delegations + + distribution :: Map StakeCred (Coin, StakePoolKeyHash) + distribution = Map.mapKeys (toStakeCred network) $ VMap.toMap $ + VMap.mapMaybe id $ VMap.mapWithKey (\k p -> (, convertStakePoolkeyHash p) <$> lookupStake k) delegationsSliced + + convertStakePoolkeyHash :: KeyHash 'StakePool c -> StakePoolKeyHash convertStakePoolkeyHash (KeyHash h) = StakePoolKeyHash $ hashToBytes h --- Is there a better way to do this? -mapBimap :: Ord k2 => (k1 -> k2) -> (a1 -> a2) -> Map k1 a1 -> Map k2 a2 -mapBimap fk fa = Map.fromAscList . map (bimap fk fa) . Map.toAscList - +-- Use Set because they guarantee unique elements. +stakeDistPoolHashKeys :: StakeSlice -> Set StakePoolKeyHash +stakeDistPoolHashKeys = Set.fromList . map snd . Map.elems . sliceDistr +stakeDistStakeCreds :: StakeSlice -> Set StakeCred +stakeDistStakeCreds = Map.keysSet . sliceDistr diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs index 009d499be..8c143263d 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs @@ -10,7 +10,6 @@ module Cardano.DbSync.Era.Shelley.Insert ( insertShelleyBlock , postEpochRewards - , postEpochStake -- These are exported for data in Shelley Genesis , insertPoolRegister @@ -137,6 +136,8 @@ insertShelleyBlock tracer lenv firstBlockOfEpoch blk lStateSnap details = do whenJust (lssNewEpoch lStateSnap) $ \ newEpoch -> do insertOnNewEpoch tracer blkId (Generic.blkSlotNo blk) (sdEpochNo details) newEpoch + insertStakeSlice tracer (leIndexCache lenv) (lssStakeSlice lStateSnap) + mbop <- liftIO . atomically $ tryReadTBQueue (leBulkOpQueue lenv) whenJust (maybeToStrict mbop) $ \ bop -> insertEpochInterleaved tracer bop diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert/Epoch.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert/Epoch.hs index 347e70ec5..dd76c0c8a 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert/Epoch.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert/Epoch.hs @@ -15,7 +15,7 @@ module Cardano.DbSync.Era.Shelley.Insert.Epoch , insertEpochInterleaved , insertPoolDepositRefunds , postEpochRewards - , postEpochStake + , insertStakeSlice ) where import Cardano.Prelude @@ -36,8 +36,8 @@ import Cardano.DbSync.Util import Cardano.Slotting.Slot (EpochNo (..)) -import Control.Monad.Class.MonadSTM.Strict (flushTBQueue, isEmptyTBQueue, readTVar, - writeTBQueue, writeTVar) +import Control.Monad.Class.MonadSTM.Strict (StrictTVar, flushTBQueue, isEmptyTBQueue, + readTVar, readTVarIO, writeTBQueue, writeTVar) import Control.Monad.Extra (mapMaybeM) import Control.Monad.Trans.Control (MonadBaseControl) import Control.Monad.Trans.Except.Extra (hoistEither) @@ -93,19 +93,7 @@ insertEpochInterleaved tracer bop = BulkRewardReport epochNo _ rewardCount total -> do liftIO $ reportRewards epochNo rewardCount lift $ insertEpochRewardTotalReceived epochNo total - BulkStakeDistChunk epochNo _ icache sDistChunk -> - insertEpochStake tracer icache epochNo sDistChunk - BulkStakeDistReport epochNo _ count -> - liftIO $ reportStakeDist epochNo count where - reportStakeDist :: EpochNo -> Int -> IO () - reportStakeDist epochNo count = - logInfo tracer $ - mconcat - [ "insertEpochInterleaved: Epoch ", textShow (unEpochNo epochNo) - , ", ", textShow count, " stake addresses" - ] - reportRewards :: EpochNo -> Int -> IO () reportRewards epochNo rewardCount = logInfo tracer $ @@ -127,18 +115,6 @@ postEpochRewards lenv rwds point = do writeTBQueue (leBulkOpQueue lenv) $ BulkRewardReport epochNo point (length $ Generic.rwdRewards rwds) (sumRewardTotal $ Generic.rwdRewards rwds) -postEpochStake - :: (MonadBaseControl IO m, MonadIO m) - => LedgerEnv -> Generic.StakeDist -> CardanoPoint - -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -postEpochStake lenv smap point = do - icache <- lift $ updateIndexCache lenv (Generic.stakeDistStakeCreds smap) (Generic.stakeDistPoolHashKeys smap) - liftIO . atomically $ do - let epochNo = Generic.sdistEpochNo smap - forM_ (chunksOf 1000 $ Map.toList (Generic.sdistStakeMap smap)) $ \stakeChunk -> - writeTBQueue (leBulkOpQueue lenv) $ BulkStakeDistChunk epochNo point icache stakeChunk - writeTBQueue (leBulkOpQueue lenv) $ BulkStakeDistReport epochNo point (length $ Generic.sdistStakeMap smap) - isEmptyEpochBulkOps :: MonadIO m => LedgerEnv @@ -159,12 +135,26 @@ insertEpochRewardTotalReceived epochNo total = , DB.epochRewardTotalReceivedAmount = Generic.coinToDbLovelace total } +insertStakeSlice + :: (MonadBaseControl IO m, MonadIO m) + => Trace IO Text -> StrictTVar IO IndexCache -> Generic.StakeSliceRes + -> ExceptT SyncNodeError (ReaderT SqlBackend m) () +insertStakeSlice _ _ Generic.NoSlices = pure () +insertStakeSlice tracer cacheVar (Generic.Slice slice finalSlice) = do + cache <- liftIO $ readTVarIO cacheVar + -- cache TVar is not updated. We just use a slice here. + cacheSlice <- lift $ modifyCache (Generic.stakeDistStakeCreds slice) (Generic.stakeDistPoolHashKeys slice) cache + insertEpochStake cacheSlice (Generic.sliceEpochNo slice) (Map.toList $ Generic.sliceDistr slice) + when finalSlice $ do + size <- lift $ DB.queryEpochStakeCount (unEpochNo $ Generic.sliceEpochNo slice) + liftIO . logInfo tracer $ mconcat ["Inserted ", show size, " EpochStake for ", show (Generic.sliceEpochNo slice)] + insertEpochStake :: (MonadBaseControl IO m, MonadIO m) - => Trace IO Text -> IndexCache -> EpochNo + => IndexCache -> EpochNo -> [(Generic.StakeCred, (Shelley.Coin, Generic.StakePoolKeyHash))] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertEpochStake _tracer icache epochNo stakeChunk = do +insertEpochStake icache epochNo stakeChunk = do dbStakes <- mapM mkStake stakeChunk lift $ DB.insertManyEpochStakes dbStakes where @@ -292,21 +282,22 @@ updateIndexCache -> ReaderT SqlBackend m IndexCache updateIndexCache lenv screds pkhs = do oldCache <- liftIO . atomically $ readTVar (leIndexCache lenv) - newIndexCache <- createNewCache oldCache + newIndexCache <- modifyCache screds pkhs oldCache liftIO . atomically $ writeTVar (leIndexCache lenv) newIndexCache pure newIndexCache - where - createNewCache - :: (MonadBaseControl IO m, MonadIO m) - => IndexCache -> ReaderT SqlBackend m IndexCache - createNewCache oldCache = do - newAddresses <- newAddressCache (icAddressCache oldCache) - newPools <- newPoolCache (icPoolCache oldCache) - pure $ IndexCache - { icAddressCache = newAddresses - , icPoolCache = newPools - } +modifyCache + :: (MonadBaseControl IO m, MonadIO m) + => Set Generic.StakeCred -> Set Generic.StakePoolKeyHash + -> IndexCache -> ReaderT SqlBackend m IndexCache +modifyCache screds pkhs oldCache = do + newAddresses <- newAddressCache (icAddressCache oldCache) + newPools <- newPoolCache (icPoolCache oldCache) + pure $ IndexCache + { icAddressCache = newAddresses + , icPoolCache = newPools + } + where newAddressCache :: (MonadBaseControl IO m, MonadIO m) => Map Generic.StakeCred DB.StakeAddressId diff --git a/cardano-db-sync/src/Cardano/DbSync/LedgerEvent.hs b/cardano-db-sync/src/Cardano/DbSync/LedgerEvent.hs index 9a1ea817a..63597c3fd 100644 --- a/cardano-db-sync/src/Cardano/DbSync/LedgerEvent.hs +++ b/cardano-db-sync/src/Cardano/DbSync/LedgerEvent.hs @@ -54,7 +54,6 @@ data LedgerEvent = LedgerNewEpoch !EpochNo !SyncState | LedgerStartAtEpoch !EpochNo | LedgerRewards !SlotDetails !Generic.Rewards - | LedgerStakeDist !Generic.StakeDist | LedgerRewardDist !Generic.Rewards | LedgerMirDist !Generic.Rewards diff --git a/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs b/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs index ce6487589..ce81f64de 100644 --- a/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs +++ b/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs @@ -4,6 +4,7 @@ {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE NoImplicitPrelude #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-} @@ -32,11 +33,11 @@ module Cardano.DbSync.LedgerState , getAlonzoPParams ) where -import Prelude (String, id) +import Prelude (String, fail, id) import Cardano.BM.Trace (Trace, logInfo, logWarning) -import Cardano.Binary (DecoderError) +import Cardano.Binary (Decoder, DecoderError, Encoding, FromCBOR (..), ToCBOR (..)) import qualified Cardano.Binary as Serialize import qualified Cardano.Db as DB @@ -130,8 +131,6 @@ import System.Mem (performMajorGC) data BulkOperation = BulkRewardChunk !EpochNo !CardanoPoint !IndexCache ![(StakeCred, Set Generic.Reward)] | BulkRewardReport !EpochNo !CardanoPoint !Int !Coin - | BulkStakeDistChunk !EpochNo !CardanoPoint !IndexCache ![(StakeCred, (Coin, StakePoolKeyHash))] - | BulkStakeDistReport !EpochNo !CardanoPoint !Int data IndexCache = IndexCache { icAddressCache :: !(Map StakeCred DB.StakeAddressId) @@ -161,21 +160,54 @@ data LedgerEnv = LedgerEnv , leStableEpochSlot :: !EpochSlot } +-- TODO this is unstable in terms of restarts and we should try to remove it. data LedgerEventState = LedgerEventState { lesInitialized :: !Bool , lesEpochNo :: !(Maybe EpochNo) , lesLastRewardsEpoch :: !(Maybe EpochNo) - , lesLastStateDistEpoch :: !(Maybe EpochNo) , lesLastAdded :: !CardanoPoint } topLevelConfig :: LedgerEnv -> TopLevelConfig CardanoBlock topLevelConfig = Consensus.pInfoConfig . leProtocolInfo -newtype CardanoLedgerState = CardanoLedgerState +data CardanoLedgerState = CardanoLedgerState { clsState :: ExtLedgerState CardanoBlock + , clsEpochBlockNo :: EpochBlockNo } +-- The height of the block in the current Epoch. We maintain this +-- data next to the ledger state and store it in the same blob file. +data EpochBlockNo = GenesisEpochBlockNo | EBBEpochBlockNo | EpochBlockNo Word64 + +instance ToCBOR EpochBlockNo where + toCBOR GenesisEpochBlockNo = toCBOR (0 :: Word8) + toCBOR EBBEpochBlockNo = toCBOR (1 :: Word8) + toCBOR (EpochBlockNo n) = + toCBOR (2 :: Word8) <> toCBOR n + +instance FromCBOR EpochBlockNo where + fromCBOR = do + tag :: Word8 <- fromCBOR + case tag of + 0 -> pure GenesisEpochBlockNo + 1 -> pure EBBEpochBlockNo + 2 -> EpochBlockNo <$> fromCBOR + n -> fail $ "unexpected EpochBlockNo value " <> show n + +encodeCardanoLedgerState :: (ExtLedgerState CardanoBlock -> Encoding) + -> CardanoLedgerState -> Encoding +encodeCardanoLedgerState encodeExt cls = mconcat + [ encodeExt (clsState cls) + , toCBOR (clsEpochBlockNo cls) + ] + +decodeCardanoLedgerState :: (forall s. Decoder s (ExtLedgerState CardanoBlock)) + -> (forall s. Decoder s CardanoLedgerState) +decodeCardanoLedgerState decodeExt = do + ldgrState <- decodeExt + CardanoLedgerState ldgrState <$> fromCBOR + data LedgerStateFile = LedgerStateFile { lsfSlotNo :: !SlotNo , lsfHash :: !ByteString @@ -189,6 +221,7 @@ data LedgerStateSnapshot = LedgerStateSnapshot , lssNewEpoch :: !(Strict.Maybe Generic.NewEpoch) -- Only Just for a single block at the epoch boundary , lssSlotDetails :: !SlotDetails , lssPoint :: !CardanoPoint + , lssStakeSlice :: !Generic.StakeSliceRes , lssEvents :: ![LedgerEvent] } @@ -259,7 +292,6 @@ mkLedgerEnv trce protocolInfo dir nw stableEpochSlot systemStart aop = do { lesInitialized = False , lesEpochNo = Nothing , lesLastRewardsEpoch = Nothing - , lesLastStateDistEpoch = Nothing , lesLastAdded = GenesisPoint } @@ -267,6 +299,7 @@ mkLedgerEnv trce protocolInfo dir nw stableEpochSlot systemStart aop = do initCardanoLedgerState :: Consensus.ProtocolInfo IO CardanoBlock -> CardanoLedgerState initCardanoLedgerState pInfo = CardanoLedgerState { clsState = Consensus.pInfoInitLedger pInfo + , clsEpochBlockNo = GenesisEpochBlockNo } -- TODO make this type safe. We make the assumption here that the first message of @@ -291,18 +324,22 @@ applyBlock env blk = do ledgerDB <- readStateUnsafe env let oldState = ledgerDbCurrent ledgerDB let !result = applyBlk (ExtLedgerCfg (topLevelConfig env)) blk (clsState oldState) - let !newState = oldState { clsState = lrResult result } - details <- getSlotDetails env (ledgerState $ clsState newState) time (cardanoBlockSlotNo blk) + let !newLedgerState = lrResult result + details <- getSlotDetails env (ledgerState newLedgerState) time (cardanoBlockSlotNo blk) + let !newEpoch = mkNewEpoch (clsState oldState) newLedgerState + let !newEpochBlockNo = applyToEpochBlockNo (isJust $ blockIsEBB blk) (isJust newEpoch) (clsEpochBlockNo oldState) + let !newState = CardanoLedgerState newLedgerState newEpochBlockNo let !ledgerDB' = pushLedgerDB ledgerDB newState writeTVar (leStateVar env) (Just ledgerDB') oldEventState <- readTVar (leEventState env) - events <- generateEvents env oldEventState details newState (blockPoint blk) + events <- generateEvents env oldEventState details newState (blockPoint blk) pure $ LedgerStateSnapshot { lssState = newState , lssOldState = oldState - , lssNewEpoch = maybeToStrict $ mkNewEpoch oldState newState + , lssNewEpoch = maybeToStrict newEpoch , lssSlotDetails = details , lssPoint = blockPoint blk + , lssStakeSlice = stakeSlice newState details , lssEvents = events ++ mapMaybe (convertAuxLedgerEvent (leNetwork env)) (lrEvents result) } where @@ -315,7 +352,7 @@ applyBlock env blk = do Left err -> panic err Right result -> result - mkNewEpoch :: CardanoLedgerState -> CardanoLedgerState -> Maybe Generic.NewEpoch + mkNewEpoch :: ExtLedgerState CardanoBlock -> ExtLedgerState CardanoBlock -> Maybe Generic.NewEpoch mkNewEpoch oldState newState = if ledgerEpochNo env newState /= ledgerEpochNo env oldState + 1 then Nothing @@ -325,16 +362,36 @@ applyBlock env blk = do { Generic.neEpoch = ledgerEpochNo env newState , Generic.neIsEBB = isJust $ blockIsEBB blk , Generic.neAdaPots = maybeToStrict $ getAdaPots newState - , Generic.neEpochUpdate = Generic.epochUpdate (clsState newState) + , Generic.neEpochUpdate = Generic.epochUpdate newState } + applyToEpochBlockNo :: Bool -> Bool -> EpochBlockNo -> EpochBlockNo + applyToEpochBlockNo True _ _ = EBBEpochBlockNo + applyToEpochBlockNo _ True _ = EpochBlockNo 0 + applyToEpochBlockNo _ _ (EpochBlockNo n) = EpochBlockNo (n + 1) + applyToEpochBlockNo _ _ GenesisEpochBlockNo = EpochBlockNo 0 + applyToEpochBlockNo _ _ EBBEpochBlockNo = EpochBlockNo 0 + + stakeSliceMinSize :: Word64 + stakeSliceMinSize = 2000 + + stakeSlice :: CardanoLedgerState -> SlotDetails -> Generic.StakeSliceRes + stakeSlice cls details = case clsEpochBlockNo cls of + EpochBlockNo n -> Generic.getStakeSlice + (leProtocolInfo env) + (leNetwork env) + (sdEpochNo details) + n + stakeSliceMinSize + (clsState cls) + _ -> Generic.NoSlices + generateEvents :: LedgerEnv -> LedgerEventState -> SlotDetails -> CardanoLedgerState -> CardanoPoint -> STM [LedgerEvent] generateEvents env oldEventState details cls pnt = do writeTVar (leEventState env) newEventState pure $ catMaybes [ newEpochEvent , LedgerRewards details <$> rewards - , LedgerStakeDist <$> stakeDist ] where currentEpochNo :: EpochNo @@ -359,22 +416,9 @@ generateEvents env oldEventState details cls pnt = do then mkRewards else Nothing - mkRewards :: Maybe Generic.Rewards mkRewards = Generic.epochRewards (leNetwork env) (sdEpochNo details) (clsState cls) - stakeDist :: Maybe Generic.StakeDist - stakeDist = - case lesLastStateDistEpoch oldEventState of - Nothing -> mkStakeDist - Just oldStakeEpoch -> - if oldStakeEpoch < currentEpochNo - then mkStakeDist - else Nothing - - mkStakeDist :: Maybe Generic.StakeDist - mkStakeDist = Generic.epochStakeDist (leNetwork env) (sdEpochNo details) (clsState cls) - newEventState :: LedgerEventState newEventState = LedgerEventState @@ -384,19 +428,15 @@ generateEvents env oldEventState details cls pnt = do if isJust rewards then Just currentEpochNo else lesLastRewardsEpoch oldEventState - , lesLastStateDistEpoch = - if isJust stakeDist - then Just currentEpochNo - else lesLastStateDistEpoch oldEventState , lesLastAdded = - if isNothing rewards && isNothing stakeDist + if isNothing rewards then lesLastAdded oldEventState else pnt } -saveCurrentLedgerState :: LedgerEnv -> ExtLedgerState CardanoBlock -> Maybe EpochNo -> IO () +saveCurrentLedgerState :: LedgerEnv -> CardanoLedgerState -> Maybe EpochNo -> IO () saveCurrentLedgerState env ledger mEpochNo = do - case mkLedgerStateFilename (leDir env) ledger mEpochNo of + case mkLedgerStateFilename (leDir env) (clsState ledger) mEpochNo of Origin -> pure () -- we don't store genesis At file -> do exists <- doesFileExist file @@ -406,11 +446,12 @@ saveCurrentLedgerState env ledger mEpochNo = do else do LBS.writeFile file $ Serialize.serializeEncoding $ - Consensus.encodeExtLedgerState - (encodeDisk codecConfig) - (encodeDisk codecConfig) - (encodeDisk codecConfig) - ledger + encodeCardanoLedgerState + (Consensus.encodeExtLedgerState + (encodeDisk codecConfig) + (encodeDisk codecConfig) + (encodeDisk codecConfig)) + ledger logInfo (leTrace env) $ mconcat ["Took a ledger snapshot at ", Text.pack file] where codecConfig :: CodecConfig CardanoBlock @@ -423,7 +464,7 @@ mkLedgerStateFilename dir ledger mEpochNo = lsfFilePath . dbPointToFileName dir saveCleanupState :: LedgerEnv -> CardanoLedgerState -> Maybe EpochNo -> IO () saveCleanupState env ledger mEpochNo = do let st = clsState ledger - saveCurrentLedgerState env st mEpochNo + saveCurrentLedgerState env ledger mEpochNo cleanupLedgerStateFiles env $ fromWithOrigin (SlotNo 0) (ledgerTipSlot $ ledgerState st) @@ -639,9 +680,9 @@ loadLedgerStateFromFile config delete lsf = do mst <- safeReadFile (lsfFilePath lsf) case mst of Left err -> when delete (safeRemoveFile $ lsfFilePath lsf) >> pure (Left err) - Right st -> pure . Right $ CardanoLedgerState { clsState = st } + Right st -> pure $ Right st where - safeReadFile :: FilePath -> IO (Either Text (ExtLedgerState CardanoBlock)) + safeReadFile :: FilePath -> IO (Either Text CardanoLedgerState) safeReadFile fp = do mbs <- Exception.try $ BS.readFile fp case mbs of @@ -654,16 +695,21 @@ loadLedgerStateFromFile config delete lsf = do codecConfig :: CodecConfig CardanoBlock codecConfig = configCodec config - decode :: ByteString -> Either DecoderError (ExtLedgerState CardanoBlock) - decode = + decode :: ByteString -> Either DecoderError CardanoLedgerState + decode = do Serialize.decodeFullDecoder "Ledger state file" - (Consensus.decodeExtLedgerState - (decodeDisk codecConfig) - (decodeDisk codecConfig) - (decodeDisk codecConfig)) + decodeState . LBS.fromStrict + decodeState :: (forall s. Decoder s CardanoLedgerState) + decodeState = + decodeCardanoLedgerState $ + Consensus.decodeExtLedgerState + (decodeDisk codecConfig) + (decodeDisk codecConfig) + (decodeDisk codecConfig) + -- Get a list of the ledger state files order most recent listLedgerStateFilesOrdered :: LedgerStateDir -> IO [LedgerStateFile] listLedgerStateFilesOrdered dir = do @@ -702,18 +748,18 @@ getPoolParamsShelley lState = -- We only compute 'AdaPots' for later eras. This is a time consuming -- function and we only want to run it on epoch boundaries. -getAdaPots :: CardanoLedgerState -> Maybe Shelley.AdaPots +getAdaPots :: ExtLedgerState CardanoBlock -> Maybe Shelley.AdaPots getAdaPots st = - case ledgerState $ clsState st of + case ledgerState st of LedgerStateByron _ -> Nothing LedgerStateShelley sts -> Just $ totalAdaPots sts LedgerStateAllegra sta -> Just $ totalAdaPots sta LedgerStateMary stm -> Just $ totalAdaPots stm LedgerStateAlonzo sta -> Just $ totalAdaPots sta -ledgerEpochNo :: LedgerEnv -> CardanoLedgerState -> EpochNo +ledgerEpochNo :: LedgerEnv -> ExtLedgerState CardanoBlock -> EpochNo ledgerEpochNo env cls = - case ledgerTipSlot (ledgerState (clsState cls)) of + case ledgerTipSlot (ledgerState cls) of Origin -> 0 -- An empty chain is in epoch 0 NotOrigin slot -> case runExcept $ epochInfoEpoch epochInfo slot of @@ -721,7 +767,7 @@ ledgerEpochNo env cls = Right en -> en where epochInfo :: EpochInfo (Except Consensus.PastHorizonException) - epochInfo = epochInfoLedger (configLedger $ topLevelConfig env) (hardForkLedgerStatePerEra . ledgerState $ clsState cls) + epochInfo = epochInfoLedger (configLedger $ topLevelConfig env) (hardForkLedgerStatePerEra $ ledgerState cls) -- Like 'Consensus.tickThenReapply' but also checks that the previous hash from the block matches -- the head hash of the ledger state. diff --git a/cardano-db/src/Cardano/Db/Query.hs b/cardano-db/src/Cardano/Db/Query.hs index 32044ae25..146f84424 100644 --- a/cardano-db/src/Cardano/Db/Query.hs +++ b/cardano-db/src/Cardano/Db/Query.hs @@ -57,6 +57,7 @@ module Cardano.Db.Query , queryTxOutCount , queryTxOutValue , queryTxOutCredentials + , queryEpochStakeCount , queryUtxoAtBlockNo , queryUtxoAtSlotNo , queryWithdrawalsUpToBlockNo @@ -742,6 +743,13 @@ queryUtxoAtBlockId blkid = do (out, Value (Just hash')) -> Just (entityVal out, hash') (_, Value Nothing) -> Nothing +queryEpochStakeCount :: MonadIO m => Word64 -> ReaderT SqlBackend m Word64 +queryEpochStakeCount epoch = do + res <- select $ do + epochStake <- from $ table @ EpochStake + where_ (epochStake ^. EpochStakeEpochNo ==. val epoch) + pure countRows + pure $ maybe 0 unValue (listToMaybe res) queryUtxoAtBlockNo :: MonadIO m => Word64 -> ReaderT SqlBackend m [(TxOut, ByteString)] queryUtxoAtBlockNo blkNo = do diff --git a/schema/migration-2-0011-20220318.sql b/schema/migration-2-0011-20220318.sql new file mode 100644 index 000000000..fd5791fab --- /dev/null +++ b/schema/migration-2-0011-20220318.sql @@ -0,0 +1,21 @@ +-- Persistent generated migration. + +CREATE FUNCTION migrate() RETURNS void AS $$ +DECLARE + next_version int ; +BEGIN + SELECT stage_two + 1 INTO next_version FROM schema_version ; + IF next_version = 11 THEN + EXECUTE 'ALTER TABLE "block" ALTER COLUMN "slot_no" TYPE word63type' ; + EXECUTE 'ALTER TABLE "ada_pots" ALTER COLUMN "slot_no" TYPE word63type' ; + EXECUTE 'ALTER TABLE "delegation" ALTER COLUMN "slot_no" TYPE word63type' ; + -- Hand written SQL statements can be added here. + UPDATE schema_version SET stage_two = next_version ; + RAISE NOTICE 'DB has been migrated to stage_two version %', next_version ; + END IF ; +END ; +$$ LANGUAGE plpgsql ; + +SELECT migrate() ; + +DROP FUNCTION migrate() ;