Merge pull request #234 from input-output-hk/erikd/cert-index

Add index for all certificates within a transaction

cardano-db-sync/CHANGELOG.md

@@ -4,6 +4,9 @@
 
 * Note that this release requires the database to be dropped and recreated.
 * Add and populate 'tx_metadata' table.
+* Insert stake deregistrations into correct table. Previously deregistrations were
+  inserted into the registration table.
+* For all transaction certificates, add index within the transaction (#230).
 
 ## 3.1.0 -- July 2020
 

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs

@@ -147,16 +147,13 @@ insertTx tracer env blkId blockIndex tx = do
       Nothing -> pure ()
       Just md -> insertTxMetadata tracer txId md
 
-    mapM_ (insertPoolCert tracer txId) $ Shelley.txPoolCertificates tx
-    mapM_ (insertDelegCert tracer env txId) $ Shelley.txDelegationCerts tx
-    mapM_ (insertMirCert tracer env txId) $ Shelley.txMirCertificates tx
+    mapM_ (insertCertificate tracer env txId) $ Shelley.txCertificates tx
     mapM_ (insertWithdrawals tracer txId) $ Shelley.txWithdrawals tx
 
     case Shelley.txParamUpdate tx of
       Nothing -> pure ()
       Just pu -> insertParamUpdate tracer txId pu
 
-
 insertTxOut
     :: (MonadBaseControl IO m, MonadIO m)
     => Trace IO Text -> DB.TxId -> (Word16, ShelleyTxOut)
@@ -184,31 +181,46 @@ insertTxIn _tracer txInId (Shelley.TxIn txId index) = do
               , DB.txInTxOutIndex = fromIntegral index
               }
 
+insertCertificate
+    :: (MonadBaseControl IO m, MonadIO m)
+    => Trace IO Text -> DbSyncEnv -> DB.TxId -> (Word16, ShelleyDCert)
+    -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
+insertCertificate tracer env txId (idx, cert) =
+  case cert of
+    Shelley.DCertDeleg deleg -> insertDelegCert tracer env txId idx deleg
+    Shelley.DCertPool pool -> insertPoolCert tracer txId idx pool
+    Shelley.DCertMir mir -> insertMirCert tracer env txId idx mir
+    Shelley.DCertGenesis _gen -> do
+        -- TODO : Low priority
+        liftIO $ logError tracer "insertCertificate: Unhandled DCertGenesis certificate"
+        pure ()
+
+
 insertPoolCert
     :: (MonadBaseControl IO m, MonadIO m)
-    => Trace IO Text -> DB.TxId -> ShelleyPoolCert
+    => Trace IO Text -> DB.TxId -> Word16 -> ShelleyPoolCert
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertPoolCert tracer txId pCert =
+insertPoolCert tracer txId idx pCert =
   case pCert of
-    Shelley.RegPool pParams -> insertPoolRegister tracer txId pParams
-    Shelley.RetirePool keyHash epochNum -> insertPoolRetire txId epochNum keyHash
+    Shelley.RegPool pParams -> insertPoolRegister tracer txId idx pParams
+    Shelley.RetirePool keyHash epochNum -> insertPoolRetire txId epochNum idx keyHash
 
 insertDelegCert
     :: (MonadBaseControl IO m, MonadIO m)
-    => Trace IO Text -> DbSyncEnv -> DB.TxId -> ShelleyDelegCert
+    => Trace IO Text -> DbSyncEnv -> DB.TxId -> Word16 -> ShelleyDelegCert
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertDelegCert tracer env txId dCert =
+insertDelegCert tracer env txId idx dCert =
   case dCert of
-    Shelley.RegKey cred -> insertStakeRegistration tracer env txId cred
-    Shelley.DeRegKey cred -> insertStakeDeregistration tracer env txId cred
-    Shelley.Delegate (Shelley.Delegation cred poolkh) -> insertDelegation tracer env txId cred poolkh
+    Shelley.RegKey cred -> insertStakeRegistration tracer env txId idx cred
+    Shelley.DeRegKey cred -> insertStakeDeregistration tracer env txId idx cred
+    Shelley.Delegate (Shelley.Delegation cred poolkh) -> insertDelegation tracer env txId idx cred poolkh
 
 
 insertPoolRegister
     :: (MonadBaseControl IO m, MonadIO m)
-    => Trace IO Text -> DB.TxId -> ShelleyPoolParams
+    => Trace IO Text -> DB.TxId -> Word16 -> ShelleyPoolParams
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertPoolRegister tracer txId params = do
+insertPoolRegister tracer txId idx params = do
   mdId <- case strictMaybeToMaybe $ Shelley._poolMD params of
             Just md -> Just <$> insertMetaData txId md
             Nothing -> pure Nothing
@@ -225,6 +237,7 @@ insertPoolRegister tracer txId params = do
   poolUpdateId <- lift . DB.insertPoolUpdate $
                     DB.PoolUpdate
                       { DB.poolUpdateHashId = poolHashId
+                      , DB.poolUpdateCertIndex = idx
                       , DB.poolUpdateVrfKey = Crypto.hashToBytes (Shelley._poolVrf params)
                       , DB.poolUpdatePledge = DbWord64 $ fromIntegral (Shelley.unCoin $ Shelley._poolPledge params)
                       , DB.poolUpdateRewardAddrId = rewardId
@@ -242,13 +255,14 @@ maxLovelace = 45000000000000000
 
 insertPoolRetire
     :: (MonadBaseControl IO m, MonadIO m)
-    => DB.TxId -> EpochNo ->  ShelleyStakePoolKeyHash
+    => DB.TxId -> EpochNo -> Word16 -> ShelleyStakePoolKeyHash
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertPoolRetire txId epochNum keyHash = do
+insertPoolRetire txId epochNum idx keyHash = do
   updateId <- firstExceptT (NELookup "insertPoolRetire") . newExceptT $ queryStakePoolKeyHash keyHash
   void . lift . DB.insertPoolRetire $
     DB.PoolRetire
       { DB.poolRetireUpdateId = updateId
+      , DB.poolRetireCertIndex = idx
       , DB.poolRetireAnnouncedTxId = txId
       , DB.poolRetireRetiringEpoch = unEpochNo epochNum
       }
@@ -289,35 +303,37 @@ insertPoolOwner poolId skh =
 
 insertStakeRegistration
     :: (MonadBaseControl IO m, MonadIO m)
-    => Trace IO Text -> DbSyncEnv -> DB.TxId -> ShelleyStakingCred
+    => Trace IO Text -> DbSyncEnv -> DB.TxId -> Word16 -> ShelleyStakingCred
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertStakeRegistration _tracer env txId cred = do
+insertStakeRegistration _tracer env txId idx cred = do
   scId <- insertStakeAddress $ Shelley.stakingCredHash env cred
   void . lift . DB.insertStakeRegistration $
     DB.StakeRegistration
       { DB.stakeRegistrationAddrId = scId
+      , DB.stakeRegistrationCertIndex = idx
       , DB.stakeRegistrationTxId = txId
       }
 
 insertStakeDeregistration
     :: (MonadBaseControl IO m, MonadIO m)
-    => Trace IO Text -> DbSyncEnv -> DB.TxId -> ShelleyStakingCred
+    => Trace IO Text -> DbSyncEnv -> DB.TxId -> Word16 -> ShelleyStakingCred
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertStakeDeregistration _tracer env txId cred = do
+insertStakeDeregistration _tracer env txId idx cred = do
   scId <- firstExceptT (NELookup "insertStakeDeregistration")
             . newExceptT
             $ queryStakeAddress (Shelley.stakingCredHash env cred)
-  void . lift . DB.insertStakeRegistration $
-    DB.StakeRegistration
-      { DB.stakeRegistrationAddrId = scId
-      , DB.stakeRegistrationTxId = txId
+  void . lift . DB.insertStakeDeregistration $
+    DB.StakeDeregistration
+      { DB.stakeDeregistrationAddrId = scId
+      , DB.stakeDeregistrationCertIndex = idx
+      , DB.stakeDeregistrationTxId = txId
       }
 
 insertDelegation
     :: (MonadBaseControl IO m, MonadIO m)
-    => Trace IO Text -> DbSyncEnv -> DB.TxId -> ShelleyStakingCred -> ShelleyStakePoolKeyHash
+    => Trace IO Text -> DbSyncEnv -> DB.TxId -> Word16 -> ShelleyStakingCred -> ShelleyStakePoolKeyHash
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertDelegation _tracer env txId cred poolkh = do
+insertDelegation _tracer env txId idx cred poolkh = do
   addrId <- firstExceptT (NELookup "insertDelegation")
                 . newExceptT
                 $ queryStakeAddress (Shelley.stakingCredHash env cred)
@@ -327,15 +343,16 @@ insertDelegation _tracer env txId cred poolkh = do
   void . lift . DB.insertDelegation $
     DB.Delegation
       { DB.delegationAddrId = addrId
+      , DB.delegationCertIndex = idx
       , DB.delegationUpdateId = updateId
       , DB.delegationTxId = txId
       }
 
 insertMirCert
     :: (MonadBaseControl IO m, MonadIO m)
-    => Trace IO Text -> DbSyncEnv -> DB.TxId -> ShelleyMIRCert
+    => Trace IO Text -> DbSyncEnv -> DB.TxId -> Word16 -> ShelleyMIRCert
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertMirCert _tracer env txId mcert = do
+insertMirCert _tracer env txId idx mcert = do
     case Shelley.mirPot mcert of
       Shelley.ReservesMIR ->
         mapM_ insertMirReserves $ Map.toList (Shelley.mirRewards mcert)
@@ -353,6 +370,7 @@ insertMirCert _tracer env txId mcert = do
       void . lift . DB.insertReserve $
         DB.Reserve
           { DB.reserveAddrId = addrId
+          , DB.reserveCertIndex = idx
           , DB.reserveTxId = txId
           , DB.reserveAmount = fromIntegral $ Shelley.unCoin coin
           }
@@ -368,6 +386,7 @@ insertMirCert _tracer env txId mcert = do
       void . lift . DB.insertTreasury $
         DB.Treasury
           { DB.treasuryAddrId = addrId
+          , DB.treasuryCertIndex = idx
           , DB.treasuryTxId = txId
           , DB.treasuryAmount = fromIntegral $ Shelley.unCoin coin
           }

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Util.hs

@@ -27,14 +27,12 @@ module Cardano.DbSync.Era.Shelley.Util
   , stakingCredHash
   , txFee
   , txHash
-  , txDelegationCerts
+  , txCertificates
   , txInputList
   , txMetadata
   , txOutputList
   , txOutputSum
-  , txMirCertificates
   , txParamUpdate
-  , txPoolCertificates
   , txWithdrawals
   , unHeaderHash
   , unitIntervalToDouble
@@ -195,15 +193,9 @@ stakingCredHash env cred =
   in Shelley.serialiseRewardAcnt $ Shelley.RewardAcnt network cred
 
 
-txDelegationCerts :: ShelleyTx -> [ShelleyDelegCert]
-txDelegationCerts tx =
-    mapMaybe extractDelegationCerts $ toList (Shelley._certs $ Shelley._body tx)
-  where
-    extractDelegationCerts :: ShelleyDCert -> Maybe ShelleyDelegCert
-    extractDelegationCerts dcert =
-      case dcert of
-        Shelley.DCertDeleg pcert -> Just pcert
-        _otherwise -> Nothing
+txCertificates :: ShelleyTx -> [(Word16, ShelleyDCert)]
+txCertificates tx =
+    zip [0 ..] (toList . Shelley._certs $ Shelley._body tx)
 
 txFee :: ShelleyTx -> Word64
 txFee = fromIntegral . unCoin . Shelley._txfee . Shelley._body
@@ -217,29 +209,9 @@ txInputList = toList . Shelley._inputs . Shelley._body
 txMetadata :: ShelleyTx -> Maybe Shelley.MetaData
 txMetadata = Shelley.strictMaybeToMaybe . Shelley._metadata
 
-txMirCertificates :: ShelleyTx -> [ShelleyMIRCert]
-txMirCertificates tx =
-    mapMaybe extractMirCert $ toList (Shelley._certs $ Shelley._body tx)
-  where
-    extractMirCert :: ShelleyDCert -> Maybe ShelleyMIRCert
-    extractMirCert dcert =
-      case dcert of
-        Shelley.DCertMir mcert -> Just mcert
-        _otherwise -> Nothing
-
 txParamUpdate :: ShelleyTx -> Maybe (Shelley.Update TPraosStandardCrypto)
 txParamUpdate = Shelley.strictMaybeToMaybe . Shelley._txUpdate . Shelley._body
 
-txPoolCertificates :: ShelleyTx -> [ShelleyPoolCert]
-txPoolCertificates tx =
-    mapMaybe extractPoolCertificate $ toList (Shelley._certs $ Shelley._body tx)
-  where
-    extractPoolCertificate :: ShelleyDCert -> Maybe ShelleyPoolCert
-    extractPoolCertificate dcert =
-      case dcert of
-        Shelley.DCertPool pcert -> Just pcert
-        _otherwise -> Nothing
-
 -- Outputs are ordered, so provide them as such with indices.
 txOutputList :: ShelleyTx -> [(Word16, ShelleyTxOut)]
 txOutputList tx =

cardano-db/src/Cardano/Db/Insert.hs

@@ -16,6 +16,7 @@ module Cardano.Db.Insert
   , insertReserve
   , insertSlotLeader
   , insertStakeAddress
+  , insertStakeDeregistration
   , insertStakeRegistration
   , insertTreasury
   , insertTx
@@ -85,6 +86,9 @@ insertSlotLeader = insertByReturnKey "SlotLeader"
 insertStakeAddress :: (MonadBaseControl IO m, MonadIO m) => StakeAddress -> ReaderT SqlBackend m StakeAddressId
 insertStakeAddress = insertByReturnKey "StakeAddress"
 
+insertStakeDeregistration :: (MonadBaseControl IO m, MonadIO m) => StakeDeregistration -> ReaderT SqlBackend m StakeDeregistrationId
+insertStakeDeregistration = insertByReturnKey "StakeDeregistration"
+
 insertStakeRegistration :: (MonadBaseControl IO m, MonadIO m) => StakeRegistration -> ReaderT SqlBackend m StakeRegistrationId
 insertStakeRegistration = insertByReturnKey "StakeRegistration"
 

cardano-db/src/Cardano/Db/Schema.hs

@@ -162,6 +162,7 @@ share
 
   PoolUpdate
     hashId              PoolHashId
+    certIndex           Word16
     vrfKey              ByteString          sqltype=hash32type
     pledge              DbWord64            sqltype=word64type
     rewardAddrId        StakeAddressId
@@ -178,6 +179,7 @@ share
 
   PoolRetire
     updateId            PoolUpdateId
+    certIndex           Word16
     announcedTxId       TxId                                    -- Slot number in which the pool announced it was retiring.
     retiringEpoch       Word64              sqltype=uinteger    -- Epoch number in which the pool will retire.
     UniquePoolRetiring  updateId
@@ -199,6 +201,7 @@ share
 
   Reserve
     addrId              StakeAddressId
+    certIndex           Word16
     -- poolId              PoolHashId
     amount              Word64              sqltype=lovelace
     txId                TxId
@@ -213,19 +216,22 @@ share
 
   Delegation
     addrId              StakeAddressId
+    certIndex           Word16
     updateId            PoolUpdateId
     txId                TxId
     UniqueDelegation    addrId updateId txId
 
   -- When was a staking key/script registered
   StakeRegistration
     addrId              StakeAddressId
+    certIndex           Word16
     txId                TxId
     UniqueStakeRegistration addrId txId
 
   -- When was a staking key/script deregistered
   StakeDeregistration
     addrId              StakeAddressId
+    certIndex           Word16
     txId                TxId
     UniqueStakeDeregistration addrId txId
 
@@ -242,6 +248,7 @@ share
     -- This design allows rewards to be discriminated based on how they are earned.
   Reward
     addrId              StakeAddressId
+    certIndex           Word16
     -- poolId              PoolHashId
     amount              Word64              sqltype=lovelace
     txId                TxId
@@ -255,6 +262,7 @@ share
 
   Treasury
     addrId              StakeAddressId
+    certIndex           Word16
     -- poolId              PoolHashId
     amount              Word64              sqltype=lovelace
     txId                TxId

doc/interesting-queries.md

@@ -72,4 +72,15 @@ cexplorer=# select pg_size_pretty (pg_database_size ('cexplorer'));
 (1 row)
 ```
 
+### Current valid pools
+In general the database is operated on in an append only manner. The means that when there is a
+pool registration, followed by a degreistration followed by a second registration, all three
+events will be recorded. To get the latest pool registrations:
+two stake pool
+```
+cexplorer=# select * from pool_update where not exists
+             (select update_id from pool_retire where update_id = pool_update.id) ;
+```
+
+
 [Query.hs]: https://github.com/input-output-hk/cardano-db-sync/blob/master/cardano-db/src/Cardano/Db/Query.hs