Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify extra migrations #1496

Merged
merged 1 commit into from
Sep 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 20 additions & 17 deletions cardano-db-sync/src/Cardano/DbSync/Api.hs
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,13 @@ runIndexMigrations env = do
initPruneConsumeMigration :: Bool -> Bool -> DB.PruneConsumeMigration
initPruneConsumeMigration consumed pruneTxOut =
DB.PruneConsumeMigration
{ DB.pcmConsume = consumed
, DB.pcmPruneTxOut = pruneTxOut
{ DB.pcmPruneTxOut = pruneTxOut
, DB.pcmConsumeOrPruneTxOut = consumed || pruneTxOut
}

runExtraMigrationsMaybe :: SyncEnv -> IO ()
runExtraMigrationsMaybe syncEnv = do
pcm <- liftIO $ readTVarIO $ envPruneConsumeMigration syncEnv
let pcm = envPruneConsumeMigration syncEnv
logInfo (getTrace syncEnv) $ textShow pcm
DB.runDbIohkNoLogging (envBackend syncEnv) $
DB.runExtraMigrations
Expand All @@ -166,23 +165,23 @@ getPruneInterval syncEnv = 10 * getSecurityParam syncEnv

whenConsumeOrPruneTxOut :: MonadIO m => SyncEnv -> m () -> m ()
whenConsumeOrPruneTxOut env action = do
extraMigr <- liftIO $ readTVarIO $ envPruneConsumeMigration env
let extraMigr = envPruneConsumeMigration env
when (DB.pcmConsumeOrPruneTxOut extraMigr) action

whenPruneTxOut :: MonadIO m => SyncEnv -> m () -> m ()
whenPruneTxOut env action = do
extraMigr <- liftIO $ readTVarIO $ envPruneConsumeMigration env
let extraMigr = envPruneConsumeMigration env
when (DB.pcmPruneTxOut extraMigr) action

getHasConsumedOrPruneTxOut :: SyncEnv -> IO Bool
getHasConsumedOrPruneTxOut :: SyncEnv -> Bool
getHasConsumedOrPruneTxOut env = do
extraMigr <- liftIO $ readTVarIO $ envPruneConsumeMigration env
pure $ DB.pcmConsumeOrPruneTxOut extraMigr
let extraMigr = envPruneConsumeMigration env
DB.pcmConsumeOrPruneTxOut extraMigr

getPrunes :: SyncEnv -> IO Bool
getPrunes :: SyncEnv -> Bool
getPrunes env = do
extraMigr <- liftIO $ readTVarIO $ envPruneConsumeMigration env
pure $ DB.pcmPruneTxOut extraMigr
let extraMigr = envPruneConsumeMigration env
DB.pcmPruneTxOut extraMigr

fullInsertOptions :: InsertOptions
fullInsertOptions = InsertOptions True True True True
Expand Down Expand Up @@ -318,7 +317,7 @@ mkSyncEnv trce connString backend syncOptions protoInfo nw nwMagic systemStart s
consistentLevelVar <- newTVarIO Unchecked
fixDataVar <- newTVarIO $ if ranMigrations then DataFixRan else NoneFixRan
indexesVar <- newTVarIO $ enpForceIndexes syncNodeParams
pcmVar <- newTVarIO $ initPruneConsumeMigration (enpMigrateConsumed syncNodeParams) (enpPruneTxOut syncNodeParams)
let pcm = initPruneConsumeMigration (enpMigrateConsumed syncNodeParams) (enpPruneTxOut syncNodeParams)
owq <- newTBQueueIO 100
orq <- newTBQueueIO 100
epochVar <- newTVarIO initEpochState
Expand Down Expand Up @@ -359,7 +358,7 @@ mkSyncEnv trce connString backend syncOptions protoInfo nw nwMagic systemStart s
, envOfflineWorkQueue = owq
, envOptions = syncOptions
, envProtocol = SyncProtocolCardano
, envPruneConsumeMigration = pcmVar
, envPruneConsumeMigration = pcm
, envRunDelayedMigration = runMigrationFnc
, envSystemStart = systemStart
}
Expand All @@ -380,16 +379,20 @@ mkSyncEnvFromConfig trce connString backend syncOptions genCfg syncNodeParams ra
case genCfg of
GenesisCardano _ bCfg sCfg _
| unProtocolMagicId (Byron.configProtocolMagicId bCfg) /= Shelley.sgNetworkMagic (scConfig sCfg) ->
pure . Left . SNErrCardanoConfig $
mconcat
pure
. Left
. SNErrCardanoConfig
$ mconcat
[ "ProtocolMagicId "
, DB.textShow (unProtocolMagicId $ Byron.configProtocolMagicId bCfg)
, " /= "
, DB.textShow (Shelley.sgNetworkMagic $ scConfig sCfg)
]
| Byron.gdStartTime (Byron.configGenesisData bCfg) /= Shelley.sgSystemStart (scConfig sCfg) ->
pure . Left . SNErrCardanoConfig $
mconcat
pure
. Left
. SNErrCardanoConfig
$ mconcat
[ "SystemStart "
, DB.textShow (Byron.gdStartTime $ Byron.configGenesisData bCfg)
, " /= "
Expand Down
2 changes: 1 addition & 1 deletion cardano-db-sync/src/Cardano/DbSync/Api/Types.hs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ data SyncEnv = SyncEnv
, envOfflineWorkQueue :: !(StrictTBQueue IO PoolFetchRetry)
, envOptions :: !SyncOptions
, envProtocol :: !SyncProtocol
, envPruneConsumeMigration :: !(StrictTVar IO DB.PruneConsumeMigration)
, envPruneConsumeMigration :: !DB.PruneConsumeMigration
, envRunDelayedMigration :: RunMigration
, envSystemStart :: !SystemStart
}
Expand Down
4 changes: 2 additions & 2 deletions cardano-db-sync/src/Cardano/DbSync/Era/Byron/Genesis.hs
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ insertValidateGenesisDist ::
insertValidateGenesisDist syncEnv (NetworkName networkName) cfg = do
-- Setting this to True will log all 'Persistent' operations which is great
-- for debugging, but otherwise *way* too chatty.
hasConsumed <- liftIO $ getHasConsumedOrPruneTxOut syncEnv
prunes <- liftIO $ getPrunes syncEnv
let hasConsumed = getHasConsumedOrPruneTxOut syncEnv
prunes = getPrunes syncEnv
if False
then newExceptT $ DB.runDbIohkLogging (envBackend syncEnv) tracer (insertAction hasConsumed prunes)
else newExceptT $ DB.runDbIohkNoLogging (envBackend syncEnv) (insertAction hasConsumed prunes)
Expand Down
2 changes: 1 addition & 1 deletion cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ insertByronTx ::
ExceptT SyncNodeError (ReaderT SqlBackend m) Word64
insertByronTx syncEnv blkId tx blockIndex = do
resolvedInputs <- mapM resolveTxInputs (toList $ Byron.txInputs (Byron.taTx tx))
hasConsumed <- liftIO $ getHasConsumedOrPruneTxOut syncEnv
let hasConsumed = getHasConsumedOrPruneTxOut syncEnv
valFee <- firstExceptT annotateTx $ ExceptT $ pure (calculateTxFee (Byron.taTx tx) resolvedInputs)
txId <-
lift . DB.insertTx $
Expand Down
4 changes: 2 additions & 2 deletions cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ insertValidateGenesisDist ::
Bool ->
ExceptT SyncNodeError IO ()
insertValidateGenesisDist syncEnv networkName cfg shelleyInitiation = do
hasConsumed <- liftIO $ getHasConsumedOrPruneTxOut syncEnv
prunes <- liftIO $ getPrunes syncEnv
let hasConsumed = getHasConsumedOrPruneTxOut syncEnv
prunes = getPrunes syncEnv
-- Setting this to True will log all 'Persistent' operations which is great
-- for debugging, but otherwise *way* too chatty.
when (not shelleyInitiation && (hasInitialFunds || hasStakes)) $ do
Expand Down
2 changes: 1 addition & 1 deletion cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ insertTx syncEnv isMember blkId epochNo slotNo depositsMap blockIndex tx grouped
let !mdeposits = if not (Generic.txValidContract tx) then Just (Coin 0) else lookupDepositsMap txHash depositsMap
let !outSum = fromIntegral $ unCoin $ Generic.txOutSum tx
!withdrawalSum = fromIntegral $ unCoin $ Generic.txWithdrawalSum tx
hasConsumed <- liftIO $ getHasConsumedOrPruneTxOut syncEnv
hasConsumed = getHasConsumedOrPruneTxOut syncEnv
-- In some txs and with specific configuration we may be able to find necessary data within the tx body.
-- In these cases we can avoid expensive queries.
(resolvedInputs, fees', deposits) <- case (mdeposits, unCoin <$> Generic.txFees tx) of
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ insertBlockGroupedData ::
BlockGroupedData ->
ExceptT SyncNodeError (ReaderT SqlBackend m) DB.MinIds
insertBlockGroupedData syncEnv grouped = do
hasConsumed <- liftIO $ getHasConsumedOrPruneTxOut syncEnv
let hasConsumed = getHasConsumedOrPruneTxOut syncEnv
txOutIds <- lift . DB.insertManyTxOutPlex hasConsumed $ etoTxOut . fst <$> groupedTxOut grouped
let maTxOuts = concatMap mkmaTxOuts $ zip txOutIds (snd <$> groupedTxOut grouped)
maTxOutIds <- lift $ DB.insertManyMaTxOut maTxOuts
Expand Down
3 changes: 1 addition & 2 deletions cardano-db/src/Cardano/Db/Types.hs
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,7 @@ wasPruneTxOutPreviouslySet :: [ExtraMigration] -> Bool
wasPruneTxOutPreviouslySet = elem PruneTxOutFlagPreviouslySet

data PruneConsumeMigration = PruneConsumeMigration
{ pcmConsume :: Bool
, pcmPruneTxOut :: Bool
{ pcmPruneTxOut :: Bool
, -- we make the assumption that if the user is using prune flag
-- they will also want consume automatically set for them.
pcmConsumeOrPruneTxOut :: Bool
Expand Down
Loading