Compare Marconi Utxo with cardano-db-sync

Stream db-sync utxos to marconi SQLite and compute delta of addresses,
addresses in Marconi but not in cardano-db-sync

[PLT-5383]

compare dbsync

marconi-chain-index/marconi-chain-index.cabal

@@ -362,7 +362,7 @@ test-suite marconi-chain-index-test-compare-cardano-db-sync
     Utxo
 
   if flag(ci)
-    buildable: False
+    buildable: True
 
   --------------------
   -- Local components
@@ -418,6 +418,7 @@ test-suite marconi-chain-index-test-compare-cardano-db-sync
     , serialise
     , sqlite-simple
     , stm
+    , stm-chans
     , streaming
     , tasty
     , tasty-golden

marconi-chain-index/test-compare-cardano-db-sync/DBUtils.hs

@@ -1,16 +1,30 @@
 {-# LANGUAGE LambdaCase        #-}
 {-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes       #-}
+{-# LANGUAGE TemplateHaskell   #-}
 
 module DBUtils where
 
+import Control.Concurrent.STM.TMChan ()
 import Control.Exception (handle)
-import Control.Monad.IO.Class (liftIO)
+import Control.Lens.TH (makeLenses)
+import Database.PostgreSQL.Simple ()
 import Database.PostgreSQL.Simple qualified as PG
+import Database.PostgreSQL.Simple.FromRow ()
+import Database.PostgreSQL.Simple.ToRow ()
 import Database.PostgreSQL.Simple.URL (parseDatabaseUrl)
 import Database.SQLite.Simple qualified as SQLite
+import GHC.Generics (Generic)
+import Streaming ()
 import System.Environment (lookupEnv)
 
-import Hedgehog qualified as H
+
+data DbConnection = DbConnection
+  { _dbcSQLite :: SQLite.Connection
+  , _dbcPG     :: PG.Connection
+  } deriving (Generic)
+
+$(makeLenses ''DbConnection)
 
 -- | Connect to cardano-db-sync postgres with password from
 -- DBSYNC_PGPASSWORD env variable.
@@ -28,8 +42,13 @@ getSQLiteConnection = do
   handle (\ (e::SQLite.SQLError) ->
             fail (show e)) (SQLite.open path)
 
+mkDbConnection :: IO DbConnection
+mkDbConnection = do
+  s <- getSQLiteConnection
+  p <- getDbSyncPgConnection
+  pure $ DbConnection s p
 -- | Get string from the environment or fail test with instruction.
 envOrFail :: String -> IO String
-envOrFail str = (lookupEnv str) >>= \case
+envOrFail str = lookupEnv str >>= \case
   Just v  -> return v
   Nothing -> fail $ str <> " environment variable not set!"

marconi-chain-index/test-compare-cardano-db-sync/EpochState.hs

@@ -12,9 +12,11 @@
 
 3. Run this test by setting the env variables:
 
-     - CARDANO_NODE_SOCKET_PATH:
+     - CARDANO_NODE_SOCKET_PATH
      - CARDANO_NODE_CONFIG_PATH
      - MARCONI_DB_DIRECTORY_PATH
+     - DBSYNC_PGPASSWORD: The default password for cardano-db-sync's postgres database is in its repo in the file: config/secrets/postgres_password
+     - NETWORK_MAGIC: "mainnet" or number
 
    And then run the command:
 
@@ -43,6 +45,7 @@ import Database.PostgreSQL.Simple.FromField qualified as PG
 import Database.PostgreSQL.Simple.ToField qualified as PG
 import System.Environment (lookupEnv)
 import System.FilePath ((</>))
+import Text.Read (readMaybe)
 
 import Cardano.Api qualified as C
 import Cardano.Api.Shelley qualified as C
@@ -59,14 +62,13 @@ import Marconi.ChainIndex.Types (epochStateDbName)
 import Marconi.ChainIndex.Utils qualified as Utils
 import Marconi.Core.Storable qualified as Storable
 
-
 import Hedgehog ((===))
 import Hedgehog qualified as H
-import Test.Tasty (TestTree, defaultMain, testGroup)
+import Test.Tasty (TestTree, testGroup)
 import Test.Tasty.Hedgehog (testPropertyNamed)
 
 tests :: TestTree
-tests = testGroup "EpochState"
+tests = testGroup "Marconi to cardano-db-sync comparisons"
   [ testPropertyNamed
       "Compare all epoch nonces between Marconi and cardano-db-sync"
       "propEpochNonce" propEpochNonce
@@ -87,10 +89,12 @@ propEpochStakepoolSize = H.withTests 1 $ H.property $ do
         dbSyncResult <- liftIO $ dbSyncStakepoolSizes conn epochNo
         marconiResult <- liftIO $ indexerStakepoolSizes epochNo indexer
         liftIO $ putStr
-          $ "\nComparing random epoch " <> show epochNo
+          $ "\nComparing epoch " <> show epochNo
           <> ", number of stakepools in epoch " <> show (Map.size dbSyncResult)
-          <> ", epoch chosen from between " <> show (coerce @_ @Word64 minEpochNo) <> " and " <> show (coerce @_ @Word64 maxEpochNo) <> ")"
         dbSyncResult === marconiResult
+  liftIO $ putStrLn
+     $ "Min and max epoch in cardano-db-sync postgres: "
+    <> show (coerce @_ @Word64 minEpochNo) <> " and " <> show (coerce @_ @Word64 maxEpochNo) <> ")"
   forM_ [minEpochNo .. maxEpochNo] compareEpoch
 
 dbSyncStakepoolSizes :: PG.Connection -> C.EpochNo -> IO (Map.Map C.PoolId C.Lovelace)
@@ -113,7 +117,7 @@ dbSyncStakepoolSizes conn epochNo = do
 indexerStakepoolSizes :: C.EpochNo -> Storable.State EpochState.EpochStateHandle -> IO (Map.Map C.PoolId C.Lovelace)
 indexerStakepoolSizes epochNo indexer = do
   let query = EpochState.SDDByEpochNoQuery epochNo
-  result <- throwIndexerError $ Storable.queryStorage dummyInterval [] (indexer ^. Storable.handle) query
+  result <- throwIndexerError $ Storable.queryStorage [] (indexer ^. Storable.handle) query
   case result of
     EpochState.SDDByEpochNoResult rows -> return $ Map.fromList $ map toPair rows
     _                                  -> return undefined
@@ -144,7 +148,7 @@ propEpochNonce = H.withTests 1 $ H.property $ do
 queryIndexerEpochNonce :: C.EpochNo -> Storable.State EpochState.EpochStateHandle -> IO (Maybe Ledger.Nonce)
 queryIndexerEpochNonce epochNo indexer = do
   let query = EpochState.NonceByEpochNoQuery epochNo
-  res' <- throwIndexerError $ Storable.queryStorage dummyInterval [] (indexer ^. Storable.handle) query
+  res' <- throwIndexerError $ Storable.queryStorage [] (indexer ^. Storable.handle) query
   case res' of
     EpochState.NonceByEpochNoResult res -> return $ EpochState.epochNonceRowNonce <$> res
     _                                   -> return Nothing
@@ -154,8 +158,14 @@ openEpochStateIndexer = do
   socketPath <- envOrFail "CARDANO_NODE_SOCKET_PATH"
   nodeConfigPath <- envOrFail "CARDANO_NODE_CONFIG_PATH"
   dbDir <- envOrFail "MARCONI_DB_DIRECTORY_PATH"
+  networkMagicStr <- envOrFail "NETWORK_MAGIC"
+  networkMagic <- case networkMagicStr of
+    "mainnet" -> return C.Mainnet
+    _ -> case readMaybe networkMagicStr of
+      Nothing     -> fail $ "Can't parse network magic: " <> networkMagicStr
+      Just word32 -> return $ C.Testnet $ C.NetworkMagic word32
   liftIO $ do
-    securityParam <- throwIndexerError $ Utils.querySecurityParamEra C.BabbageEraInCardanoMode C.Mainnet socketPath
+    securityParam <- throwIndexerError $ Utils.querySecurityParamEra C.BabbageEraInCardanoMode networkMagic socketPath
     topLevelConfig <- topLevelConfigFromNodeConfig nodeConfigPath
     let dbPath = dbDir </> epochStateDbName
         ledgerStateDirPath = dbDir </> "ledgerStates"
@@ -179,7 +189,7 @@ getDbSyncPgConnection = do
 
 -- | Get string from the environment or fail test with instruction.
 envOrFail :: String -> H.PropertyT IO String
-envOrFail str = liftIO $ (lookupEnv str) >>= \case
+envOrFail str = liftIO (lookupEnv str) >>= \case
   Just v  -> return v
   Nothing -> fail $ str <> " environment variable not set!"
 
@@ -218,6 +228,3 @@ instance PG.FromField C.PoolId where
 
 deriving newtype instance Real C.EpochNo
 deriving newtype instance Integral C.EpochNo
-
-dummyInterval :: Storable.QueryInterval C.ChainPoint
-dummyInterval = error "dummyInterval: The interval parameter will be removed in the future, don't use it!"

marconi-chain-index/test-compare-cardano-db-sync/Spec.hs

@@ -28,17 +28,22 @@
 
 module Main where
 
+import DBUtils (mkDbConnection)
 import EpochState qualified
 import Utxo qualified
 
+import DBUtils qualified as Utxo
 import Test.Tasty (TestTree, defaultMain, testGroup)
 
 main :: IO ()
-main = Utxo.createUtxosTable >> defaultMain tests
+main = do
+  dbs <- mkDbConnection
+  Utxo.initDBs dbs
+  defaultMain $ tests dbs
 
-tests :: TestTree
-tests = testGroup "Marconi to cardano-db-sync comparisons"
+tests :: Utxo.DbConnection -> TestTree
+tests dbs = testGroup "Marconi to cardano-db-sync comparisons"
   [
     EpochState.tests
-  , Utxo.tests
+  , Utxo.tests dbs
   ]