1588 - remove jsonb from Schema & add config

cardano-chain-gen/cardano-chain-gen.cabal

@@ -156,6 +156,7 @@ test-suite cardano-chain-gen
                         Test.Cardano.Db.Mock.Unit.Conway.CommandLineArg.ConfigFile
                         Test.Cardano.Db.Mock.Unit.Conway.CommandLineArg.EpochDisabled
                         Test.Cardano.Db.Mock.Unit.Conway.CommandLineArg.ForceIndex
+                        Test.Cardano.Db.Mock.Unit.Conway.Config.JsonbInSchema
                         Test.Cardano.Db.Mock.Unit.Conway.Config.Parse
                         Test.Cardano.Db.Mock.Unit.Conway.Config.MigrateConsumedPruneTxOut
                         Test.Cardano.Db.Mock.Unit.Conway.Governance
@@ -200,7 +201,6 @@ test-suite cardano-chain-gen
                       , stm
                       , strict-stm
                       , tasty
-                      , tasty-expected-failure
                       , tasty-quickcheck
                       , text
                       , transformers

cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

@@ -37,6 +37,7 @@ module Test.Cardano.Db.Mock.Config (
   withFullConfig,
   withFullConfigAndDropDB,
   withFullConfigAndLogs,
+  withCustomConfigAndLogsAndDropDB,
   withCustomConfig,
   withCustomConfigAndDropDB,
   withCustomConfigAndLogs,
@@ -438,6 +439,27 @@ withCustomConfigAndLogs =
         }
     )
 
+withCustomConfigAndLogsAndDropDB ::
+  CommandLineArgs ->
+  -- | custom SyncNodeConfig
+  Maybe SyncNodeConfig ->
+  -- | config filepath
+  FilePath ->
+  -- | test label
+  FilePath ->
+  (Interpreter -> ServerHandle IO CardanoBlock -> DBSyncEnv -> IO a) ->
+  IOManager ->
+  [(Text, Text)] ->
+  IO a
+withCustomConfigAndLogsAndDropDB =
+  withFullConfig'
+    ( WithConfigArgs
+        { hasFingerprint = True
+        , shouldLog = True
+        , shouldDropDB = True
+        }
+    )
+
 withFullConfig' ::
   WithConfigArgs ->
   CommandLineArgs ->

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Config.hs

@@ -31,6 +31,7 @@ insertConfig = do
           , sioGovernance = GovernanceConfig False
           , sioOffchainPoolData = OffchainPoolDataConfig False
           , sioJsonType = JsonTypeDisable
+          , sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
           }
 
   dncInsertOptions cfg @?= expected

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage.hs

@@ -9,7 +9,6 @@ module Test.Cardano.Db.Mock.Unit.Babbage (
 import Cardano.Mock.ChainSync.Server (IOManager)
 import Data.Text (Text)
 import Test.Tasty (TestTree, testGroup)
-import Test.Tasty.ExpectedFailure (expectFail)
 import Test.Tasty.HUnit (Assertion, testCase)
 
 import qualified Test.Cardano.Db.Mock.Unit.Babbage.CommandLineArg.ConfigFile as ConfigFile
@@ -25,6 +24,7 @@ import qualified Test.Cardano.Db.Mock.Unit.Babbage.Rollback as BabRollback
 import qualified Test.Cardano.Db.Mock.Unit.Babbage.Simple as BabSimple
 import qualified Test.Cardano.Db.Mock.Unit.Babbage.Stake as BabStake
 import qualified Test.Cardano.Db.Mock.Unit.Babbage.Tx as BabTx
+import Test.Cardano.Db.Mock.Validate (expectFailSilent)
 
 unitTests :: IOManager -> [(Text, Text)] -> TestTree
 unitTests iom knownMigrations =
@@ -45,9 +45,9 @@ unitTests iom knownMigrations =
             , test "no pruning and rollback" MigrateConsumedPruneTxOut.noPruneAndRollBack
             , test "prune same block" MigrateConsumedPruneTxOut.pruneSameBlock
             , test "no pruning same block" MigrateConsumedPruneTxOut.noPruneSameBlock
-            , expectFail $ test "restart with new consumed set to false" MigrateConsumedPruneTxOut.migrateAndPruneRestart
-            , expectFail $ test "set prune flag, restart missing prune flag" MigrateConsumedPruneTxOut.pruneRestartMissingFlag
-            , expectFail $ test "set bootstrap flag, restart missing bootstrap flag" MigrateConsumedPruneTxOut.bootstrapRestartMissingFlag
+            , expectFailSilent "restart with new consumed set to false" $ MigrateConsumedPruneTxOut.migrateAndPruneRestart iom knownMigrations
+            , expectFailSilent "set prune flag, restart missing prune flag" $ MigrateConsumedPruneTxOut.pruneRestartMissingFlag iom knownMigrations
+            , expectFailSilent "set bootstrap flag, restart missing bootstrap flag" $ MigrateConsumedPruneTxOut.bootstrapRestartMissingFlag iom knownMigrations
             ]
         ]
     , testGroup
@@ -63,7 +63,7 @@ unitTests iom knownMigrations =
         "Command Line Arguments"
         [ testGroup
             "config"
-            [ expectFail $ test "fails if incorrect config file given" ConfigFile.checkConfigFileArg
+            [ expectFailSilent "fails if incorrect config file given" $ ConfigFile.checkConfigFileArg iom knownMigrations
             ]
         , testGroup
             "disable-epoch"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/Parse.hs

@@ -31,6 +31,7 @@ insertConfig = do
           , sioGovernance = GovernanceConfig False
           , sioOffchainPoolData = OffchainPoolDataConfig False
           , sioJsonType = JsonTypeDisable
+          , sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
           }
 
   dncInsertOptions cfg @?= expected

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs

@@ -5,6 +5,7 @@ import Cardano.Prelude
 import qualified Test.Cardano.Db.Mock.Unit.Conway.CommandLineArg.ConfigFile as ConfigFile
 import qualified Test.Cardano.Db.Mock.Unit.Conway.CommandLineArg.EpochDisabled as EpochDisabled
 import qualified Test.Cardano.Db.Mock.Unit.Conway.CommandLineArg.ForceIndex as ForceIndex
+import qualified Test.Cardano.Db.Mock.Unit.Conway.Config.JsonbInSchema as Config
 import qualified Test.Cardano.Db.Mock.Unit.Conway.Config.MigrateConsumedPruneTxOut as MigrateConsumedPruneTxOut
 import qualified Test.Cardano.Db.Mock.Unit.Conway.Config.Parse as Config
 import qualified Test.Cardano.Db.Mock.Unit.Conway.Governance as Governance
@@ -16,8 +17,8 @@ import qualified Test.Cardano.Db.Mock.Unit.Conway.Rollback as Rollback
 import qualified Test.Cardano.Db.Mock.Unit.Conway.Simple as Simple
 import qualified Test.Cardano.Db.Mock.Unit.Conway.Stake as Stake
 import qualified Test.Cardano.Db.Mock.Unit.Conway.Tx as Tx
+import Test.Cardano.Db.Mock.Validate (expectFailSilent)
 import Test.Tasty (TestTree (), testGroup)
-import Test.Tasty.ExpectedFailure (expectFail)
 import Test.Tasty.HUnit (Assertion (), testCase)
 import Prelude (String ())
 
@@ -34,6 +35,14 @@ unitTests iom knownMigrations =
         , testCase "mismatched conway genesis hash" Config.wrongConwayGenesisHash
         , testCase "default insert config" Config.defaultInsertConfig
         , testCase "insert config" Config.insertConfig
+        , testGroup
+            "jsonb-in-schema"
+            [ test "jsonb in schema true" Config.configRemoveJsonbFromSchemaEnabled
+            , test "jsonb in schema false" Config.configRemoveJsonbFromSchemaDisabled
+            , test
+                "remove jsonb from schema and add back"
+                Config.configJsonbInSchemaShouldRemoveThenAdd
+            ]
         , testGroup
             "tx-out"
             [ test "consumed_by_tx_id column check" MigrateConsumedPruneTxOut.txConsumedColumnCheck
@@ -45,18 +54,15 @@ unitTests iom knownMigrations =
             , test "no pruning and rollback" MigrateConsumedPruneTxOut.noPruneAndRollBack
             , test "prune same block" MigrateConsumedPruneTxOut.pruneSameBlock
             , test "no pruning same block" MigrateConsumedPruneTxOut.noPruneSameBlock
-            , expectFail $
-                test
-                  "restart with new consumed set to false"
-                  MigrateConsumedPruneTxOut.migrateAndPruneRestart
-            , expectFail $
-                test
-                  "set prune flag, restart missing prune flag"
-                  MigrateConsumedPruneTxOut.pruneRestartMissingFlag
-            , expectFail $
-                test
-                  "set bootstrap flag, restart missing bootstrap flag"
-                  MigrateConsumedPruneTxOut.bootstrapRestartMissingFlag
+            , expectFailSilent
+                "restart with new consumed set to false"
+                $ MigrateConsumedPruneTxOut.migrateAndPruneRestart iom knownMigrations
+            , expectFailSilent
+                "set prune flag, restart missing prune flag"
+                $ MigrateConsumedPruneTxOut.pruneRestartMissingFlag iom knownMigrations
+            , expectFailSilent
+                "set bootstrap flag, restart missing bootstrap flag"
+                $ MigrateConsumedPruneTxOut.bootstrapRestartMissingFlag iom knownMigrations
             ]
         ]
     , testGroup
@@ -72,10 +78,9 @@ unitTests iom knownMigrations =
         "Command Line Arguments"
         [ testGroup
             "config"
-            [ expectFail $
-                test
-                  "fails if incorrect config file given"
-                  ConfigFile.checkConfigFileArg
+            [ expectFailSilent
+                "fails if incorrect config file given"
+                $ ConfigFile.checkConfigFileArg iom knownMigrations
             ]
         , testGroup
             "disable-epoch"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/JsonbInSchema.hs

@@ -0,0 +1,123 @@
+{-# LANGUAGE NumericUnderscores #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+module Test.Cardano.Db.Mock.Unit.Conway.Config.JsonbInSchema (
+  configRemoveJsonbFromSchemaEnabled,
+  configRemoveJsonbFromSchemaDisabled,
+  configJsonbInSchemaShouldRemoveThenAdd,
+) where
+
+import qualified Cardano.Db as DB
+import Cardano.DbSync.Config (SyncNodeConfig (..))
+import Cardano.DbSync.Config.Types (RemoveJsonbFromSchemaConfig (..), SyncInsertOptions (..))
+import Cardano.Mock.ChainSync.Server (IOManager ())
+import Cardano.Prelude hiding (head)
+import Test.Cardano.Db.Mock.Config
+import Test.Cardano.Db.Mock.Validate
+import Test.Tasty.HUnit (Assertion ())
+
+configRemoveJsonbFromSchemaEnabled :: IOManager -> [(Text, Text)] -> Assertion
+configRemoveJsonbFromSchemaEnabled ioManager metadata = do
+  syncNodeConfig <- mksNodeConfig
+  withCustomConfigAndDropDB args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata
+  where
+    action = \_interpreter _mockServer dbSync -> do
+      startDBSync dbSync
+      threadDelay 7_000_000
+      assertEqQuery
+        dbSync
+        DB.queryJsonbInSchemaExists
+        False
+        "There should be no jsonb data types in database if option is enabled"
+      checkStillRuns dbSync
+
+    args = initCommandLineArgs {claFullMode = False}
+    testLabel = "conwayConfigRemoveJsonbFromSchemaEnabled"
+
+    cfgDir = conwayConfigDir
+
+    mksNodeConfig :: IO SyncNodeConfig
+    mksNodeConfig = do
+      initConfigFile <- mkSyncNodeConfig cfgDir args
+      let dncInsertOptions' = dncInsertOptions initConfigFile
+      pure $
+        initConfigFile
+          { dncInsertOptions = dncInsertOptions' {sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig True}
+          }
+
+configRemoveJsonbFromSchemaDisabled :: IOManager -> [(Text, Text)] -> Assertion
+configRemoveJsonbFromSchemaDisabled ioManager metadata = do
+  syncNodeConfig <- mksNodeConfig
+  withCustomConfigAndDropDB args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata
+  where
+    action = \_interpreter _mockServer dbSync -> do
+      startDBSync dbSync
+      threadDelay 7_000_000
+      assertEqQuery
+        dbSync
+        DB.queryJsonbInSchemaExists
+        True
+        "There should be jsonb types in database if option is disabled"
+      checkStillRuns dbSync
+
+    args = initCommandLineArgs {claFullMode = False}
+    testLabel = "conwayConfigRemoveJsonbFromSchemaDisabled"
+
+    cfgDir = conwayConfigDir
+
+    mksNodeConfig :: IO SyncNodeConfig
+    mksNodeConfig = do
+      initConfigFile <- mkSyncNodeConfig cfgDir args
+      let dncInsertOptions' = dncInsertOptions initConfigFile
+      pure $
+        initConfigFile
+          { dncInsertOptions = dncInsertOptions' {sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False}
+          }
+
+configJsonbInSchemaShouldRemoveThenAdd :: IOManager -> [(Text, Text)] -> Assertion
+configJsonbInSchemaShouldRemoveThenAdd ioManager metadata = do
+  syncNodeConfig <- mksNodeConfig
+  withCustomConfigAndDropDB args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata
+  where
+    action = \_interpreter _mockServer dbSync -> do
+      startDBSync dbSync
+      threadDelay 7_000_000
+      assertEqQuery
+        dbSync
+        DB.queryJsonbInSchemaExists
+        False
+        "There should be no jsonb types in database if option has been enabled"
+      stopDBSync dbSync
+      let newDbSyncEnv =
+            dbSync
+              { dbSyncConfig =
+                  (dbSyncConfig dbSync)
+                    { dncInsertOptions =
+                        (dncInsertOptions $ dbSyncConfig dbSync)
+                          { sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
+                          }
+                    }
+              }
+      startDBSync newDbSyncEnv
+      threadDelay 7_000_000
+      assertEqQuery
+        dbSync
+        DB.queryJsonbInSchemaExists
+        True
+        "There should be jsonb types in database if option has been disabled"
+      -- Expected to fail
+      checkStillRuns dbSync
+
+    args = initCommandLineArgs {claFullMode = False}
+    testLabel = "configJsonbInSchemaShouldRemoveThenAdd"
+
+    cfgDir = conwayConfigDir
+
+    mksNodeConfig :: IO SyncNodeConfig
+    mksNodeConfig = do
+      initConfigFile <- mkSyncNodeConfig cfgDir args
+      let dncInsertOptions' = dncInsertOptions initConfigFile
+      pure $
+        initConfigFile
+          { dncInsertOptions = dncInsertOptions' {sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig True}
+          }

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs

@@ -101,6 +101,7 @@ insertConfig = do
           , sioGovernance = GovernanceConfig False
           , sioOffchainPoolData = OffchainPoolDataConfig False
           , sioJsonType = JsonTypeDisable
+          , sioRemoveJsonbFromSchema = RemoveJsonbFromSchemaConfig False
           }
 
   dncInsertOptions cfg @?= expected