Add option for nightly, ci and dev test

The option iohk-test-mode can be set to nightly, ci or dev mode to run different number of quickcheck tests - Nightly mode runs 1_000_000 quickcheck tests - CI mode runs 1_00_000 quickcheck tests - Dev mode runs 100 quickcheck tests Some tests override above numbers to be able to keep running time to a reasonable limit. An existing mode iohk-enable-nightly-tests is deprecated. However, if this flag is set, it sets iohk-test-mode to nightly.
IntersectMBO · Aug 10, 2022 · 63f58bc · 63f58bc
1 parent 548af5b
commit 63f58bc
Show file tree

Hide file tree

Showing 15 changed files with 153 additions and 36 deletions.
diff --git a/ouroboros-consensus-byron-test/test/Main.hs b/ouroboros-consensus-byron-test/test/Main.hs
@@ -6,9 +6,10 @@ import qualified Test.Consensus.Byron.Golden (tests)
 import qualified Test.Consensus.Byron.Serialisation (tests)
 import qualified Test.ThreadNet.Byron (tests)
 import qualified Test.ThreadNet.DualByron (tests)
+import           Test.Util.TestMode (defaultMainWithIohkTestMode)
 
 main :: IO ()
-main = defaultMain tests
+main = defaultMainWithIohkTestMode tests
 
 tests :: TestTree
 tests =

diff --git a/ouroboros-consensus-cardano-test/test/Main.hs b/ouroboros-consensus-cardano-test/test/Main.hs
@@ -6,7 +6,7 @@ import           System.IO (BufferMode (LineBuffering), hSetBuffering,
 import           Cardano.Crypto.Libsodium (sodiumInit)
 
 import           Test.Tasty
-import           Test.Util.Nightly
+import           Test.Util.TestMode
 
 import qualified Test.Consensus.Cardano.ByronCompatibility (tests)
 import qualified Test.Consensus.Cardano.Golden (tests)
@@ -21,7 +21,7 @@ main = do
   hSetBuffering stdout LineBuffering
   hSetEncoding stdout utf8
   sodiumInit
-  defaultMainWithIohkNightly tests
+  defaultMainWithIohkTestMode tests
 
 tests :: TestTree
 tests =

diff --git a/ouroboros-consensus-cardano-test/test/Test/ThreadNet/AllegraMary.hs b/ouroboros-consensus-cardano-test/test/Test/ThreadNet/AllegraMary.hs
@@ -67,9 +67,10 @@ import           Test.ThreadNet.Util.NodeToNodeVersion (genVersionFiltered)
 import           Test.ThreadNet.Util.Seed (runGen)
 import qualified Test.Util.BoolProps as BoolProps
 import           Test.Util.HardFork.Future (EraSize (..), Future (..))
-import           Test.Util.Nightly (askIohkNightlyEnabled)
 import           Test.Util.Orphans.Arbitrary ()
 import           Test.Util.Slots (NumSlots (..))
+import           Test.Util.TestMode (IohkTestMode (..), askIohkTestMode,
+                     resetQuickCheckTests)
 
 import           Test.Consensus.Shelley.MockCrypto (MockCrypto)
 import qualified Test.ThreadNet.Infra.Shelley as Shelley
@@ -164,12 +165,18 @@ oneTenthTestCount (QuickCheckTests n) = QuickCheckTests $
 tests :: TestTree
 tests = testGroup "AllegraMary ThreadNet" $
     [ let name = "simple convergence" in
-      askIohkNightlyEnabled $ \enabled ->
-      (if enabled then id else adjustOption oneTenthTestCount) $
+      askIohkTestMode $ flip adjustTestMode $
       testProperty name $ \setup ->
         prop_simple_allegraMary_convergence setup
     ]
 
+    where
+      adjustTestMode :: IohkTestMode -> TestTree -> TestTree
+      adjustTestMode = \case
+        Nightly -> resetQuickCheckTests id
+        CI      -> resetQuickCheckTests oneTenthTestCount
+        Dev     -> resetQuickCheckTests oneTenthTestCount
+
 prop_simple_allegraMary_convergence :: TestSetup -> Property
 prop_simple_allegraMary_convergence TestSetup
   { setupD

diff --git a/ouroboros-consensus-cardano-test/test/Test/ThreadNet/Cardano.hs b/ouroboros-consensus-cardano-test/test/Test/ThreadNet/Cardano.hs
@@ -1,5 +1,6 @@
 {-# LANGUAGE FlexibleContexts    #-}
 {-# LANGUAGE GADTs               #-}
+{-# LANGUAGE LambdaCase          #-}
 {-# LANGUAGE NamedFieldPuns      #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications    #-}
@@ -70,14 +71,14 @@ import           Test.ThreadNet.Util.NodeToNodeVersion (genVersionFiltered)
 import           Test.ThreadNet.Util.Seed (runGen)
 import qualified Test.Util.BoolProps as BoolProps
 import           Test.Util.HardFork.Future
-import           Test.Util.Nightly
 import           Test.Util.Orphans.Arbitrary ()
 import           Test.Util.Slots (NumSlots (..))
 
 import           Ouroboros.Consensus.Protocol.Praos.Translate ()
 import           Ouroboros.Consensus.Shelley.Node.Praos
                      (ProtocolParamsBabbage (..))
 import           Test.ThreadNet.Infra.TwoEras
+import           Test.Util.TestMode
 
 -- | Use 'MockCryptoCompatByron' so that bootstrap addresses and
 -- bootstrap witnesses are supported.
@@ -163,15 +164,20 @@ twoFifthsTestCount (QuickCheckTests n) = QuickCheckTests $
 tests :: TestTree
 tests = testGroup "Cardano ThreadNet" $
     [ let name = "simple convergence" in
-      askIohkNightlyEnabled $ \enabled ->
-      if enabled
-      then testProperty name $ \setup ->
-             prop_simple_cardano_convergence setup
-      else adjustOption twoFifthsTestCount $
-           testProperty name $ \setup ->
-             prop_simple_cardano_convergence setup
+      askIohkTestMode $ flip adjustTestMode $
+      testProperty name $ \setup ->
+                      prop_simple_cardano_convergence setup
+
     ]
 
+    where
+      adjustTestMode :: IohkTestMode -> TestTree -> TestTree
+      adjustTestMode = \case
+        Nightly -> resetQuickCheckTests id
+        CI      -> resetQuickCheckTests twoFifthsTestCount
+        Dev     -> resetQuickCheckTests twoFifthsTestCount
+
+
 prop_simple_cardano_convergence :: TestSetup -> Property
 prop_simple_cardano_convergence TestSetup
   { setupD

diff --git a/ouroboros-consensus-cardano-test/test/Test/ThreadNet/MaryAlonzo.hs b/ouroboros-consensus-cardano-test/test/Test/ThreadNet/MaryAlonzo.hs
@@ -65,9 +65,10 @@ import           Test.ThreadNet.Util.NodeToNodeVersion (genVersionFiltered)
 import           Test.ThreadNet.Util.Seed (runGen)
 import qualified Test.Util.BoolProps as BoolProps
 import           Test.Util.HardFork.Future (EraSize (..), Future (..))
-import           Test.Util.Nightly (askIohkNightlyEnabled)
 import           Test.Util.Orphans.Arbitrary ()
 import           Test.Util.Slots (NumSlots (..))
+import           Test.Util.TestMode (IohkTestMode (..), askIohkTestMode,
+                     resetQuickCheckTests)
 
 import           Test.Consensus.Shelley.MockCrypto (MockCrypto)
 import qualified Test.ThreadNet.Infra.Alonzo as Alonzo
@@ -163,12 +164,18 @@ oneTenthTestCount (QuickCheckTests n) = QuickCheckTests $
 tests :: TestTree
 tests = testGroup "MaryAlonzo ThreadNet" $
     [ let name = "simple convergence" in
-      askIohkNightlyEnabled $ \enabled ->
-      (if enabled then id else adjustOption oneTenthTestCount) $
+      askIohkTestMode $ flip adjustTestMode $
       testProperty name $ \setup ->
         prop_simple_allegraAlonzo_convergence setup
     ]
 
+    where
+      adjustTestMode :: IohkTestMode -> TestTree -> TestTree
+      adjustTestMode = \case
+        Nightly -> resetQuickCheckTests id
+        CI      -> resetQuickCheckTests oneTenthTestCount
+        Dev     -> resetQuickCheckTests oneTenthTestCount
+
 prop_simple_allegraAlonzo_convergence :: TestSetup -> Property
 prop_simple_allegraAlonzo_convergence TestSetup
   { setupD

diff --git a/ouroboros-consensus-cardano-test/test/Test/ThreadNet/ShelleyAllegra.hs b/ouroboros-consensus-cardano-test/test/Test/ThreadNet/ShelleyAllegra.hs
@@ -66,9 +66,10 @@ import           Test.ThreadNet.Util.NodeToNodeVersion (genVersionFiltered)
 import           Test.ThreadNet.Util.Seed (runGen)
 import qualified Test.Util.BoolProps as BoolProps
 import           Test.Util.HardFork.Future (EraSize (..), Future (..))
-import           Test.Util.Nightly (askIohkNightlyEnabled)
 import           Test.Util.Orphans.Arbitrary ()
 import           Test.Util.Slots (NumSlots (..))
+import           Test.Util.TestMode (IohkTestMode (..), askIohkTestMode,
+                     resetQuickCheckTests)
 
 import           Test.Consensus.Shelley.MockCrypto (MockCrypto)
 import qualified Test.ThreadNet.Infra.Shelley as Shelley
@@ -164,12 +165,17 @@ oneTenthTestCount (QuickCheckTests n) = QuickCheckTests $
 tests :: TestTree
 tests = testGroup "ShelleyAllegra ThreadNet" $
     [ let name = "simple convergence" in
-      askIohkNightlyEnabled $ \enabled ->
-      (if enabled then id else adjustOption oneTenthTestCount) $
-      testProperty name $ \setup ->
+      askIohkTestMode $ flip adjustTestMode $ testProperty name $ \setup ->
         prop_simple_shelleyAllegra_convergence setup
     ]
 
+    where
+      adjustTestMode :: IohkTestMode -> TestTree -> TestTree
+      adjustTestMode = \case
+        Nightly -> resetQuickCheckTests id
+        CI      -> resetQuickCheckTests oneTenthTestCount
+        Dev     -> resetQuickCheckTests oneTenthTestCount
+
 prop_simple_shelleyAllegra_convergence :: TestSetup -> Property
 prop_simple_shelleyAllegra_convergence TestSetup
   { setupD

diff --git a/ouroboros-consensus-mock-test/test/Main.hs b/ouroboros-consensus-mock-test/test/Main.hs
@@ -7,9 +7,10 @@ import qualified Test.ThreadNet.BFT (tests)
 import qualified Test.ThreadNet.LeaderSchedule (tests)
 import qualified Test.ThreadNet.PBFT (tests)
 import qualified Test.ThreadNet.Praos (tests)
+import           Test.Util.TestMode (defaultMainWithIohkTestMode)
 
 main :: IO ()
-main = defaultMain tests
+main = defaultMainWithIohkTestMode tests
 
 tests :: TestTree
 tests =

diff --git a/ouroboros-consensus-shelley-test/test/Main.hs b/ouroboros-consensus-shelley-test/test/Main.hs
@@ -3,15 +3,15 @@ module Main (main) where
 import           Cardano.Crypto.Libsodium (sodiumInit)
 
 import           Test.Tasty
-import           Test.Util.Nightly
 
 import qualified Test.Consensus.Shelley.Coherence (tests)
 import qualified Test.Consensus.Shelley.Golden (tests)
 import qualified Test.Consensus.Shelley.Serialisation (tests)
 import qualified Test.ThreadNet.Shelley (tests)
+import           Test.Util.TestMode (defaultMainWithIohkTestMode)
 
 main :: IO ()
-main = sodiumInit >> defaultMainWithIohkNightly tests
+main = sodiumInit >> defaultMainWithIohkTestMode tests
 
 tests :: TestTree
 tests =

diff --git a/ouroboros-consensus-shelley-test/test/Test/ThreadNet/Shelley.hs b/ouroboros-consensus-shelley-test/test/Test/ThreadNet/Shelley.hs
@@ -1,3 +1,4 @@
+{-# LANGUAGE LambdaCase       #-}
 {-# LANGUAGE NamedFieldPuns   #-}
 {-# LANGUAGE TypeApplications #-}
 
@@ -28,9 +29,10 @@ import           Test.ThreadNet.Network (TestNodeInitialization (..),
                      nodeOutputFinalLedger)
 
 import           Test.Util.HardFork.Future (singleEraFuture)
-import           Test.Util.Nightly
 import           Test.Util.Orphans.Arbitrary ()
 import           Test.Util.Slots (NumSlots (..))
+import           Test.Util.TestMode (IohkTestMode (..), askIohkTestMode,
+                     resetQuickCheckTests)
 
 import qualified Cardano.Ledger.BaseTypes as SL (UnitInterval,
                      mkNonceFromNumber, unboundRational)
@@ -160,13 +162,13 @@ fifthTestCount (QuickCheckTests n) = QuickCheckTests $
 tests :: TestTree
 tests = testGroup "Shelley ThreadNet"
     [ let name = "simple convergence" in
-      askIohkNightlyEnabled $ \enabled ->
-      if enabled
-      then testProperty name $ \(NightlyTestSetup setup) ->
-             prop_simple_real_tpraos_convergence setup
-      else adjustOption fifthTestCount $
-           testProperty name $ \setup ->
-             prop_simple_real_tpraos_convergence setup
+      askIohkTestMode $ \case
+          Nightly -> resetQuickCheckTests id $ testProperty name $ \(NightlyTestSetup setup) ->
+            prop_simple_real_tpraos_convergence setup
+          CI      -> resetQuickCheckTests fifthTestCount $ testProperty name $ \setup ->
+            prop_simple_real_tpraos_convergence setup
+          Dev     -> resetQuickCheckTests fifthTestCount $ testProperty name $ \setup ->
+            prop_simple_real_tpraos_convergence setup
     ]
 
 prop_simple_real_tpraos_convergence :: TestSetup -> Property

diff --git a/ouroboros-consensus-test/ouroboros-consensus-test.cabal b/ouroboros-consensus-test/ouroboros-consensus-test.cabal
@@ -72,6 +72,7 @@ library
                        Test.Util.Split
                        Test.Util.Stream
                        Test.Util.TestBlock
+                       Test.Util.TestMode
                        Test.Util.Time
                        Test.Util.Tracer
                        Test.Util.WithEq
@@ -95,6 +96,7 @@ library
                      , graphviz
                      , mtl               >=2.2   && <2.3
                      , nothunks
+                     , optparse-applicative
                      , QuickCheck
                      , quickcheck-state-machine
                      , quiet             >=0.2   && <0.3

diff --git a/ouroboros-consensus-test/src/Test/Util/Nightly.hs b/ouroboros-consensus-test/src/Test/Util/Nightly.hs
@@ -35,7 +35,8 @@ instance IsOption IohkNightlyEnabled where
   defaultValue = IohkNightlyEnabled False
   parseValue = fmap IohkNightlyEnabled . safeReadBool
   optionName = pure "iohk-enable-nightly-tests"
-  optionHelp = pure "Enable more expensive tests (specific to IOHK)"
+  optionHelp = pure "DEPRECATED Enable more expensive tests (specific to IOHK)\n\
+                    \ This option will be removed in future. Use iohk-test-mode instead."
 
   -- Use typical Un*x syntax for Boolean flags
   optionCLParser = flagCLParser Nothing (IohkNightlyEnabled True)
diff --git a/ouroboros-consensus-test/src/Test/Util/TestMode.hs b/ouroboros-consensus-test/src/Test/Util/TestMode.hs
@@ -0,0 +1,81 @@
+{-# LANGUAGE LambdaCase #-}
+-- | A @tasty@ command-line option for enabling nightly tests
+module Test.Util.TestMode (
+    IohkTestMode (..)
+  , askIohkTestMode
+  , defaultMainWithIohkTestMode
+  , iohkTestModeIngredient
+  , resetQuickCheckTests
+  ) where
+
+import           Data.Proxy (Proxy (..))
+import           Options.Applicative (metavar)
+import           Test.Tasty
+import           Test.Tasty.Ingredients
+import           Test.Tasty.Options
+import           Test.Tasty.QuickCheck
+import           Test.Util.Nightly
+
+-- | 'defaultMain' extended with 'iohkTestEnvIngredient'
+defaultMainWithIohkTestMode :: TestTree -> IO ()
+defaultMainWithIohkTestMode testTree =
+    defaultMainWithIngredients (iohkNightlyIngredient : iohkTestModeIngredient : defaultIngredients) $
+    askIohkTestMode (defaultTestEnv $ askOption iohkNightlyToTestMode)
+
+    where
+      -- If iohk-nightly is already set, change the mode to nightly
+      iohkNightlyToTestMode :: IohkNightlyEnabled -> TestTree
+      iohkNightlyToTestMode (IohkNightlyEnabled True) = adjustOption (const Nightly) testTree
+      iohkNightlyToTestMode _                         = testTree
+
+
+-- | This ingredient merely adds the 'IohkTestEnv' 'Option' to the
+-- @tasty@ command-line parser.
+iohkTestModeIngredient :: Ingredient
+iohkTestModeIngredient =
+    TestManager [Option (Proxy :: Proxy IohkTestMode)] $
+    \_optionSet _testTree -> Nothing
+
+-- | Query and adjust options for `IohkTestMode`
+askIohkTestMode :: (IohkTestMode -> TestTree) -> TestTree
+askIohkTestMode = askOption
+
+-- | Setup the default test environment
+defaultTestEnv :: TestTree -> IohkTestMode -> TestTree
+defaultTestEnv testTree = \case
+  Nightly -> adjustOption (const (QuickCheckTests 1000000)) testTree
+  CI      -> adjustOption (const (QuickCheckTests 10000)) testTree
+  Dev     -> testTree
+
+-- | Reset quickcheck tests
+resetQuickCheckTests :: (QuickCheckTests -> QuickCheckTests) -> TestTree -> TestTree
+resetQuickCheckTests f = adjustOption $ const (f (QuickCheckTests 100))
+
+-- | An 'Option' that indicates the environment in which to run tests.
+data IohkTestMode =
+  -- | Run more expensive tests
+    Nightly
+  -- | Run tests for the continuous integration, slightly more expensive
+  | CI
+  -- | A fast tests for developers to run before committin
+  | Dev
+
+safeReadTestMode :: String -> Maybe IohkTestMode
+safeReadTestMode "nightly" = Just Nightly
+safeReadTestMode "ci"      = Just CI
+safeReadTestMode "dev"     = Just Dev
+safeReadTestMode _         = Nothing
+
+instance IsOption IohkTestMode where
+  defaultValue = Dev
+  parseValue = safeReadTestMode
+  optionName = pure "iohk-test-mode"
+  optionHelp = pure "Enable a test mode (specific to IOHK). \
+      \ The 'dev' mode runs default number of quickcheck tests (100), \
+      \ 'nightly' mode runs 1_000_000 quickcheck tests, and \
+      \ 'ci' mode runs 100_000 quickcheck tests. \
+      \ Individual tests can be adjusted to run less/more depending \
+      \ upon time taken to run them"
+
+  -- Use typical Un*x syntax for Boolean flags
+  optionCLParser = mkOptionCLParser $ metavar "nightly|ci|dev"
diff --git a/ouroboros-consensus-test/test-consensus/Main.hs b/ouroboros-consensus-test/test-consensus/Main.hs
@@ -15,9 +15,10 @@ import qualified Test.Consensus.Node (tests)
 import qualified Test.Consensus.ResourceRegistry (tests)
 import qualified Test.Consensus.Util.MonadSTM.RAWLock (tests)
 import qualified Test.Consensus.Util.Versioned (tests)
+import           Test.Util.TestMode (defaultMainWithIohkTestMode)
 
 main :: IO ()
-main = defaultMain tests
+main = defaultMainWithIohkTestMode tests
 
 tests :: TestTree
 tests =

diff --git a/ouroboros-consensus-test/test-infra/Main.hs b/ouroboros-consensus-test/test-infra/Main.hs
@@ -6,9 +6,10 @@ import qualified Test.ThreadNet.Util.Tests (tests)
 import qualified Test.Util.ChainUpdates.Tests (tests)
 import qualified Test.Util.Schedule.Tests (tests)
 import qualified Test.Util.Split.Tests (tests)
+import           Test.Util.TestMode (defaultMainWithIohkTestMode)
 
 main :: IO ()
-main = defaultMain tests
+main = defaultMainWithIohkTestMode tests
 
 tests :: TestTree
 tests =

diff --git a/ouroboros-consensus-test/test-storage/Main.hs b/ouroboros-consensus-test/test-storage/Main.hs
@@ -5,12 +5,13 @@ import           System.IO.Temp
 import           Test.Tasty
 
 import qualified Test.Ouroboros.Storage
+import           Test.Util.TestMode (defaultMainWithIohkTestMode)
 
 main :: IO ()
 main = do
   sysTmpDir <- Dir.getTemporaryDirectory
   withTempDirectory sysTmpDir "cardano-s-m" $ \tmpDir ->
-    defaultMain (tests tmpDir)
+    defaultMainWithIohkTestMode (tests tmpDir)
 
 tests :: FilePath -> TestTree
 tests tmpDir =