wip: Add Shelley ff testnet to nightly restore tests

.buildkite/bench-restore.sh

@@ -3,23 +3,27 @@
 
 set -euo pipefail
 
-target=byron
-artifact_name=restore-$target-$NETWORK
+if [ "$#" -lt 2 ]; then
+  echo "usage: $0 TARGET NETWORK"
+  exit 1
+fi
+
+target=$1
+network=$2
+artifact_name=restore-$target-$network
 log=restore.log
-results=restore-$target-$NETWORK.txt
+results=restore-$target-$network.txt
 total_time=restore-time.txt
-export NODE_DB=$HOME/node-db-$NETWORK
 
-echo "--- Build"
+node_db=$HOME/node-db-$target-$network
 
-echo "pwd=$(pwd)"
-echo "NODE_DB=$NODE_DB"
+echo "--- Build"
 
 nix-build -A benchmarks.cardano-wallet-$target.restore -o bench-$target-restore
 
-bench="./bench-$target-restore/bin/restore --$NETWORK"
+bench="./bench-$target-restore/bin/restore $network"
 
-echo "--- Run benchmarks - $target - $NETWORK"
+echo "--- Run benchmarks - $target - $network"
 
 if [ -n "${SCRATCH_DIR:-}" ]; then
   mkdir -p "$SCRATCH_DIR"
@@ -30,7 +34,7 @@ command time -o $total_time -v $bench +RTS -N2 -qg -A1m -I0 -T -M8G -h -RTS 2>&1
 
 grep -v INFO $log | awk '/All results/,EOF { print $0 }' > $results
 
-echo "+++ Results - $target - $NETWORK"
+echo "+++ Results - $target - $network"
 
 cat $results
 
@@ -49,7 +53,7 @@ show ylabel;
 
 set terminal svg dynamic size 1200,700 background rgb 'white';
 set output "plot.svg";
-set title "Restoring byron wallets on $NETWORK";
+set title "Restoring byron wallets on $network";
 
 set key left top;
 

.buildkite/nightly.yml

@@ -7,25 +7,32 @@ env:
 
 steps:
   - label: 'Restore benchmark - mainnet'
-    command: "./.buildkite/bench-restore.sh"
+    command: "./.buildkite/bench-restore.sh byron mainnet"
     env:
       HOME: "/cache/cardano-wallet.home"
-      NETWORK: mainnet
     timeout_in_minutes: 600
     agents:
       system: x86_64-linux
       queue: benchmark
 
   - label: 'Restore benchmark - testnet'
-    command: "./.buildkite/bench-restore.sh"
+    command: "./.buildkite/bench-restore.sh byron testnet"
     env:
       HOME: "/cache/cardano-wallet.home"
-      NETWORK: testnet
     timeout_in_minutes: 600
     agents:
       system: x86_64-linux
       queue: benchmark
 
+  - label: 'Restore benchmark - ff'
+    command: "./.buildkite/bench-restore.sh shelley ff"
+    env:
+      HOME: "/cache/cardano-wallet.home"
+    timeout_in_minutes: 60
+    agents:
+      system: x86_64-linux
+      queue: benchmark
+
   - label: 'Database benchmark'
     command: "./.buildkite/bench-db.sh"
     timeout_in_minutes: 120

lib/byron/bench/Restore.hs

@@ -181,30 +181,96 @@ import qualified Data.Text.Encoding as T
 
 main :: IO ()
 main = do
-    configs <- getEnv "CARDANO_NODE_CONFIGS"
-    let testnetGenesis = configs </> "testnet" </> "genesis.json"
-    let opts = info (fmap exec (networkConfigurationOption testnetGenesis)) mempty
-    join $ execParser opts
-
--- | --mainnet | --testnet
-networkConfigurationOption
-    :: FilePath
-    -> Parser NetworkConfiguration
-networkConfigurationOption testnetGenesis =
-    mainnetFlag <|> testnetFlag
+    (networkConfig, nodeConfig, cleanup) <- getNetworkConfiguration
+
+    exec networkConfig nodeConfig `finally` cleanup
+
+data Args = Args
+    { networkName :: Maybe String
+    , configsDir :: Maybe FilePath
+    , nodeDatabaseDir :: Maybe FilePath
+    } deriving (Show, Eq)
+
+getNetworkConfiguration :: IO (NetworkConfiguration, CardanoNodeConfig, IO ())
+getNetworkConfiguration = do
+    let opts = info argsParser mempty
+    args <- addEnvs =<< execParser opts
+
+    configs <- maybe (die "--cardano-node-configs arg not set") pure (configsDir args)
+
+    networkConfig <- case networkName args of
+        Nothing ->
+            die "NETWORK arg not set"
+        Just "mainnet" ->
+            pure $ MainnetConfig (SomeNetworkDiscriminant $ Proxy @'Mainnet, mainnetVersionData)
+        Just networkName -> do
+            let testnetGenesis = configs </> networkName </> "genesis.json"
+            pure $ TestnetConfig testnetGenesis
+
+    (dbDir, cleanup) <- case nodeDatabaseDir args of
+        Nothing -> do
+            -- Temporary directory for storing socket and node database
+            tmpDir <- getCanonicalTemporaryDirectory
+                >>= \tmpRoot -> createTempDirectory tmpRoot "cw-byron"
+            pure (tmpDir, removeDirectoryRecursive tmpDir)
+        Just d -> pure (d, pure ())
+
+    let networkDir = configs </> networkName
+    let nodeConfig = CardanoNodeConfig
+            { nodeConfigFile   = networkDir </> "configuration.json"
+            , nodeDatabaseDir  = dbDir
+            , nodeDlgCertFile  = ""
+            , nodeSignKeyFile  = ""
+            , nodeSocketFile   = dbDir </> "cardano-node.socket"
+            , nodeTopologyFile = networkDir </> "topology.json"
+            }
+
+    pure (networkConfig, nodeConfig, cleanup)
+
+argsParser :: Parser Args
+argsParser = Args
+    <$> strArgument (metavar "NETWORK" <> help "Blockchain to use. Defaults to $NETWORK.")
+    <*> strOption
+        ( long "cardano-node-configs"
+          <> short 'c'
+          <> metavar "DIR"
+          <> help "Directory containing configurations for each network. Defaults to $CARDANO_NODE_CONFIGS")
+    <*> strOption
+        ( long "node-db"
+          <> metavar "DB"
+          <> help "Directory to put cardano-node state. Defaults to $NODE_DB, falls back to temporary directory")
+
+cardanoNodeCommand :: CardanoNodeConfig -> Int -> Command
+cardanoNodeCommand cfg port = Command "cardano-node" args (return ()) Inherit Inherit
   where
-    mainnetFlag = flag'
-        (MainnetConfig (SomeNetworkDiscriminant $ Proxy @'Mainnet, mainnetVersionData))
-        (long "mainnet")
-
-    testnetFlag = flag'
-        (TestnetConfig testnetGenesis)
-        (long "testnet")
+    args =
+        [ "run"
+        , "--database-path", nodeDatabaseDir cfg
+        , "--topology", nodeTopologyFile cfg
+        , "--socket-path", nodeSocketPath cfg
+        , "--config", nodeConfigFile cfg
+        , "--port", show port
+        ]
+
+
+-- Environment variables set by nix/haskell.nix (or manually)
+-- Environment variables set by ./buildkite/bench-restore.sh (or manually)
+addEnvs :: Args -> IO Args
+addEnvs (Args n c d) = update
+    <$> lookupEnv' "NETWORK"
+    <*> lookupEnv' "CARDANO_NODE_CONFIGS"
+    <*> lookupEnv' "NODE_DB"
+  where
+    update ne ce de = Args (n <|> ne) (c <|> ce) (d <|> de)
+    lookupEnv' k = lookupEnv k <&> \case
+        Just "" -> Nothing
+        Just v -> Just v
+        Nothing -> Nothing
 
 -- | Run all available benchmarks. Can accept one argument that is a target
 -- network against which benchmarks below should be ran
-exec :: NetworkConfiguration -> IO ()
-exec c = do
+exec :: NetworkConfiguration -> CardanoNodeConfig -> IO ()
+exec c nodeConfig = do
     hSetBuffering stdout NoBuffering
     hSetBuffering stderr NoBuffering
 
@@ -214,49 +280,22 @@ exec c = do
     (SomeNetworkDiscriminant networkProxy, np, vData, _b)
         <- unsafeRunExceptT $ parseGenesisData c
 
-    ----------------------------------------------------------------------------
-    -- Environment variables set by nix/haskell.nix (or manually)
-    configs <- getEnv "CARDANO_NODE_CONFIGS"
-    let networkDir = case c of
-            MainnetConfig _ -> "mainnet"
-            TestnetConfig _ -> "testnet"
-        topology = configs </> networkDir </> "topology.json"
-        config = configs </> networkDir </> "configuration.json"
-
-    ----------------------------------------------------------------------------
-    -- Environment variables set by ./buildkite/bench-restore.sh (or manually)
-    nodeDB <- getEnv "NODE_DB"
-    ----------------------------------------------------------------------------
-
-    -- Temporary directory for storing socket
-    tmpDir <- getCanonicalTemporaryDirectory
-        >>= \tmpRoot -> createTempDirectory tmpRoot "cw-byron"
-
     let network = networkDescription networkProxy
     sayErr $ "Network: " <> network
 
-    let socketPath = tmpDir </> "cardano-node.socket"
-    let args =
-            [ "run"
-            , "--database-path", nodeDB
-            , "--topology", topology
-            , "--socket-path", socketPath
-            , "--config", config
-            , "--port", "7776"
-            ]
-    let cmd = Command "cardano-node" args (return ()) Inherit Inherit
+    cmd <- cardanoNodeCommand nodeConfig <$> getRandomPort
 
     sayErr "Starting node with command:"
     sayErr $ pretty cmd
 
     void $ withBackendProcess nullTracer cmd $ do
-            prepareNode networkProxy socketPath np vData
+            prepareNode networkProxy (nodeSocketFile nodeConfig) np vData
             runBenchmarks
                 [ bench ("restore " <> network <> " seq")
                     (bench_restoration @_ @ByronKey
                         networkProxy
                         tr
-                        socketPath
+                        (nodeSocketFile nodeConfig)
                         np
                         vData
                         "seq.timelog"
@@ -266,7 +305,7 @@ exec c = do
                     (bench_restoration @_ @IcarusKey
                         networkProxy
                         tr
-                        socketPath
+                        (nodeSocketFile nodeConfig)
                         np
                         vData
                         "1-percent.timelog"
@@ -276,7 +315,7 @@ exec c = do
                     (bench_restoration @_ @IcarusKey
                         networkProxy
                         tr
-                        socketPath
+                        (nodeSocketFile nodeConfig)
                         np
                         vData
                         "2-percent.timelog"