CAD-2069 locli: log analysis

Makefile

@@ -25,7 +25,7 @@ help:
 EXES=cardano-node cardano-cli cardano-tx-generator
 
 ###
-###
+### Dev env
 ###
 nix:                      MODE=nix
 cabal:                    MODE=cabal
@@ -54,11 +54,23 @@ setup:
 build: setup
 	cabal v2-build ${CABAL_OPTIONS} $(foreach exe,${EXES},${exe}:exe:${exe})
 
+###
+### Cluster
+###
+genesis:
+	cd ./benchmarks/shelley3pools; ./prepare-genesis.sh --${MODE}
+
 recluster: EXTRA_OPTS+=--reuse-genesis
 recluster: cluster
 cluster: build
 	cd ./benchmarks/shelley3pools; ./start.sh --${MODE} ${EXTRA_OPTS}
 
+###
+### Analyses
+###
+leads leaderships:
+	./benchmarks/shelley3pools/analyse-leaderships.sh
+
 ###
 ###
 ###

benchmarks/shelley3pools/analyse-leaderships.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090
+
+basedir=$(realpath "$(dirname "$0")")
+. "$basedir"/../../scripts/common.sh
+. "$basedir"/configuration/parameters
+
+gendir=$basedir/$GENESISDIR_shelley
+logdir="$basedir"/logs
+
+prebuild 'locli' || exit 1
+
+set -eo pipefail
+
+machines=(node-1 node-2 node-3)
+
+leadership_analysis_args=(
+        analyse leadership
+        --slot-length  "$(jq .slotLength  "$gendir"/genesis.json -r)"
+        --epoch-slots  "$(jq .epochLength "$gendir"/genesis.json -r)"
+        --system-start "$(jq .systemStart "$gendir"/genesis.json -r)"
+)
+
+keyfile=$(mktemp -t XXXXXXXXXX.keys)
+run locli analyse substring-keys > "$keyfile"
+
+mkdir -p "$logdir"/analysis
+for mach in ${machines[*]}
+do grep -hFf "$keyfile" "$logdir"/"$mach"*.json \
+        > "$logdir"/analysis/logs-"$mach".json
+   run locli ${leadership_analysis_args[*]} \
+         --dump-leaderships     "$logdir"/analysis/logs-"$mach".leaderships.json \
+         --dump-pretty-timeline "$logdir"/analysis/logs-"$mach".leaderships.pretty.json \
+         > "$logdir"/analysis/logs-"$mach".leadership-analysis.json \
+         "$logdir"/analysis/logs-"$mach".json
+done
+
+rm -f "$keyfile"
+
+oprint "leadership analyses at:  $logdir/analysis"

benchmarks/shelley3pools/benchmark.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# shellcheck disable=SC1090
 
 # preparation
 BASEDIR=$(realpath $(dirname "$0"))

benchmarks/shelley3pools/prepare-genesis.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090
+
+# preparation
+BASEDIR=$(realpath "$(dirname "$0")")
+. "${BASEDIR}"/../../scripts/common.sh
+
+cd "$BASEDIR" || exit 1
+
+rm -f 'configuration/start-time'
+. "$BASEDIR"/configuration/parameters
+./prepare_genesis_byron.sh
+if test -z "$reuse_genesis"
+then ./prepare_genesis_shelley_staked.sh
+else sed -i 's/"systemStart": ".*"/"systemStart": "'"$(date \
+       --iso-8601=seconds \
+       --date=@$(cat "$BASEDIR"/configuration/start-time))"'"/
+       ' "$GENESISDIR_shelley"/genesis.json
+     ./hash_genesis.sh
+fi

locli/NOTICE

@@ -0,0 +1,13 @@
+Copyright 2020 Input Output (Hong Kong) Ltd.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

locli/app/locli.hs

@@ -0,0 +1,18 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+import           Cardano.Prelude hiding (option)
+
+import           Control.Monad.Trans.Except.Exit (orDie)
+import qualified Options.Applicative as Opt
+
+import           Cardano.Unlog.Parsers (opts, pref)
+import           Cardano.Unlog.Run (renderClientCommandError, runClientCommand)
+import           Cardano.TopHandler
+
+
+main :: IO ()
+main = toplevelExceptionHandler $ do
+
+  co <- Opt.customExecParser pref opts
+
+  orDie renderClientCommandError $ runClientCommand co

locli/locli.cabal

@@ -0,0 +1,87 @@
+cabal-version: 2.4
+
+name:                  locli
+version:               1.21.1
+description:           Log parsing CLI.
+author:                IOHK
+maintainer:            operations@iohk.io
+license:               Apache-2.0
+license-files:
+  NOTICE
+build-type:            Simple
+
+library
+
+  hs-source-dirs:      src
+
+  exposed-modules:     Data.Accum
+                       Data.Distribution
+
+                       Cardano.Config.Git.Rev
+                       Cardano.Config.Git.RevFromGit
+
+                       Cardano.TopHandler
+
+                       Cardano.Unlog.Analysis
+                       Cardano.Unlog.Commands
+                       Cardano.Unlog.LogObject
+                       Cardano.Unlog.Parsers
+                       Cardano.Unlog.Resources
+                       Cardano.Unlog.Run
+
+  other-modules:       Paths_locli
+
+  build-depends:       base
+                     , aeson
+                     , aeson-pretty
+                     , attoparsec
+                     , attoparsec-iso8601
+                     , bytestring
+                     , cardano-prelude
+                     , containers
+                     , directory
+                     , filepath
+                     , file-embed
+                     , iohk-monitoring
+                     , optparse-applicative
+                     , process
+                     , scientific
+                     , split
+                     , template-haskell
+                     , text
+                     , time
+                     , transformers
+                     , transformers-except
+                     , unordered-containers
+                     , utf8-string
+                     , vector
+
+  default-language:    Haskell2010
+  default-extensions:  NoImplicitPrelude
+                       OverloadedStrings
+                       TupleSections
+
+  ghc-options:         -Wall
+                       -Wincomplete-record-updates
+                       -Wincomplete-uni-patterns
+                       -Wredundant-constraints
+                       -Wpartial-fields
+                       -Wcompat
+                       -Wno-all-missed-specialisations
+
+executable locli
+  hs-source-dirs:      app
+  main-is:             locli.hs
+  default-language:    Haskell2010
+  ghc-options:         -threaded
+                       -Wall
+                       -rtsopts
+                       "-with-rtsopts=-T"
+  build-depends:       base
+                     , cardano-prelude
+                     , locli
+                     , optparse-applicative
+                     , text
+                     , transformers
+                     , transformers-except
+  default-extensions:  NoImplicitPrelude

locli/src/Cardano/Config/Git/Rev.hs

@@ -0,0 +1,39 @@
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TemplateHaskell #-}
+
+module Cardano.Config.Git.Rev (
+      gitRev
+    ) where
+
+import           Cardano.Prelude
+
+import           Data.FileEmbed (dummySpaceWith)
+import           Data.Text (Text)
+import qualified Data.Text as T
+import           Data.Text.Encoding (decodeUtf8)
+
+import           Cardano.Config.Git.RevFromGit (gitRevFromGit)
+
+gitRev :: Text
+gitRev | gitRevEmbed /= zeroRev = gitRevEmbed
+       | T.null fromGit         = zeroRev
+       | otherwise              = fromGit
+    where
+        -- Git revision embedded after compilation using
+        -- Data.FileEmbed.injectWith. If nothing has been injected,
+        -- this will be filled with 0 characters.
+        gitRevEmbed :: Text
+        gitRevEmbed = decodeUtf8 $(dummySpaceWith "gitrev" 40)
+
+        -- Git revision found during compilation by running git. If
+        -- git could not be run, then this will be empty.
+#if defined(arm_HOST_ARCH)
+        -- cross compiling to arm fails; due to a linker bug
+        fromGit = ""
+#else
+        fromGit = T.strip (T.pack $(gitRevFromGit))
+#endif
+
+zeroRev :: Text
+zeroRev = "0000000000000000000000000000000000000000"

locli/src/Cardano/Config/Git/RevFromGit.hs

@@ -0,0 +1,27 @@
+module Cardano.Config.Git.RevFromGit (
+      gitRevFromGit
+    ) where
+
+import           Cardano.Prelude
+import           Prelude (String)
+
+import qualified Language.Haskell.TH as TH
+import           System.Exit (ExitCode (..))
+import           System.IO.Error (ioeGetErrorType, isDoesNotExistErrorType)
+import           System.Process (readProcessWithExitCode)
+
+-- | Git revision found by running git rev-parse. If git could not be
+-- executed, then this will be an empty string.
+gitRevFromGit :: TH.Q TH.Exp
+gitRevFromGit = TH.LitE . TH.StringL <$> TH.runIO runGitRevParse
+    where
+        runGitRevParse :: IO String
+        runGitRevParse = handleJust missingGit (const $ pure "") $ do
+            (exitCode, output, _) <-
+                readProcessWithExitCode "git" ["rev-parse", "--verify", "HEAD"] ""
+            pure $ case exitCode of
+                ExitSuccess -> output
+                _           -> ""
+
+        missingGit e = if isDoesNotExistErrorType (ioeGetErrorType e) then Just () else Nothing
+