locli: report improvements: autogen list, pass through workload & tag…

… charts with runs

bench/locli/src/Cardano/Analysis/API.hs

@@ -340,11 +340,11 @@ data PropSubset
 
 bpFieldSelectForger :: Field DSelect p a -> Bool
 bpFieldSelectForger Field{fId} = elem fId
-  [ "fStarted", "fLeading", "fForged", "fAnnounced", "fAdopted", "fSendStart" ]
+  [ "bfStarted", "bfLeading", "bfForged", "bfAnnounced", "fSending", "bfAdopted" ]
 
 bpFieldSelectPeers :: Field DSelect p a -> Bool
 bpFieldSelectPeers Field{fId} = elem fId
-  [ "pNoticed", "pRequested", "pFetched", "pAnnounced", "pAdopted", "pSendStart" ]
+  [ "boNoticed", "boRequested", "boFetched", "boAnnounced", "boSending", "boAdopted" ]
 
 bpFieldSelectEndToEnd :: Field DSelect p a -> Bool
 bpFieldSelectEndToEnd Field{fHead2} = elem fHead2 adoptionCentilesRendered
@@ -449,37 +449,37 @@ instance CDFFields BlockProp p where
 instance TimelineFields BlockEvents where
   timelineFields _ =
     --  Width LeftPad
-    [ Field 5 0 "block"        "block" "no."    (IWord64 (unBlockNo . beBlockNo)) ""
-    , Field 5 0 "abs.slot"     "abs."  "slot#"  (IWord64 (unSlotNo  . beSlotNo)) ""
-    , Field 6 0 "hash"         "block" "hash"   (IText   (shortHash . beBlock)) ""
-    , Field 6 0 "hashPrev"     "prev"  "hash"   (IText   (shortHash . beBlockPrev)) ""
-    , Field 7 0 "forger"       "forger" "host"  (IText   (toText . unHost . bfForger . beForge)) ""
-    , Field 6 0 "blockSize"    "size"  "bytes"  (IInt    (bfBlockSize . beForge)) ""
-    , Field 5 0 "blockGap"     "block" "gap"    (IDeltaT (bfBlockGap  . beForge)) ""
-    , Field 3 0 "forks"         "for"  "-ks"    (IInt    (count bpeIsFork . beErrors)) ""
-    , Field 4 0 "fStarted"      (f!!0) "Start"  (IDeltaT (bfStarted   . beForge)) ""
-    , Field 4 0 "fBlkCtx"       (f!!1) "BlkCtx" (IText (maybe "?" show.bfBlkCtx  .beForge)) ""
-    , Field 4 0 "fLgrState"     (f!!2) "LgrSta" (IText (maybe "?" show.bfLgrState.beForge)) ""
-    , Field 4 0 "fLgrView"      (f!!3) "LgrVie" (IText (maybe "?" show.bfLgrView .beForge)) ""
-    , Field 4 0 "fLeading"      (f!!4) "Lead"   (IDeltaT (bfLeading   . beForge)) ""
-    , Field 4 0 "fForged"       (f!!5) "Forge"  (IDeltaT (bfForged    . beForge)) ""
-    , Field 4 0 "fAnnounced"    (f!!6) "Announ" (IDeltaT (bfAnnounced . beForge)) ""
-    , Field 4 0 "fSendStart"    (f!!7) "Sendin" (IDeltaT (bfSending   . beForge)) ""
-    , Field 4 0 "fAdopted"      (f!!8) "Adopt"  (IDeltaT (bfAdopted   . beForge)) ""
-    , Field 4 0 "noticedVal"    (p!!0) "Notic"  (IDeltaT (af  boNoticed   . valids)) ""
-    , Field 4 0 "requestedVal"  (p!!1) "Requd"  (IDeltaT (af  boRequested . valids)) ""
-    , Field 4 0 "fetchedVal"    (p!!2) "Fetch"  (IDeltaT (af  boFetched   . valids)) ""
-    , Field 4 0 "pAnnouncedVal" (p!!3) "Annou"  (IDeltaT (af' boAnnounced . valids)) ""
-    , Field 4 0 "pSendStartVal" (p!!4) "Send"   (IDeltaT (af' boSending   . valids)) ""
-    , Field 4 0 "pAdoptedVal"   (p!!5) "Adopt"  (IDeltaT (af' boAdopted   . valids)) ""
-    , Field 4 0 "pPropag0.5"    (r!!0) "0.5"    (IDeltaT (percSpec 0.5  . bePropagation)) ""
-    , Field 4 0 "pPropag0.8"    (r!!1) "0.8"    (IDeltaT (percSpec 0.8  . bePropagation)) ""
-    , Field 4 0 "pPropag0.96"   (r!!2) "0.96"   (IDeltaT (percSpec 0.96 . bePropagation)) ""
-    , Field 4 0 "pPropag1.0"    (r!!3) "1.0"    (IDeltaT (snd . cdfRange . bePropagation)) ""
-    , Field 3 0 "valid"         "va-"  "lid"    (IText   (bool "-" "+" . (== 0) . length
+    [ Field 5 0 "beBlockNo"    "block" "no."    (IWord64 (unBlockNo . beBlockNo)) ""
+    , Field 5 0 "beSlotNo"     "abs."  "slot#"  (IWord64 (unSlotNo  . beSlotNo)) ""
+    , Field 6 0 "beBlock"      "block" "hash"   (IText   (shortHash . beBlock)) ""
+    , Field 6 0 "beBlockPrev"  "prev"  "hash"   (IText   (shortHash . beBlockPrev)) ""
+    , Field 7 0 "bfForger"     "forger" "host"  (IText   (toText . unHost . bfForger . beForge)) ""
+    , Field 6 0 "bfBlockSize"  "size"  "bytes"  (IInt    (bfBlockSize . beForge)) ""
+    , Field 5 0 "bfBlockGap"   "block" "gap"    (IDeltaT (bfBlockGap  . beForge)) ""
+    , Field 3 0 "bpeIsFork"    "for"  "-ks"    (IInt    (count bpeIsFork . beErrors)) ""
+    , Field 4 0 "bfStarted"     (f!!0) "Start"  (IDeltaT (bfStarted   . beForge)) ""
+    , Field 4 0 "bfBlkCtx"      (f!!1) "BlkCtx" (IText (maybe "?" show.bfBlkCtx  .beForge)) ""
+    , Field 4 0 "bfLgrState"    (f!!2) "LgrSta" (IText (maybe "?" show.bfLgrState.beForge)) ""
+    , Field 4 0 "bfLgrView"     (f!!3) "LgrVie" (IText (maybe "?" show.bfLgrView .beForge)) ""
+    , Field 4 0 "bfLeading"     (f!!4) "Lead"   (IDeltaT (bfLeading   . beForge)) ""
+    , Field 4 0 "bfForged"      (f!!5) "Forge"  (IDeltaT (bfForged    . beForge)) ""
+    , Field 4 0 "bfAnnounced"   (f!!6) "Announ" (IDeltaT (bfAnnounced . beForge)) ""
+    , Field 4 0 "bfSending"     (f!!7) "Sendin" (IDeltaT (bfSending   . beForge)) ""
+    , Field 4 0 "bfAdopted"     (f!!8) "Adopt"  (IDeltaT (bfAdopted   . beForge)) ""
+    , Field 4 0 "boNoticed"     (p!!0) "Notic"  (IDeltaT (af  boNoticed   . valids)) ""
+    , Field 4 0 "boRequested"   (p!!1) "Requd"  (IDeltaT (af  boRequested . valids)) ""
+    , Field 4 0 "boFetched"     (p!!2) "Fetch"  (IDeltaT (af  boFetched   . valids)) ""
+    , Field 4 0 "boAnnounced"   (p!!3) "Annou"  (IDeltaT (af' boAnnounced . valids)) ""
+    , Field 4 0 "boSending"     (p!!4) "Send"   (IDeltaT (af' boSending   . valids)) ""
+    , Field 4 0 "pAdopted"      (p!!5) "Adopt"  (IDeltaT (af' boAdopted   . valids)) ""
+    , Field 4 0 "bePropagation.5" (r!!0) "0.5"  (IDeltaT (percSpec 0.5  . bePropagation)) ""
+    , Field 4 0 "bePropagation.8" (r!!1) "0.8"  (IDeltaT (percSpec 0.8  . bePropagation)) ""
+    , Field 4 0 "bePropagation.96"(r!!2) "0.96" (IDeltaT (percSpec 0.96 . bePropagation)) ""
+    , Field 4 0 "bePropagation.1" (r!!3) "1.0"  (IDeltaT (snd . cdfRange . bePropagation)) ""
+    , Field 3 0 "beAcceptance"  "va-"  "lid"    (IText   (bool "-" "+" . (== 0) . length
                                                           . filter (not . snd) . beAcceptance)) ""
     , Field 3 0 "valid.observ" "good"  "obsv"   (IInt    (length          . valids)) ""
-    , Field 5 0 "errors"        "all"  "errs"   (IInt    (length . beErrors)) ""
+    , Field 5 0 "beErrors"      "all"  "errs"   (IInt    (length . beErrors)) ""
     , Field 3 0 "missNotic"     (m!!0) "ntc"    (IInt    (count (bpeIsMissing Notice) . beErrors)) ""
     , Field 3 0 "missReque"     (m!!1) "req"    (IInt    (count (bpeIsMissing Request) . beErrors)) ""
     , Field 3 0 "missFetch"     (m!!2) "fch"    (IInt    (count (bpeIsMissing Fetch) . beErrors)) ""
@@ -538,7 +538,7 @@ data PerfSubset
 
 mtFieldsReport :: Field DSelect p a -> Bool
 mtFieldsReport Field{fId} = elem fId
-  [ "cpuProcess", "cpuGC", "cpuMutator", "cpuSpanLenAll", "memRSS", "rtsHeap", "rtsLive", "rtsAllocation" ]
+  [ "rCentiCpu", "rCentiGC", "rCentiMut", "cdfSpanLensCpu", "rRSS", "rHeap", "rLive", "rAlloc" ]
 
 perfSubsetFn :: PerfSubset -> (Field DSelect p a -> Bool)
 perfSubsetFn = \case

bench/locli/src/Cardano/Analysis/Context.hs

@@ -46,11 +46,13 @@ data PParams
 
 data GeneratorProfile
   = GeneratorProfile
-  { add_tx_size     :: Word64
-  , inputs_per_tx   :: Word64
-  , outputs_per_tx  :: Word64
-  , tps             :: Double
-  , tx_count        :: Word64
+  { add_tx_size      :: Word64
+  , inputs_per_tx    :: Word64
+  , outputs_per_tx   :: Word64
+  , tps              :: Double
+  , tx_count         :: Word64
+  , plutusMode       :: Maybe Bool
+  , plutusLoopScript :: Maybe FilePath
   }
   deriving (Generic, Show, FromJSON, ToJSON)
 

bench/locli/src/Cardano/Report.hs

@@ -16,9 +16,10 @@ import Data.List (last)
 import Data.Text qualified as T
 import Data.Text.Lazy qualified as LT
 import Data.Time.Clock
+import System.FilePath as FS
 import System.Posix.User
 
-import Text.EDE
+import Text.EDE hiding (Id)
 
 import Data.CDF
 
@@ -61,18 +62,20 @@ getReport rTarget mrev = do
 
 data Workload
   = WValue
-  | WPlutus
+  | WPlutusLoopCountdown
+  | WPlutusLoopSECP
 
 instance ToJSON Workload where
   toJSON = \case
-    WValue  -> "Value"
-    WPlutus -> "Plutus"
+    WValue               -> "value-only"
+    WPlutusLoopCountdown -> "Plutus countdown loop"
+    WPlutusLoopSECP      -> "Plutus SECP loop"
 
 data Section where
   STable :: CDFFields a p =>
     { sData          :: !(a p)
     , sFieldSelector :: !(Field DSelect p a -> Bool)
-    , sDataId        :: !Text
+    , sDataRef       :: !Text
     , sOrgTableSrc   :: !Text
     , sTitle         :: !Text
     } -> Section
@@ -107,7 +110,7 @@ instance ToJSON TmplRun where
   toJSON TmplRun{trManifest=Manifest{..},..} =
     object
       [ "meta"       .= trMeta
-      , "id"         .= trShortId
+      , "shortId"    .= trShortId
       , "workload"   .= trWorkload
       , "branch"     .= mNodeBranch
       , "ver"        .= mNodeApproxVer
@@ -124,26 +127,33 @@ instance ToJSON TmplRun where
       ]
 
 liftTmplRun :: Run -> TmplRun
-liftTmplRun Run{..} =
+liftTmplRun Run{generatorProfile=GeneratorProfile{..}, ..} =
   TmplRun
   { trMeta      = metadata
   , trShortId   = ShortId (batch metadata)
-  , trWorkload  = WValue
   , trManifest  = manifest metadata & unsafeShortenManifest 5
+  , trWorkload  =
+    case ( plutusMode       & fromMaybe False
+         , plutusLoopScript & fromMaybe "" & FS.takeFileName & FS.dropExtension ) of
+         (False, _)                       -> WValue
+         (True, "loop")                   -> WPlutusLoopCountdown
+         (True, "schnorr-secp256k1-loop") -> WPlutusLoopSECP
+         (_, scr) ->
+           error $ "Unknown Plutus script:  " <> scr
   }
 
 data TmplSection
   = TmplTable
     { tsTitle       :: !Text
-    , tsDataId      :: !Text
+    , tsDataRef     :: !Text
     , tsOrgTableSrc :: !Text
     , tsNRows       :: !Int
     }
 
 instance ToJSON TmplSection where
   toJSON TmplTable{..} = object
     [ "title"     .= tsTitle
-    , "dataId"    .= tsDataId
+    , "dataRef"   .= tsDataRef
     , "orgFile"   .= tsOrgTableSrc
     -- Yes, strange as it is, this is the encoding to ease iteration in ED-E.
     , "rows"      .= T.replicate tsNRows "."
@@ -155,9 +165,11 @@ liftTmplSection =
     STable{..} ->
       TmplTable
       { tsTitle       = sTitle
-      , tsDataId      = sDataId
+      , tsDataRef     = sDataRef
       , tsOrgTableSrc = sOrgTableSrc
-      , tsNRows       = length $ filterFields sFieldSelector
+      , tsNRows       =
+        -- trace (printf "nrows: %d" $ length $ filterFields sFieldSelector :: String) $
+        length $ filterFields sFieldSelector
       }
 
 generate :: InputDir -> Maybe TextInputFile

bench/locli/src/Cardano/Unlog/LogObject.hs

@@ -56,8 +56,8 @@ readLogObjectStream f okDErr anyOks =
     <&>
     (if okDErr then id else
         filter ((\case
-                    LODecodeError err -> error
-                      (printf "Decode error while parsing %s -- %s" f (show err))
+                    LODecodeError input err -> error
+                      (printf "Decode error while parsing %s:\n%s\non input:\n>>>  %s" f (Text.toString err) (Text.toString input))
                     _ -> True)
                . loBody)) .
     filter ((\case
@@ -67,13 +67,21 @@ readLogObjectStream f okDErr anyOks =
                     (printf "Unexpected LOAny while parsing %s -- %s: %s" f (show laty) (show obj))
                 _ -> True)
              . loBody) .
-    filter ((/= eofError) . loBody) .
-    fmap (either (LogObject zeroUTCTime "Cardano.Analysis.DecodeError" "DecodeError" "" (TId "0") . LODecodeError . Text.fromText . LText.pack)
-                 id
-          . AE.eitherDecode)
+    filter (not . isDecodeError "Error in $: not enough input" . loBody) .
+    fmap (\bs ->
+            AE.eitherDecode bs &
+            either
+            (LogObject zeroUTCTime "Cardano.Analysis.DecodeError" "DecodeError" "" (TId "0")
+             . LODecodeError (Text.fromByteString (LBS.toStrict bs)
+                               & fromMaybe "#<ERROR decoding input fromByteString>")
+              . Text.fromText
+              . LText.pack)
+            id)
     . LBS.split (fromIntegral $ fromEnum '\n')
  where
-   eofError = LODecodeError "Error in $: not enough input"
+   isDecodeError x = \case
+     LODecodeError _ x' -> x == x'
+     _ -> False
 
 data LogObject
   = LogObject
@@ -325,7 +333,10 @@ data LOBody
   | LOGeneratorSummary !Bool !Word64 !NominalDiffTime (Vector Double)
   -- Everything else:
   | LOAny !LOAnyType !Object
-  | LODecodeError !ShortText
+  | LODecodeError
+    { loRawText :: !ShortText
+    , loError   :: !ShortText
+    }
   deriving (Eq, Generic, Show)
   deriving anyclass NFData
 

nix/workbench/ede/chart.ede

@@ -14,9 +14,9 @@ set yrange [*:*]
 set ylabel "{{ args.ylabel }}"
 {% endif %}
 eval cdfI_{{ (runs | length) + 1 }}("{{ args.metric }}", "{{ args.title }}", \
-            "{{ base.ver }}/{{ base.meta.era | toTitle }}",   "{{ base.meta.tag }}", \
+            "{{ base.meta.tag }}/{{ base.shortId }}/{{ base.meta.era | toTitle }}",   "{{ base.meta.tag }}", \
 {% for run in runs %}
-            "{{ run.value.ver }}/{{ run.value.meta.era | toTitle }}", "{{ run.value.meta.tag }}"{% if !run.last %},{% endif %} \
+            "{{ run.value.meta.tag }}/{{ run.value.shortId }}/{{ run.value.meta.era | toTitle }}", "{{ run.value.meta.tag }}"{% if !run.last %},{% endif %} \
 {% endfor %}
             )
 #+end_src

nix/workbench/ede/report.ede

@@ -1,4 +1,4 @@
-#+CONSTANTS:{% for sec in sections %} {{ sec.value.dataId }}={{ sec.value.orgFile }} {% endfor %}
+#+CONSTANTS:{% for sec in sections %} {{ sec.value.dataRef }}={{ sec.value.orgFile }} {% endfor %}
 #+CONSTANTS: base=../{{ base.meta.tag }}/analysis
 {% for run in runs %}
 #+CONSTANTS: run{{ run.index }}=../{{ run.value.meta.tag }}/analysis
@@ -18,7 +18,7 @@
 
 *** Manifest
 
-We compare ... relative to ={{ base.ver }}= /{{ base.meta.era | toTitle }}, under {{ base.workload }} workload.
+We compare {% for run in runs %}{%if !run.first%}{%if !run.last%}, {%else%} and {%endif%}{%endif%}{{ run.value.ver }}/{{ run.value.meta.era | toTitle }}{% endfor %} relative to ={{ base.ver }}=/{{ base.meta.era | toTitle }}, under {{ base.workload }} workload.
 
 {% include "manifest.ede" %}
 

nix/workbench/ede/table.ede

@@ -4,7 +4,7 @@
 |                           |     {% for run in runs %} |      |      |     {% endfor %} |
 {% endfor %}
 |---------------------------+-----{% for run in runs %}-+------+------+-----{% endfor %}-|
-#+TBLFM: $2=remote(file:$base/${{ table.dataId }},@@#$average);p3::$1=remote(file:$base/${{ table.dataId }},@@#$metric)
+#+TBLFM: $2=remote(file:$base/${{ table.dataRef }},@@#$average);p3::$1=remote(file:$base/${{ table.dataRef }},@@#$metric)
 {% for run in runs %}
-#+TBLFM: ${{ run.index * 3 }}=remote(file:$run{{ run.index }}/${{ table.dataId }},@@#$average);p3::${{ run.index * 3 + 1 }}=${{ run.index * 3 }}-$2;p3::${{ run.index * 3 + 2 }}=round(100*${{ run.index * 3 + 1 }}/$2)
+#+TBLFM: ${{ run.index * 3 }}=remote(file:$run{{ run.index }}/${{ table.dataRef }},@@#$average);p3::${{ run.index * 3 + 1 }}=${{ run.index * 3 }}-$2;p3::${{ run.index * 3 + 2 }}=round(100*${{ run.index * 3 + 1 }}/$2)
 {% endfor %}

nix/workbench/run.sh

@@ -18,13 +18,13 @@ usage_run() {
                           A unique name would be allocated for this run,
                             and a run alias $(green current) will be created for it.
 
-    $(helpcmd list-aws)             List AWS runs
-     $(blk lsaws)
-    $(helpcmd fetch-aws RUN)
+    $(helpcmd list-remote)          List AWS runs
+     $(blk lsaws lsr)
+    $(helpcmd fetch-run RUN)
                           Fetch an AWS run
-     $(blk fetch)
-    $(helpcmd analysis-from-aws RUN..)
-     $(blk fetch-analysis fa)    Fetch analyses of AWS runs
+     $(blk fetch fr)
+    $(helpcmd fetch-analysis RUN..)
+     $(blk fa)                   Fetch analyses of AWS runs
     $(helpcmd analyse-aws RUN..)     Run analyses of AWS runs, remotely
 
     $(helpcmd describe RUN)
@@ -95,7 +95,7 @@ case "$op" in
                  sh -c "'$(run_ls_cmd $(jq <<<$r .depl)/runs)'"
         fi;;
 
-    list-remote | lsr )
+    list-remote | lsaws | lsr )
         run "${sargs[@]}" list --on-remote;;
 
     list-sets | lss )
@@ -524,7 +524,7 @@ case "$op" in
 
         echo $dir;;
 
-    fetch-run | fr )
+    fetch-run | fetch | fr )
         local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]"
         local run=${1:?$usage}
         local mach=${2:-all-hosts}