Add/debug logging mode (#74)

* add support for debug logging mode
Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/main.yml

@@ -60,6 +60,7 @@ jobs:
                ["lammps", "ghcr.io/rse-ops/lammps:flux-sched-focal-v0.24.0", 30],
                ["pokemon", "ghcr.io/rse-ops/pokemon:app-latest", 30],
                ["timed", "ghcr.io/flux-framework/flux-restful-api:latest", 30],               
+               ["debug", "ghcr.io/flux-framework/flux-restful-api:latest", 30],               
                ["flux-resource-list", "ghcr.io/flux-framework/flux-restful-api:latest", 30],
                ["volumes", "ghcr.io/flux-framework/flux-restful-api:latest", 30]]
 

api/v1alpha1/minicluster_types.go

@@ -88,6 +88,11 @@ type LoggingSpec struct {
 	// +optional
 	QuietMode bool `json:"quiet"`
 
+	// Debug mode adds extra verbosity to Flux
+	// +kubebuilder:default=false
+	// +optional
+	DebugMode bool `json:"debug"`
+
 	// Timed mode adds timing to Flux commands
 	// +kubebuilder:default=false
 	// +optional

config/crd/bases/flux-framework.org_miniclusters.yaml

@@ -196,6 +196,10 @@ spec:
                 description: Logging modes determine the output you see in the job
                   log
                 properties:
+                  debug:
+                    default: false
+                    description: Debug mode adds extra verbosity to Flux
+                    type: boolean
                   quiet:
                     default: false
                     description: Quiet mode silences all output so the job only shows

controllers/flux/minicluster.go

@@ -220,8 +220,7 @@ func generateWaitScript(cluster *api.MiniCluster, containerIndex int) (string, e
 		Cores:             cores,
 		FluxRestfulPort:   cluster.Spec.FluxRestful.Port,
 		FluxRestfulBranch: cluster.Spec.FluxRestful.Branch,
-		QuietMode:         cluster.Spec.Logging.QuietMode,
-		TimedMode:         cluster.Spec.Logging.TimedMode,
+		Logging:           cluster.Spec.Logging,
 	}
 	t, err := template.New("wait-sh").Parse(waitToStartTemplate)
 	if err != nil {

controllers/flux/templates.go

@@ -11,6 +11,8 @@ SPDX-License-Identifier: Apache-2.0
 package controllers
 
 import (
+	api "flux-framework/flux-operator/api/v1alpha1"
+
 	_ "embed"
 )
 
@@ -38,9 +40,8 @@ type WaitTemplate struct {
 	Tasks             int32
 	Size              int32 // size of the Minicluster (nodes / pods in indexed jobs)
 
-	// Logging Modes
-	QuietMode    bool // Don't print additional output
-	TimedMode    bool // Add times when appropriate
+	// Logging Modes (FluxLogLevel is per container)
+	Logging      api.LoggingSpec
 	FluxLogLevel int32
 }
 

controllers/flux/templates/wait.sh

@@ -11,7 +11,7 @@ asFlux="sudo -u flux -E"
 # If any preCommand logic is defined
 {{ .PreCommand}}
 
-{{ if not .QuietMode }}# Show asFlux directive once
+{{ if not .Logging.QuietMode }}# Show asFlux directive once
 printf "\nAs Flux prefix for flux commands: ${asFlux}\n"{{ end }}
 
 
@@ -20,7 +20,7 @@ which sudo > /dev/null 2>&1 || (echo "sudo is required to be installed" && exit
 which flux > /dev/null 2>&1 || (echo "flux is required to be installed" && exit 1);
 
 # We use the actual time command and not the wrapper, otherwise we get there is no argument -f
-{{ if .TimedMode }}which /usr/bin/time > /dev/null 2>&1 || (echo "/usr/bin/time is required to use logging.timed true" && exit 1);{{ end }}
+{{ if .Logging.TimedMode }}which /usr/bin/time > /dev/null 2>&1 || (echo "/usr/bin/time is required to use logging.timed true" && exit 1);{{ end }}
 
 # Broker Options: important!
 # The local-uri setting places the unix domain socket in rundir 
@@ -32,7 +32,7 @@ brokerOptions="-Scron.directory=/etc/flux/system/cron.d \
   -Srundir=/run/flux \
   -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \
   -Slocal-uri=local:///run/flux/local \
-{{ if not .QuietMode }}  -Slog-stderr-level={{or .FluxLogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \
+{{ if not .Logging.QuietMode }}  -Slog-stderr-level={{or .FluxLogLevel 6}} {{ else }} -Slog-stderr-level=0 {{ end }} \
   -Slog-stderr-mode=local"
 
 # quorum settings influence how the instance treats missing ranks
@@ -76,7 +76,7 @@ mainHost="{{ .MainHost}}"
 # The working directory should be set by the CRD or the container
 workdir=${PWD}
 
-{{ if not .QuietMode }}
+{{ if not .Logging.QuietMode }}
 printf "\n👋 Hello, I'm $(hostname)\n"
 printf "The main host is ${mainHost}\n"
 printf "The working directory is ${workdir}\n"
@@ -87,14 +87,14 @@ ls ${workdir}{{ end }}
 mkdir -p /etc/flux/system
 
 # --cores=IDS Assign cores with IDS to each rank in R, so we  assign 0-(N-1) to each host
-{{ if not .QuietMode }}echo "flux R encode --hosts={{ .Hosts}} {{if .Cores}}--cores=0-{{.Cores}}{{ end }}"{{ end }}
+{{ if not .Logging.QuietMode }}echo "flux R encode --hosts={{ .Hosts}} {{if .Cores}}--cores=0-{{.Cores}}{{ end }}"{{ end }}
 flux R encode --hosts={{ .Hosts}} {{if .Cores}}--cores=0-{{.Cores}}{{ end }} > /etc/flux/system/R
-{{ if not .QuietMode }}printf "\n📦 Resources\n"
+{{ if not .Logging.QuietMode }}printf "\n📦 Resources\n"
 cat /etc/flux/system/R{{ end }}
 
 # Do we want to run diagnostics instead of regular entrypoint?
 diagnostics="{{ .Diagnostics}}"
-{{ if not .QuietMode }}printf "\n🐸 Diagnostics: ${diagnostics}\n"{{ end }}
+{{ if not .Logging.QuietMode }}printf "\n🐸 Diagnostics: ${diagnostics}\n"{{ end }}
 
 # Flux option flags
 option_flags="{{ .FluxOptionFlags}}"
@@ -108,7 +108,7 @@ if [ "${option_flags}" != "" ]; then
     else 
         export FLUX_OPTION_FLAGS="${option_flags}"
     fi
-{{ if not .QuietMode }}    echo "🚩️ Flux Option Flags defined"{{ end }}
+{{ if not .Logging.QuietMode }}    echo "🚩️ Flux Option Flags defined"{{ end }}
 fi
 
 mkdir -p /etc/flux/imp/conf.d/
@@ -118,21 +118,21 @@ allowed-users = [ "flux", "root" ]
 allowed-shells = [ "/usr/libexec/flux/flux-shell" ]	
 EOT
 
-{{ if not .QuietMode }}printf "\n🦊 Independent Minister of Privilege\n"
+{{ if not .Logging.QuietMode }}printf "\n🦊 Independent Minister of Privilege\n"
 cat /etc/flux/imp/conf.d/imp.toml
 
 printf "\n🐸 Broker Configuration\n"
 cat /etc/flux/config/broker.toml{{ end }}
 
 # Add a flux user (required)
-sudo adduser --disabled-password --uid 1000 --gecos "" flux > /dev/null 2>&1 || {{ if not .QuietMode }} printf "flux user is already added.\n"{{ else }}true{{ end }}
+sudo adduser --disabled-password --uid 1000 --gecos "" flux > /dev/null 2>&1 || {{ if not .Logging.QuietMode }} printf "flux user is already added.\n"{{ else }}true{{ end }}
 
 # The rundir needs to be created first, and owned by user flux
 # Along with the state directory and curve certificate
 mkdir -p /run/flux /etc/curve
 
 # Show generated curve certificate - the munge.key should already be equivalent (and exist)
-{{ if not .QuietMode }}cat /mnt/curve/curve.cert{{ end }}
+{{ if not .Logging.QuietMode }}cat /mnt/curve/curve.cert{{ end }}
 cp /mnt/curve/curve.cert /etc/curve/curve.cert
 
 # Remove group and other reead
@@ -144,7 +144,7 @@ chmod g-r /etc/curve/curve.cert
 fluxuid=$(id -u flux)
 chown -R ${fluxuid} /run/flux ${STATE_DIR} /etc/curve/curve.cert ${workdir}
 
-{{ if not .QuietMode }}
+{{ if not .Logging.QuietMode }}
 printf "\n✨ Curve certificate generated by helper pod\n"
 cat /etc/curve/curve.cert{{ end }}
 
@@ -157,7 +157,7 @@ else
     if [ $(hostname) == "${mainHost}" ]; then
 
         # No command - use default to start server
-{{ if not .QuietMode }}        echo "Extra arguments are: $@"{{ end }}
+{{ if not .Logging.QuietMode }}        echo "Extra arguments are: $@"{{ end }}
         if [ "$@" == "" ]; then
 
             # Start restful API server
@@ -174,30 +174,30 @@ else
             FLUX_NUMBER_NODES={{ .Size}}
             export FLUX_TOKEN FLUX_USER FLUX_REQUIRE_AUTH FLUX_NUMBER_NODES
 
-{{ if not .QuietMode }}
+{{ if not .Logging.QuietMode }}
             printf "\n 🔑 Your Credentials! These will allow you to control your MiniCluster with flux-framework/flux-restful-api\n"
             printf "export FLUX_TOKEN=${FLUX_TOKEN}\n"
             printf "export FLUX_USER=${FLUX_USER}\n"
 
             # -o is an "option" for the broker
             # -S corresponds to a shortened --setattr=ATTR=VAL
             printf "\n🌀 flux start -o --config /etc/flux/config ${brokerOptions} ${startServer}\n"{{ end }}
-            {{ if .TimedMode }}/usr/bin/time -f "FLUXTIME fluxstart wall time %E" {{ end }}${asFlux} flux start -o --config /etc/flux/config ${brokerOptions} ${startServer}
+            {{ if .Logging.TimedMode }}/usr/bin/time -f "FLUXTIME fluxstart wall time %E" {{ end }}${asFlux} flux start -o --config /etc/flux/config ${brokerOptions} ${startServer}
 
         # Case 2: Fall back to provided command
         else
-{{ if not .QuietMode }} 
-            printf "\n🌀 flux start -o --config /etc/flux/config ${brokerOptions} flux mini submit {{ if gt .Tasks .Size }} -N {{.Size}}{{ end }} -n {{.Tasks}} --quiet {{ if .FluxOptionFlags }}{{ .FluxOptionFlags}}{{ end }} --watch $@\n"{{ end }}
-            {{ if .TimedMode }}/usr/bin/time -f "FLUXTIME fluxsubmit wall time %E" {{ end }}${asFlux} flux start -o --config /etc/flux/config ${brokerOptions} flux mini submit {{ if gt .Tasks .Size }} -N {{.Size}}{{ end }} -n {{.Tasks}} --quiet {{ if .FluxOptionFlags }}{{ .FluxOptionFlags}}{{ end }} --watch $@
+{{ if not .Logging.QuietMode }} 
+            printf "\n🌀 flux start -o --config /etc/flux/config ${brokerOptions} flux mini submit {{ if gt .Tasks .Size }} -N {{.Size}}{{ end }} -n {{.Tasks}} --quiet {{ if .FluxOptionFlags }}{{ .FluxOptionFlags}}{{ end }} --watch{{ if .Logging.DebugMode }} -vvv{{ end }} $@\n"{{ end }}
+            {{ if .Logging.TimedMode }}/usr/bin/time -f "FLUXTIME fluxsubmit wall time %E" {{ end }}${asFlux} flux start -o --config /etc/flux/config ${brokerOptions} flux mini submit {{ if gt .Tasks .Size }} -N {{.Size}}{{ end }} -n {{.Tasks}} --quiet {{ if .FluxOptionFlags }}{{ .FluxOptionFlags}}{{ end }} --watch{{ if .Logging.DebugMode }} -vvv{{ end }} $@
         fi
     else
         # Sleep until the broker is ready
-{{ if not .QuietMode }}
+{{ if not .Logging.QuietMode }}
         printf "\n🌀 flux start -o --config /etc/flux/config ${brokerOptions}\n"{{ end }}
         while true
         do
-            {{ if .TimedMode }}/usr/bin/time -f "FLUXTIME fluxstart wall time %E" {{ end }}${asFlux} flux start -o --config /etc/flux/config ${brokerOptions}
-            {{ if not .QuietMode }}printf "\n😪 Sleeping 15s until broker is ready..."{{ end }}
+            {{ if .Logging.TimedMode }}/usr/bin/time -f "FLUXTIME fluxstart wall time %E" {{ end }}${asFlux} flux start -o --config /etc/flux/config ${brokerOptions}
+            {{ if not .Logging.QuietMode }}printf "\n😪 Sleeping 15s until broker is ready..."{{ end }}
             sleep 15
         done
     fi

docs/getting_started/custom-resource-definition.md

@@ -167,6 +167,15 @@ FLUXTIME fluxsubmit wall time 0:04.73
 All timed command lines are prefixed with `FLUXTIME` and the main submit will be `fluxsubmit`
 and the worker pods flux start will have `fluxstart`.
 
+#### debug
+
+Debug mode adds verbosity to flux to see additional information about the job submission.
+
+```yaml
+logging:
+  debug: true
+```
+
 ### pod
 
 Variables and attributes for each pod in the Indexed job.

examples/dist/flux-operator.yaml

@@ -202,6 +202,10 @@ spec:
                 description: Logging modes determine the output you see in the job
                   log
                 properties:
+                  debug:
+                    default: false
+                    description: Debug mode adds extra verbosity to Flux
+                    type: boolean
                   quiet:
                     default: false
                     description: Quiet mode silences all output so the job only shows

examples/tests/debug/minicluster.yaml

@@ -0,0 +1,21 @@
+apiVersion: flux-framework.org/v1alpha1
+kind: MiniCluster
+metadata:
+  name: flux-sample
+  namespace: flux-operator
+spec:
+  # Set to true to use volume mounts instead of volume claims
+  localDeploy: true
+
+  # suppress all output except for test run
+  logging:
+    quiet: true
+    debug: true
+
+  # Number of pods to create for MiniCluster
+  size: 2
+
+  # This is a list because a pod can support multiple containers
+  containers:
+    - image: ghcr.io/flux-framework/flux-restful-api:latest
+      command: echo hello world