src: deploy: Add verbose mode

kw deploy generates a very polluted output, this commit cleanup the
output by:

* Hide unnecessary command log
* Use progress bar for modules_install
* Add messages per deploy milestone
* Add --verbose in case user want to know every step

Note: This is part of the issue kworkflow#179

Signed-off-by: Rodrigo Siqueira <siqueirajordao@riseup.net>

documentation/dependencies/arch.dependencies

@@ -22,3 +22,4 @@ bc
 perl-authen-sasl
 perl-io-socket-ssl
 sqlite3
+pv

documentation/dependencies/debian.dependencies

@@ -23,3 +23,4 @@ zstd
 bc
 git-email
 sqlite3
+pv

src/bash_autocomplete.sh

@@ -29,8 +29,10 @@ function _kw_autocomplete()
   kw_options['debug']='--local --remote --event --ftrace --dmesg --cmd
                        --history --disable --list --follow --reset --help'
 
+  kw_options['bd']='--verbose'
+
   kw_options['deploy']='--force --list --list-all --local --ls-line --modules
-                        --reboot --remote --uninstall --vm --setup'
+                        --reboot --remote --uninstall --vm --setup --verbose'
   kw_options['d']="${kw_options['deploy']}"
 
   kw_options['device']='--local --remote --vm'

src/deploy.sh

@@ -102,6 +102,8 @@ function deploy_main()
   uninstall_force="${options_values['UNINSTALL_FORCE']}"
   setup="${options_values['SETUP']}"
 
+  [[ -n "${options_values['VERBOSE']}" ]] && flag='VERBOSE'
+
   update_deploy_variables
 
   # Let's ensure that the target machine is ready for the deploy
@@ -172,7 +174,7 @@ function deploy_main()
   # new kernel version we also update all modules; maybe one day we can change
   # it, but for now this looks the safe option.
   start=$(date +%s)
-  modules_install '' "$target"
+  modules_install "$flag" "$target"
   modules_install_status="$?"
   end=$(date +%s)
   runtime=$((end - start))
@@ -187,7 +189,7 @@ function deploy_main()
     # Update name: release + alias
     name=$(make kernelrelease)
 
-    run_kernel_install "$reboot" "$name" '' "$target" '' "$build_and_deploy"
+    run_kernel_install "$reboot" "$name" "$flag" "$target" '' "$build_and_deploy"
     end=$(date +%s)
     runtime=$((runtime + (end - start)))
     statistics_manager 'deploy' "$runtime"
@@ -496,8 +498,9 @@ function prepare_remote_dir()
   # not exits
   if [[ -z "$first_deploy" ]]; then
     cp2remote "$flag" "$files_to_send" "$REMOTE_KW_DEPLOY" \
-      '--archive' "$remote" "$port" "$user"
+      '--archive' "$remote" "$port" "$user" 'quiet'
   else
+    say '* Sending kw to the remote'
     cmd_remotely "mkdir -p $REMOTE_KW_DEPLOY" "$flag" "$remote" "$port" "$user"
     cmd="scp -q $files_to_send $user@$remote:$REMOTE_KW_DEPLOY"
     cmd_manager "$flag" "$cmd"
@@ -589,8 +592,14 @@ function collect_target_info_for_deploy()
   local distro_info
   local distro
   local data
+  local verbose
 
   flag=${flag:-'SILENT'}
+  # We cannot have any sort of extra output inside collect info
+  if [[ "$flag" == 'VERBOSE' ]]; then
+    flag='SILENT'
+    verbose=1
+  fi
 
   case "$target" in
     1) # VM_TARGET
@@ -623,6 +632,8 @@ function collect_target_info_for_deploy()
     complain 'Unfortunately, there is no support for the target distro'
     exit 95 # ENOTSUP
   fi
+
+  [[ -n "$verbose" ]] && printf '%s\n' "$data"
 }
 
 # This function handles the kernel uninstall process for different targets.
@@ -729,15 +740,14 @@ function modules_install()
   local cmd
   local compression_type="${configurations[deploy_default_compression]}"
 
-  flag=${flag:-''}
+  flag=${flag:-'SILENT'}
 
   case "$target" in
     1) # VM_TARGET
       modules_install_to "${configurations[mount_point]}" "$flag"
       ;;
     2) # LOCAL_TARGET
-      cmd='sudo -E make modules_install'
-      cmd_manager "$flag" "$cmd"
+      modules_install_to '/lib/modules' "$flag" 'local'
       ;;
     3) # REMOTE_TARGET
       remote="${remote_parameters['REMOTE_IP']}"
@@ -748,12 +758,12 @@ function modules_install()
       modules_install_to "$KW_CACHE_DIR/$LOCAL_REMOTE_DIR/" "$flag"
 
       release=$(get_kernel_release "$flag")
-      success "Kernel: $release"
       generate_tarball "$KW_CACHE_DIR/$LOCAL_REMOTE_DIR/lib/modules/" \
         "$KW_CACHE_DIR/$LOCAL_TO_DEPLOY_DIR/$release.tar" \
         "$compression_type" "$release" "$flag"
 
       local tarball_for_deploy_path="$KW_CACHE_DIR/$LOCAL_TO_DEPLOY_DIR/$release.tar"
+      say "* Sending kernel modules ($release) to the remote"
       cp2remote "$flag" "$tarball_for_deploy_path" "$KW_DEPLOY_TMP_FILE"
 
       # 3. Deploy: Execute script
@@ -775,11 +785,29 @@ function modules_install_to()
 {
   local install_to="$1"
   local flag="$2"
+  local local_deploy="$3"
+  local total_lines
+  local pv_cmd
+  local cmd=''
+  local sign_extra_line=2
 
   flag=${flag:-'SILENT'}
 
-  local cmd="make INSTALL_MOD_PATH=$install_to modules_install"
-  set +e
+  if [[ "$local_deploy" == 'local' ]]; then
+    cmd='sudo -E make modules_install'
+    sign_extra_line=1
+  else
+    cmd="make INSTALL_MOD_PATH=$install_to modules_install"
+  fi
+
+  if [[ "$flag" != 'VERBOSE' && -f './modules.order' ]]; then
+    total_lines=$(wc -l < './modules.order')
+    # Multiply by two because we have the driver name and the signing line
+    total_lines=$((total_lines * "$sign_extra_line"))
+    cmd+=" | pv -p --line-mode --size $total_lines > /dev/null"
+  fi
+
+  say '* Preparing modules'
   cmd_manager "$flag" "$cmd"
 }
 
@@ -817,7 +845,7 @@ function run_kernel_install()
   kernel_name=${kernel_name:-'nothing'}
   mkinitcpio_name=${mkinitcpio_name:-'nothing'}
   name=${name:-'kw'}
-  flag=${flag:-''}
+  flag=${flag:-'SILENT'}
 
   if [[ "$reboot" == 0 ]]; then
     reboot_default="${configurations[reboot_after_deploy]}"
@@ -904,6 +932,7 @@ function run_kernel_install()
       distro_info=$(which_distro "$remote" "$port" "$user")
       distro=$(detect_distro '/' "$distro_info")
 
+      say '* Sending kernel image and config file to the remote'
       cp2remote "$flag" \
         "arch/$arch_target/boot/$kernel_img_name" "$KW_DEPLOY_TMP_FILE/vmlinuz-$name"
 
@@ -938,8 +967,8 @@ function parse_deploy_options()
   local remote
   local options
   local long_options='remote:,local,vm,reboot,modules,list,ls-line,uninstall:'
-  long_options+=',list-all,force,setup'
-  local short_options='r,m,l,s,u:,a,f'
+  long_options+=',list-all,force,setup,verbose'
+  local short_options='r,m,l,s,u:,a,f,v'
 
   options="$(kw_parse "$short_options" "$long_options" "$@")"
 
@@ -959,6 +988,7 @@ function parse_deploy_options()
   options_values['MENU_CONFIG']='nconfig'
   options_values['LS_ALL']=''
   options_values['SETUP']=''
+  options_values['VERBOSE']=''
 
   remote_parameters['REMOTE_IP']=''
   remote_parameters['REMOTE_PORT']=''
@@ -1035,6 +1065,10 @@ function parse_deploy_options()
         options_values['UNINSTALL']+="$2"
         shift 2
         ;;
+      --verbose | -v)
+        options_values['VERBOSE']=1
+        shift
+        ;;
       --force | -f)
         options_values['UNINSTALL_FORCE']=1
         shift
@@ -1075,6 +1109,7 @@ function deploy_help()
     '  deploy - installs kernel and modules:' \
     '  deploy (--remote <remote>:<port> | --local | --vm) - choose target' \
     '  deploy (--reboot | -r) - reboot machine after deploy' \
+    '  deploy (--verbose | -v) - Show a detailed output' \
     '  deploy (--setup) - Set up target machine for deploy' \
     '  deploy (--modules | -m) - install only modules' \
     '  deploy (--uninstall | -u) [(--force | -f)] <kernel-name>,... - uninstall given kernels' \

src/kwlib.sh

@@ -106,7 +106,7 @@ function cmd_manager()
       say "$command_for_eval"
       return 0
       ;;
-    *)
+    *) # VERBOSE
       say "$command_for_eval"
       ;;
   esac
@@ -180,7 +180,7 @@ function get_kernel_release()
   local flag="$1"
   local cmd='make kernelrelease'
 
-  flag=${flag:-'SILENT'}
+  [[ "$flag" != 'TEST_MODE' ]] && flag='SILENT'
 
   cmd_manager "$flag" "$cmd"
 }

src/plugins/kernel_install/arch.sh

@@ -9,6 +9,7 @@
 declare -ga required_packages=(
   'rsync'
   'screen'
+  'pv'
 )
 
 # ArchLinux package manager

src/plugins/kernel_install/debian.sh

@@ -10,6 +10,7 @@
 declare -ag required_packages=(
   'rsync'
   'screen'
+  'pv'
 )
 
 # Debian package manager command

src/plugins/kernel_install/grub.sh

@@ -12,18 +12,44 @@ function run_bootloader_update()
   local flag="$1"
   local target="$2"
   local cmd_grub
+  local cmd_sudo
+  local total_count
 
   if [[ "$target" == 'local' ]]; then
-    cmd_grub='sudo -E '
+    cmd_sudo='sudo -E '
+    cmd_grub+="$cmd_sudo"
   elif [[ "$target" == 'vm' ]]; then
     run_bootloader_for_vm "$flag"
     return "$?"
   fi
 
   cmd_grub+="$DEFAULT_GRUB_CMD_UPDATE"
+
+  if [[ "$flag" != 'VERBOSE' ]]; then
+    total_count=$(total_of_installed_kernels "$flag" "$target")
+    total_count=$((total_count * 2 + 7))
+    # TODO: For some reason, this is not working via ssh
+    #cmd_grub+=" |& pv -p --line-mode --size $total_count > /dev/null"
+  fi
+
   cmd_manager "$flag" "$cmd_grub"
 }
 
+function total_of_installed_kernels()
+{
+  local flag="$1"
+  local target="$2"
+  local total_count
+  local find_cmd="find /boot -name 'vmlinuz*' | wc -l"
+
+  [[ "$target" == 'local' ]] && find_cmd="sudo -E $find_cmd"
+
+  [[ "$flag" != 'TEST_MODE' ]] && total_count=$(eval "$find_cmd")
+  total_count=$((total_count * 2 + 7))
+
+  printf '%d' "$total_count"
+}
+
 # After configuring the handle (adding a disk image in write mode), the
 # guestfish performes the followed steps: (1) mount image; (2) updates kernel
 # images list; (3) create a dummy device.map that tells Grub to look for

src/plugins/kernel_install/utils.sh

@@ -26,7 +26,11 @@ function cmd_manager()
       printf '%s\n' "$@"
       return 0
       ;;
-    *)
+    VERBOSE)
+      shift 1
+      printf '%s\n' "$@"
+      ;;
+    *) # VERBOSE
       printf '%s\n' "$@"
       ;;
   esac
@@ -196,6 +200,8 @@ function install_modules()
   local ret
   local tar_cmd
 
+  flag=${flag:-'SILENT'}
+
   modules_path="$KW_DEPLOY_TMP_FILE/$module_name"
 
   if [[ ! -f "$modules_path" ]]; then
@@ -287,9 +293,6 @@ function do_uninstall()
     "$prefix/var/lib/initramfs-tools/$target"
   )
 
-
-
-
   if [[ -z "$target" ]]; then
     printf '%s\n' 'No parameter, nothing to do'
     exit 22 # EINVAL
@@ -372,10 +375,13 @@ function install_kernel()
   local sudo_cmd=''
   local cmd=''
   local path_prefix=''
+  local verbose_cp
 
   flag=${flag:-'SILENT'}
   target=${target:-'remote'}
 
+  [[ "$flag" == 'VERBOSE' ]] && verbose_cp='-v'
+
   if [[ "$target" == 'local' ]]; then
     sudo_cmd='sudo -E'
   fi
@@ -391,7 +397,7 @@ function install_kernel()
       path_prefix="${configurations[mount_point]}"
       INSTALLED_KERNELS_PATH="$path_prefix/$INSTALLED_KERNELS_PATH"
       # Copy config file
-      cmd_manager "$flag" "cp -v .config $path_prefix/boot/config-$name"
+      cmd_manager "$flag" "cp $verbose_cp .config $path_prefix/boot/config-$name"
     else
       complain 'Did you check if your VM is mounted?'
       return 125 # ECANCELED
@@ -406,16 +412,17 @@ function install_kernel()
 
   if [[ "$target" != 'remote' ]]; then
     [[ -z "$architecture" ]] && architecture='x86_64'
-    cmd="$sudo_cmd cp -v arch/$architecture/boot/$kernel_image_name $path_prefix/boot/vmlinuz-$name"
+    cmd="$sudo_cmd cp $verbose_cp arch/$architecture/boot/$kernel_image_name $path_prefix/boot/vmlinuz-$name"
     cmd_manager "$flag" "$cmd"
   else
-    cmd="$sudo_cmd cp -v $KW_DEPLOY_TMP_FILE/vmlinuz-$name $path_prefix/boot/vmlinuz-$name"
+    cmd="$sudo_cmd cp $verbose_cp $KW_DEPLOY_TMP_FILE/vmlinuz-$name $path_prefix/boot/vmlinuz-$name"
     cmd_manager "$flag" "$cmd"
   fi
 
   # Each distro has their own way to generate their temporary root file system.
   # For example, Debian uses update-initramfs, Arch uses mkinitcpio, etc
   cmd="generate_${distro}_temporary_root_file_system"
+
   eval "$cmd" "$name" "$target" "$flag" "$path_prefix"
   ret="$?"
   if [[ "$ret" != 0 ]]; then

src/remote.sh

@@ -105,19 +105,23 @@ function cp2remote()
   local remote=${5:-${remote_parameters['REMOTE_IP']}}
   local port=${6:-${remote_parameters['REMOTE_PORT']}}
   local user=${7:-${remote_parameters['REMOTE_USER']}}
+  local quiet="$8"
+  local progress_flag='--info=progress2'
 
   if [[ -v configurations['ssh_configfile'] && -v configurations['hostname'] ]]; then
     rsync_target="'ssh -F ${configurations['ssh_configfile']}' $src ${configurations['hostname']}:$dst"
   else
     rsync_target="'ssh -p $port' $src $user@$remote:$dst"
   fi
 
+  [[ -n "$quiet" ]] && progress_flag=''
+
   # The -LrlptD flags for rsync are similar to the -La flags used for archiving
   # files and resolving symlinks the diference lies on the absence of the -og
   # flags that preserve group and user ownership. We don't want to preserve
   # ownership in order to automatically transfer the files to the root user and
-  # group.
-  cmd_manager "$flag" "rsync -e $rsync_target -LrlptD --rsync-path='sudo rsync' $rsync_params"
+  # group. --partial --progress
+  cmd_manager "$flag" "rsync $progress_flag -e $rsync_target -LrlptD --rsync-path='sudo rsync' $rsync_params"
 }
 
 # This function copies files from the remote machine to the local host.
@@ -136,7 +140,7 @@ function remote2host()
   local port=${5:-${configurations[ssh_port]}}
   local user=${6:-${configurations[ssh_user]}}
 
-  cmd_manager "$flag" "rsync -e \"ssh -p $port\" $user@$ip:$src $dst"
+  cmd_manager "$flag" "rsync --info=progress2 -e \"ssh -p $port\" $user@$ip:$src $dst"
 }
 
 # Access the target device and query the distro name.