e-m-b-a · m-1-k-3 · Feb 28, 2023 · Feb 26, 2023 · Feb 26, 2023 · Feb 26, 2023
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -24,5 +24,6 @@ jobs:
       - name: EMBA container build
         uses: Wandalen/wretry.action@master
         with:
-          command: docker-compose build --no-cache --pull && ./emba -d -y -j
+          # command: docker-compose build --no-cache --pull && ./emba -d -y -j
+          command: docker-compose build --no-cache --pull
           attempt_limit: 3
diff --git a/emba b/emba
@@ -166,12 +166,25 @@ run_modules()
         # module_start_log "$MODULE_MAIN"
         if [[ "$RESTART" -eq 1 ]]; then
           if [[ $(grep -i -c "$MODULE_MAIN finished" "$LOG_DIR"/"$MAIN_LOG_FILE") -gt 0 ]]; then
-            if [[ "$MODULE_MAIN" == "P99_"* ]] || [[ "$MODULE_MAIN" == "L10_"* ]]; then
-              print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished but essential - rerun it" "no_log"
+            if [[ "$MODULE_MAIN" == "P99_"* ]] || [[ "$MODULE_MAIN" == "L1"* ]]; then
+              print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished but essential - rerun it" "main"
               MOD_FIN=0
             else
-              print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished ... skipping" "no_log"
+              print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished ... skipping" "main"
               MOD_FIN=1
+              if [[ $HTML -eq 1 ]] ; then
+                # we need to build the web reporter links for skipping modules
+                mapfile -t LOG_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -iname "$MODULE_MAIN*.txt" | sort)
+                for LOG_FILE_ in "${LOG_FILES[@]}"; do
+                  if grep -q "nothing reported" "$LOG_FILE_"; then
+                    continue
+                  fi
+                  MODULE_NAME_=$(basename -s .txt "$LOG_FILE_")
+                  MODUL_NAME="$( strip_color_tags "$(grep -a -E -B 1 '[=]{65}' "$LOG_FILE_" | head -n 1 )" | cut -d" " -f2- )"
+                  HTML_FILE="$MODULE_NAME_.html"
+                  add_link_to_index "$HTML_FILE" "$MODUL_NAME"
+                done
+              fi
             fi
           fi
         fi
@@ -185,14 +198,6 @@ run_modules()
           else
             "$MODULE_MAIN"
           fi
-        else
-          local ENABLE=1
-          FILE_NAME=$(echo "$MODULE_MAIN" | sed -e 's/\(.*\)/\L\1/' | tr " " _ | tr '[:upper:]' '[:lower:]')
-          LOG_FILE="$LOG_DIR""/""$FILE_NAME"".txt"
-          if grep -i -q "$MODULE_MAIN nothing reported" "$LOG_FILE"; then
-            ENABLE=0
-          fi
-          module_end_log "$MODULE_MAIN" "$ENABLE"
         fi
         reset_module_count
       fi
@@ -214,12 +219,25 @@ run_modules()
           # module_start_log "$MODULE_MAIN"
           if [[ "$RESTART" -eq 1 ]]; then
             if [[ $(grep -i -c "$MODULE_MAIN finished" "$LOG_DIR"/"$MAIN_LOG_FILE") -gt 0 ]]; then
-              if [[ "$MODULE_MAIN" == "P99_"* ]] || [[ "$MODULE_MAIN" == "L10_"* ]]; then
-                print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished but essential - rerun it" "no_log"
+              if [[ "$MODULE_MAIN" == "P99_"* ]] || [[ "$MODULE_MAIN" == "L1"* ]]; then
+                print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished but essential - rerun it" "main"
                 MOD_FIN=0
               else
-                print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished ... skipping" "no_log"
+                print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished ... skipping" "main"
                 MOD_FIN=1
+                if [[ $HTML -eq 1 ]] ; then
+                  # we need to build the web reporter links for skipping modules
+                  mapfile -t LOG_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -iname "$MODULE_MAIN*.txt" | sort)
+                  for LOG_FILE_ in "${LOG_FILES[@]}"; do
+                    if grep -q "nothing reported" "$LOG_FILE_"; then
+                      continue
+                    fi
+                    MODULE_NAME_=$(basename -s .txt "$LOG_FILE_")
+                    MODUL_NAME="$( strip_color_tags "$(grep -a -E -B 1 '[=]{65}' "$LOG_FILE_" | head -n 1 )" | cut -d" " -f2- )"
+                    HTML_FILE="$MODULE_NAME_.html"
+                    add_link_to_index "$HTML_FILE" "$MODUL_NAME"
+                  done
+                fi
               fi
             fi
           fi
@@ -233,14 +251,6 @@ run_modules()
             else
               "$MODULE_MAIN"
             fi
-          else
-            local ENABLE=1
-            FILE_NAME=$(echo "$MODULE_MAIN" | sed -e 's/\(.*\)/\L\1/' | tr " " _ )
-            LOG_FILE="$LOG_DIR""/""$FILE_NAME"".txt"
-            if grep -i -q "$MODULE_MAIN nothing reported" "$LOG_FILE"; then
-              ENABLE=0
-            fi
-            module_end_log "$MODULE_MAIN" "$ENABLE"
           fi
           reset_module_count
         fi
@@ -280,12 +290,25 @@ run_modules()
             # module_start_log "$MODULE_MAIN"
             if [[ "$RESTART" -eq 1 ]]; then
               if [[ $(grep -i -c "$MODULE_MAIN finished" "$LOG_DIR"/"$MAIN_LOG_FILE") -gt 0 ]]; then
-                if [[ "$MODULE_MAIN" == "P99_"* ]] || [[ "$MODULE_MAIN" == "L10_"* ]]; then
+                if [[ "$MODULE_MAIN" == "P99_"* ]] || [[ "$MODULE_MAIN" == "L1"* ]]; then
                   print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished but essential - rerun it" "main"
                   MOD_FIN=0
                 else
                   print_output "[*] Module $ORANGE$MODULE_MAIN$NC already finished ... skipping" "main"
                   MOD_FIN=1
+                  if [[ $HTML -eq 1 ]] ; then
+                    # we need to build the web reporter links for skipping modules
+                    mapfile -t LOG_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -iname "$MODULE_MAIN*.txt" | sort)
+                    for LOG_FILE_ in "${LOG_FILES[@]}"; do
+                      if grep -q "nothing reported" "$LOG_FILE_"; then
+                        continue
+                      fi
+                      MODULE_NAME_=$(basename -s .txt "$LOG_FILE_")
+                      MODUL_NAME="$( strip_color_tags "$(grep -a -E -B 1 '[=]{65}' "$LOG_FILE_" | head -n 1 )" | cut -d" " -f2- )"
+                      HTML_FILE="$MODULE_NAME_.html"
+                      add_link_to_index "$HTML_FILE" "$MODUL_NAME"
+                    done
+                  fi
                 fi
               fi
             fi
@@ -299,14 +322,6 @@ run_modules()
               else
                 "$MODULE_MAIN"
               fi
-            else
-              local ENABLE=1
-              FILE_NAME=$(echo "$MODULE_MAIN" | sed -e 's/\(.*\)/\L\1/' | tr " " _ )
-              LOG_FILE="$LOG_DIR""/""$FILE_NAME"".txt"
-              if grep -i -q "$MODULE_MAIN nothing reported" "$LOG_FILE"; then
-                ENABLE=0
-              fi
-              module_end_log "$MODULE_MAIN" "$ENABLE"
             fi
             reset_module_count
           fi

diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh
@@ -21,6 +21,7 @@ run_web_reporter_mod_name() {
     # usually we should only find one file:
     mapfile -t LOG_FILES < <(find "$LOG_DIR" -maxdepth 1 -type f -iname "$MOD_NAME*.txt" | sort)
     for LOG_FILE in "${LOG_FILES[@]}"; do
+      MOD_NAME=$(basename -s .txt "$LOG_FILE")
       generate_report_file "$LOG_FILE"
       sed -i -E '/^\[REF\]|\[ANC\].*/d' "$LOG_FILE"
     done

diff --git a/helpers/helpers_emba_prepare.sh b/helpers/helpers_emba_prepare.sh
@@ -96,10 +96,6 @@ log_folder()
             if ! [[ -d "$TMP_DIR" ]]; then
               mkdir "$TMP_DIR"
             fi
-            if [[ -d "$LOG_DIR"/html-report ]]; then
-              print_output "[*] EMBA needs to remove and re-create the current HTML report" "no_log"
-              rm -r "$LOG_DIR""/html-report" && mkdir "$LOG_DIR""/html-report"
-            fi
             touch "$TMP_DIR"/restart
           else
             echo -e "\\n${RED}Terminate EMBA${NC}\\n"

diff --git a/helpers/helpers_emba_print.sh b/helpers/helpers_emba_print.sh
@@ -63,7 +63,9 @@ module_log_init()
 
   if [[ -f "$LOG_FILE" ]]; then
     print_output "[*] Found old module log file $ORANGE$LOG_FILE$NC... creating a backup" "no_log"
-    mv "$LOG_FILE" "$LOG_FILE".bak."$RANDOM" || true
+    export OLD_LOG_FILE=""
+    OLD_LOG_FILE="$LOG_FILE".bak."$RANDOM"
+    mv "$LOG_FILE" "$OLD_LOG_FILE" || true
   fi
 
   module_start_log "${FILE_NAME^}"
@@ -290,6 +292,7 @@ write_pid_log() {
     return
   fi
 
+  # shellcheck disable=SC2153
   echo "$LOG_MESSAGE" >> "$TMP_DIR"/"$PID_LOG_FILE" || true
 }
 
@@ -665,7 +668,9 @@ module_start_log() {
   LOG_PATH_MODULE=$(abs_path "$LOG_DIR""/""$(echo "$MODULE_MAIN_NAME" | tr '[:upper:]' '[:lower:]')")
   if [[ -d "$LOG_PATH_MODULE" ]] ; then
     print_output "[*] Found old module log path for $ORANGE$MODULE_MAIN_NAME$NC ... creating a backup" "no_log"
-    mv "$LOG_PATH_MODULE" "$LOG_PATH_MODULE".bak."$RANDOM" || true
+    export OLD_LOG_DIR=""
+    OLD_LOG_DIR="$LOG_PATH_MODULE".bak."$RANDOM" || true
+    mv "$LOG_PATH_MODULE" "$OLD_LOG_DIR" || true
   fi
   if ! [[ -d "$LOG_PATH_MODULE" ]]; then
     mkdir "$LOG_PATH_MODULE" || true
@@ -696,7 +701,10 @@ module_end_log() {
 
   if [[ "$MODULE_REPORT_STATE" -eq 0 ]]; then
     print_output "[-] $(date) - $MODULE_MAIN_NAME nothing reported"
-  else
+  fi
+
+  # we do not report the templates on restarted tests
+  if [[ "$MODULE_REPORT_STATE" -ne 0 ]]; then
     REPORT_TEMPLATE="$(basename -s ".sh" "$MODULE_MAIN_NAME")-post"
     # We handle .txt and .sh files in report_template folder.
     # .txt are just echoed on cli and report

diff --git a/helpers/helpers_emba_system_emulation.sh b/helpers/helpers_emba_system_emulation.sh
@@ -17,6 +17,8 @@
 restart_emulation() {
   local IP_ADDRESS_="${1:-}"
   local IMAGE_NAME_="${2:-}"
+  # restart_scan is used to indicate a restarted scan. There we do not need to restart the network
+  local RESTART_SCAN="${3:-0}"
 
   if ping -c 1 "$IP_ADDRESS_" &> /dev/null; then
     print_output "[+] System with $ORANGE$IP_ADDRESS_$NC responding again - probably it recovered automatically."
@@ -32,7 +34,7 @@ restart_emulation() {
   print_output "[*] Trying to auto-maintain emulated system now ..."
 
   stopping_emulation_process "$IMAGE_NAME_"
-  reset_network_emulation 2
+  [[ "$RESTART_SCAN" -eq 0 ]] && reset_network_emulation 2
 
   # what an ugly hack - probably we are going to improve this later on
   local HOME_PATH=""
@@ -48,6 +50,8 @@ restart_emulation() {
     [[ "$COUNTER" -gt 50 ]] && (print_output "[-] System not recovered" && return)
     sleep 6
   done
-  print_output "[*] System automatically maintained and should be available again in a few moments ..."
+  print_output "[*] System automatically maintained and should be available again in a few moments ... check ip address $ORANGE$IP_ADDRESS_$NC"
   sleep 60
+  export SYS_ONLINE=1
+  export TCP="ok"
 }
diff --git a/modules/L10_system_emulation.sh b/modules/L10_system_emulation.sh
@@ -49,92 +49,92 @@ L10_system_emulation() {
       LOG_PATH_MODULE=$(abs_path "$LOG_PATH_MODULE")
       R_PATH_CNT=1
 
-      # handling old emulation processes:
+      # handling restarted scans with old emulation processes:
       if [[ -f "$LOG_DIR"/emulator_online_results.log ]] && grep -q "L10_system_emulation finished" "$LOG_DIR"/emba.log; then
+        print_ln
         print_output "[*] Found finished emulation process - trying to recover old emulation process"
 
-        local IP_ADDRESS_=""
-        local IMAGE_NAME_=""
+        export IP_ADDRESS_=""
+        export IMAGE_NAME=""
         export ARCHIVE_PATH=""
 
-        IP_ADDRESS=$(grep "TCP ok" "$LOG_DIR"/emulator_online_results.log | sort -k 7 -t ';' | tail -1 | cut -d\; -f8 | awk '{print $3}')
+        IP_ADDRESS_=$(grep "TCP ok" "$LOG_DIR"/emulator_online_results.log | sort -k 7 -t ';' | tail -1 | cut -d\; -f8 | awk '{print $3}')
         IMAGE_NAME="$(grep "TCP ok" "$LOG_DIR"/emulator_online_results.log | sort -k 7 -t ';' | tail -1 | cut -d\; -f10)"
-        ARCHIVE_PATH="$LOG_PATH_MODULE""/""$IMAGE_NAME"
-        print_output "[*] Recovered IP address: $ORANGE$IP_ADDRESS$NC"
+        ARCHIVE_PATH="$OLD_LOG_DIR""/""$IMAGE_NAME"
+        print_output "[*] Recovered IP address: $ORANGE$IP_ADDRESS_$NC"
         print_output "[*] Recovered IMAGE_NAME: $ORANGE$IMAGE_NAME$NC"
         print_output "[*] Recovered ARCHIVE_PATH: $ORANGE$ARCHIVE_PATH$NC"
 
-        if [[ -f "$ARCHIVE_PATH" ]]; then
-          print_output "[+] Archive path found in old logs ... restarting emulation process"
+        if [[ -f "$ARCHIVE_PATH""/run.sh" ]]; then
+          print_output "[+] Startup script (run.sh) found in old logs ... restarting emulation process now"
 
-          restart_emulation "$IP_ADDRESS_" "$IMAGE_NAME"
-          module_end_log "${FUNCNAME[0]}" 1
-          return
+          restart_emulation "$IP_ADDRESS_" "$IMAGE_NAME" 1
+          # we should get TCP="ok" and SYS_ONLINE=1 back
         else
           print_output "[-] No archive path found in old logs ... restarting emulation process not possible"
         fi
       fi
 
-      for R_PATH in "${ROOT_PATH[@]}" ; do
-        print_output "[*] Testing root path ($ORANGE$R_PATH_CNT$NC/$ORANGE${#ROOT_PATH[@]}$NC): $ORANGE$R_PATH$NC"
-        write_link "p59"
-
-        if [[ -n "$D_END" ]]; then
-          TAPDEV_0="tap0_0"
-          D_END="$(echo "$D_END" | tr '[:upper:]' '[:lower:]')"
-          ARCH_END="$(echo "$ARCH" | tr '[:upper:]' '[:lower:]')$(echo "$D_END" | tr '[:upper:]' '[:lower:]')"
+      if [[ "$SYS_ONLINE" -ne 1 ]] && [[ "$TCP" != "ok" ]]; then
+        for R_PATH in "${ROOT_PATH[@]}" ; do
+          print_output "[*] Testing root path ($ORANGE$R_PATH_CNT$NC/$ORANGE${#ROOT_PATH[@]}$NC): $ORANGE$R_PATH$NC"
+          write_link "p59"
+
+          if [[ -n "$D_END" ]]; then
+            TAPDEV_0="tap0_0"
+            D_END="$(echo "$D_END" | tr '[:upper:]' '[:lower:]')"
+            ARCH_END="$(echo "$ARCH" | tr '[:upper:]' '[:lower:]')$(echo "$D_END" | tr '[:upper:]' '[:lower:]')"
+
+            # default is ARM_SF -> we only need to check if it is HF
+            # The information is based on the results of architecture_check()
+            if [[ -n "$ARM_HF" ]] && [[ "$ARM_HF" -gt "${ARM_SF:-0}" ]]; then
+              print_output "[*] ARM hardware floating detected"
+              ARCH_END="$ARCH_END""hf"
+            fi
 
-          # default is ARM_SF -> we only need to check if it is HF
-          # The information is based on the results of architecture_check()
-          if [[ -n "$ARM_HF" ]] && [[ "$ARM_HF" -gt "${ARM_SF:-0}" ]]; then
-            print_output "[*] ARM hardware floating detected"
-            ARCH_END="$ARCH_END""hf"
-          fi
+            if [[ "$ARCH_END" == "armbe"* ]] || [[ "$ARCH_END" == "mips64r2"* ]] || [[ "$ARCH_END" == "mips64_3"* ]]; then
+              print_output "[-] Found NOT supported architecture $ORANGE$ARCH_END$NC"
+              write_link "p99"
+              print_output "[-] Please open a new issue here: https://github.com/e-m-b-a/emba/issues"
+              UNSUPPORTED_ARCH=1
+              return
+            fi
 
-          if [[ "$ARCH_END" == "armbe"* ]] || [[ "$ARCH_END" == "mips64r2"* ]] || [[ "$ARCH_END" == "mips64_3"* ]]; then
-            print_output "[-] Found NOT supported architecture $ORANGE$ARCH_END$NC"
-            write_link "p99"
-            print_output "[-] Please open a new issue here: https://github.com/e-m-b-a/emba/issues"
-            UNSUPPORTED_ARCH=1
-            return
-          fi
+            # just in case we remove the return in the unsupported arch checker for testing:
+            if [[ "$UNSUPPORTED_ARCH" -ne 1 ]]; then
+              print_output "[*] Found supported architecture $ORANGE$ARCH_END$NC"
+              write_link "p99"
+            fi
 
-          # just in case we remove the return in the unsupported arch checker for testing:
-          if [[ "$UNSUPPORTED_ARCH" -ne 1 ]]; then
-            print_output "[*] Found supported architecture $ORANGE$ARCH_END$NC"
-            write_link "p99"
-          fi
+            pre_cleanup_emulator
 
-          pre_cleanup_emulator
+            main_emulation "$R_PATH" "$ARCH_END"
 
-          main_emulation "$R_PATH" "$ARCH_END"
+            if [[ -d "$MNT_POINT" ]]; then
+              rm -r "$MNT_POINT"
+            fi
 
-          if [[ -d "$MNT_POINT" ]]; then
-            rm -r "$MNT_POINT"
-          fi
+            if [[ "$SYS_ONLINE" -eq 1 ]] && [[ "$TCP" == "ok" ]]; then
+              # do not test other root paths if we are already online (some ports are available)
+              break
+            fi
+            # if [[ -f "$LOG_DIR"/emulator_online_results.log ]]; then
+            #  if [[ $(grep "TCP ok" "$LOG_DIR"/emulator_online_results.log | sort -t ';' -k6 -n -r | head -1 || true) -gt 1 ]]; then
+            #    print_output "[+] Identified the following system emulation results:"
+            #    print_output "$(indent "$(orange "$(grep "TCP ok" "$LOG_DIR"/emulator_online_results.log | sort -t ';' -k6 -n -r | head -1 || true)")")"
+            #    print_ln
+            #    print_output "[*] Restarting emulation for further analysis ..."
+            #    break
+            #  fi
+            # fi
 
-          if [[ "$SYS_ONLINE" -eq 1 ]] && [[ "$TCP" == "ok" ]]; then
-            # do not test other root paths if we are already online (some ports are available)
-            break
+          else
+            print_output "[!] No supported architecture detected"
           fi
-          # if [[ -f "$LOG_DIR"/emulator_online_results.log ]]; then
-          #  if [[ $(grep "TCP ok" "$LOG_DIR"/emulator_online_results.log | sort -t ';' -k6 -n -r | head -1 || true) -gt 1 ]]; then
-          #    print_output "[+] Identified the following system emulation results:"
-          #    print_output "$(indent "$(orange "$(grep "TCP ok" "$LOG_DIR"/emulator_online_results.log | sort -t ';' -k6 -n -r | head -1 || true)")")"
-          #    print_ln
-          #    print_output "[*] Restarting emulation for further analysis ..."
-          #    break
-          #  fi
-          # fi
-
-        else
-          print_output "[!] No supported architecture detected"
-        fi
-        ((R_PATH_CNT+=1))
-      done
-
-      print_system_emulation_results
-
+          ((R_PATH_CNT+=1))
+        done
+        print_system_emulation_results
+      fi
       MODULE_END=1
     else
       print_output "[!] No supported architecture found.\\n"
@@ -1940,6 +1940,11 @@ reset_network_emulation() {
     return
   fi
 
+  # Todo: handle network shutdown also on restarted tests
+  if [[ "$RESTART" -ne 0 ]]; then
+    return
+  fi
+
   if [[ "$EXECUTE_" -ne 0 ]]; then
     print_output "[*] Stopping Qemu emulation ..."
     pkill -9 -f "qemu-system-.*$IMAGE_NAME.*" || true &>/dev/null