WIP

IntersectMBO · May 31, 2023 · 0beafda · 0beafda
1 parent 42b2bd8
commit 0beafda
Showing 1 changed file with 47 additions and 13 deletions.
diff --git a/nix/workbench/service/healthcheck.nix b/nix/workbench/service/healthcheck.nix
@@ -102,6 +102,8 @@ let
               if test -s "''${exit_code_path}"
               then
                 # The node exited!
+                local exit_code
+                exit_code="$(cat "''${exit_code_path}")"
                 # Node failed with non-zero exit code?
                 if test "''${exit_code}" != "0"
                 then
@@ -268,15 +270,18 @@ let
           # With all these, we can efficiently select the first occurrence like
           # this: "nth(0; inputs | select( INSERT_BLOCK_QUERY ))"
           #
-          # Note that the log file is composed of one JSON object per line and
-          # we are using `--compact-output` to make sure every output log
-          # message matched by `jq` is contained within a line.
+          # Note that the log file is composed of one JSON object per line were
+          # 'inputs' read one by one and we are using `--compact-output` to make
+          # sure every output log message matched by `jq` is contained within a
+          # line.
           #
           # Also, the stdout of the node starts with some text that's echoed by
           # node's start.sh script that is not valid JSON, so we `grep` for
           # "{"at": ... }". We still need to check the exit code of these
           # functions just in case because if it fails a query may have failed
-          # and the healthcheck should fail.
+          # and the healthcheck should fail. This is tricky because when 'jq'
+          # finishes and exists `tac` or `grep` throw the following error:
+          # "writing output failed: Broken pipe"
           #
           # Finally the "at" time has format "2023-05-16 19:57:19.0231Z" and I
           # can't parse it using any standard `date` format so I'm stripping the
@@ -290,7 +295,9 @@ let
             local node=$1
             local select=$2
             local stdout_path="../''${node}/stdout"
-            if !  ${coreutils}/bin/tac "''${stdout_path}" \
+            local ans return_code=0 pipe_status=(0 0 0 0)
+            ans="$( \
+                ${coreutils}/bin/tac "''${stdout_path}" \
                 | \
                   ${grep}/bin/grep -E "^{\"at\":.*}$" \
                 | \
@@ -308,9 +315,19 @@ let
                           |=
                              (.[:20] | strptime("%Y-%m-%d %H:%M:%S.") | mktime)
                         )
-                    '
+                    ' \
+              || \
+                { return_code="$?"; pipe_status="''${PIPESTATUS[@]}"; } \
+            )"
+            if test "''${return_code}" == 0
             then
-              exit_22 "jq error: jq_node_stdout_last: ''${node}"
+              echo "''${ans}"
+            else
+              # Ignore "writing output failed: Broken pipe"
+              if test "''${pipe_status[2]}" != 0 || test "''${pipe_status[3]}" != 0 || test "''${return_code}" != 141
+              then
+                exit_22 "jq error: jq_node_stdout_last: ''${node}"
+              fi
             fi
           }
 
@@ -326,11 +343,16 @@ let
             local node=$1
             local select=$2
             local stdout_path="../''${node}/stdout"
-            if !  ${coreutils}/bin/tac "''${stdout_path}" \
+            local ans return_code=0 pipe_status=(0 0 0 0 0)
+            ans="$( \
+                  ${coreutils}/bin/tac "''${stdout_path}" \
                 | \
                   ${grep}/bin/grep -E "^{\"at\":.*}$" \
                 | \
-                  ${jq}/bin/jq --compact-output --unbuffered "select(''${select})" \
+                  ${jq}/bin/jq            \
+                    --compact-output      \
+                    --unbuffered          \
+                    "select(''${select})" \
                 | \
                   ${coreutils}/bin/head --lines=+1 \
                 | \
@@ -339,9 +361,19 @@ let
                         .at
                       |=
                         (.[:20] | strptime("%Y-%m-%d %H:%M:%S.") | mktime)
-                    '
+                    ' \
+              || \
+                { return_code="$?"; pipe_status="''${PIPESTATUS[@]}"; } \
+            )"
+            if test "''${return_code}" == 0
             then
-              exit_22 "jq error: jq_node_stdout_last_v0: ''${node}"
+              echo "''${ans}"
+            else
+              # Ignore "writing output failed: Broken pipe"
+              if test "''${pipe_status[2]}" != 0 || test "''${pipe_status[4]}" != 0 || test "''${return_code}" != 141
+              then
+                exit_22 "jq error: jq_node_stdout_last_v0: ''${node}"
+              fi
             fi
           }
 
@@ -547,18 +579,20 @@ let
 
           function exit_healthcheck {
             # Unbuffered stderr, if not the error message may be lost!
-            ${coreutils}/bin/stdbuf -o0 -e0 ${coreutils}/bin/echo -e "$(${coreutils}/bin/date --rfc-3339=seconds): $1" >&2
+            ${coreutils}/bin/stdbuf -o0 -e0 { ${coreutils}/bin/echo -e "$(${coreutils}/bin/date --rfc-3339=seconds): $1" >&2; }
             exit 1
           }
 
           function exit_22 {
             # Unbuffered stderr, if not the error message may be lost!
-            ${coreutils}/bin/stdbuf -o0 -e0 ${coreutils}/bin/echo -e "$(${coreutils}/bin/date --rfc-3339=seconds): $1" >&2
+            ${coreutils}/bin/stdbuf -o0 -e0 { ${coreutils}/bin/echo -e "$(${coreutils}/bin/date --rfc-3339=seconds): $1" >&2; }
             exit 22
           }
 
           if test -n "''${NOMAD_DEBUG:-}"
           then
+            # DEBUG_FILE="$(${coreutils}/bin/dirname "$(${coreutils}/bin/readlink -f "$0")")"/"$0".debug
+            # echo "Using debug file ''${DEBUG_FILE}" >&2
             exec 5> "$(${coreutils}/bin/dirname "$(${coreutils}/bin/readlink -f "$0")")"/start.sh.debug
             BASH_XTRACEFD="5"
             PS4='$LINENO: '