test: [NPM] fail cyclonus if unfinished

.pipelines/npm/npm-conformance-tests.yaml

@@ -26,7 +26,8 @@ jobs:
           mkdir -p '$(GOBIN)'
           mkdir -p '$(GOPATH)/pkg'
           BUILD_NUMBER=$(Build.BuildNumber)
-          RG=e2e-$(echo "npm-`date "+%Y-%m-%d-%S"`")
+          # format: npm-<year>-<month>-<day>-<minute>-<second>
+          RG=e2e-$(echo "npm-`date "+%Y-%m-%d-%M-%S"`")
           TAG=$(make npm-version)
           echo "Resource group: $RG"
           echo "Image tag: $TAG"
@@ -267,11 +268,9 @@ jobs:
           mkdir -p $npmLogsFolder
           cp ./kubeconfig $npmLogsFolder/kubeconfig
 
-          ## write to all NPM pod logs in the background (do this in the background instead of after to make sure the logs aren't truncated)
           npmPodList=`kubectl --kubeconfig=./kubeconfig get pods -n kube-system | grep npm | awk '{print $1}'`
           echo "Found NPM pods: $npmPodList"
 
-
           ## Run all Conformance tests in the background
           echo $FQDN
           chmod +x $(Pipeline.Workspace)/Test/e2e.test
@@ -345,12 +344,25 @@ jobs:
               exitCode=$?
             fi
           fi
-          # kill the background processes (the logs) that have this process' pid (i.e. $$) as a parent
 
+          # get all current npm pods
+          kubectl --kubeconfig=./kubeconfig get pods -n kube-system | grep npm
+          npmPodList=`kubectl --kubeconfig=./kubeconfig get pods -n kube-system | grep npm | awk '{print $1}'`
+          # capture all logs
           for npmPod in $npmPodList; do
               ./kubectl --kubeconfig=./kubeconfig logs -n kube-system $npmPod > $npmLogsFolder/$npmPod-logs.txt
           done
 
+          # capture any previous logs in case there was a crash
+          for npmPod in $npmPodList; do
+              previousLogFile=$npmLogsFolder/previous-$npmPod-logs.txt
+              ./kubectl --kubeconfig=./kubeconfig logs -n kube-system $npmPod -p > $previousLogFile
+              if [[ $? -ne 0 ]]; then
+                  # remove the empty file if kubectl logs failed (e.g. there was no previous terminated container)
+                  rm $previousLogFile
+              fi
+          done
+
           exit $exitCode
         displayName: "Run Test Suite and Get Logs"
         failOnStderr: false
@@ -479,7 +491,18 @@ jobs:
           cp cyclonus-$CLUSTER_NAME $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/cyclonus-$CLUSTER_NAME
           echo "Getting cluster state for $CLUSTER_NAME"
           mkdir -p $(System.DefaultWorkingDirectory)/$CLUSTER_NAME
+          kubectl get pods -n kube-system | grep npm
           kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/npm-logs_$(PROFILE).txt
+          # capture any previous logs in case there was a crash
+          npmPodList=`kubectl get pods -n kube-system | grep npm | awk '{print $1}'`
+          for npmPod in $npmPodList; do
+              previousLogFile=$(System.DefaultWorkingDirectory)/$CLUSTER_NAME/previous-npm-logs_$(PROFILE).txt
+              kubectl logs -n kube-system $npmPod -p > $previousLogFile
+              if [[ $? -ne 0 ]]; then
+                  # remove the empty file if kubectl logs failed (e.g. there was no previous terminated container)
+                  rm $previousLogFile
+              fi
+          done
           cp ./kubeconfig $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/.kubeconfig
         condition: always()
 

test/cyclonus/test-cyclonus-windows.sh

@@ -1,4 +1,5 @@
 curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_linux_amd64.tar.gz | tar -zxv
+LOG_FILE=cyclonus-$CLUSTER_NAME
 ./cyclonus_linux_amd64/cyclonus generate \
     --noisy=true \
     --retries=7 \
@@ -8,12 +9,21 @@ curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_lin
     --pod-creation-timeout-seconds=480 \
     --job-timeout-seconds=15 \
     --server-protocol=TCP,UDP \
-    --exclude sctp,named-port,ip-block-with-except,multi-peer,upstream-e2e,example,end-port,namespaces-by-default-label,update-policy | tee cyclonus-$CLUSTER_NAME
+    --exclude sctp,named-port,ip-block-with-except,multi-peer,upstream-e2e,example,end-port,namespaces-by-default-label,update-policy | tee $LOG_FILE
+
+# might need to redirect to /dev/null 2>&1 instead of just grepping with -q to avoid "cat: write error: Broken pipe"
+rc=999
+cat $LOG_FILE | grep "SummaryTable:" > /dev/null 2>&1 && rc=$?
+echo $rc
+if [ $rc -ne 0 ]; then
+    echo "FAILING because cyclonus tests did not complete"
+    exit 2
+fi
 
 rc=0
-cat cyclonus-$CLUSTER_NAME | grep "failed" > /dev/null 2>&1 || rc=$?
+cat $LOG_FILE | grep "failed" > /dev/null 2>&1 || rc=$?
 echo $rc
 if [ $rc -eq 0 ]; then
-    echo "failures detected"
-    exit 1
+    echo "FAILING because cyclonus completed but failures detected"
+    exit 3
 fi

test/cyclonus/test-cyclonus.sh

@@ -54,11 +54,19 @@ kubectl delete --ignore-not-found=true clusterrolebinding cyclonus
 kubectl delete --ignore-not-found=true sa cyclonus -n kube-system
 kubectl delete --ignore-not-found=true -f $cyclonusProfile
 
-# if 'failure' is in the logs, fail; otherwise succeed
-rc=0
+# might need to redirect to /dev/null 2>&1 instead of just grepping with -q to avoid "cat: write error: Broken pipe"
+rc=999
+cat $LOG_FILE | grep "SummaryTable:" > /dev/null 2>&1 && rc=$?
+echo $rc
+if [ $rc -ne 0 ]; then
+    echo "FAILING because cyclonus tests did not complete"
+    exit 2
+fi
 
-cat "$LOG_FILE" | grep "failed" > /dev/null 2>&1 || rc=$?
+rc=0
+cat $LOG_FILE | grep "failed" > /dev/null 2>&1 || rc=$?
 echo $rc
 if [ $rc -eq 0 ]; then
-    exit 1
+    echo "FAILING because cyclonus completed but failures detected"
+    exit 3
 fi