Merge ff2c6be into 9110296

kubeflow · Nov 30, 2018 · e390d9e · e390d9e
2 parents 9110296 + ff2c6be
commit e390d9e
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 29 deletions.
diff --git a/test/check-argo-status.sh b/test/check-argo-status.sh
@@ -15,38 +15,37 @@
 # limitations under the License.
 
 
-echo "check status of argo workflow $ARGO_WORKFLOW...."
-# probing the argo workflow status until it completed. Timeout after 30 minutes
-for i in $(seq 1 ${PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT})
-do
-  WORKFLOW_STATUS=`kubectl get workflow $ARGO_WORKFLOW -n ${NAMESPACE} --show-labels`
-  echo $WORKFLOW_STATUS | grep ${WORKFLOW_COMPLETE_KEYWORD} && s=0 && break || s=$? && printf "Workflow ${ARGO_WORKFLOW} is not finished.\n${WORKFLOW_STATUS}\nSleep for 20 seconds...\n" && sleep 20
+#Wait for the workflow to appear
+while [ "$(kubectl get workflow/$ARGO_WORKFLOW --namespace "$NAMESPACE" -o=jsonpath='{.status.phase}')" == "" ]; do
+  echo "Workflow has not started yet"
+  sleep 10s
 done
 
-# Check whether the argo workflow finished or not and exit if not.
-if [[ $s != 0 ]]; then
- echo "Prow job Failed: Argo workflow timeout.."
- argo logs -w ${ARGO_WORKFLOW} -n ${NAMESPACE}
- exit $s
-fi
+#Wait for the workflow to start Running (to stop Pending)
+#TODO: Remove after https://github.com/argoproj/argo/issues/989 is fixed.
+while [ "$(kubectl get workflow/$ARGO_WORKFLOW --namespace "$NAMESPACE" -o=jsonpath='{.status.phase}')" == "Pending" ]; do
+  echo "Workflow is still Pending"
+  sleep 10s
+done
+sleep 2m #Preventing errors: "warning msg="container 'main' of pod 'build-images-xxxx-yyyyyyy' has not started within expected timeout". See https://github.com/argoproj/argo/issues/1104
+
+#Stream workflow logs
+#kubectl logs --follow --pod-running-timeout 30m workflow/$ARGO_WORKFLOW --namespace "$NAMESPACE" #Currently we cannot use kubectl for Workflow. See https://github.com/argoproj/argo/issues/1108
+argo logs --follow --workflow $ARGO_WORKFLOW --namespace "$NAMESPACE" --timestamps
 
-echo "Argo workflow finished."
+WORKFLOW_STATUS=$(kubectl get workflow/$ARGO_WORKFLOW --namespace "$NAMESPACE" -o=jsonpath='{.status.phase}')
+echo Workflow finished with status: $WORKFLOW_STATUS.
+
+#Print workflow object summary
+argo get $ARGO_WORKFLOW
+
+if [ "$WORKFLOW_STATUS" != "Succeeded" ]; then
+  exit 1
+fi
 
 if [[ ! -z "$TEST_RESULT_FOLDER" ]]
 then
   echo "Copy test result"
   mkdir -p $ARTIFACT_DIR
   gsutil cp -r "${TEST_RESULTS_GCS_DIR}"/* "${ARTIFACT_DIR}" || true
 fi
-
-if [[ $WORKFLOW_STATUS = *"${WORKFLOW_FAILED_KEYWORD}"* ]]; then
-  echo "Test workflow failed."
-  echo "=========Argo Workflow Logs========="
-  argo logs -w ${ARGO_WORKFLOW} -n ${NAMESPACE}
-  echo "===================================="
-  argo get ${ARGO_WORKFLOW} -n ${NAMESPACE}
-  exit 1
-else
-  argo get ${ARGO_WORKFLOW} -n ${NAMESPACE}
-  exit 0
-fi
diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh
@@ -31,7 +31,6 @@ PROJECT=ml-pipeline-test
 TEST_RESULT_BUCKET=ml-pipeline-test
 GCR_IMAGE_BASE_DIR=gcr.io/ml-pipeline-test/${PULL_PULL_SHA}
 CLUSTER_TYPE=create-gke
-TIMEOUT_SECONDS=1800
 NAMESPACE=default
 
 while [ "$1" != "" ]; do
@@ -62,9 +61,6 @@ done
 
 TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${PULL_PULL_SHA}/${TEST_RESULT_FOLDER}
 ARTIFACT_DIR=$WORKSPACE/_artifacts
-WORKFLOW_COMPLETE_KEYWORD="completed=true"
-WORKFLOW_FAILED_KEYWORD="phase=Failed"
-PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=$(expr $TIMEOUT_SECONDS / 20 )
 
 echo "presubmit test starts"