Skip to content

Commit

Permalink
Switching test to kubeflow deployment (#351)
Browse files Browse the repository at this point in the history
* test

* fix

* fix

* fix

* fix

* fix

* update

* cleanup

* fix

* coopy test

* chmod

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* update

* fix

* fix

* fix

* fix

* fix

* fix

* fix sample test

* fix

* fix

* merge

* update image builder image

* update script

* mount permission
  • Loading branch information
IronPan authored and k8s-ci-robot committed Nov 29, 2018
1 parent d534103 commit 9b77d4a
Show file tree
Hide file tree
Showing 16 changed files with 989 additions and 98 deletions.
44 changes: 8 additions & 36 deletions backend/test/test_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (

"testing"

"net/http"

"github.com/cenkalti/backoff"
experimentparams "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_client/experiment_service"
jobparams "github.com/kubeflow/pipelines/backend/api/go_http_client/job_client/job_service"
Expand All @@ -31,59 +33,29 @@ import (
"github.com/kubeflow/pipelines/backend/src/common/client/api_server"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

const (
// ML pipeline API server root URL
mlPipelineAPIServerBase = "/api/v1/namespaces/%s/services/ml-pipeline:8888/proxy/apis/v1beta1/%s"
)

var namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
var initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")

func getKubernetesClient() (*kubernetes.Clientset, error) {
// use the current context in kubeconfig
config, err := rest.InClusterConfig()
if err != nil {
return nil, errors.Wrapf(err, "Failed to get cluster config during K8s client initialization")
}
// create the clientset
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, errors.Wrapf(err, "Failed to create client set during K8s client initialization")
}

return clientSet, nil
}

func waitForReady(namespace string, initializeTimeout time.Duration) error {
clientSet, err := getKubernetesClient()
if err != nil {
return errors.Wrapf(err, "Failed to get K8s client set when waiting for ML pipeline to be ready")
}

var operation = func() error {
response := clientSet.RESTClient().Get().
AbsPath(fmt.Sprintf(mlPipelineAPIServerBase, namespace, "healthz")).Do()
if response.Error() == nil {
response, err := http.Get(fmt.Sprintf("http://ml-pipeline.%s.svc.cluster.local:8888/apis/v1beta1/healthz", namespace))
if err == nil {
return nil
}
var code int
response.StatusCode(&code)
// we wait only on 503 service unavailable. Stop retry otherwise.
if code != 503 {
return backoff.Permanent(errors.Wrapf(response.Error(), "Waiting for ml pipeline failed with non retriable error."))
if response.StatusCode != 503 {
return backoff.Permanent(errors.Wrapf(err, "Waiting for ml pipeline failed with non retriable error."))
}
return response.Error()
return err
}

b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = initializeTimeout
err = backoff.Retry(operation, b)
err := backoff.Retry(operation, b)
return errors.Wrapf(err, "Waiting for ml pipeline failed after all attempts.")
}

Expand Down
4 changes: 4 additions & 0 deletions test/api-integration-test/run_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ if [ -z "$RESULTS_GCS_DIR" ]; then
exit 1
fi

if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
fi

GITHUB_REPO=kubeflow/pipelines
BASE_DIR=/go/src/github.com/${GITHUB_REPO}
JUNIT_TEST_RESULT=junit_ApiIntegrationTestOutput.xml
Expand Down
2 changes: 1 addition & 1 deletion test/build_image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ spec:
valueFrom:
path: /outputs/strict-image-name/file
container:
image: gcr.io/ml-pipeline-staging/image-builder:v20181120-0.1.3-rc.1-21-gb7c8af0-dirty-9786df
image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
imagePullPolicy: 'Always'
args: [
"--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",
Expand Down
52 changes: 52 additions & 0 deletions test/check-argo-status.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#!/bin/bash
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


echo "check status of argo workflow $ARGO_WORKFLOW...."
# probing the argo workflow status until it completed. Timeout after 30 minutes
for i in $(seq 1 ${PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT})
do
WORKFLOW_STATUS=`kubectl get workflow $ARGO_WORKFLOW -n ${NAMESPACE} --show-labels`
echo $WORKFLOW_STATUS | grep ${WORKFLOW_COMPLETE_KEYWORD} && s=0 && break || s=$? && printf "Workflow ${ARGO_WORKFLOW} is not finished.\n${WORKFLOW_STATUS}\nSleep for 20 seconds...\n" && sleep 20
done

# Check whether the argo workflow finished or not and exit if not.
if [[ $s != 0 ]]; then
echo "Prow job Failed: Argo workflow timeout.."
argo logs -w ${ARGO_WORKFLOW} -n ${NAMESPACE}
exit $s
fi

echo "Argo workflow finished."

if [[ ! -z "$TEST_RESULT_FOLDER" ]]
then
echo "Copy test result"
mkdir -p $ARTIFACT_DIR
gsutil cp -r "${TEST_RESULTS_GCS_DIR}"/* "${ARTIFACT_DIR}" || true
fi

if [[ $WORKFLOW_STATUS = *"${WORKFLOW_FAILED_KEYWORD}"* ]]; then
echo "Test workflow failed."
echo "=========Argo Workflow Logs========="
argo logs -w ${ARGO_WORKFLOW} -n ${NAMESPACE}
echo "===================================="
argo get ${ARGO_WORKFLOW} -n ${NAMESPACE}
exit 1
else
argo get ${ARGO_WORKFLOW} -n ${NAMESPACE}
exit 0
fi
3 changes: 1 addition & 2 deletions test/e2e_test_gke.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ spec:
valueFrom:
path: /outputs/strict-image-name/file
container:
image: gcr.io/ml-pipeline-staging/image-builder:v20181120-0.1.3-rc.1-21-gb7c8af0-dirty-9786df
image: gcr.io/ml-pipeline-test/image-builder:v20181128-0.1.3-rc.1-109-ga5a14dc-e3b0c4
imagePullPolicy: 'Always'
args: [
"--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",
Expand Down Expand Up @@ -267,4 +267,3 @@ spec:
"--namespace", "{{inputs.parameters.namespace}}",
"--test-name", "{{inputs.parameters.test-name}}",
]

Loading

0 comments on commit 9b77d4a

Please sign in to comment.