diff --git a/components/dataproc/xgboost/common/_utils.py b/components/dataproc/xgboost/common/_utils.py index 0c4a5c136db..110afb2e1cb 100644 --- a/components/dataproc/xgboost/common/_utils.py +++ b/components/dataproc/xgboost/common/_utils.py @@ -134,6 +134,7 @@ def copy_resources_to_gcs(file_paths, gcs_path): dest_files = [] for file_name in file_paths: dest_file = os.path.join(gcs_path, tmpdir, os.path.basename(file_name)) + subprocess.call(['gcloud', 'auth', 'activate-service-account', '--key-file', os.environ['GOOGLE_APPLICATION_CREDENTIALS']]) subprocess.call(['gsutil', 'cp', file_name, dest_file]) dest_files.append(dest_file) diff --git a/components/kubeflow/deployer/src/deploy.sh b/components/kubeflow/deployer/src/deploy.sh index f13c8e26e40..8a310c74052 100755 --- a/components/kubeflow/deployer/src/deploy.sh +++ b/components/kubeflow/deployer/src/deploy.sh @@ -26,16 +26,6 @@ while (($#)); do MODEL_PATH="$1" shift ;; - "--project") - shift - PROJECT="$1" - shift - ;; - "--zone") - shift - ZONE="$1" - shift - ;; "--cluster-name") shift CLUSTER_NAME="$1" @@ -65,27 +55,22 @@ fi echo "Deploying the model '${MODEL_PATH}'" -if [ -z "${PROJECT}" ]; then - PROJECT=$(wget -q -O- --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id) -fi - if [ -z "${CLUSTER_NAME}" ]; then CLUSTER_NAME=$(wget -q -O- --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name) fi - -if [ -z "${ZONE}" ]; then - ZONE=$(wget -q -O- --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/zone | cut -d '/' -f 4) -fi # Ensure the server name is not more than 63 characters. SERVER_NAME="${SERVER_NAME:0:63}" # Trim any trailing hyphens from the server name. while [[ "${SERVER_NAME:(-1)}" == "-" ]]; do SERVER_NAME="${SERVER_NAME::-1}"; done -echo "Deploying ${SERVER_NAME} to the cluster ${CLUSTER_NAME} in the project ${PROJECT} and the zone ${ZONE}..." +echo "Deploying ${SERVER_NAME} to the cluster ${CLUSTER_NAME}" -# Connect kubectl to the cluster -gcloud --project "${PROJECT}" container clusters get-credentials "${CLUSTER_NAME}" --zone "${ZONE}" +# Connect kubectl to the local cluster +kubectl config set-cluster "${CLUSTER_NAME}" --server=https://kubernetes.default --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt +kubectl config set-credentials pipeline --token "$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" +kubectl config set-context kubeflow --cluster "${CLUSTER_NAME}" --user pipeline +kubectl config use-context kubeflow # Configure and deploy the TF serving app cd /src/github.com/kubeflow/kubeflow diff --git a/pipeline/pipeline/pipeline-apiserver.libsonnet b/pipeline/pipeline/pipeline-apiserver.libsonnet index d562b990ed5..bb0224a2630 100644 --- a/pipeline/pipeline/pipeline-apiserver.libsonnet +++ b/pipeline/pipeline/pipeline-apiserver.libsonnet @@ -255,6 +255,7 @@ resources: [ "pods", "pods/exec", + "pods/log", "services", ], verbs: [ diff --git a/samples/kubeflow-tf/kubeflow-training-classification.py b/samples/kubeflow-tf/kubeflow-training-classification.py index 43911e52a95..c01a1ddcb73 100755 --- a/samples/kubeflow-tf/kubeflow-training-classification.py +++ b/samples/kubeflow-tf/kubeflow-training-classification.py @@ -15,6 +15,7 @@ import kfp.dsl as dsl +import kfp.gcp as gcp import datetime def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'): @@ -31,7 +32,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc '--output', transform_output, ], file_outputs = {'transformed': '/output.txt'} - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training'): @@ -49,7 +50,7 @@ def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', l '--job-dir', training_output, ], file_outputs = {'train': '/output.txt'} - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'): return dsl.ContainerOp( @@ -65,7 +66,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json] '--output', prediction_output, ], file_outputs = {'prediction': '/output.txt'} - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def confusion_matrix_op(predictions, output, step_name='confusionmatrix'): return dsl.ContainerOp( @@ -75,7 +76,7 @@ def confusion_matrix_op(predictions, output, step_name='confusionmatrix'): '--predictions', predictions, '--output', output, ] - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) @dsl.pipeline( name='Pipeline TFJob', diff --git a/samples/tfx/taxi-cab-classification-pipeline.py b/samples/tfx/taxi-cab-classification-pipeline.py index 3c219b688d1..abfaf67f39b 100755 --- a/samples/tfx/taxi-cab-classification-pipeline.py +++ b/samples/tfx/taxi-cab-classification-pipeline.py @@ -15,6 +15,7 @@ import kfp.dsl as dsl +import kfp.gcp as gcp import datetime def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'GcsUri', column_names: 'GcsUri[text/json]', key_columns, project: 'GcpProject', mode, validation_output: 'GcsUri[Directory]', step_name='validation'): @@ -34,7 +35,7 @@ def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'G 'output': '/output.txt', 'schema': '/output_schema.json', } - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'): return dsl.ContainerOp( @@ -50,7 +51,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc '--output', transform_output, ], file_outputs = {'transformed': '/output.txt'} - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target: str, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training'): @@ -68,7 +69,7 @@ def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate '--job-dir', training_output, ], file_outputs = {'train': '/output.txt'} - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', analyze_mode, analyze_slice_column, analysis_output: 'GcsUri', step_name='analysis'): return dsl.ContainerOp( @@ -84,7 +85,7 @@ def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'Gc '--output', analysis_output, ], file_outputs = {'analysis': '/output.txt'} - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'): @@ -101,7 +102,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json] '--output', prediction_output, ], file_outputs = {'prediction': '/output.txt'} - ) + ).apply(gcp.use_gcp_secret('user-gcp-sa')) def kubeflow_deploy_op(model: 'TensorFlow model', tf_server_name, step_name='deploy'): return dsl.ContainerOp( diff --git a/samples/xgboost-spark/xgboost-training-cm.py b/samples/xgboost-spark/xgboost-training-cm.py index 8a1ed3ba448..6d22b28d5a4 100755 --- a/samples/xgboost-spark/xgboost-training-cm.py +++ b/samples/xgboost-spark/xgboost-training-cm.py @@ -15,6 +15,7 @@ import kfp.dsl as dsl +import kfp.gcp as gcp # ================================================================ @@ -179,26 +180,26 @@ def xgb_train_pipeline( ): delete_cluster_op = DeleteClusterOp('delete-cluster', project, region) with dsl.ExitHandler(exit_op=delete_cluster_op): - create_cluster_op = CreateClusterOp('create-cluster', project, region, output) + create_cluster_op = CreateClusterOp('create-cluster', project, region, output).apply(gcp.use_gcp_secret('user-gcp-sa')) analyze_op = AnalyzeOp('analyze', project, region, create_cluster_op.output, schema, - train_data, '%s/{{workflow.name}}/analysis' % output) + train_data, '%s/{{workflow.name}}/analysis' % output).apply(gcp.use_gcp_secret('user-gcp-sa')) transform_op = TransformOp('transform', project, region, create_cluster_op.output, train_data, eval_data, target, analyze_op.output, - '%s/{{workflow.name}}/transform' % output) + '%s/{{workflow.name}}/transform' % output).apply(gcp.use_gcp_secret('user-gcp-sa')) train_op = TrainerOp('train', project, region, create_cluster_op.output, transform_op.outputs['train'], transform_op.outputs['eval'], target, analyze_op.output, workers, - rounds, '%s/{{workflow.name}}/model' % output) + rounds, '%s/{{workflow.name}}/model' % output).apply(gcp.use_gcp_secret('user-gcp-sa')) predict_op = PredictOp('predict', project, region, create_cluster_op.output, transform_op.outputs['eval'], - train_op.output, target, analyze_op.output, '%s/{{workflow.name}}/predict' % output) + train_op.output, target, analyze_op.output, '%s/{{workflow.name}}/predict' % output).apply(gcp.use_gcp_secret('user-gcp-sa')) confusion_matrix_op = ConfusionMatrixOp('confusion-matrix', predict_op.output, - '%s/{{workflow.name}}/confusionmatrix' % output) + '%s/{{workflow.name}}/confusionmatrix' % output).apply(gcp.use_gcp_secret('user-gcp-sa')) - roc_op = RocOp('roc', predict_op.output, true_label, '%s/{{workflow.name}}/roc' % output) + roc_op = RocOp('roc', predict_op.output, true_label, '%s/{{workflow.name}}/roc' % output).apply(gcp.use_gcp_secret('user-gcp-sa')) if __name__ == '__main__': import kfp.compiler as compiler diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh index 18de0240afa..c8d1b0fc1cb 100755 --- a/test/presubmit-tests-with-pipeline-deployment.sh +++ b/test/presubmit-tests-with-pipeline-deployment.sh @@ -115,7 +115,7 @@ function clean_up { cd ${KFAPP} ${KUBEFLOW_SRC}/scripts/kfctl.sh delete all } -trap clean_up EXIT +# trap clean_up EXIT ${KUBEFLOW_SRC}/scripts/kfctl.sh init ${KFAPP} --platform gcp --project ${PROJECT} --skipInitProject