Merge 2a21822 into 76bcd1a

kubeflow · Dec 5, 2018 · 76da964 · 76da964
2 parents 76bcd1a + 2a21822
commit 76da964
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 38 deletions.
diff --git a/components/dataproc/xgboost/common/_utils.py b/components/dataproc/xgboost/common/_utils.py
@@ -134,6 +134,7 @@ def copy_resources_to_gcs(file_paths, gcs_path):
   dest_files = []
   for file_name in file_paths:
     dest_file = os.path.join(gcs_path, tmpdir, os.path.basename(file_name))
+    subprocess.call(['gcloud', 'auth', 'activate-service-account', '--key-file', os.environ['GOOGLE_APPLICATION_CREDENTIALS']])
     subprocess.call(['gsutil', 'cp', file_name, dest_file])
     dest_files.append(dest_file)
 

diff --git a/components/kubeflow/deployer/src/deploy.sh b/components/kubeflow/deployer/src/deploy.sh
@@ -26,16 +26,6 @@ while (($#)); do
        MODEL_PATH="$1"
        shift
        ;;
-     "--project")
-       shift
-       PROJECT="$1"
-       shift
-       ;;
-     "--zone")
-       shift
-       ZONE="$1"
-       shift
-       ;;
      "--cluster-name")
        shift
        CLUSTER_NAME="$1"
@@ -65,27 +55,22 @@ fi
 
 echo "Deploying the model '${MODEL_PATH}'"
 
-if [ -z "${PROJECT}" ]; then
-  PROJECT=$(wget -q -O- --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id)
-fi
-
 if [ -z "${CLUSTER_NAME}" ]; then
   CLUSTER_NAME=$(wget -q -O- --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name)
 fi
-
-if [ -z "${ZONE}" ]; then
-  ZONE=$(wget -q -O- --header="Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/zone | cut -d '/' -f 4)
-fi
 
 # Ensure the server name is not more than 63 characters.
 SERVER_NAME="${SERVER_NAME:0:63}"
 # Trim any trailing hyphens from the server name.
 while [[ "${SERVER_NAME:(-1)}" == "-" ]]; do SERVER_NAME="${SERVER_NAME::-1}"; done
 
-echo "Deploying ${SERVER_NAME} to the cluster ${CLUSTER_NAME} in the project ${PROJECT} and the zone ${ZONE}..."
+echo "Deploying ${SERVER_NAME} to the cluster ${CLUSTER_NAME}"
 
-# Connect kubectl to the cluster
-gcloud --project "${PROJECT}" container clusters get-credentials "${CLUSTER_NAME}" --zone "${ZONE}"
+# Connect kubectl to the local cluster
+kubectl config set-cluster "${CLUSTER_NAME}" --server=https://kubernetes.default --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+kubectl config set-credentials pipeline --token "$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+kubectl config set-context kubeflow --cluster "${CLUSTER_NAME}" --user pipeline
+kubectl config use-context kubeflow
 
 # Configure and deploy the TF serving app
 cd /src/github.com/kubeflow/kubeflow

diff --git a/pipeline/pipeline/pipeline-apiserver.libsonnet b/pipeline/pipeline/pipeline-apiserver.libsonnet
@@ -255,6 +255,7 @@
           resources: [
             "pods",
             "pods/exec",
+            "pods/log",
             "services",
           ],
           verbs: [

diff --git a/samples/kubeflow-tf/kubeflow-training-classification.py b/samples/kubeflow-tf/kubeflow-training-classification.py
@@ -15,6 +15,7 @@
 
 
 import kfp.dsl as dsl
+import kfp.gcp as gcp
 import datetime
 
 def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'):
@@ -31,7 +32,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc
             '--output', transform_output,
         ],
         file_outputs = {'transformed': '/output.txt'}
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 
 def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training'):
@@ -49,7 +50,7 @@ def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', l
             '--job-dir', training_output,
         ],
         file_outputs = {'train': '/output.txt'}
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'):
     return dsl.ContainerOp(
@@ -65,7 +66,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]
             '--output', prediction_output,
         ],
         file_outputs = {'prediction': '/output.txt'}
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 def confusion_matrix_op(predictions, output, step_name='confusionmatrix'):
     return dsl.ContainerOp(
@@ -75,7 +76,7 @@ def confusion_matrix_op(predictions, output, step_name='confusionmatrix'):
           '--predictions', predictions,
           '--output', output,
         ]
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 @dsl.pipeline(
   name='Pipeline TFJob',

diff --git a/samples/tfx/taxi-cab-classification-pipeline.py b/samples/tfx/taxi-cab-classification-pipeline.py
@@ -15,6 +15,7 @@
 
 
 import kfp.dsl as dsl
+import kfp.gcp as gcp
 import datetime
 
 def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'GcsUri', column_names: 'GcsUri[text/json]', key_columns, project: 'GcpProject', mode, validation_output: 'GcsUri[Directory]', step_name='validation'):
@@ -34,7 +35,7 @@ def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'G
             'output': '/output.txt',
             'schema': '/output_schema.json',
         }
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'):
     return dsl.ContainerOp(
@@ -50,7 +51,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc
             '--output', transform_output,
         ],
         file_outputs = {'transformed': '/output.txt'}
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 
 def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target: str, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training'):
@@ -68,7 +69,7 @@ def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate
             '--job-dir', training_output,
         ],
         file_outputs = {'train': '/output.txt'}
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', analyze_mode, analyze_slice_column, analysis_output: 'GcsUri', step_name='analysis'):
     return dsl.ContainerOp(
@@ -84,7 +85,7 @@ def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'Gc
             '--output', analysis_output,
         ],
         file_outputs = {'analysis': '/output.txt'}
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 
 def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'):
@@ -101,7 +102,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]
             '--output', prediction_output,
         ],
         file_outputs = {'prediction': '/output.txt'}
-    )
+    ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 def kubeflow_deploy_op(model: 'TensorFlow model', tf_server_name, step_name='deploy'):
     return dsl.ContainerOp(

diff --git a/samples/xgboost-spark/xgboost-training-cm.py b/samples/xgboost-spark/xgboost-training-cm.py
@@ -15,6 +15,7 @@
 
 
 import kfp.dsl as dsl
+import kfp.gcp as gcp
 
 
 # ================================================================
@@ -179,26 +180,26 @@ def xgb_train_pipeline(
 ):
   delete_cluster_op = DeleteClusterOp('delete-cluster', project, region)
   with dsl.ExitHandler(exit_op=delete_cluster_op):
-    create_cluster_op = CreateClusterOp('create-cluster', project, region, output)
+    create_cluster_op = CreateClusterOp('create-cluster', project, region, output).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
     analyze_op = AnalyzeOp('analyze', project, region, create_cluster_op.output, schema,
-                           train_data, '%s/{{workflow.name}}/analysis' % output)
+                           train_data, '%s/{{workflow.name}}/analysis' % output).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
     transform_op = TransformOp('transform', project, region, create_cluster_op.output,
                                train_data, eval_data, target, analyze_op.output,
-                               '%s/{{workflow.name}}/transform' % output)
+                               '%s/{{workflow.name}}/transform' % output).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
     train_op = TrainerOp('train', project, region, create_cluster_op.output, transform_op.outputs['train'],
                          transform_op.outputs['eval'], target, analyze_op.output, workers,
-                         rounds, '%s/{{workflow.name}}/model' % output)
+                         rounds, '%s/{{workflow.name}}/model' % output).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
     predict_op = PredictOp('predict', project, region, create_cluster_op.output, transform_op.outputs['eval'],
-                           train_op.output, target, analyze_op.output, '%s/{{workflow.name}}/predict' % output)
+                           train_op.output, target, analyze_op.output, '%s/{{workflow.name}}/predict' % output).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
     confusion_matrix_op = ConfusionMatrixOp('confusion-matrix', predict_op.output,
-                                            '%s/{{workflow.name}}/confusionmatrix' % output)
+                                            '%s/{{workflow.name}}/confusionmatrix' % output).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
-    roc_op = RocOp('roc', predict_op.output, true_label, '%s/{{workflow.name}}/roc' % output)
+    roc_op = RocOp('roc', predict_op.output, true_label, '%s/{{workflow.name}}/roc' % output).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
 if __name__ == '__main__':
   import kfp.compiler as compiler

diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh
@@ -115,7 +115,7 @@ function clean_up {
   cd ${KFAPP}
   ${KUBEFLOW_SRC}/scripts/kfctl.sh delete all
 }
-trap clean_up EXIT
+# trap clean_up EXIT
 
 ${KUBEFLOW_SRC}/scripts/kfctl.sh init ${KFAPP} --platform gcp --project ${PROJECT} --skipInitProject