Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use kubeflow-ci as the test cluster #411

Merged
merged 12 commits into from
Mar 15, 2018
Merged

Use kubeflow-ci as the test cluster #411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
8dee1d0
use kubeflow-ci as the test cluster
lluunn Mar 12, 2018
4515a73
change cluster to kubeflow-ci
lluunn Mar 12, 2018
ee7576b
remove param
lluunn Mar 12, 2018
d3b1875
also change bucket
lluunn Mar 12, 2018
21a132e
use github token in checkout
lluunn Mar 14, 2018
d5c1ef9
fix
lluunn Mar 14, 2018
21292e3
fix
lluunn Mar 14, 2018
bc5d501
fix
lluunn Mar 14, 2018
2ad7a09
fix
lluunn Mar 14, 2018
e7c846b
fix
lluunn Mar 14, 2018
f3dc48f
fix
lluunn Mar 14, 2018
f783bad
fix
lluunn Mar 14, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions components/k8s-model-server/images/releaser/components/params.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,10 @@
// Component-level parameters, defined initially from 'ks prototype use ...'
// Each object below should correspond to a component in the components/ directory
workflows: {
bucket: "mlkube-testing_temp",
commit: "master",
name: "new9",
namespace: "kubeflow-test-infra",
prow_env: "REPO_OWNER=kubeflow,REPO_NAME=kubeflow,PULL_BASE_SHA=master",
serving_image: "gcr.io/kubeflow-ci/model-server",
testing_image: "gcr.io/mlkube-testing/test-worker:latest",
tf_testing_image: "gcr.io/kubeflow-ci/tf-test-worker:1.0",
project: "mlkube-testing",
cluster: "kubeflow-testing",
zone: "us-east1-d",
},
// Test deploying a GPU model.
gpu_model: {
Expand Down
35 changes: 11 additions & 24 deletions components/k8s-model-server/images/releaser/components/workflows.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
//
// TODO(jlewi): Use camelCase consistently.
defaultParams:: {
bucket: "mlkube-testing_temp",
bucket: "kubeflow-ci_temp",
commit: "master",
// Name of the secret containing GCP credentials.
gcpCredentialsSecretName: "kubeflow-testing-credentials",
Expand All @@ -38,7 +38,7 @@
// The default image to use for the steps in the Argo workflow.
testing_image: "gcr.io/mlkube-testing/kubeflow-testing",
tf_testing_image: "gcr.io/kubeflow-ci/tf-test-worker:1.0",
project: "mlkube-testing",
project: "kubeflow-ci",
cluster: "kubeflow-testing",
zone: "us-east1-d",
build_image: false,
Expand Down Expand Up @@ -283,28 +283,15 @@
}],
],
},
{
name: "checkout",
container: {
command: [
"/usr/local/bin/checkout.sh",
],
args: [
srcRootDir,
],
env: prow_env + [{
name: "EXTRA_REPOS",
value: "kubeflow/testing@HEAD",
}],
image: testing_image,
volumeMounts: [
{
name: dataVolume,
mountPath: mountPath,
},
],
},
}, // checkout
buildTemplate(
"checkout",
["/usr/local/bin/checkout.sh", srcRootDir],
[{
name: "EXTRA_REPOS",
value: "kubeflow/testing@HEAD",
}],
[], // no sidecars
),

buildImageTemplate("build-tf-serving-cpu", "Dockerfile.cpu", cpuImage),

Expand Down
2 changes: 1 addition & 1 deletion testing/workflows/components/params.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
// Component-level parameters, defined initially from 'ks prototype use ...'
// Each object below should correspond to a component in the components/ directory
workflows: {
bucket: "mlkube-testing_temp",
bucket: "kubeflow-ci_temp",
name: "jlewi-kubeflow-kubeflow-presubmit-test-227-643b",
namespace: "kubeflow-test-infra",
prow: "JOB_NAME=kubeflow-presubmit-test,JOB_TYPE=presubmit,PULL_NUMBER=209,REPO_NAME=kubeflow,REPO_OWNER=kubeflow,BUILD_NUMBER=997a",
Expand Down
131 changes: 59 additions & 72 deletions testing/workflows/components/workflows.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,56 +48,56 @@
local tfOperatorRoot = srcRootDir + "/kubeflow/tf-operator";
local tfOperatorPy = tfOperatorRoot;

local project = "mlkube-testing";
local project = "kubeflow-ci";
// GKE cluster to use
local cluster = "kubeflow-testing";
local zone = "us-east1-d";
{
// Build an Argo template to execute a particular command.
// step_name: Name for the template
// command: List to pass as the container command.
buildTemplate(step_name, command):: {
name: step_name,
container: {
command: command,
image: image,
env: [
{
// Add the source directories to the python path.
name: "PYTHONPATH",
value: kubeflowPy + ":" + kubeflowTestingPy + ":" + tfOperatorPy,
},
{
name: "GOOGLE_APPLICATION_CREDENTIALS",
value: "/secret/gcp-credentials/key.json",
},
{
name: "GITHUB_TOKEN",
valueFrom: {
secretKeyRef: {
name: "github-token",
key: "github_token",
},
// Build an Argo template to execute a particular command.
// step_name: Name for the template
// command: List to pass as the container command.
local buildTemplate(step_name, command, env_vars=[], sidecars=[]) = {
name: step_name,
container: {
command: command,
image: image,
env: [
{
// Add the source directories to the python path.
name: "PYTHONPATH",
value: kubeflowPy + ":" + kubeflowTestingPy + ":" + tfOperatorPy,
},
{
name: "GOOGLE_APPLICATION_CREDENTIALS",
value: "/secret/gcp-credentials/key.json",
},
{
name: "GITHUB_TOKEN",
valueFrom: {
secretKeyRef: {
name: "github-token",
key: "github_token",
},
},
] + prow_env,
volumeMounts: [
{
name: dataVolume,
mountPath: mountPath,
},
{
name: "github-token",
mountPath: "/secret/github-token",
},
{
name: "gcp-credentials",
mountPath: "/secret/gcp-credentials",
},
],
},
}, // buildTemplate

},
] + prow_env + env_vars,
volumeMounts: [
{
name: dataVolume,
mountPath: mountPath,
},
{
name: "github-token",
mountPath: "/secret/github-token",
},
{
name: "gcp-credentials",
mountPath: "/secret/gcp-credentials",
},
],
},
sidecars: sidecars,
}; // buildTemplate
{
apiVersion: "argoproj.io/v1alpha1",
kind: "Workflow",
metadata: {
Expand Down Expand Up @@ -170,29 +170,16 @@
}],
],
},
{
name: "checkout",
container: {
command: [
"/usr/local/bin/checkout.sh",
],
args: [
srcRootDir,
],
env: prow_env + [{
name: "EXTRA_REPOS",
value: "kubeflow/tf-operator@HEAD;kubeflow/testing@HEAD",
}],
image: image,
volumeMounts: [
{
name: dataVolume,
mountPath: mountPath,
},
],
},
}, // checkout
$.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("setup", [
buildTemplate(
"checkout",
["/usr/local/bin/checkout.sh", srcRootDir],
[{
name: "EXTRA_REPOS",
value: "kubeflow/tf-operator@HEAD;kubeflow/testing@HEAD",
}],
[], // no sidecars
),
buildTemplate("setup", [
"python",
"-m",
"testing.test_deploy",
Expand All @@ -204,7 +191,7 @@
"--artifacts_dir=" + artifactsDir,
"setup",
]), // setup
$.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("teardown", [
buildTemplate("teardown", [
"python",
"-m",
"testing.test_deploy",
Expand All @@ -216,23 +203,23 @@
"--artifacts_dir=" + artifactsDir,
"teardown",
]), // teardown
$.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("create-pr-symlink", [
buildTemplate("create-pr-symlink", [
"python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + outputDir,
"create_pr_symlink",
"--bucket=" + bucket,
]), // create-pr-symlink
$.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("copy-artifacts", [
buildTemplate("copy-artifacts", [
"python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + outputDir,
"copy_artifacts",
"--bucket=" + bucket,
]), // copy-artifacts
$.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("tfjob-test", [
buildTemplate("tfjob-test", [
"python",
"-m",
"py.test_runner",
Expand Down