From bdb17630451a2e3528d91b90383fb178dea5685e Mon Sep 17 00:00:00 2001 From: Johnu George Date: Sun, 10 Jun 2018 13:39:05 +0530 Subject: [PATCH 1/2] Adding pytorch to nightly builds --- releasing/prow_config_release.yaml | 3 ++ releasing/pytorch-operator/.gitignore | 4 +++ releasing/pytorch-operator/app.yaml | 31 +++++++++++++++++++ .../components/params.libsonnet | 26 ++++++++++++++++ .../components/pytorch-operator.jsonnet | 6 ++++ .../environments/base.libsonnet | 4 +++ .../environments/default/globals.libsonnet | 2 ++ .../environments/default/main.jsonnet | 8 +++++ .../environments/default/params.libsonnet | 17 ++++++++++ 9 files changed, 101 insertions(+) create mode 100644 releasing/pytorch-operator/.gitignore create mode 100644 releasing/pytorch-operator/app.yaml create mode 100644 releasing/pytorch-operator/components/params.libsonnet create mode 100644 releasing/pytorch-operator/components/pytorch-operator.jsonnet create mode 100644 releasing/pytorch-operator/environments/base.libsonnet create mode 100644 releasing/pytorch-operator/environments/default/globals.libsonnet create mode 100644 releasing/pytorch-operator/environments/default/main.jsonnet create mode 100644 releasing/pytorch-operator/environments/default/params.libsonnet diff --git a/releasing/prow_config_release.yaml b/releasing/prow_config_release.yaml index a380f2e6a1f..f4392c99f39 100644 --- a/releasing/prow_config_release.yaml +++ b/releasing/prow_config_release.yaml @@ -26,3 +26,6 @@ workflows: registry: "gcr.io/kubeflow-images-public" testing_image: "gcr.io/kubeflow-releasing/worker:latest" zone: "us-central1-a" + - app_dir: kubeflow/kubeflow/releasing/pytorch-operator + component: release + name: pytorch-operator-release diff --git a/releasing/pytorch-operator/.gitignore b/releasing/pytorch-operator/.gitignore new file mode 100644 index 00000000000..f8714d3a3b4 --- /dev/null +++ b/releasing/pytorch-operator/.gitignore @@ -0,0 +1,4 @@ +/lib +/.ksonnet/registries +/app.override.yaml +/.ks_environment diff --git a/releasing/pytorch-operator/app.yaml b/releasing/pytorch-operator/app.yaml new file mode 100644 index 00000000000..1057590f59f --- /dev/null +++ b/releasing/pytorch-operator/app.yaml @@ -0,0 +1,31 @@ +apiVersion: 0.1.0 +environments: + default: + destination: + namespace: default + server: https://35.200.179.178 + k8sVersion: v1.10.2 + path: default +kind: ksonnet.io/app +libraries: + automation: + gitVersion: + commitSha: 076305508ad2bedaa364b1b4cdbc3945d8ba9842 + refSpec: master + name: automation + registry: kubeflow +name: pytorch-operator +registries: + incubator: + gitVersion: + commitSha: 40285d8a14f1ac5787e405e1023cf0c07f6aa28c + refSpec: master + protocol: github + uri: github.com/ksonnet/parts/tree/master/incubator + kubeflow: + gitVersion: + commitSha: 076305508ad2bedaa364b1b4cdbc3945d8ba9842 + refSpec: master + protocol: github + uri: github.com/kubeflow/kubeflow/tree/master/kubeflow +version: 0.0.1 diff --git a/releasing/pytorch-operator/components/params.libsonnet b/releasing/pytorch-operator/components/params.libsonnet new file mode 100644 index 00000000000..08a916495b3 --- /dev/null +++ b/releasing/pytorch-operator/components/params.libsonnet @@ -0,0 +1,26 @@ +{ + global: { + // User-defined global parameters; accessible to all component and environments, Ex: + // replicas: 4, + }, + components: { + // Component-level parameters, defined initially from 'ks prototype use ...' + // Each object below should correspond to a component in the components/ directory + "pytorch-operator": { + bucket: "kubeflow-releasing-artifacts", + cluster: "kubeflow-releasing", + extra_args: "Extra", + gcpCredentialsSecretName: "gcp-credentials", + image: "pytorch-operator", + name: "pytorch-operator", + namespace: "kubeflow-releasing", + nfsVolumeClaim: "nfs-external", + project: "kubeflow-releasing", + prow_env: "REPO_OWNER=kubeflow,REPO_NAME=kubeflow,PULL_BASE_SHA=master", + registry: "gcr.io/kubeflow-images-public", + testing_image: "gcr.io/kubeflow-releasing/worker:latest", + versionTag: "latest", + zone: "us-central1-a", + }, + }, +} diff --git a/releasing/pytorch-operator/components/pytorch-operator.jsonnet b/releasing/pytorch-operator/components/pytorch-operator.jsonnet new file mode 100644 index 00000000000..049ebf6e3ba --- /dev/null +++ b/releasing/pytorch-operator/components/pytorch-operator.jsonnet @@ -0,0 +1,6 @@ +local env = std.extVar("__ksonnet/environments"); +local params = std.extVar("__ksonnet/params").components["pytorch-operator"]; +local k = import "k.libsonnet"; +local release = import "kubeflow/automation/release.libsonnet"; + +std.prune(k.core.v1.list.new(release.parts(params.namespace, params.name, overrides=params).release)) diff --git a/releasing/pytorch-operator/environments/base.libsonnet b/releasing/pytorch-operator/environments/base.libsonnet new file mode 100644 index 00000000000..a129affb1f3 --- /dev/null +++ b/releasing/pytorch-operator/environments/base.libsonnet @@ -0,0 +1,4 @@ +local components = std.extVar("__ksonnet/components"); +components + { + // Insert user-specified overrides here. +} diff --git a/releasing/pytorch-operator/environments/default/globals.libsonnet b/releasing/pytorch-operator/environments/default/globals.libsonnet new file mode 100644 index 00000000000..7a73a41bfdf --- /dev/null +++ b/releasing/pytorch-operator/environments/default/globals.libsonnet @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/releasing/pytorch-operator/environments/default/main.jsonnet b/releasing/pytorch-operator/environments/default/main.jsonnet new file mode 100644 index 00000000000..58695a80cbf --- /dev/null +++ b/releasing/pytorch-operator/environments/default/main.jsonnet @@ -0,0 +1,8 @@ +local base = import "base.libsonnet"; +// uncomment if you reference ksonnet-lib +// local k = import "k.libsonnet"; + +base + { + // Insert user-specified overrides here. For example if a component is named \"nginx-deployment\", you might have something like:\n") + // "nginx-deployment"+: k.deployment.mixin.metadata.labels({foo: "bar"}) +} diff --git a/releasing/pytorch-operator/environments/default/params.libsonnet b/releasing/pytorch-operator/environments/default/params.libsonnet new file mode 100644 index 00000000000..b6eb32db55a --- /dev/null +++ b/releasing/pytorch-operator/environments/default/params.libsonnet @@ -0,0 +1,17 @@ +local params = std.extVar("__ksonnet/params"); +local globals = import "globals.libsonnet"; +local envParams = params + { + components +: { + // Insert component parameter overrides here. Ex: + // guestbook +: { + // name: "guestbook-dev", + // replicas: params.global.replicas, + // }, + }, +}; + +{ + components: { + [x]: envParams.components[x] + globals, for x in std.objectFields(envParams.components) + }, +} From bcf5b8f86c5e645a9973e5cfbbcfbebd97b3307e Mon Sep 17 00:00:00 2001 From: Johnu George Date: Thu, 14 Jun 2018 09:48:04 +0530 Subject: [PATCH 2/2] Changes added for new release app --- releasing/pytorch-operator/app.yaml | 4 ++-- releasing/pytorch-operator/components/params.libsonnet | 5 ++++- .../pytorch-operator/components/pytorch-operator.jsonnet | 6 +++++- releasing/pytorch-operator/environments/base.libsonnet | 2 +- .../environments/default/globals.libsonnet | 2 +- .../pytorch-operator/environments/default/main.jsonnet | 2 +- .../pytorch-operator/environments/default/params.libsonnet | 7 ++++--- 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/releasing/pytorch-operator/app.yaml b/releasing/pytorch-operator/app.yaml index 1057590f59f..16d666fb320 100644 --- a/releasing/pytorch-operator/app.yaml +++ b/releasing/pytorch-operator/app.yaml @@ -10,7 +10,7 @@ kind: ksonnet.io/app libraries: automation: gitVersion: - commitSha: 076305508ad2bedaa364b1b4cdbc3945d8ba9842 + commitSha: f6377404e032d036a8d78a91a25501dc9933499a refSpec: master name: automation registry: kubeflow @@ -24,7 +24,7 @@ registries: uri: github.com/ksonnet/parts/tree/master/incubator kubeflow: gitVersion: - commitSha: 076305508ad2bedaa364b1b4cdbc3945d8ba9842 + commitSha: f6377404e032d036a8d78a91a25501dc9933499a refSpec: master protocol: github uri: github.com/kubeflow/kubeflow/tree/master/kubeflow diff --git a/releasing/pytorch-operator/components/params.libsonnet b/releasing/pytorch-operator/components/params.libsonnet index 08a916495b3..37ea78540fd 100644 --- a/releasing/pytorch-operator/components/params.libsonnet +++ b/releasing/pytorch-operator/components/params.libsonnet @@ -9,7 +9,10 @@ "pytorch-operator": { bucket: "kubeflow-releasing-artifacts", cluster: "kubeflow-releasing", - extra_args: "Extra", + dockerfile: "Dockerfile", + dockerfileDir: "kubeflow/pytorch-operator", + extra_args: "/mnt", + extra_repos: "kubeflow/testing@HEAD;kubeflow/pytorch-operator@HEAD", gcpCredentialsSecretName: "gcp-credentials", image: "pytorch-operator", name: "pytorch-operator", diff --git a/releasing/pytorch-operator/components/pytorch-operator.jsonnet b/releasing/pytorch-operator/components/pytorch-operator.jsonnet index 049ebf6e3ba..381471f5191 100644 --- a/releasing/pytorch-operator/components/pytorch-operator.jsonnet +++ b/releasing/pytorch-operator/components/pytorch-operator.jsonnet @@ -1,6 +1,10 @@ local env = std.extVar("__ksonnet/environments"); local params = std.extVar("__ksonnet/params").components["pytorch-operator"]; + local k = import "k.libsonnet"; local release = import "kubeflow/automation/release.libsonnet"; +local updatedParams = params { + extra_args: if params.extra_args == "null" then "" else " " + params.extra_args, +}; -std.prune(k.core.v1.list.new(release.parts(params.namespace, params.name, overrides=params).release)) +std.prune(k.core.v1.list.new(release.parts(updatedParams.namespace, updatedParams.name, overrides=updatedParams).release)) diff --git a/releasing/pytorch-operator/environments/base.libsonnet b/releasing/pytorch-operator/environments/base.libsonnet index a129affb1f3..dee3168de3b 100644 --- a/releasing/pytorch-operator/environments/base.libsonnet +++ b/releasing/pytorch-operator/environments/base.libsonnet @@ -1,4 +1,4 @@ local components = std.extVar("__ksonnet/components"); -components + { +components { // Insert user-specified overrides here. } diff --git a/releasing/pytorch-operator/environments/default/globals.libsonnet b/releasing/pytorch-operator/environments/default/globals.libsonnet index 7a73a41bfdf..2c63c085104 100644 --- a/releasing/pytorch-operator/environments/default/globals.libsonnet +++ b/releasing/pytorch-operator/environments/default/globals.libsonnet @@ -1,2 +1,2 @@ { -} \ No newline at end of file +} diff --git a/releasing/pytorch-operator/environments/default/main.jsonnet b/releasing/pytorch-operator/environments/default/main.jsonnet index 58695a80cbf..1a44c481d5a 100644 --- a/releasing/pytorch-operator/environments/default/main.jsonnet +++ b/releasing/pytorch-operator/environments/default/main.jsonnet @@ -2,7 +2,7 @@ local base = import "base.libsonnet"; // uncomment if you reference ksonnet-lib // local k = import "k.libsonnet"; -base + { +base { // Insert user-specified overrides here. For example if a component is named \"nginx-deployment\", you might have something like:\n") // "nginx-deployment"+: k.deployment.mixin.metadata.labels({foo: "bar"}) } diff --git a/releasing/pytorch-operator/environments/default/params.libsonnet b/releasing/pytorch-operator/environments/default/params.libsonnet index b6eb32db55a..876f969726d 100644 --- a/releasing/pytorch-operator/environments/default/params.libsonnet +++ b/releasing/pytorch-operator/environments/default/params.libsonnet @@ -1,7 +1,7 @@ local params = std.extVar("__ksonnet/params"); local globals = import "globals.libsonnet"; -local envParams = params + { - components +: { +local envParams = params { + components+: { // Insert component parameter overrides here. Ex: // guestbook +: { // name: "guestbook-dev", @@ -12,6 +12,7 @@ local envParams = params + { { components: { - [x]: envParams.components[x] + globals, for x in std.objectFields(envParams.components) + [x]: envParams.components[x] + globals + for x in std.objectFields(envParams.components) }, }