From 60843c4dd44431e55130e2c6208c0896acf580ea Mon Sep 17 00:00:00 2001 From: kunmingg <37601826+kunmingg@users.noreply.github.com> Date: Wed, 23 May 2018 17:41:30 -0700 Subject: [PATCH] Use yaml config to manage k8s resources deployed by bootstrapper (#853) * Use yaml config to manage k8s resources deployed by bootstrapper * handle review feedback --- bootstrap/Dockerfile | 1 + bootstrap/bootstrapper.yaml | 17 +- .../cmd/bootstrap/app/options/options.go | 10 +- bootstrap/cmd/bootstrap/app/server.go | 208 +++++++++--------- bootstrap/config/default.yaml | 11 + bootstrap/config/gcp_prototype.yaml | 39 ++++ testing/e2e_env.yaml | 12 + .../workflows/components/workflows.libsonnet | 1 + 8 files changed, 178 insertions(+), 121 deletions(-) create mode 100644 bootstrap/config/default.yaml create mode 100644 bootstrap/config/gcp_prototype.yaml create mode 100644 testing/e2e_env.yaml diff --git a/bootstrap/Dockerfile b/bootstrap/Dockerfile index e96f2bd96a2..1d7b9216878 100644 --- a/bootstrap/Dockerfile +++ b/bootstrap/Dockerfile @@ -51,6 +51,7 @@ RUN cd /opt/ksonnet && \ COPY --from=builder /opt/kubeflow/bootstrapper /opt/kubeflow/ COPY start.sh /opt/kubeflow/ +COPY config/default.yaml /opt/kubeflow/ RUN chmod a+rx /opt/kubeflow/bootstrapper RUN chmod a+rx /opt/kubeflow/start.sh diff --git a/bootstrap/bootstrapper.yaml b/bootstrap/bootstrapper.yaml index 8c86439ae6a..6c1c5ab256a 100644 --- a/bootstrap/bootstrapper.yaml +++ b/bootstrap/bootstrapper.yaml @@ -19,15 +19,6 @@ roleRef: name: cluster-admin apiGroup: rbac.authorization.k8s.io --- -# Required by StatefulSet -kind: Service -apiVersion: v1 -metadata: - name: kubeflow-bootstrapper - namespace: kubeflow-admin -spec: - clusterIP: "None" ---- # Store ksonnet apps apiVersion: v1 kind: PersistentVolumeClaim @@ -64,12 +55,8 @@ spec: image: gcr.io/kubeflow-images-public/bootstrapper:latest workingDir: /opt/bootstrap command: [ "/opt/kubeflow/bootstrapper"] - args: ["--in-cluster", "--namespace=kubeflow"] - env: - - name: NAMESPACE - value: "kubeflow" - - name: DEPLOY_JOB - value: "TRUE" + # app-dir: path you wish to store ks apps + args: ["--in-cluster", "--namespace=kubeflow", "--app-dir=/opt/bootstrap/default"] volumeMounts: - name: kubeflow-ksonnet-pvc mountPath: /opt/bootstrap diff --git a/bootstrap/cmd/bootstrap/app/options/options.go b/bootstrap/cmd/bootstrap/app/options/options.go index 43bff0ac1c3..7a321c3b9b6 100644 --- a/bootstrap/cmd/bootstrap/app/options/options.go +++ b/bootstrap/cmd/bootstrap/app/options/options.go @@ -26,11 +26,9 @@ type ServerOption struct { InCluster bool KeepAlive bool AppDir string - KfVersion string - NameSpace string - Project string + Config string Email string - IpName string + NameSpace string RegistryUri string } @@ -45,13 +43,11 @@ func (s *ServerOption) AddFlags(fs *flag.FlagSet) { fs.BoolVar(&s.PrintVersion, "version", false, "Show version and quit") fs.BoolVar(&s.JsonLogFormat, "json-log-format", true, "Set true to use json style log format. Set false to use plaintext style log format") fs.StringVar(&s.AppDir, "app-dir", "/opt/bootstrap/default", "The directory for the ksonnet application.") - fs.StringVar(&s.KfVersion, "kubeflow-version", "v0.1.0-rc.4", "The Kubeflow version to use.") fs.StringVar(&s.NameSpace, "namespace", "kubeflow", "The namespace where all resources for kubeflow will be created") fs.BoolVar(&s.Apply, "apply", false, "Whether or not to apply the configuration.") - fs.StringVar(&s.Project, "project", "", "The GCP project where kubeflow will be installed") fs.StringVar(&s.Email, "email", "", "Your Email address for GCP account, if you are using GKE.") - fs.StringVar(&s.IpName, "ip-name", "kubeflow", "Name of the ip you reserved on GCP project") fs.BoolVar(&s.InCluster, "in-cluster", false, "Whether bootstrapper is executed inside a pod") fs.StringVar(&s.RegistryUri, "registry-uri", "/opt/kubeflow/kubeflow", "Location of kubeflow registry.") fs.BoolVar(&s.KeepAlive, "keep-alive", true, "Whether bootstrapper will stay alive after setup resources.") + fs.StringVar(&s.Config, "config", "/opt/kubeflow/default.yaml", "Path to bootstrapper components config.") } diff --git a/bootstrap/cmd/bootstrap/app/server.go b/bootstrap/cmd/bootstrap/app/server.go index 990398df58f..e05d8cd4d0d 100644 --- a/bootstrap/cmd/bootstrap/app/server.go +++ b/bootstrap/cmd/bootstrap/app/server.go @@ -24,11 +24,11 @@ import ( "path/filepath" "regexp" "strconv" - "strings" "time" "os/exec" "bytes" + "github.com/ghodss/yaml" "github.com/ksonnet/ksonnet/actions" kApp "github.com/ksonnet/ksonnet/metadata/app" "github.com/kubeflow/kubeflow/bootstrap/cmd/bootstrap/app/options" @@ -45,19 +45,62 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "io/ioutil" ) // RecommendedConfigPathEnvVar is a environment variable for path configuration const RecommendedConfigPathEnvVar = "KUBECONFIG" -// DefaultStorageAnnotation is the name of the default annotation used to indicate +// DefaultStorageAnnotation is the Name of the default annotation used to indicate // whether a storage class is the default. const DefaultStorageAnnotation = "storageclass.beta.kubernetes.io/is-default-class" // Assume gcloud is on the path. const GcloudPath = "gcloud" -const Kubectl = "/usr/local/bin/kubectl" +const RegistryName = "kubeflow" + +type KsComponent struct{ + Name string + Prototype string +} + +type KsPackage struct{ + Name string +} + +type KsParameter struct{ + Component string + Name string + Value string +} + +// +type AppConfig struct { + Packages []KsPackage + Components []KsComponent + Parameters []KsParameter +} + +type BootConfig struct { + App AppConfig +} + +// Load yaml config +func LoadConfig(path string) (*BootConfig, error) { + if path == "" { + return nil, errors.New("empty path") + } + var c BootConfig + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + if err = yaml.Unmarshal(data, &c); err != nil { + return nil, err + } + return &c, nil +} // TODO(jlewi): If we use the same userid and groupid when running in a container then // we shoiuld be able to map in a user's home directory which could be useful e.g for @@ -221,6 +264,57 @@ func createComponent(opt *options.ServerOption, kfApp *kApp.App, fs *afero.Fs, a } } +func appGenerate(opt *options.ServerOption, kfApp *kApp.App, fs *afero.Fs, appConfig *AppConfig) { + libs, err := (*kfApp).Libraries() + + if err != nil { + log.Fatalf("Could not list libraries for app; error %v", err) + } + // Install packages. + for _, p := range appConfig.Packages { + pkgName := p.Name + _, err = (*fs).Stat(path.Join(opt.RegistryUri, pkgName)) + if err != nil { + log.Fatalf("Package %v didn't exist in registry %v", pkgName, opt.RegistryUri) + } + full := fmt.Sprintf("kubeflow/%v", pkgName) + log.Infof("Installing package %v", full) + + if _, found := libs[pkgName]; found { + log.Infof("Package %v already exists", pkgName) + continue + } + err := actions.RunPkgInstall(map[string]interface{}{ + actions.OptionApp: *kfApp, + actions.OptionLibName: full, + actions.OptionName: pkgName, + }) + + if err != nil { + log.Fatalf("There was a problem installing package %v; error %v", full, err) + } + } + + // Create Components + for _, c := range appConfig.Components { + createComponent(opt, kfApp, fs, []string{c.Prototype, c.Name}) + } + + // Apply Params + for _, p := range appConfig.Parameters { + err = actions.RunParamSet(map[string]interface{}{ + actions.OptionApp: *kfApp, + actions.OptionName: p.Component, + actions.OptionPath: p.Name, + actions.OptionValue: p.Value, + }) + + if err != nil { + log.Fatalf("Error when setting Parameters %v for Component %v: %v", p.Name, p.Component, err) + } + } +} + // Run the tool. func Run(opt *options.ServerOption) error { // Check if the -version flag was passed and, if so, print the version and exit. @@ -233,6 +327,11 @@ func Run(opt *options.ServerOption) error { return err } + bootConfig, err := LoadConfig(opt.Config) + if err != nil { + return err + } + kubeClient, err := clientset.NewForConfig(rest.AddUserAgent(config, "kubeflow-bootstrapper")) if err != nil { return err @@ -322,11 +421,9 @@ func Run(opt *options.ServerOption) error { log.Fatalf("There was a problem loading the app: %v", err) } - registryName := "kubeflow" - options := map[string]interface{}{ actions.OptionApp: kfApp, - actions.OptionName: registryName, + actions.OptionName: RegistryName, actions.OptionURI: opt.RegistryUri, // Version doesn't actually appear to be used by the add function. actions.OptionVersion: "", @@ -341,8 +438,8 @@ func Run(opt *options.ServerOption) error { log.Fatal("There was a problem listing registries; %v", err) } - if _, found := registries[registryName]; found { - log.Infof("App already has registry %v", registryName) + if _, found := registries[RegistryName]; found { + log.Infof("App already has registry %v", RegistryName) } else { err = actions.RunRegistryAdd(options) @@ -351,46 +448,10 @@ func Run(opt *options.ServerOption) error { } } - libs, err := kfApp.Libraries() - - if err != nil { - log.Fatalf("Could not list libraries for app; error %v", err) - } - - // Install packages. - for _, p := range []string{"kubeflow/core", "kubeflow/tf-serving", "kubeflow/tf-job", "kubeflow/pytorch-job"} { - pieces := strings.Split(p, "/") - _, err = fs.Stat(path.Join(opt.RegistryUri, pieces[1])) - if err != nil { - continue - } - full := fmt.Sprintf("%v@%v", p, opt.KfVersion) - log.Infof("Installing package %v", full) - - pkgName := pieces[1] + // Load default kubeflow apps + appGenerate(opt, &kfApp, &fs, &bootConfig.App) - if _, found := libs[pkgName]; found { - log.Infof("Package %v already exists", pkgName) - continue - } - err := actions.RunPkgInstall(map[string]interface{}{ - actions.OptionApp: kfApp, - actions.OptionLibName: full, - actions.OptionName: pkgName, - }) - - if err != nil { - log.Fatalf("There was a problem installing package %v; error %v", full, err) - } - } - - // Create the Kubeflow component kubeflowCoreName := "kubeflow-core" - createComponent(opt, &kfApp, &fs, []string{kubeflowCoreName, kubeflowCoreName}) - - // Create the pytorch-operator component - pytorchName := "pytorch-operator" - createComponent(opt, &kfApp, &fs, []string{pytorchName, pytorchName}) pvcMount := "" if hasDefault { @@ -408,57 +469,6 @@ func Run(opt *options.ServerOption) error { return err } - if isGke(clusterVersion) && opt.Project != "" { - log.Infof("Prepare Https access ...") - - if !isGke(clusterVersion) { - return errors.New("Currently https auto setup only available on GKE.") - } - endpointsArgs := []string{ - "cloud-endpoints", - "cloud-endpoints", - "--namespace", - opt.NameSpace, - "--secretName", - "cloudep-sa", - } - createComponent(opt, &kfApp, &fs, endpointsArgs) - - certManagerArgs := []string{ - "cert-manager", - "cert-manager", - "--namespace", - opt.NameSpace, - "--acmeEmail", - opt.Email, - } - createComponent(opt, &kfApp, &fs, certManagerArgs) - - FQDN := fmt.Sprintf("kubeflow.endpoints.%v.cloud.goog", opt.Project) - iapIngressArgs := []string{ - "iap-ingress", - "iap-ingress", - "--namespace", - opt.NameSpace, - "--ipName", - opt.IpName, - "--hostname", - FQDN, - } - - createComponent(opt, &kfApp, &fs, iapIngressArgs) - - err = actions.RunParamSet(map[string]interface{}{ - actions.OptionApp: kfApp, - actions.OptionName: kubeflowCoreName, - actions.OptionPath: "jupyterHubAuthenticator", - actions.OptionValue: "iap", - }) - if err != nil { - return err - } - } - if err := os.Chdir(kfApp.Root()); err != nil { return err } @@ -470,8 +480,8 @@ func Run(opt *options.ServerOption) error { // (05092018): why not use API: // ks runApply API expects clientcmd.ClientConfig, which kind of have soft dependency on existence of ~/.kube/config // if use k8s client-go API, would be quite verbose if we create all resources one by one. - // TODO: use API to create ks components - log.Infof("Apply kubeflow components...") + // TODO: use API to create ks Components + log.Infof("Apply kubeflow Components...") rawCmd := "ks show default | kubectl apply -f -" applyCmd := exec.Command("bash", "-c", rawCmd) diff --git a/bootstrap/config/default.yaml b/bootstrap/config/default.yaml new file mode 100644 index 00000000000..21df10ccef9 --- /dev/null +++ b/bootstrap/config/default.yaml @@ -0,0 +1,11 @@ +# Default config for kubeflow bootstrapper, included in bootstrapper image +--- +# App that always apply +app: + packages: + - name: core + - name: tf-serving + - name: tf-job + components: + - name: kubeflow-core + prototype: kubeflow-core diff --git a/bootstrap/config/gcp_prototype.yaml b/bootstrap/config/gcp_prototype.yaml new file mode 100644 index 00000000000..2c987008f7e --- /dev/null +++ b/bootstrap/config/gcp_prototype.yaml @@ -0,0 +1,39 @@ +# Sample config for kubeflow bootstrapper +--- +# App only apply if on GKE +app: + packages: + - name: core + - name: tf-serving + - name: tf-job + - name: pytorch-job + components: + - name: kubeflow-core + prototype: kubeflow-core + - name: pytorch-operator + prototype: pytorch-operator + - name: cloud-endpoints + prototype: cloud-endpoints + - name: cert-manager + prototype: cert-manager + - name: iap-ingress + prototype: iap-ingress + parameters: + - component: cloud-endpoints + name: secretname + value: cloudep-sa + - component: cert-manager + name: acmeemail + # todo: use your email for ssl cert + value: + - component: iap-ingress + name: ipname + # todo: change ip name here + value: + - component: iap-ingress + name: hostname + # todo: replace with name of gcp project where kubeflow will be installed + value: kubeflow.endpoints..cloud.goog + - component: kubeflow-core + name: jupyterhubauthenticator + value: iap diff --git a/testing/e2e_env.yaml b/testing/e2e_env.yaml new file mode 100644 index 00000000000..c854af50528 --- /dev/null +++ b/testing/e2e_env.yaml @@ -0,0 +1,12 @@ +# App for e2e test +app: + packages: + - name: core + - name: tf-serving + - name: tf-job + - name: pytorch-job + components: + - name: kubeflow-core + prototype: kubeflow-core + - name: pytorch-operator + prototype: pytorch-operator diff --git a/testing/workflows/components/workflows.libsonnet b/testing/workflows/components/workflows.libsonnet index 74cf98c1c91..2646fc91db4 100644 --- a/testing/workflows/components/workflows.libsonnet +++ b/testing/workflows/components/workflows.libsonnet @@ -467,6 +467,7 @@ "--namespace=" + stepsNamespace, "--registry-uri=" + srcDir + "/kubeflow", "--app-dir=" + testDir + "/app", + "--config=" + srcDir + "/testing/e2e_env.yaml", "--keep-alive=false", ]), // bootstrap-kubeflow ], // templates