Use yaml config to manage k8s resources deployed by bootstrapper (kub…

…eflow#853)

* Use yaml config to manage k8s resources deployed by bootstrapper

* handle review feedback

bootstrap/Dockerfile

@@ -51,6 +51,7 @@ RUN cd /opt/ksonnet && \
 
 COPY --from=builder /opt/kubeflow/bootstrapper /opt/kubeflow/
 COPY start.sh /opt/kubeflow/
+COPY config/default.yaml /opt/kubeflow/
 
 RUN chmod a+rx /opt/kubeflow/bootstrapper
 RUN chmod a+rx /opt/kubeflow/start.sh

bootstrap/bootstrapper.yaml

@@ -19,15 +19,6 @@ roleRef:
   name: cluster-admin
   apiGroup: rbac.authorization.k8s.io
 ---
-# Required by StatefulSet
-kind: Service
-apiVersion: v1
-metadata:
-  name: kubeflow-bootstrapper
-  namespace: kubeflow-admin
-spec:
-  clusterIP: "None"
----
 # Store ksonnet apps
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -64,12 +55,8 @@ spec:
         image: gcr.io/kubeflow-images-public/bootstrapper:latest
         workingDir: /opt/bootstrap
         command: [ "/opt/kubeflow/bootstrapper"]
-        args: ["--in-cluster", "--namespace=kubeflow"]
-        env:
-        - name: NAMESPACE
-          value: "kubeflow"
-        - name: DEPLOY_JOB
-          value: "TRUE"
+        # app-dir: path you wish to store ks apps
+        args: ["--in-cluster", "--namespace=kubeflow", "--app-dir=/opt/bootstrap/default"]
         volumeMounts:
         - name: kubeflow-ksonnet-pvc
           mountPath: /opt/bootstrap

bootstrap/cmd/bootstrap/app/options/options.go

@@ -26,11 +26,9 @@ type ServerOption struct {
 	InCluster     bool
 	KeepAlive     bool
 	AppDir        string
-	KfVersion     string
-	NameSpace     string
-	Project       string
+	Config        string
 	Email         string
-	IpName        string
+	NameSpace     string
 	RegistryUri   string
 }
 
@@ -45,13 +43,11 @@ func (s *ServerOption) AddFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&s.PrintVersion, "version", false, "Show version and quit")
 	fs.BoolVar(&s.JsonLogFormat, "json-log-format", true, "Set true to use json style log format. Set false to use plaintext style log format")
 	fs.StringVar(&s.AppDir, "app-dir", "/opt/bootstrap/default", "The directory for the ksonnet application.")
-	fs.StringVar(&s.KfVersion, "kubeflow-version", "v0.1.0-rc.4", "The Kubeflow version to use.")
 	fs.StringVar(&s.NameSpace, "namespace", "kubeflow", "The namespace where all resources for kubeflow will be created")
 	fs.BoolVar(&s.Apply, "apply", false, "Whether or not to apply the configuration.")
-	fs.StringVar(&s.Project, "project", "", "The GCP project where kubeflow will be installed")
 	fs.StringVar(&s.Email, "email", "", "Your Email address for GCP account, if you are using GKE.")
-	fs.StringVar(&s.IpName, "ip-name", "kubeflow", "Name of the ip you reserved on GCP project")
 	fs.BoolVar(&s.InCluster, "in-cluster", false, "Whether bootstrapper is executed inside a pod")
 	fs.StringVar(&s.RegistryUri, "registry-uri", "/opt/kubeflow/kubeflow", "Location of kubeflow registry.")
 	fs.BoolVar(&s.KeepAlive, "keep-alive", true, "Whether bootstrapper will stay alive after setup resources.")
+	fs.StringVar(&s.Config, "config", "/opt/kubeflow/default.yaml", "Path to bootstrapper components config.")
 }

bootstrap/cmd/bootstrap/app/server.go

@@ -24,11 +24,11 @@ import (
 	"path/filepath"
 	"regexp"
 	"strconv"
-	"strings"
 	"time"
 	"os/exec"
 	"bytes"
 
+	"github.com/ghodss/yaml"
 	"github.com/ksonnet/ksonnet/actions"
 	kApp "github.com/ksonnet/ksonnet/metadata/app"
 	"github.com/kubeflow/kubeflow/bootstrap/cmd/bootstrap/app/options"
@@ -45,19 +45,62 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"io/ioutil"
 )
 
 // RecommendedConfigPathEnvVar is a environment variable for path configuration
 const RecommendedConfigPathEnvVar = "KUBECONFIG"
 
-// DefaultStorageAnnotation is the name of the default annotation used to indicate
+// DefaultStorageAnnotation is the Name of the default annotation used to indicate
 // whether a storage class is the default.
 const DefaultStorageAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
 
 // Assume gcloud is on the path.
 const GcloudPath = "gcloud"
 
-const Kubectl = "/usr/local/bin/kubectl"
+const RegistryName = "kubeflow"
+
+type KsComponent struct{
+	Name      string
+	Prototype string
+}
+
+type KsPackage struct{
+	Name string
+}
+
+type KsParameter struct{
+	Component string
+	Name      string
+	Value     string
+}
+
+//
+type AppConfig struct {
+	Packages   []KsPackage
+	Components []KsComponent
+	Parameters []KsParameter
+}
+
+type BootConfig struct {
+	App  AppConfig
+}
+
+// Load yaml config
+func LoadConfig(path string) (*BootConfig, error) {
+	if path == "" {
+		return nil, errors.New("empty path")
+	}
+	var c BootConfig
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	if err = yaml.Unmarshal(data, &c); err != nil {
+		return nil, err
+	}
+	return &c, nil
+}
 
 // TODO(jlewi): If we use the same userid and groupid when running in a container then
 // we shoiuld be able to map in a user's home directory which could be useful e.g for
@@ -221,6 +264,57 @@ func createComponent(opt *options.ServerOption, kfApp *kApp.App, fs *afero.Fs, a
 	}
 }
 
+func appGenerate(opt *options.ServerOption, kfApp *kApp.App, fs *afero.Fs, appConfig *AppConfig) {
+	libs, err := (*kfApp).Libraries()
+
+	if err != nil {
+		log.Fatalf("Could not list libraries for app; error %v", err)
+	}
+	// Install packages.
+	for _, p := range appConfig.Packages {
+		pkgName := p.Name
+		_, err = (*fs).Stat(path.Join(opt.RegistryUri, pkgName))
+		if err != nil {
+			log.Fatalf("Package %v didn't exist in registry %v", pkgName, opt.RegistryUri)
+		}
+		full := fmt.Sprintf("kubeflow/%v", pkgName)
+		log.Infof("Installing package %v", full)
+
+		if _, found := libs[pkgName]; found {
+			log.Infof("Package %v already exists", pkgName)
+			continue
+		}
+		err := actions.RunPkgInstall(map[string]interface{}{
+			actions.OptionApp:     *kfApp,
+			actions.OptionLibName: full,
+			actions.OptionName:    pkgName,
+		})
+
+		if err != nil {
+			log.Fatalf("There was a problem installing package %v; error %v", full, err)
+		}
+	}
+
+	// Create Components
+	for _, c := range appConfig.Components {
+		createComponent(opt, kfApp, fs, []string{c.Prototype, c.Name})
+	}
+
+	// Apply Params
+	for _, p := range appConfig.Parameters {
+		err = actions.RunParamSet(map[string]interface{}{
+			actions.OptionApp:   *kfApp,
+			actions.OptionName:  p.Component,
+			actions.OptionPath:  p.Name,
+			actions.OptionValue: p.Value,
+		})
+
+		if err != nil {
+			log.Fatalf("Error when setting Parameters %v for Component %v: %v", p.Name, p.Component, err)
+		}
+	}
+}
+
 // Run the tool.
 func Run(opt *options.ServerOption) error {
 	// Check if the -version flag was passed and, if so, print the version and exit.
@@ -233,6 +327,11 @@ func Run(opt *options.ServerOption) error {
 		return err
 	}
 
+	bootConfig, err := LoadConfig(opt.Config)
+	if err != nil {
+		return err
+	}
+
 	kubeClient, err := clientset.NewForConfig(rest.AddUserAgent(config, "kubeflow-bootstrapper"))
 	if err != nil {
 		return err
@@ -322,11 +421,9 @@ func Run(opt *options.ServerOption) error {
 		log.Fatalf("There was a problem loading the app: %v", err)
 	}
 
-	registryName := "kubeflow"
-
 	options := map[string]interface{}{
 		actions.OptionApp:  kfApp,
-		actions.OptionName: registryName,
+		actions.OptionName: RegistryName,
 		actions.OptionURI:  opt.RegistryUri,
 		// Version doesn't actually appear to be used by the add function.
 		actions.OptionVersion: "",
@@ -341,8 +438,8 @@ func Run(opt *options.ServerOption) error {
 		log.Fatal("There was a problem listing registries; %v", err)
 	}
 
-	if _, found := registries[registryName]; found {
-		log.Infof("App already has registry %v", registryName)
+	if _, found := registries[RegistryName]; found {
+		log.Infof("App already has registry %v", RegistryName)
 	} else {
 
 		err = actions.RunRegistryAdd(options)
@@ -351,46 +448,10 @@ func Run(opt *options.ServerOption) error {
 		}
 	}
 
-	libs, err := kfApp.Libraries()
-
-	if err != nil {
-		log.Fatalf("Could not list libraries for app; error %v", err)
-	}
-
-	// Install packages.
-	for _, p := range []string{"kubeflow/core", "kubeflow/tf-serving", "kubeflow/tf-job", "kubeflow/pytorch-job"} {
-		pieces := strings.Split(p, "/")
-		_, err = fs.Stat(path.Join(opt.RegistryUri, pieces[1]))
-		if err != nil {
-			continue
-		}
-		full := fmt.Sprintf("%v@%v", p, opt.KfVersion)
-		log.Infof("Installing package %v", full)
-
-		pkgName := pieces[1]
+	// Load default kubeflow apps
+	appGenerate(opt, &kfApp, &fs, &bootConfig.App)
 
-		if _, found := libs[pkgName]; found {
-			log.Infof("Package %v already exists", pkgName)
-			continue
-		}
-		err := actions.RunPkgInstall(map[string]interface{}{
-			actions.OptionApp:     kfApp,
-			actions.OptionLibName: full,
-			actions.OptionName:    pkgName,
-		})
-
-		if err != nil {
-			log.Fatalf("There was a problem installing package %v; error %v", full, err)
-		}
-	}
-
-	// Create the Kubeflow component
 	kubeflowCoreName := "kubeflow-core"
-	createComponent(opt, &kfApp, &fs, []string{kubeflowCoreName, kubeflowCoreName})
-
-	// Create the pytorch-operator component
-	pytorchName := "pytorch-operator"
-	createComponent(opt, &kfApp, &fs, []string{pytorchName, pytorchName})
 
 	pvcMount := ""
 	if hasDefault {
@@ -408,57 +469,6 @@ func Run(opt *options.ServerOption) error {
 		return err
 	}
 
-	if isGke(clusterVersion) && opt.Project != "" {
-		log.Infof("Prepare Https access ...")
-
-		if !isGke(clusterVersion) {
-			return errors.New("Currently https auto setup only available on GKE.")
-		}
-		endpointsArgs := []string{
-			"cloud-endpoints",
-			"cloud-endpoints",
-			"--namespace",
-			opt.NameSpace,
-			"--secretName",
-			"cloudep-sa",
-		}
-		createComponent(opt, &kfApp, &fs, endpointsArgs)
-
-		certManagerArgs := []string{
-			"cert-manager",
-			"cert-manager",
-			"--namespace",
-			opt.NameSpace,
-			"--acmeEmail",
-			opt.Email,
-		}
-		createComponent(opt, &kfApp, &fs, certManagerArgs)
-
-		FQDN := fmt.Sprintf("kubeflow.endpoints.%v.cloud.goog", opt.Project)
-		iapIngressArgs := []string{
-			"iap-ingress",
-			"iap-ingress",
-			"--namespace",
-			opt.NameSpace,
-			"--ipName",
-			opt.IpName,
-			"--hostname",
-			FQDN,
-		}
-
-		createComponent(opt, &kfApp, &fs, iapIngressArgs)
-
-		err = actions.RunParamSet(map[string]interface{}{
-			actions.OptionApp:   kfApp,
-			actions.OptionName:  kubeflowCoreName,
-			actions.OptionPath:  "jupyterHubAuthenticator",
-			actions.OptionValue: "iap",
-		})
-		if err != nil {
-			return err
-		}
-	}
-
 	if err := os.Chdir(kfApp.Root()); err != nil {
 		return err
 	}
@@ -470,8 +480,8 @@ func Run(opt *options.ServerOption) error {
 		// (05092018): why not use API:
 		// ks runApply API expects clientcmd.ClientConfig, which kind of have soft dependency on existence of ~/.kube/config
 		// if use k8s client-go API, would be quite verbose if we create all resources one by one.
-		// TODO: use API to create ks components
-		log.Infof("Apply kubeflow components...")
+		// TODO: use API to create ks Components
+		log.Infof("Apply kubeflow Components...")
 		rawCmd := "ks show default | kubectl apply -f -"
 		applyCmd := exec.Command("bash", "-c", rawCmd)
 

bootstrap/config/default.yaml

@@ -0,0 +1,11 @@
+# Default config for kubeflow bootstrapper, included in bootstrapper image
+---
+# App that always apply
+app:
+  packages:
+    - name: core
+    - name: tf-serving
+    - name: tf-job
+  components:
+    - name: kubeflow-core
+      prototype: kubeflow-core