part of 'add the gcp platform to kfctl (golang)' (#2440)

* revert to prior version

* fixes kfctl (golang) rename 'ks' directory and app to ksonnet and ksonnet.go respectively

* fixes 'kfctl - Fetch registry automatically'

* fixes 'refactor gcp, minikube, docker-for-desktop, ack to be kfctl plugins'

* change DefaultDevRepo to point to just the repo not repo/kubeflow

* fixes 'refactor gcp, minikube, docker-for-desktop, ack to be kfctl plugins'

* plugins for all existing KfApp instances {ksonnet, minikube, foo}.

* update golang version

* fixes 'docker-for-desktop'

* delete meta-controller-cluster-role-binding

* update README.md

* remove unused var DEBUG

* remove bootstrap/cmd/plugins/foo.go

* fixes kfctl (golang) rename 'ks' directory and app to ksonnet and ksonnet.go respectively

* fixes 'add the gcp platform to kfctl (golang)'

* fix merge conflict

* cli option --project is an init flag

* fix to write out project to app.yaml

* fix for nil project check

* include fix for #2367

* added updateDM, createSecrets to Apply

* fixes to macros

* merge changes from upstream master

* remove old files, update README.md

* fix for init app-name where app-name is not a path

* fix for gcp apply

* code to create a Gcp Secret, doesn't do correct auth right now

* snapshot

* update on downloadK8sManifests, gcpInitProject, added TODO's

* additional work on gcpInitProject

* gcpInitProject now works but requires GOOGLE_APPLICATION_CREDENTIALS env var

* initial pass on updateDeployment

* update README.md to described static vs dynamic platforms

* updates to Gcp.updateDM, added GOOGLE_APPLICATION_CREDENTIALS to the Makefile tests

* minor changes on init, generate usage

* change default platform to ksonnet

* default --version to master until 0.5.0

* address comments from Jeremy

* address comments from Jeremy

bootstrap/Makefile

@@ -15,6 +15,7 @@
 GCLOUD_PROJECT ?= kubeflow-images-public
 GOLANG_VERSION ?= 1.11.5
 GOPATH ?= $(HOME)/go
+GOOGLE_APPLICATION_CREDENTIALS ?= $(HOME)/auth.json
 # To build without the cache set the environment variable
 # export DOCKER_BUILD_OPTS=--no-cache
 IMG ?= gcr.io/$(GCLOUD_PROJECT)/bootstrapper
@@ -87,23 +88,26 @@ run-local-docker:
 
 test-known-platforms-init: install build-dockerfordesktop-plugin
 	@rm -rf $(HOME)/dockerfordesktop && \
-	PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl init $(HOME)/dockerfordesktop -V --platform docker-for-desktop --version master && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl init $(HOME)/dockerfordesktop -V --platform docker-for-desktop && \
 	rm -rf $(HOME)/ksonnet && \
-	kfctl init $(HOME)/ksonnet -V --platform ksonnet --version master && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl init $(HOME)/ksonnet -V --platform ksonnet && \
 	rm -rf $(HOME)/minikube && \
-	kfctl init $(HOME)/minikube -V --platform minikube --version master && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl init $(HOME)/minikube -V --platform minikube && \
+	rm -rf $(HOME)/gcp && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl init $(HOME)/gcp -V --platform gcp --project $(GCLOUD_PROJECT) && \
 	echo SUCCESS
 
 test-known-platforms-generate: test-known-platforms-init
 	@cd ~/dockerfordesktop && \
-	PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl generate all -V && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl generate all -V && \
 	cd ~/ksonnet && \
-	kfctl generate all -V && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl generate all -V && \
 	cd ~/minikube && \
-	kfctl generate all -V --mount-local && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl generate all -V --mount-local && \
+	cd ~/gcp && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl generate all -V --email jsmith@acme.com && \
 	echo SUCCESS
 
-
 # static and plugins toogle whether we try and load a platform as a .so.
 # There is an example plugin 'dockerfordesktop' that can be loaded as a .so.
 # By default we disable loading plugins and link dockerfordesktop into the kfctl binary (static)

bootstrap/cmd/kfctl/README.md

@@ -20,7 +20,7 @@ Note: Additional issues have been opened so this README.md will have additional
 
 ## API and Packaging
 
-New directories (`cmd/kfctl, pkg`):
+New directories
 
 ```sh
 bootstrap/cmd/kfctl
@@ -42,15 +42,17 @@ The `KfApp` golang Interface
 
 ```golang
 type ResourceEnum string
+
 const (
 	ALL      ResourceEnum = "all"
 	K8S      ResourceEnum = "k8s"
 	PLATFORM ResourceEnum = "platform"
+	NONE     ResourceEnum = "none"
 )
 
 //
 // KfApp is used by commands under bootstrap/cmd/{bootstrap,kfctl}. KfApp provides a common
-// API for different implementations like Ksonnet, GcpApp, MinikubeApp, etc.
+// API for different implementations like ksonnet, gcp, minikube, docker-for-desktop, etc.
 //
 type KfApp interface {
 	Apply(resources ResourceEnum, options map[string]interface{}) error
@@ -60,14 +62,20 @@ type KfApp interface {
 }
 ```
 
-kfctl includes platforms that implement the KfApp interface. (gcp will be added in the next phase)
+kfctl will statically include platforms that implement the KfApp interface. 
+These include:
 
 - platform: **ksonnet**
   - bootstrap/pkg/client/ksonnet/ksonnet.go
 - platform: **minikube**
   - bootstrap/pkg/client/minikube/minikube.go
-- platform: **docker-for-desktop**
-  - bootstrap/pkg/client/dockerfordesktop/dockerfordesktop.go
+- platform: **gcp** 
+  - bootstrap/pkg/client/gcp/gcp.go
+- platform: **ack** (in progress)
+  - bootstrap/pkg/client/ack/ack.go
+
+kfctl can also dynamically load platforms that are not statically linked, as 
+described below in [Extending kfctl](#extending-kfctl).
 
 ## Usage
 
@@ -102,7 +110,9 @@ kfctl apply
 
 ## Subcommands
 
-### **init** (kubeflow/bootstrap/cmd/kfctl/cmd/init.go)
+### **init** 
+
+(kubeflow/bootstrap/cmd/kfctl/cmd/init.go)
 
 ```
 Create a kubeflow application under <[path/]name>. The <[path/]name> argument can either be a full path
@@ -116,14 +126,16 @@ Flags:
       --debug              debug debug default is false
   -h, --help               help for init
   -n, --namespace string   namespace where kubeflow will be deployed (default "kubeflow")
-  -p, --platform string    one of 'gcp|minikube|docker-for-desktop|ack' (default "none")
+  -p, --platform string    one of 'gcp|minikube|ksonnet' (default=ksonnet)
       --project string     name of the gcp project if --platform gcp
   -r, --repo string        local github kubeflow repo
   -V, --verbose            verbose output default is false
   -v, --version string     desired version Kubeflow or latest tag if not provided by user  (default "v0.4.1")
 ```
 
-### **generate** (kubeflow/bootstrap/cmd/kfctl/cmd/generate.go)
+### **generate**
+
+(kubeflow/bootstrap/cmd/kfctl/cmd/generate.go)
 
 ```
 Generate a kubeflow application where resources is one of 'platform | k8s | all'.
@@ -145,7 +157,9 @@ Flags:
   -V, --verbose         verbose output default is false
 ```
 
-### **apply** (kubeflow/bootstrap/cmd/kfctl/cmd/apply.go)
+### **apply** 
+
+(kubeflow/bootstrap/cmd/kfctl/cmd/apply.go)
 
 ```
 Deploy a generated kubeflow application.
@@ -158,7 +172,9 @@ Flags:
   -V, --verbose   verbose output default is false
 ```
 
-### **delete** (kubeflow/bootstrap/cmd/kfctl/cmd/delete.go)
+### **delete** 
+
+(kubeflow/bootstrap/cmd/kfctl/cmd/delete.go)
 
 ```
 Delete a kubeflow application.
@@ -273,19 +289,20 @@ type KsonnetSpec struct {
 }
 ```
 
-#### app.yaml example for --platform none
+#### app.yaml example for --platform ksonnet
 
 ```
 apiVersion: ksonnet.apps.kubeflow.org/v1alpha1
 kind: Ksonnet
 metadata:
   creationTimestamp: null
-  name: ks-app
+  name: ksonnet
   namespace: kubeflow
 spec:
-  platform: none
-  repo: /Users/kdkasrav/go/src/github.com/kubeflow/kubeflow/kubeflow
-  version: v0.4.1
+  appdir: /Users/kdkasrav/ksonnet
+  platform: ksonnet
+  repo: /Users/kdkasrav/ksonnet/.cache/master/kubeflow
+  version: master
 status: {}
 ```
 

bootstrap/cmd/kfctl/cmd/generate.go

@@ -26,8 +26,8 @@ var generateCfg = viper.New()
 // generateCmd represents the generate command
 var generateCmd = &cobra.Command{
 	Use:   "generate [all(=default)|k8s|platform]",
-	Short: "Generate a kubeflow application where resources is one of 'platform | k8s | all'.",
-	Long: `Generate a kubeflow application where resources is one of 'platform | k8s | all'.
+	Short: "Generate a kubeflow application where resources is one of 'platform|k8s|all'.",
+	Long: `Generate a kubeflow application where resources is one of 'platform|k8s|all'.
 
   platform: non kubernetes resources (eg --platform gcp)
   k8s: kubernetes resources
@@ -49,10 +49,14 @@ The default is 'all' for any selected platform.`,
 		}
 		email := generateCfg.GetString(string(kftypes.EMAIL))
 		ipName := generateCfg.GetString(string(kftypes.IPNAME))
-		mountLocal := generateCfg.GetBool("mount-local")
+		hostName := generateCfg.GetString(string(kftypes.HOSTNAME))
+		zone := generateCfg.GetString(string(kftypes.ZONE))
+		mountLocal := generateCfg.GetBool(string(kftypes.MOUNT_LOCAL))
 		options := map[string]interface{}{
 			string(kftypes.EMAIL):       email,
 			string(kftypes.IPNAME):      ipName,
+			string(kftypes.HOSTNAME):    hostName,
+			string(kftypes.ZONE):        zone,
 			string(kftypes.MOUNT_LOCAL): mountLocal,
 		}
 		kfApp, kfAppErr := loadKfApp(options)
@@ -83,6 +87,15 @@ func init() {
 		return
 	}
 
+	// platform gcp
+	generateCmd.Flags().String(string(kftypes.ZONE), "us-east1-d",
+		string(kftypes.ZONE)+" if '--platform gcp'")
+	bindErr = generateCfg.BindPFlag(string(kftypes.ZONE), generateCmd.Flags().Lookup(string(kftypes.ZONE)))
+	if bindErr != nil {
+		log.Errorf("couldn't set flag --%v: %v", string(kftypes.ZONE), bindErr)
+		return
+	}
+
 	// platform gcp
 	generateCmd.Flags().String(string(kftypes.IPNAME), "",
 		string(kftypes.IPNAME)+" if '--platform gcp'")
@@ -92,6 +105,15 @@ func init() {
 		return
 	}
 
+	// platform gcp
+	generateCmd.Flags().String(string(kftypes.HOSTNAME), "",
+		string(kftypes.HOSTNAME)+" if '--platform gcp'")
+	bindErr = generateCfg.BindPFlag(string(kftypes.HOSTNAME), generateCmd.Flags().Lookup(string(kftypes.HOSTNAME)))
+	if bindErr != nil {
+		log.Errorf("couldn't set flag --%v: %v", string(kftypes.HOSTNAME), bindErr)
+		return
+	}
+
 	// platforms minikube, docker-for-desktop
 	generateCmd.Flags().Bool(string(kftypes.MOUNT_LOCAL), false,
 		string(kftypes.MOUNT_LOCAL)+" if '--platform minikube || --platform docker-for-desktop'")

bootstrap/cmd/kfctl/cmd/init.go

@@ -29,8 +29,7 @@ var initCmd = &cobra.Command{
 	Use:   "init <[path/]name>",
 	Short: "Create a kubeflow application under <[path/]name>",
 	Long: `Create a kubeflow application under <[path/]name>. The <[path/]name> argument can either be a full path
-or a name where the kubeflow application will be initialized in $PWD/name if <name> is not a path or in the parent
-directory is name is a path.`,
+or a name where the kubeflow application will be initialized in the current directory.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		log.SetLevel(log.InfoLevel)
 		log.Info("initializing kubeflow application")
@@ -79,7 +78,7 @@ func init() {
 	initCfg.SetConfigType("yaml")
 
 	initCmd.Flags().StringP(string(kftypes.PLATFORM), "p", kftypes.DefaultPlatform,
-		"one of 'gcp|minikube|docker-for-desktop|ack'")
+		"one of 'gcp|minikube|ksonnet'")
 	bindErr := initCfg.BindPFlag(string(kftypes.PLATFORM), initCmd.Flags().Lookup(string(kftypes.PLATFORM)))
 	if bindErr != nil {
 		log.Errorf("couldn't set flag --%v: %v", string(kftypes.PLATFORM), bindErr)

bootstrap/cmd/kfctl/cmd/root.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ksonnet/ksonnet/pkg/app"
 	kftypes "github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps"
 	kstypes "github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps/ksonnet/v1alpha1"
+	"github.com/kubeflow/kubeflow/bootstrap/pkg/client/gcp"
 	// STATIC
 	"github.com/kubeflow/kubeflow/bootstrap/pkg/client/dockerfordesktop"
 	// -STATIC //
@@ -44,6 +45,8 @@ func GetPlatform(options map[string]interface{}) (kftypes.KfApp, error) {
 		return ksonnet.GetKfApp(options), nil
 	case "minikube":
 		return minikube.GetKfApp(options), nil
+	case "gcp":
+		return gcp.GetKfApp(options), nil
 	// STATIC
 	case "docker-for-desktop":
 		return dockerfordesktop.GetKfApp(options), nil
@@ -77,12 +80,12 @@ func newKfApp(options map[string]interface{}) (kftypes.KfApp, error) {
 	//appName can be a path
 	appName := options[string(kftypes.APPNAME)].(string)
 	appDir := path.Dir(appName)
-	if appDir == "" {
+	if appDir == "" || appDir == "." {
 		cwd, err := os.Getwd()
 		if err != nil {
 			return nil, fmt.Errorf("could not get current directory %v", err)
 		}
-		appDir = cwd
+		appDir = path.Join(cwd, appName)
 	} else {
 		if appDir == "~" {
 			home, homeErr := homedir.Dir()
@@ -131,23 +134,26 @@ func loadKfApp(options map[string]interface{}) (kftypes.KfApp, error) {
 	if kAppErr != nil {
 		return nil, fmt.Errorf("there was a problem loading app %v. Error: %v", appName, kAppErr)
 	}
-	ksApp := &kstypes.Ksonnet{}
-	dat, datErr := ioutil.ReadFile(cfgfile)
-	if datErr != nil {
-		return nil, fmt.Errorf("couldn't read %v. Error: %v", cfgfile, datErr)
+	buf, bufErr := ioutil.ReadFile(cfgfile)
+	if bufErr != nil {
+		return nil, fmt.Errorf("couldn't read %v. Error: %v", cfgfile, bufErr)
 	}
-	specErr := yaml.Unmarshal(dat, ksApp)
-	if specErr != nil {
-		return nil, fmt.Errorf("couldn't unmarshall Ksonnet. Error: %v", specErr)
-	}
-	options[string(kftypes.PLATFORM)] = ksApp.Spec.Platform
+	var v interface{}
+	yaml.Unmarshal(buf, &v)
+	data := v.(map[string]interface{})
+	metadata := data["metadata"].(map[string]interface{})
+	spec := data["spec"].(map[string]interface{})
+	platform := spec["platform"].(string)
+	appName = metadata["name"].(string)
+	appDir = spec["appdir"].(string)
+	options[string(kftypes.PLATFORM)] = platform
 	options[string(kftypes.APPNAME)] = appName
 	options[string(kftypes.APPDIR)] = appDir
 	options[string(kftypes.KAPP)] = kApp
-	options[string(kftypes.KSAPP)] = ksApp
+	options[string(kftypes.DATA)] = buf
 	pApp, pAppErr := GetPlatform(options)
 	if pAppErr != nil {
-		return nil, fmt.Errorf("unable to load platform %v Error: %v", ksApp.Spec.Platform, pAppErr)
+		return nil, fmt.Errorf("unable to load platform %v Error: %v", platform, pAppErr)
 	}
 	return pApp, nil
 }

bootstrap/cmd/kfctl/config/app.yaml

@@ -1,61 +1,11 @@
-apiversion: apps.kubeflow.org/v1alpha1
+apiVersion: ksonnet.apps.kubeflow.org/v1alpha1
 kind: Ksonnet
 metadata:
-  name: kf-app
+  creationTimestamp: null
+  name: ksonnet
   namespace: kubeflow
-namespace: kubeflow
 spec:
-  app:
-    packages:
-    - name: argo
-      registry: kubeflow
-    - name: pipeline
-      registry: kubeflow
-    - name: common
-      registry: kubeflow
-    - name: examples
-      registry: kubeflow
-    - name: jupyter
-      registry: kubeflow
-    - name: katib
-      registry: kubeflow
-    - name: mpi-job
-      registry: kubeflow
-    - name: pytorch-job
-      registry: kubeflow
-    - name: seldon
-      registry: kubeflow
-    - name: tf-serving
-      registry: kubeflow
-    - name: openvino
-      registry: kubeflow
-    - name: tensorboard
-      registry: kubeflow
-    - name: tf-training
-      registry: kubeflow
-    - name: metacontroller
-      registry: kubeflow
-    - name: profiles
-      registry: kubeflow
-    - name: application
-      registry: kubeflow
-    - name: modeldb
-      registry: kubeflow
-    parameters:
-    - component: spartakus
-      name: usageId
-      value: 43606238
-    - component: spartakus
-      name: reportUsage
-      value: 1
-    registries:
-    - RegUri: /Users/fred/go/src/github.com/kubeflow/kubeflow/kubeflow
-      name: kubeflow
-      path: kubeflow
-      repo: https://github.com/kubeflow/kubeflow.git
-      version: v0.4.1
-  components:
-  - all
-  platform: none
-  repo: /Users/fred/go/src/github.com/kubeflow/kubeflow/kubeflow
-  version: v0.4.1
+  platform: ksonnet
+  repo: /Users/kdkasrav/ksonnet/.cache/master/kubeflow
+  version: master
+status: {}

bootstrap/cmd/plugins/gcp/gcp.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	kftypes "github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps"
+	"github.com/kubeflow/kubeflow/bootstrap/pkg/client/gcp"
+)
+
+func GetKfApp(options map[string]interface{}) kftypes.KfApp {
+	return gcp.GetKfApp(options)
+}