kubeflow · kkasravi · Jun 29, 2018 · Aug 3, 2018 · Aug 24, 2018 · Aug 27, 2018
diff --git a/bootstrap/Makefile b/bootstrap/Makefile
@@ -13,46 +13,77 @@
 # limitations under the License.
 #
 GCLOUD_PROJECT ?= kubeflow-images-public
-GOLANG_VERSION ?= 1.11.5
+GOLANG_VERSION ?= 1.12
 GOPATH ?= $(HOME)/go
 GOOGLE_APPLICATION_CREDENTIALS ?= $(HOME)/auth.json
 # To build without the cache set the environment variable
 # export DOCKER_BUILD_OPTS=--no-cache
 IMG ?= gcr.io/$(GCLOUD_PROJECT)/bootstrapper
 TAG ?= $(shell git describe --tags --always --dirty)
-PORT ?= 2345
+# set to -V
+VERBOSE ?= 
 PLUGINS_ENVIRONMENT ?= $(GOPATH)/src/github.com/kubeflow/kubeflow/bootstrap/bin
 export GO111MODULE = on
 export GO = go
 
+%.so:
+	cd cmd/plugins/$* && \
+	$(GO) build -i -gcflags 'all=-N -l' -o ../../../bin/$*.so -buildmode=plugin $*.go
+
+%.init:
+	@echo kfctl init test/$* $(VERBOSE) --platform $* --project $(GCLOUD_PROJECT) && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl init $(PWD)/test/$* $(VERBOSE) --platform $* --project $(GCLOUD_PROJECT)
+
+%.generate:
+	@echo kfctl generate all $(VERBOSE) '(--platform '$*')' && \
+	cd test/$* && \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl generate all $(VERBOSE) --mount-local --email gcp-deploy@$(GCLOUD_PROJECT).iam.gserviceaccount.com
+
 all: build
 
 auth:
 	gcloud auth configure-docker
 
 # Run go fmt against code
 fmt:
-	$(GO) fmt ./pkg/... ./cmd/... ./config/...
+	@$(GO) fmt ./config/... ./pkg/apis/apps/client/... ./pkg/apis/apps/gcp/... ./pkg/utils/... ./pkg/client/minikube ./pkg/client/gcp/... ./cmd/kfctl/...
 
 # Run go vet against code
 vet:
-	$(GO) vet ./pkg/... ./cmd/... ./config/...
+	@$(GO) vet ./config/... ./pkg/apis/apps/client/... ./pkg/apis/apps/gcp/... ./pkg/utils/... ./pkg/client/minikube ./pkg/client/gcp/... ./cmd/kfctl/...
 
 generate:
-	$(GO) generate ./pkg/... ./cmd/... ./config/...
-
-deepcopy:
-	@deepcopy-gen -i github.com/kubeflow/kubeflow/bootstrap/pkg/apis/... -O zz_generated.deepcopy
+	@$(GO) generate ./config/... ./pkg/apis/apps/client/... ./pkg/apis/apps/gcp/... ./pkg/utils/... ./pkg/client/minikube ./pkg/client/gcp/... ./cmd/kfctl/...
+
+/tmp/v2:
+	@[ ! -d /tmp/v2 ] && unzip -q -d /tmp hack/v2.zip
+
+$(GOPATH)/bin/deepcopy-gen:
+	GO111MODULE=off $(GO) get k8s.io/code-generator/cmd/deepcopy-gen
+
+config/zz_generated.deepcopy.go: config/types.go
+	$(GOPATH)/bin/deepcopy-gen -i github.com/kubeflow/kubeflow/bootstrap/config/... -O zz_generated.deepcopy
+
+pkg/apis/apps/gcp/v1alpha1/zz_generated.deepcopy.go: pkg/apis/apps/gcp/v1alpha1/application_types.go
+	$(GOPATH)/bin/deepcopy-gen -i github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps/gcp/... -O zz_generated.deepcopy
+
+pkg/apis/apps/ksonnet/v1alpha1/zz_generated.deepcopy.go: pkg/apis/apps/ksonnet/v1alpha1/application_types.go
+	$(GOPATH)/bin/deepcopy-gen -i github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps/ksonnet/... -O zz_generated.deepcopy
+
+pkg/apis/apps/client/v1alpha1/zz_generated.deepcopy.go: pkg/apis/apps/client/v1alpha1/application_types.go
+	$(GOPATH)/bin/deepcopy-gen -i github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps/client/... -O zz_generated.deepcopy
+
+v2/pkg/apis/apps/client/v1alpha1/zz_generated.deepcopy.go: v2/pkg/apis/apps/client/v1alpha1/application_types.go
+	$(GOPATH)/bin/deepcopy-gen -i github.com/kubeflow/kubeflow/bootstrap/v2/pkg/apis/apps/client/... -O zz_generated.deepcopy
+
+deepcopy: $(GOPATH)/bin/deepcopy-gen config/zz_generated.deepcopy.go pkg/apis/apps/ksonnet/v1alpha1/zz_generated.deepcopy.go pkg/apis/apps/gcp/v1alpha1/zz_generated.deepcopy.go pkg/apis/apps/client/v1alpha1/zz_generated.deepcopy.go 
 
 build-bootstrap: generate fmt vet
 	$(GO) build -gcflags 'all=-N -l' -o bin/bootstrapper cmd/bootstrap/main.go
 
-build-kfctl: generate fmt vet
+build-kfctl: /tmp/v2 deepcopy generate fmt vet
 	$(GO) build -i -gcflags 'all=-N -l' -o bin/kfctl cmd/kfctl/main.go
 
-build-dockerfordesktop-plugin: generate fmt vet
-	$(GO) build -i -gcflags 'all=-N -l' -o bin/dockerfordesktop.so -buildmode=plugin cmd/plugins/dockerfordesktop/dockerfordesktop.go
-
 build-local: build-bootstrap build-kfctl
 
 # To edit which registries to add to bootstrapper, edit config (eg. config/default.yaml)
@@ -79,7 +110,7 @@ push-latest: push
 	gcloud container images add-tag --quiet $(IMG):$(TAG) $(IMG):latest --verbosity=info
 	echo created $(IMG):latest
 
-install: build-kfctl
+install: build-kfctl dockerfordesktop.so kustomize.so ksonnet.so
 	@echo copying bin/kfctl to /usr/local/bin
 	@cp bin/kfctl /usr/local/bin
 
@@ -88,47 +119,9 @@ run-local-docker:
 	  --mount type=bind,source=${HOME}/kf_app,target=/home/kubeflow \
 	  --entrypoint /bin/bash $(IMG):$(TAG)
 
-# static and plugins toogle whether we try and load a platform as a .so.
-# There is an example plugin 'dockerfordesktop' that can be loaded as a .so.
-# By default we disable loading plugins and link dockerfordesktop into the kfctl binary (static)
-static:
-	@ex pkg/apis/apps/group.go <hack/static
-	@ex cmd/kfctl/cmd/root.go <hack/static || echo Plugin functionality disabled
-
-plugins:
-	@ex pkg/apis/apps/group.go <hack/plugins
-	@ex cmd/kfctl/cmd/root.go <hack/plugins || echo Plugin functionality enabled
-
-clean:
-	@rm -rf test
-
-test-known-platforms-init: clean install build-dockerfordesktop-plugin
-	@mkdir test && \
-	echo kfctl init test/dockerfordesktop -V --platform docker-for-desktop && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl init `pwd`/test/dockerfordesktop -V --platform docker-for-desktop && \
-	echo kfctl init test/none -V && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl init `pwd`/test/none -V && \
-	echo kfctl init test/minikube -V --platform minikube && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl init $(PWD)/test/minikube -V --platform minikube && \
-	echo kfctl init test/gcp -V --platform gcp --project $(GCLOUD_PROJECT) && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl init $(PWD)/test/gcp -V --platform gcp --project $(GCLOUD_PROJECT) && \
-	echo SUCCESS
-
-test-known-platforms-generate: test-known-platforms-init
-	@cd test/dockerfordesktop && \
-	echo 'kfctl generate all (--platform dockerfordesktop)' && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) PLUGINS_ENVIRONMENT=$(PLUGINS_ENVIRONMENT) kfctl generate all && \
-	cd ../none && \
-	echo 'kfctl generate all (--platform none)' && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl generate all && \
-	cd ../minikube && \
-	echo 'kfctl generate all (--platform minikube)' && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl generate all --mount-local && \
-	cd ../gcp && \
-	echo 'kfctl generate all (--platform gcp)' && \
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) kfctl generate all --email jsmith@acme.com && \
-	echo SUCCESS
-
-test-plugins-static: plugins test-known-platforms-generate static test-known-platforms-generate
-
-test-kfctl: test-plugins-static
+clean: 
+	rm -rf test && mkdir test
+
+test-init: clean install dockerfordesktop.init minikube.init gcp.init
+
+test-generate: test-init dockerfordesktop.generate minikube.generate gcp.generate
diff --git a/bootstrap/cmd/kfctl/README.md b/bootstrap/cmd/kfctl/README.md
@@ -12,6 +12,8 @@ Note: Additional issues have been opened so this README.md will have additional
 
  - Separate different platform implementations of the [KfApp Interface](#kfapp-interface).
 
+ - Separate different package manager implementations of the [KfApp Interface](#kfapp-interface).
+
  - Allow new platforms to be added to kfctl without rebuilding or reshipping kfctl (see [Extending kfctl](#extending-kfctl) below).
 
  - Do not change existing `REST` entrypoints or the `KsService` interface in `ksServer.go` at this time.
@@ -20,25 +22,9 @@ Note: Additional issues have been opened so this README.md will have additional
 
 ## API and Packaging
 
-New directories
-
-```sh
-bootstrap/cmd/kfctl
-bootstrap/cmd/kfctl/cmd
-bootstrap/cmd/plugins/dockerfordesktop
-bootstrap/pkg/apis/apps
-bootstrap/pkg/apis/apps/ksonnet/v1alpha1
-bootstrap/pkg/utils
-bootstrap/pkg/client/dockerfordesktop
-bootstrap/pkg/client/ksonnet
-bootstrap/pkg/client/minikube
-```
-
 ### KfApp Interface
 
-Definition: github/kubeflow/kubeflow/bootstrap/pkg/client/kfapi/typed/apps/group.go
-
-The `KfApp` golang Interface
+The `KfApp` golang Interface is defined below:
 
 ```golang
 type ResourceEnum string
@@ -47,12 +33,12 @@ const (
 	ALL      ResourceEnum = "all"
 	K8S      ResourceEnum = "k8s"
 	PLATFORM ResourceEnum = "platform"
-	NONE     ResourceEnum = "none"
 )
 
 //
 // KfApp is used by commands under bootstrap/cmd/{bootstrap,kfctl}. KfApp provides a common
-// API for different implementations like gcp and minikube
+// API for different platforms implementations like gcp and minikube. 
+// KfApp is also implemented by different package managers (ksonnet, kustomize).
 //
 type KfApp interface {
 	Apply(resources ResourceEnum, options map[string]interface{}) error
@@ -65,45 +51,43 @@ type KfApp interface {
 kfctl will statically include platforms that implement the KfApp interface. 
 These include:
 
-- platform: **none**
-  - bootstrap/pkg/client/ksonnet/ksonnet.go
 - platform: **minikube**
   - bootstrap/pkg/client/minikube/minikube.go
 - platform: **gcp** 
   - bootstrap/pkg/client/gcp/gcp.go
 
-kfctl can also dynamically load platforms that are not statically linked, as 
+kfctl can dynamically load platforms and package managers that are not statically linked, as 
 described below in [Extending kfctl](#extending-kfctl).
 
 ## Usage
 
-```man
-kubeflow client tool
-
-Usage:
-  kfctl [command]
-
-Available Commands:
-  apply       Deploy a generated kubeflow application.
-  delete      Delete a kubeflow application.
-  generate    Generate a kubeflow application where resources is one of 'platform | k8s | all'.
-  help        Help about any command
-  init        Create a kubeflow application under <[path/]name>
-  version     Prints the version of kfctl.
-
-Flags:
-  -h, --help   help for kfctl
-
-Use "kfctl [command] --help" for more information about a command.
+```kubeflow client tool
+   
+   Usage:
+     kfctl [command]
+   
+   Available Commands:
+     apply       Deploy a generated kubeflow application.
+     delete      Delete a kubeflow application.
+     generate    Generate a kubeflow application where resources is one of 'platform|k8s|all'.
+     help        Help about any command
+     init        Create a kubeflow application under <[path/]name>
+     show        Deploy a generated kubeflow application.
+     version     Prints the version of kfctl.
+   
+   Flags:
+     -h, --help   help for kfctl
+   
+   Use "kfctl [command] --help" for more information about a command.
 ```
 
 Typical use-case, non-platform specific.
 
 ```sh
-kfctl init ~/myapp
-cd ~/myapp
-kfctl generate
-kfctl apply
+kfctl init ~/myapp && \
+cd ~/myapp && \
+kfctl generate all && \
+kfctl apply all
 ```
 
 ## Subcommands
@@ -152,7 +136,7 @@ Flags:
       --ipName string     ipName if '--platform gcp'
       --mount-local       mount-local if '--platform minikube'
   -V, --verbose           verbose output default is false
-      --zone string       zone if '--platform gcp' (default "us-east1-d")
+      --zone string       zone if '--platform gcp' (default "us-east1-d")```
 ```
 
 ### **apply** 
@@ -166,9 +150,10 @@ Usage:
   kfctl apply [all(=default)|k8s|platform] [flags]
 
 Flags:
-  -h, --help      help for apply
-  -V, --verbose   verbose output default is false
-```
+  -h, --help                  help for apply
+      --oauth_id string       OAuth Client ID, GCP only. Required if ENV CLIENT_ID is not set. Value passed will take precedence to ENV.
+      --oauth_secret string   OAuth Client ID, GCP only. Required if ENV CLIENT_SECRET is not set. Value passed will take precedence to ENV.
+  -V, --verbose               verbose output default is false```
 
 ### **delete** 
 
@@ -182,14 +167,13 @@ Usage:
 
 Flags:
   -h, --help      help for delete
-  -V, --verbose   verbose output default is false
-```
+  -V, --verbose   verbose output default is false```
 
 ---
 
 ## Extending kfctl
 
-`kfctl` can be extended to work with new platforms without requiring recompilation.
+`kfctl` can be extended to work with new platforms or package managers without requiring recompilation.
 An example is under bootstrap/cmd/plugins/dockerfordesktop/dockerfordesktop.go. A particular platform
 provides a shared library (.so) under the env var `PLUGINS_ENVIRONMENT`
 that kfctl would load and execute. The shared library needs to define
@@ -235,35 +219,6 @@ make test-known-platforms-init
 make test-known-platforms-generate
 ```
 
-## Debugging
-
-In order to debug in goland, the plugin code must be disabled.
-See https://github.com/golang/go/issues/23733.
-This is expected to be resolved with golang 1.12.X
-To disable the plugin code (which will cause dockerfordesktop.go to be linked statically in kfctl)
-and allow debugging in goland run:
-
-```
-make static
-```
-
-otherwise run
-
-```
-make plugins
-```
-
-Note: the default is `make static`. Do not checkin code after doing `make plugins`.
-
-Note: static and plugins make targets result in 2 files being changed:
-- pkg/apis/apps/group.go
-- cmd/kfctl/cmd/root.go
-
-These files have comments that are toggled (effectively a golang macro hack).
-This will go away when the fix noted above is available and we've moved to
-this version of go.
-
-
 ## KfApp Types used in app.yaml
 
 ### ksonnet related types (originally under bootstrap/cmd/bootstrap, moved to pkg/apis/apps/ksonnet/v1alpha1)
@@ -314,3 +269,36 @@ status: {}
 
 Ksonnet types have been moved to `github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps/ksonnet/v1alpha1`
 
+## golang modules and versioned packages
+
+Both ksonnet and kustomize package managers are loaded as .so's.
+(They will be statically linked soon see [#2635](https://github.com/kubeflow/kubeflow/issues/2635))
+The complication is that ksonnet and kustomize do not have an overlap of kubernetes versions as well 
+as client-go, and controller-runtime.
+
+- **ksonnet** (highest version)
+  - k8s.io/api kubernetes-1.10.4
+  - k8s.io/apimachinery kubernetes-1.10.4
+  - k8s.io/client-go v7.0.0
+  - sigs.k8s.io/controller-runtime v0.1.1
+
+- **kustomize** (cannot downgrade to kubernetes-1.10.4, client-go v7.0.0)
+  - k8s.io/api kubernetes-1.13.4
+  - k8s.io/apimachinery kubernetes-1.13.4
+  - k8s.io/client-go v10.0.0
+  - sigs.k8s.io/controller-runtime v0.1.10
+
+kustomize leverages golang modules by using 'v2' versions of 
+the above libraries.
+
+We insert golang package versioning for these libraries despite the fact that kubernetes has yet to move to golang modules. Updating these libraries to use golang modules is straight-forward and can be done using local git clones. Background information on how this is done can be found [here](https://github.com/golang/go/wiki/Modules#releasing-modules-v2-or-higher).
+
+Currently a build artifact was hand built and checked into PR #2548 as bootstrap/hack/v2.zip.
+This build artifact holds versioned packages for the above libraries. This hand-built artifact needs to be removed and replaced with a golang tool such as [mod](https://github.com/marwan-at-work/mod) that can do this algorithmically and as part of building kfctl.  
+
+The  [mod](https://github.com/marwan-at-work/mod) tool needs to be modified in the following ways:
+1. Update literal references to versioned packages as found [here](https://github.com/kubernetes/api/blob/kubernetes-1.13.4/apps/v1/generated.pb.go#L62) and [here](https://github.com/kubernetes/api/blob/kubernetes-1.13.4/apps/v1/generated.pb.go#L63).
+2. Move a subset of libraries tagged as [+incompatible](https://groups.google.com/forum/#!topic/golang-codereviews/t-xcPhCn3FI) within the go.mod file, instead of all of them.
+3. Create a local archive of these versioned packages 
+4. Update go.mod files that reference the versioned packages with a replace directive pointing to the the local archive location. 
+5. Do not update the git repos or push changes back to git
diff --git a/bootstrap/cmd/kfctl/cmd/generate.go b/bootstrap/cmd/kfctl/cmd/generate.go
@@ -114,7 +114,7 @@ func init() {
 		return
 	}
 
-	// platforms minikube, docker-for-desktop
+	// platforms minikube
 	generateCmd.Flags().Bool(string(kftypes.MOUNT_LOCAL), false,
 		string(kftypes.MOUNT_LOCAL)+" if '--platform minikube'")
 	bindErr = generateCfg.BindPFlag(string(kftypes.MOUNT_LOCAL), generateCmd.Flags().Lookup(string(kftypes.MOUNT_LOCAL)))