Create kfctl command

bootstrap/.dockerignore

@@ -1,2 +1 @@
-vendor/**
-
+**/.git

bootstrap/Makefile

@@ -13,14 +13,15 @@
 # limitations under the License.
 #
 GCLOUD_PROJECT ?= kubeflow-images-public
-GOLANG_VERSION ?= 1.11.2
+GOLANG_VERSION ?= 1.11.5
+GOPATH ?= $(HOME)/go
 # To build without the cache set the environment variable
 # export DOCKER_BUILD_OPTS=--no-cache
-# set to GOLANG_GCFLAGS=-gcflags "all=-N -l" to debug
-GOLANG_GCFLAGS ?= ""
 IMG ?= gcr.io/$(GCLOUD_PROJECT)/bootstrapper
-TAG ?= $(eval TAG := $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty)-$(shell git diff | shasum -a256 | cut -c -6))$(TAG)
+TAG ?= $(shell git describe --tags --always --dirty)
 PORT ?= 2345
+export GO111MODULE=on
+export GO=go
 
 all: build
 
@@ -29,14 +30,26 @@ auth:
 
 # Run go fmt against code
 fmt:
-	go fmt ./cmd/...
+	$(GO) fmt ./pkg/... ./cmd/...
 
 # Run go vet against code
 vet:
-	go vet ./cmd/...
+	$(GO) vet ./pkg/... ./cmd/...
 
-build-local: fmt vet
-	GO111MODULE=on go build -gcflags 'all=-N -l' -o bin/bootstrapper cmd/bootstrap/main.go
+generate:
+	$(GO) generate ./pkg/... ./cmd/...
+
+
+build-bootstrap: generate fmt vet
+	$(GO) build -gcflags 'all=-N -l' -o bin/bootstrapper cmd/bootstrap/main.go
+
+build-kfctl: generate fmt vet
+	$(GO) build -i -gcflags 'all=-N -l' -o bin/kfctl cmd/kfctl/main.go
+
+build-foo-plugin: generate fmt vet
+	$(GO) build -i -gcflags 'all=-N -l' -o bin/fooapp.so -buildmode=plugin cmd/plugins/fooapp.go
+
+build-local: build-bootstrap build-kfctl
 
 # To edit which registries to add to bootstrapper, edit config (eg. config/default.yaml)
 build: build-local
@@ -58,21 +71,35 @@ push-latest: push
 	gcloud container images add-tag --quiet $(IMG):$(TAG) $(IMG):latest --verbosity=info
 	echo created $(IMG):latest
 
-run-local:
-	KUBERNETES_SERVICE_HOST=https://35.233.240.120 KUBERNETES_SERVICE_PORT=80 bin/bootstrapper --app-dir=$$HOME/apps --registries-config-file=$$PWD/local_registries.yaml --config=$$PWD/config/default.yaml --in-cluster=false
+install: build-kfctl
+	@echo copying bin/kfctl to /usr/local/bin
+	@cp bin/kfctl /usr/local/bin
 
 run-local-docker:
 	docker run -d -it --name bootstrapper \
-	  --mount type=bind,source=$$HOME/kf_app,target=/home/kubeflow \
+	  --mount type=bind,source=${HOME}/kf_app,target=/home/kubeflow \
 	  --entrypoint /bin/bash $(IMG):$(TAG)
 
-debug: push cleanup
-	@echo debugging $(IMG):$(TAG) using port $(PORT)
-	@./debug.sh $(IMG) $(TAG) $(PORT)
+# init ~/myapp --platform none
+test-known-platforms-init: install build-foo-plugin
+	@rm -rf $(HOME)/ks-app && \
+	kfctl init $(HOME)/ks-app -V --platform none && \
+	rm -rf $(HOME)/minikube-app && \
+	kfctl init $(HOME)/minikube-app -V --platform minikube && \
+	rm -rf $(HOME)/foo-app && \
+	PLUGINS_ENVIRONMENT=$(GOPATH)/src/github.com/kubeflow/kubeflow/bootstrap/bin kfctl init $(HOME)/foo-app -V --platform foo && \
+	echo SUCCESS
 
-debug-latest: debug
-	@echo debugging $(IMG):latest using port $(PORT)
-	@./debug.sh $(IMG) latest $(PORT)
+# generate all --email john@foo.com --ipName 35.233.240.120
+test-known-platforms-generate: test-known-platforms-init
+	@cd ~/ks-app && \
+	kfctl generate all -V && \
+	cd ~/minikube-app && \
+	kfctl generate all -V --mount-local && \
+	cd ~/foo-app && \
+	PLUGINS_ENVIRONMENT=$(GOPATH)/src/github.com/kubeflow/kubeflow/bootstrap/bin kfctl generate all -V && \
+	echo SUCCESS
 
-cleanup:
-	@./cleanup.sh kubeflow-admin
+test-foo-plugin:
+	rm -rf $(HOME)/foo-app
+	PLUGINS_ENVIRONMENT=$(GOPATH)/src/github.com/kubeflow/kubeflow/bootstrap/bin kfctl init $(HOME)/foo-app --platform foo

bootstrap/cmd/bootstrap/app/gcpUtils.go

@@ -69,7 +69,7 @@ func (s *ksServer) InsertDeployment(ctx context.Context, req CreateRequest, dmSp
 		dmconf.Resources[0].Name = req.Name
 		dmconf.Resources[0].Properties["zone"] = req.Zone
 		dmconf.Resources[0].Properties["ipName"] = req.IpName
-		dmconf.Resources[0].Properties["createPipelinePersistentStorage"]=req.StorageOption.CreatePipelinePersistentStorage
+		dmconf.Resources[0].Properties["createPipelinePersistentStorage"] = req.StorageOption.CreatePipelinePersistentStorage
 		// https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters
 		if s.gkeVersionOverride != "" {
 			dmconf.Resources[0].Properties["cluster-version"] = s.gkeVersionOverride

bootstrap/cmd/bootstrap/app/ksServer.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ksonnet/ksonnet/pkg/actions"
 	kApp "github.com/ksonnet/ksonnet/pkg/app"
 	"github.com/ksonnet/ksonnet/pkg/client"
+	kstypes "github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps/ksapp/v1alpha1"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
@@ -110,7 +111,7 @@ type ksServer struct {
 	// This can be used to map the name of a registry to info about the registry.
 	// This allows apps to specify a registry by name without having to know any
 	// other information about the regisry.
-	knownRegistries map[string]RegistryConfig
+	knownRegistries map[string]*kstypes.RegistryConfig
 
 	//gkeVersionOverride allows overriding the GKE version specified in DM config. If not set the value in DM config is used.
 	// https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters
@@ -149,15 +150,15 @@ func (m MultiError) ToError() error {
 }
 
 // NewServer constructs a ksServer.
-func NewServer(appsDir string, registries []RegistryConfig, gkeVersionOverride string, installIstio bool) (*ksServer, error) {
+func NewServer(appsDir string, registries []*kstypes.RegistryConfig, gkeVersionOverride string, installIstio bool) (*ksServer, error) {
 	if appsDir == "" {
 		return nil, fmt.Errorf("appsDir can't be empty")
 	}
 
 	s := &ksServer{
 		appsDir:            appsDir,
 		projectLocks:       make(map[string]*sync.Mutex),
-		knownRegistries:    make(map[string]RegistryConfig),
+		knownRegistries:    make(map[string]*kstypes.RegistryConfig),
 		gkeVersionOverride: gkeVersionOverride,
 		fs:                 afero.NewOsFs(),
 		installIstio:       installIstio,
@@ -195,7 +196,7 @@ type CreateRequest struct {
 	// Name for the app.
 	Name string
 	// AppConfig is the config for the app.
-	AppConfig AppConfig
+	AppConfig kstypes.AppConfig
 
 	// Namespace for the app.
 	Namespace string
@@ -506,7 +507,7 @@ func (s *ksServer) CreateApp(ctx context.Context, request CreateRequest, dmDeplo
 
 	// Add the registries to the app.
 	for idx, registry := range request.AppConfig.Registries {
-		RegUri, err := s.getRegistryUri(&registry)
+		RegUri, err := s.getRegistryUri(registry)
 		if err != nil {
 			log.Errorf("There was a problem getRegistryUri for registry %v. Error: %v", registry.Name, err)
 			return err
@@ -589,17 +590,17 @@ func (s *ksServer) CreateApp(ctx context.Context, request CreateRequest, dmDeplo
 
 // fetch remote registry to local disk, or use baked-in registry if version not specified in user request.
 // Then return registry's RegUri.
-func (s *ksServer) getRegistryUri(registry *RegistryConfig) (string, error) {
+func (s *ksServer) getRegistryUri(registry *kstypes.RegistryConfig) (string, error) {
 	if registry.Name == "" ||
-			registry.Path == "" ||
-			registry.Repo == "" ||
-			registry.Version == "" ||
-			registry.Version == "default" {
+		registry.Path == "" ||
+		registry.Repo == "" ||
+		registry.Version == "" ||
+		registry.Version == "default" {
 
 		v, ok := s.knownRegistries[registry.Name]
 		if !ok {
 			return "", fmt.Errorf("Create request uses registry %v but some "+
-					"required fields are not specified and this is not a known registry.", registry.Name)
+				"required fields are not specified and this is not a known registry.", registry.Name)
 		}
 		log.Infof("No remote registry provided for registry %v; setting URI to local %v.", registry.Name, v.RegUri)
 		return v.RegUri, nil
@@ -651,7 +652,7 @@ func runCmd(rawcmd string) error {
 }
 
 // appGenerate installs packages and creates components.
-func (s *ksServer) appGenerate(kfApp kApp.App, appConfig *AppConfig) error {
+func (s *ksServer) appGenerate(kfApp kApp.App, appConfig *kstypes.AppConfig) error {
 	libs, err := kfApp.Libraries()
 
 	if err != nil {
@@ -676,7 +677,7 @@ func (s *ksServer) appGenerate(kfApp kApp.App, appConfig *AppConfig) error {
 		_, err = s.fs.Stat(regFile)
 		if err == nil {
 			log.Infof("processing registry file %v ", regFile)
-			var ksRegistry KsRegistry
+			var ksRegistry kstypes.KsRegistry
 			if LoadConfig(regFile, &ksRegistry) == nil {
 				for pkgName, _ := range ksRegistry.Libraries {
 					_, err = s.fs.Stat(path.Join(registry.RegUri, pkgName))
@@ -783,7 +784,7 @@ func (s *ksServer) createComponent(kfApp kApp.App, args []string) error {
 
 // autoConfigureApp attempts to automatically optimize the Kubeflow application
 // based on the cluster setup.
-func (s *ksServer) autoConfigureApp(kfApp *kApp.App, appConfig *AppConfig, namespace string, config *rest.Config) error {
+func (s *ksServer) autoConfigureApp(kfApp *kApp.App, appConfig *kstypes.AppConfig, namespace string, config *rest.Config) error {
 
 	kubeClient, err := clientset.NewForConfig(rest.AddUserAgent(config, "kubeflow-bootstrapper"))
 	if err != nil {
@@ -1200,7 +1201,7 @@ func checkDeploymentFinished(svc KsService, req CreateRequest, deployName string
 }
 
 func finishDeployment(svc KsService, req CreateRequest,
-		clusterDmDeploy *deploymentmanager.Deployment, storageDmDeploy *deploymentmanager.Deployment) {
+	clusterDmDeploy *deploymentmanager.Deployment, storageDmDeploy *deploymentmanager.Deployment) {
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, StartTime, time.Now())
 
@@ -1297,13 +1298,13 @@ func makeDeployEndpoint(svc KsService) endpoint.Endpoint {
 			}
 			req.AppConfig.Parameters = append(
 				req.AppConfig.Parameters,
-				KsParameter{
+				kstypes.KsParameter{
 					Component: "pipeline",
 					Name:      "mysqlPd",
 					Value:     req.Name + StorageDmSpec.DmNameSuffix + PipelineDbDiskSuffix})
 			req.AppConfig.Parameters = append(
 				req.AppConfig.Parameters,
-				KsParameter{
+				kstypes.KsParameter{
 					Component: "pipeline",
 					Name:      "nfsPd",
 					Value:     req.Name + StorageDmSpec.DmNameSuffix + PipelineNfsDiskSuffix})

bootstrap/cmd/bootstrap/app/options/options.go

@@ -27,6 +27,7 @@ type ServerOption struct {
 	KeepAlive            bool
 	InstallIstio         bool
 	Port                 int
+	AppName              string
 	AppDir               string
 	Config               string
 	Email                string

bootstrap/cmd/bootstrap/app/server.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	"github.com/kubeflow/kubeflow/bootstrap/cmd/bootstrap/app/options"
+	kstypes "github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps/ksapp/v1alpha1"
 	"github.com/kubeflow/kubeflow/bootstrap/version"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/api/storage/v1"
@@ -49,72 +50,18 @@ const GcloudPath = "gcloud"
 
 const RegistriesRoot = "/opt/registries"
 
-type KsComponent struct {
-	Name      string
-	Prototype string
-}
-
-type KsPackage struct {
-	Name string
-	// Registry should be the name of the registry containing the package.
-	Registry string
-}
-
-type KsParameter struct {
-	Component string
-	Name      string
-	Value     string
-}
-
-// RegistryConfig is used for two purposes:
-// 1. used during image build, to configure registries that should be baked into the bootstrapper docker image.
-//  (See: https://github.com/kubeflow/kubeflow/blob/master/bootstrap/image_registries.yaml)
-// 2. used during app create rpc call, specifies a registry to be added to an app.
-//	required info for registry: Name, Repo, Version, Path
-//  Additionally if any of required fields is blank we will try to map with one of
-//  the registries baked into the Docker image using the name.
-type RegistryConfig struct {
-	Name    string
-	Repo    string
-	Version string
-	Path    string
-	RegUri  string
-}
-
-type AppConfig struct {
-	Registries []RegistryConfig
-	Packages   []KsPackage
-	Components []KsComponent
-	Parameters []KsParameter
-}
-
-// RegistriesConfigFile corresponds to a YAML file specifying information
-// about known registries.
-type RegistriesConfigFile struct {
-	// Registries provides information about known registries.
-	Registries []RegistryConfig
-}
-
 // AppConfigFile corresponds to a YAML file specifying information
 // about the app to create.
 type AppConfigFile struct {
 	// App describes a ksonnet application.
-	App AppConfig
+	App kstypes.AppConfig
 }
 
 type LibrarySpec struct {
 	Version string
 	Path    string
 }
 
-// KsRegistry corresponds to ksonnet.io/registry
-// which is the registry.yaml file found in every registry.
-type KsRegistry struct {
-	ApiVersion string
-	Kind       string
-	Libraries  map[string]LibrarySpec
-}
-
 // Load yaml config
 func LoadConfig(path string, o interface{}) error {
 	if path == "" {
@@ -300,7 +247,7 @@ func Run(opt *options.ServerOption) error {
 	}
 
 	// Load information about the default registries.
-	var regConfig RegistriesConfigFile
+	var regConfig kstypes.RegistriesConfigFile
 
 	if opt.RegistriesConfigFile != "" {
 		log.Infof("Loading registry info in file %v", opt.RegistriesConfigFile)

bootstrap/cmd/bootstrap/main.go

@@ -16,7 +16,6 @@ package main
 
 import (
 	"flag"
-
 	"github.com/onrik/logrus/filename"
 	log "github.com/sirupsen/logrus"
 

bootstrap/cmd/kfctl/.gitignore

@@ -0,0 +1,24 @@
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Kubernetes Generated files - skip generated files, except for vendored files
+
+!vendor/**/zz_generated.*
+
+# editor and IDE paraphernalia
+.idea
+*.swp
+*.swo
+*~

bootstrap/cmd/kfctl/OWNERS

@@ -0,0 +1,6 @@
+approvers:
+  - jlewi
+  - kunmingg
+  - ashahba
+  - kkasravi
+reviewers: