Implement KEP-19 by distinguishing between app and operator version (#…

…1257)

The app version of an operator package is taken into account using operations that apply to packages. E.g., when installing a package, a user is able to provide a specific app version to install. To further distinguish app version and operator version, the 'version' field in operator packages and the package index has been renamed to 'operatorVersion'.
Package version resolution has been updated to take the app version into account; operator packages are first sorted by app version, then sorted by operator version. Furthermore, the app version is added to the package tarball name to avoid conflicts.

Co-authored-by: Aleksey Dukhovniy <adukhovniy@mesosphere.io>
Co-authored-by: Ken Sipe <kensipe@gmail.com>

pkg/kudoctl/cmd/generate/operator_test.go

@@ -35,9 +35,9 @@ func TestOperatorGenSafe(t *testing.T) {
 
 var (
 	op1 = packages.OperatorFile{
-		Name:       "foo",
-		APIVersion: reader.APIVersion,
-		Version:    "0.1.0",
+		Name:            "foo",
+		APIVersion:      reader.APIVersion,
+		OperatorVersion: "0.1.0",
 	}
 	opFilename    = path.Join("operator", "operator.yaml")
 	paramFilename = path.Join("operator", "params.yaml")

pkg/kudoctl/cmd/generate/testdata/maintainer.golden

@@ -12,6 +12,7 @@ maintainers:
 - email: c@hat.com
   name: Cat in the hat
 name: zookeeper
+operatorVersion: 0.1.0
 plans:
   deploy:
     phases:
@@ -50,4 +51,3 @@ tasks:
     resources:
     - validation.yaml
 url: https://zookeeper.apache.org/
-version: 0.1.0

pkg/kudoctl/cmd/generate/testdata/plan.golden

@@ -10,6 +10,7 @@ maintainers:
 - email: kensipe@gmail.com
   name: Ken Sipe
 name: zookeeper
+operatorVersion: 0.1.0
 plans:
   deploy:
     phases:
@@ -57,4 +58,3 @@ tasks:
     resources:
     - validation.yaml
 url: https://zookeeper.apache.org/
-version: 0.1.0

pkg/kudoctl/cmd/generate/testdata/task.golden

@@ -10,6 +10,7 @@ maintainers:
 - email: kensipe@gmail.com
   name: Ken Sipe
 name: zookeeper
+operatorVersion: 0.1.0
 plans:
   deploy:
     phases:
@@ -53,4 +54,3 @@ tasks:
     resources:
     - bar.yaml
 url: https://zookeeper.apache.org/
-version: 0.1.0

pkg/kudoctl/cmd/install.go

@@ -26,8 +26,8 @@ var (
   # Install operator from tarball at URL
   kubectl kudo install http://kudo.dev/zk.tgz
 
-  # Specify a package version of Kafka to install to your cluster
-  kubectl kudo install kafka --version=1.1.1`
+  # Specify an operator version of Kafka to install to your cluster
+  kubectl kudo install kafka --operator-version=1.1.1`
 )
 
 // newInstallCmd creates the install command for the CLI
@@ -54,7 +54,8 @@ func newInstallCmd(fs afero.Fs) *cobra.Command {
 	installCmd.Flags().StringVar(&options.InstanceName, "instance", "", "The Instance name. (defaults to Operator name appended with -instance)")
 	installCmd.Flags().StringArrayVarP(&parameters, "parameter", "p", nil, "The parameter name and value separated by '='")
 	installCmd.Flags().StringVar(&options.RepoName, "repo", "", "Name of repository configuration to use. (default defined by context)")
-	installCmd.Flags().StringVar(&options.PackageVersion, "version", "", "A specific package version on the official GitHub repo. (default to the most recent)")
+	installCmd.Flags().StringVar(&options.AppVersion, "app-version", "", "A specific app version in the official GitHub repo. (default to the most recent)")
+	installCmd.Flags().StringVar(&options.OperatorVersion, "operator-version", "", "A specific operator version int the official GitHub repo. (default to the most recent)")
 	installCmd.Flags().BoolVar(&options.SkipInstance, "skip-instance", false, "If set, install will install the Operator and OperatorVersion, but not an Instance. (default \"false\")")
 	return installCmd
 }

pkg/kudoctl/cmd/install/install.go

@@ -20,11 +20,12 @@ type RepositoryOptions struct {
 // Options defines configuration options for the install command
 type Options struct {
 	RepositoryOptions
-	InstanceName   string
-	Parameters     map[string]string
-	PackageVersion string
-	SkipInstance   bool
-	RequestTimeout int64
+	InstanceName    string
+	Parameters      map[string]string
+	AppVersion      string
+	OperatorVersion string
+	SkipInstance    bool
+	RequestTimeout  int64
 }
 
 // DefaultOptions initializes the install command options to its defaults
@@ -69,7 +70,7 @@ func installOperator(operatorArgument string, options *Options, fs afero.Fs, set
 	clog.V(3).Printf("getting package crds")
 
 	resolver := pkgresolver.New(repository)
-	pkg, err := resolver.Resolve(operatorArgument, options.PackageVersion)
+	pkg, err := resolver.Resolve(operatorArgument, options.AppVersion, options.OperatorVersion)
 	if err != nil {
 		return fmt.Errorf("failed to resolve package CRDs for operator: %s %w", operatorArgument, err)
 	}

pkg/kudoctl/cmd/package_list.go

@@ -61,16 +61,16 @@ func newPackageParamsCmd(fs afero.Fs, out io.Writer) *cobra.Command {
 }
 
 // packageDiscovery is used by all list cmds to "discover" the packages
-func packageDiscovery(fs afero.Fs, settings *env.Settings, repoName, pathOrName, packageVersion string) (*packages.Package, error) {
+func packageDiscovery(fs afero.Fs, settings *env.Settings, repoName, pathOrName, appVersion, operatorVersion string) (*packages.Package, error) {
 	repository, err := repo.ClientFromSettings(fs, settings.Home, repoName)
 	if err != nil {
 		return nil, fmt.Errorf("could not build operator repository: %w", err)
 	}
 	clog.V(3).Printf("repository used %s", repository)
 
-	clog.V(3).Printf("getting package pkg files for %v with version: %v", pathOrName, packageVersion)
+	clog.V(3).Printf("getting package pkg files for %v with version: %v_%v", pathOrName, appVersion, operatorVersion)
 	resolver := pkgresolver.New(repository)
-	pf, err := resolver.Resolve(pathOrName, packageVersion)
+	pf, err := resolver.Resolve(pathOrName, appVersion, operatorVersion)
 	if err != nil {
 		return nil, fmt.Errorf("failed to resolve package files for operator: %s: %w", pathOrName, err)
 	}

pkg/kudoctl/cmd/package_list_params.go

@@ -16,14 +16,15 @@ import (
 )
 
 type packageListParamsCmd struct {
-	fs             afero.Fs
-	out            io.Writer
-	pathOrName     string
-	descriptions   bool
-	namesOnly      bool
-	requiredOnly   bool
-	RepoName       string
-	PackageVersion string
+	fs              afero.Fs
+	out             io.Writer
+	pathOrName      string
+	descriptions    bool
+	namesOnly       bool
+	requiredOnly    bool
+	RepoName        string
+	AppVersion      string
+	OperatorVersion string
 }
 
 const (
@@ -66,7 +67,8 @@ func newPackageListParamsCmd(fs afero.Fs, out io.Writer) *cobra.Command {
 	f.BoolVarP(&list.requiredOnly, "required", "r", false, "Show only parameters which have no defaults but are required.")
 	f.BoolVar(&list.namesOnly, "names", false, "Display only names.")
 	f.StringVar(&list.RepoName, "repo", "", "Name of repository configuration to use. (default defined by context)")
-	f.StringVar(&list.PackageVersion, "version", "", "A specific package version on the official GitHub repo. (default to the most recent)")
+	f.StringVar(&list.AppVersion, "app-version", "", "A specific app version in the official GitHub repo. (default to the most recent)")
+	f.StringVar(&list.OperatorVersion, "operator-version", "", "A specific operator version in the official GitHub repo. (default to the most recent)")
 
 	return cmd
 }
@@ -79,7 +81,7 @@ func (c *packageListParamsCmd) run(settings *env.Settings) error {
 	if !onlyOneSet(c.requiredOnly, c.namesOnly, c.descriptions) {
 		return fmt.Errorf("only one of the flags 'required', 'names', 'descriptions' can be set")
 	}
-	pf, err := packageDiscovery(c.fs, settings, c.RepoName, c.pathOrName, c.PackageVersion)
+	pf, err := packageDiscovery(c.fs, settings, c.RepoName, c.pathOrName, c.AppVersion, c.OperatorVersion)
 	if err != nil {
 		return err
 	}

pkg/kudoctl/cmd/package_list_plans.go

@@ -20,7 +20,8 @@ type packageListPlansCmd struct {
 	out                io.Writer
 	pathOrName         string
 	RepoName           string
-	PackageVersion     string
+	AppVersion         string
+	OperatorVersion    string
 	WithTasksResources bool
 }
 
@@ -50,14 +51,15 @@ func newPackageListPlansCmd(fs afero.Fs, out io.Writer) *cobra.Command {
 
 	f := cmd.Flags()
 	f.StringVar(&lc.RepoName, "repo", "", "Name of repository configuration to use. (default defined by context)")
-	f.StringVar(&lc.PackageVersion, "version", "", "A specific package version on the official GitHub repo. (default to the most recent)")
+	f.StringVar(&lc.AppVersion, "app-version", "", "A specific app version in the official GitHub repo. (default to the most recent)")
+	f.StringVar(&lc.OperatorVersion, "operator-version", "", "A specific operator version in the official GitHub repo. (default to the most recent)")
 	f.BoolVarP(&lc.WithTasksResources, "with-tasks", "t", false, "Display task resources with plans")
 
 	return cmd
 }
 
 func (c *packageListPlansCmd) run(settings *env.Settings) error {
-	pf, err := packageDiscovery(c.fs, settings, c.RepoName, c.pathOrName, c.PackageVersion)
+	pf, err := packageDiscovery(c.fs, settings, c.RepoName, c.pathOrName, c.AppVersion, c.OperatorVersion)
 	if err != nil {
 		return err
 	}

pkg/kudoctl/cmd/package_list_tasks.go

@@ -13,11 +13,12 @@ import (
 )
 
 type packageListTasksCmd struct {
-	fs             afero.Fs
-	out            io.Writer
-	pathOrName     string
-	RepoName       string
-	PackageVersion string
+	fs              afero.Fs
+	out             io.Writer
+	pathOrName      string
+	RepoName        string
+	AppVersion      string
+	OperatorVersion string
 }
 
 const (
@@ -46,14 +47,15 @@ func newPackageListTasksCmd(fs afero.Fs, out io.Writer) *cobra.Command {
 
 	f := cmd.Flags()
 	f.StringVar(&lc.RepoName, "repo", "", "Name of repository configuration to use. (default defined by context)")
-	f.StringVar(&lc.PackageVersion, "version", "", "A specific package version on the official GitHub repo. (default to the most recent)")
+	f.StringVar(&lc.AppVersion, "app-version", "", "A specific app version in the official GitHub repo. (default to the most recent)")
+	f.StringVar(&lc.OperatorVersion, "operator-version", "", "A specific operator version in the official GitHub repo. (default to the most recent)")
 
 	return cmd
 }
 
 // run provides a table listing the tasks for an operator.
 func (c *packageListTasksCmd) run(settings *env.Settings) error {
-	pf, err := packageDiscovery(c.fs, settings, c.RepoName, c.pathOrName, c.PackageVersion)
+	pf, err := packageDiscovery(c.fs, settings, c.RepoName, c.pathOrName, c.AppVersion, c.OperatorVersion)
 	if err != nil {
 		return err
 	}

pkg/kudoctl/cmd/package_new.go

@@ -62,10 +62,10 @@ func (pkg *packageNewCmd) run() error {
 	// defaults
 	pathDefault := "operator"
 	opDefault := packages.OperatorFile{
-		Name:        pkg.name,
-		APIVersion:  reader.APIVersion,
-		Version:     "0.1.0",
-		KUDOVersion: version.Get().GitVersion,
+		Name:            pkg.name,
+		APIVersion:      reader.APIVersion,
+		OperatorVersion: "0.1.0",
+		KUDOVersion:     version.Get().GitVersion,
 	}
 
 	if !pkg.interactive {

pkg/kudoctl/cmd/prompt/operator.go

@@ -49,12 +49,19 @@ func ForOperator(fs afero.Fs, pathDefault string, overwrite bool, operatorDefaul
 		_, err := semver.NewVersion(input)
 		return err
 	}
-	opVersion, err := WithValidator("Operator Version", operatorDefault.Version, versionValid)
+	opVersion, err := WithValidator("Operator Version", operatorDefault.OperatorVersion, versionValid)
 	if err != nil {
 		return nil, "", err
 	}
 
-	appVersion, err := WithDefault("Application Version", "")
+	optionalVersionValid := func(input string) error {
+		if len(input) < 1 {
+			return nil
+		}
+		_, err := semver.NewVersion(input)
+		return err
+	}
+	appVersion, err := WithValidator("Application Version", "", optionalVersionValid)
 	if err != nil {
 		return nil, "", err
 	}
@@ -70,12 +77,12 @@ func ForOperator(fs afero.Fs, pathDefault string, overwrite bool, operatorDefaul
 	}
 
 	op := packages.OperatorFile{
-		Name:        name,
-		APIVersion:  operatorDefault.APIVersion,
-		Version:     opVersion,
-		AppVersion:  appVersion,
-		KUDOVersion: kudoVersion,
-		URL:         url,
+		Name:            name,
+		APIVersion:      operatorDefault.APIVersion,
+		OperatorVersion: opVersion,
+		AppVersion:      appVersion,
+		KUDOVersion:     kudoVersion,
+		URL:             url,
 	}
 	return &op, path, nil
 }

pkg/kudoctl/cmd/repo_index_test.go

@@ -113,6 +113,6 @@ func TestRepoIndexCmd_MergeIndex(t *testing.T) {
 	}
 
 	// local operator takes precedence
-	o, _ := indexFile.GetByNameAndVersion("mysql", "0.1.0")
+	o, _ := indexFile.GetByNameAndVersion("mysql", "5.7", "0.1.0")
 	assert.Equal(t, o.Maintainers[0].Name, "Ken Sipe")
 }

pkg/kudoctl/cmd/testdata/index.yaml

@@ -7,9 +7,9 @@ entries:
         - email: nick@dischord.org
           name: Ken Sipe
       name: mysql
+      operatorVersion: 0.1.0
       urls:
         - https://kudo-repository.storage.googleapis.com/0.7.0/mysql-0.1.0.tgz
-      version: 0.1.0
   redis:
     - appVersion: 5.0.1
       digest: 24012b833b8c864f24c03f5d792c986143b031c6783dd410538040d672813cd1
@@ -19,7 +19,7 @@ entries:
         - email: fabian@mesosphere.io
           name: Fabian Baier
       name: redis
+      operatorVersion: 0.1.0
       urls:
         - https://kudo-repository.storage.googleapis.com/0.7.0/redis-0.1.0.tgz
-      version: 0.1.0
 generated: "2019-09-12T15:04:34.675941-05:00"

pkg/kudoctl/cmd/testdata/index.yaml.golden

@@ -2,7 +2,7 @@ apiVersion: v1
 entries:
   zookeeper:
   - appVersion: 3.4.10
-    digest: fa9ba293e60d0770e686eab2618dadce896bdb556e1afd4554705487863f2089
+    digest: 050c5e67279c41e40e054d0e93d1361187ba8c9a289b5a00eb1c68cef4f0b18e
     maintainers:
     - email: avarkockova@mesosphere.com
       name: Alena Varkockova
@@ -11,7 +11,7 @@ entries:
     - email: kensipe@gmail.com
       name: Ken Sipe
     name: zookeeper
+    operatorVersion: 0.1.0
     urls:
-    - http://localhost/zookeeper-0.1.0.tgz
-    version: 0.1.0
+    - http://localhost/zookeeper-3.4.10_0.1.0.tgz
 generated: "2019-10-25T00:00:00Z"

pkg/kudoctl/cmd/testdata/merge-index.yaml.golden

@@ -6,19 +6,19 @@ entries:
     - email: michael.beisiegel@gmail.com
       name: Michael Beisiegel
     name: elastic
+    operatorVersion: 0.1.0
     urls:
     - https://kudo-repository.storage.googleapis.com/0.7.0/elastic-0.1.0.tgz
-    version: 0.1.0
   mysql:
   - appVersion: "5.7"
     digest: aafe419e62065c5a3dc2f5d1458f1b5977137918236c3812ba90b5c7f6b8fb20
     maintainers:
     - email: nick@dischord.org
       name: Ken Sipe
     name: mysql
+    operatorVersion: 0.1.0
     urls:
     - https://kudo-repository.storage.googleapis.com/0.7.0/mysql-0.1.0.tgz
-    version: 0.1.0
   redis:
   - appVersion: 5.0.1
     digest: 24012b833b8c864f24c03f5d792c986143b031c6783dd410538040d672813cd1
@@ -28,9 +28,9 @@ entries:
     - email: fabian@mesosphere.io
       name: Fabian Baier
     name: redis
+    operatorVersion: 0.1.0
     urls:
     - https://kudo-repository.storage.googleapis.com/0.7.0/redis-0.1.0.tgz
-    version: 0.1.0
   zookeeper:
   - appVersion: 3.4.10
     digest: 9d46ad80dd8869253e646276fb737b6d5ce0881ea05bf60c5ccdbf3a90b24a61
@@ -40,9 +40,9 @@ entries:
     - email: runyontr@gmail.com
       name: Tom Runyon
     name: zookeeper
+    operatorVersion: 0.2.0
     urls:
     - https://kudo-repository.storage.googleapis.com/0.7.0/zookeeper-0.2.0.tgz
-    version: 0.2.0
   - appVersion: 3.4.10
     digest: 048f2fca158af7c6d7b5c0cba3577047f1bc960feabb4b4ecb5d8f9625e58f9f
     maintainers:
@@ -51,7 +51,7 @@ entries:
     - email: runyontr@gmail.com
       name: Tom Runyon
     name: zookeeper
+    operatorVersion: 0.1.0
     urls:
     - https://kudo-repository.storage.googleapis.com/0.7.0/zookeeper-0.1.0.tgz
-    version: 0.1.0
 generated: "2019-09-12T15:04:34.675941-05:00"