Bundle Feature: Create Package Bundle (#677)

What type of PR is this?

Uncomment only one /kind <> line, hit enter to put that in a new line, and remove leading whitespaces from that line:
/component kudoctl
/component operator
/kind feature

What this PR does / why we need it:
This adds 1 feature identified in draft kep-15. It bundles a package into a tarball. It provides a small amount of validation in that the operator and params file must be present. We should add linting at some point. It allows for an output location and opt-in overwrite. It creates bundles based on the expected naming convention which is <operator.name>-<operator-version>.tgz.
It provides error handling around valid destination, valid operator folder and tgz exists.

Which issue(s) this PR fixes:
Fixes #

Special notes for your reviewer:

Does this PR introduce a user-facing change?:

`kudo bundle <operator folder> creates a tgz file for the operator named based on <operator.name>-<operator-version>.tgz`

docs/cli.md

@@ -24,6 +24,7 @@ This document demonstrates how to use the CLI but also shows what happens in `KU
             * [Get the Status of an Instance](#get-the-status-of-an-instance)
             * [Delete an Instance](#delete-an-instance)
             * [Get the History to PlanExecutions](#get-the-history-to-planexecutions)
+            * [Package an Operator](#package-an-operator)
 
 
 ## Setup the KUDO Kubectl Plugin
@@ -51,11 +52,12 @@ Or you can use compile and install the plugin from your `$GOPATH/src/github.com/
 | ------------------------------------------ | --------------------------------------------------------------------------------------------- |
 | `kubectl kudo install <name> [flags]`      | Install a Operator from the official [KUDO repo](https://github.com/kudobuilder/operators). |
 | `kubectl kudo get instances [flags]`       | Show all available instances.                                                                 |
+| `kubectl kudo package <operator_folder> [flags]` | Packages an operator in a folder into a tgz file                                        |
 | `kubectl kudo plan status [flags]`         | View all available plans.                                                                     |
 | `kubectl kudo plan history <name> [flags]` | View all available plans.                                                                     |
 | `kubectl kudo version`                     | Print the current KUDO package version.                                                       |
-| `kubectl kudo update`                      | Update installed operator parameters. 
-| `kubectl kudo upgrade`                     | Upgrade installed operator from one version to another. 
+| `kubectl kudo update`                      | Update installed operator parameters.
+| `kubectl kudo upgrade`                     | Upgrade installed operator from one version to another.
 
 ## Flags
 
@@ -78,7 +80,7 @@ Flags:
 ### Install a Package
 
 There are four options how to install a package. For development you are able to install packages from your local filesystem or local tgz file.
-For testing or working without a repository it is possible to install via a url location. The last option is installation from the package repository. 
+For testing or working without a repository it is possible to install via a url location. The last option is installation from the package repository.
 
 Installation during development can use a relative or absolute path to the package folder.
 ```bash
@@ -371,6 +373,19 @@ $ kubectl kudo plan history --instance=up
 
 This includes the previous history but also all OperatorVersions that have been applied to the selected instance.
 
+### Package an Operator
+
+We can use the `package` command to package an operator into a tarball. The package name will be determined by the operator metadata in the package files.  The folder of the operator is passed as an argument. It is possible to pass a `--destination` location to build the tgz file into.
+
+`kubectl kudo package zookeeper --destination=target`
+
+Example:
+
+```bash
+$ kubectl kudo package ../operators/repository/zookeeper/operator/ --destination=~
+  Package created: /Users/kensipe/zookeeper-0.1.0.tgz
+```
+
 ### Update parameters on running operator
 
 Every operator can define overridable parameters in `params.yaml`. When installing an operator, you can use the defined defaults or override them with `-p` parameters for `kudo install`.

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/pkg/errors v0.8.1
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/rogpeppe/go-internal v1.2.2
+	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3
 	github.com/spf13/viper v1.4.0

pkg/kudoctl/bundle/bundle.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 )
 
 // This is an abstraction which abstracts the underlying bundle, which is likely file system or compressed file.
@@ -27,34 +28,35 @@ type tarBundle struct {
 
 type fileBundle struct {
 	path string
+	fs   afero.Fs
 }
 
 // NewBundle creates the implementation of the bundle based on the path.   The expectation is the bundle
 // is always local .  The path can be relative or absolute location of the bundle.
-func NewBundle(path string) (Bundle, error) {
+func NewBundle(fs afero.Fs, path string) (Bundle, error) {
 	//	make sure file exists
-	fi, err := os.Stat(path)
+	fi, err := fs.Stat(path)
 	if err != nil {
 		return nil, fmt.Errorf("unsupported file system format %v. Expect either a tar.gz file or a folder", path)
 	}
 	// order of discovery
 	// 1. tarball
 	// 2. file based
 	if fi.Mode().IsRegular() && strings.HasSuffix(path, ".tar.gz") {
-		r, err := getFileReader(path)
+		r, err := getFileReader(fs, path)
 		if err != nil {
 			return nil, err
 		}
 		return tarBundle{r}, nil
 	} else if fi.IsDir() {
-		return fileBundle{path}, nil
+		return fileBundle{path, fs}, nil
 	} else {
 		return nil, fmt.Errorf("unsupported file system format %v. Expect either a tar.gz file or a folder", path)
 	}
 }
 
-func getFileReader(path string) (io.Reader, error) {
-	f, err := os.Open(path)
+func getFileReader(fs afero.Fs, path string) (io.Reader, error) {
+	f, err := fs.Open(path)
 	if err != nil {
 		return nil, err
 	}
@@ -76,16 +78,74 @@ func (b tarBundle) GetCRDs() (*PackageCRDs, error) {
 }
 
 func (b fileBundle) GetCRDs() (*PackageCRDs, error) {
-	p, err := fromFolder(b.path)
+	p, err := fromFolder(b.fs, b.path)
 	if err != nil {
 		return nil, errors.Wrap(err, "while reading package from the file system")
 	}
 	return p.getCRDs()
 }
 
-func fromFolder(packagePath string) (*PackageFiles, error) {
+// ToTarBundle takes a path to operator files and creates a tgz of those files with the destination and name provided
+func ToTarBundle(fs afero.Fs, path string, destination string, overwrite bool) (string, error) {
+	pkg, err := fromFolder(fs, path)
+	if err != nil {
+		return "", fmt.Errorf("invalid operator in path: %v", path)
+	}
+
+	name := packageVersionedName(pkg)
+	target, e := getFullPathToTarget(fs, destination, name, overwrite)
+	if e != nil {
+		return "", e
+	}
+
+	if _, err := fs.Stat(path); err != nil {
+		return "", fmt.Errorf("unable to package files - %v", err.Error())
+	}
+	file, err := fs.Create(target)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	err = tarballWriter(fs, path, file)
+	return target, err
+}
+
+// getFullPathToTarget takes destination path and file name and provides a clean full path while ensure the file does not exist.
+func getFullPathToTarget(fs afero.Fs, destination string, name string, overwrite bool) (string, error) {
+	if destination == "." {
+		destination = ""
+	}
+	if destination != "" {
+		if strings.Contains(destination, "~") {
+			userHome, _ := os.UserHomeDir()
+			destination = strings.Replace(destination, "~", userHome, 1)
+		}
+		fi, err := fs.Stat(destination)
+		if err != nil || !fi.Mode().IsDir() {
+			return "", fmt.Errorf("destination \"%v\" is not a proper directory", destination)
+		}
+		name = filepath.Join(destination, name)
+	}
+	target := filepath.Clean(fmt.Sprintf("%v.tgz", name))
+	if exists, _ := afero.Exists(fs, target); exists {
+		if !overwrite {
+			return "", fmt.Errorf("target file exists. Remove or --overwrite. File:%v", target)
+		}
+	}
+	return target, nil
+}
+
+// packageVersionedName provides the version name of a package provided a set of PackageFiles.  Ex. "zookeeper-0.1.0"
+func packageVersionedName(pkg *PackageFiles) string {
+	return fmt.Sprintf("%v-%v", pkg.Operator.Name, pkg.Operator.Version)
+}
+
+// fromFolder walks the path provided and returns CRD package files or an error
+func fromFolder(fs afero.Fs, packagePath string) (*PackageFiles, error) {
 	result := newPackageFiles()
-	err := filepath.Walk(packagePath, func(path string, file os.FileInfo, err error) error {
+
+	err := afero.Walk(fs, packagePath, func(path string, file os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -97,7 +157,7 @@ func fromFolder(packagePath string) (*PackageFiles, error) {
 			// skip the root folder, as Walk always starts there
 			return nil
 		}
-		bytes, err := ioutil.ReadFile(path)
+		bytes, err := afero.ReadFile(fs, path)
 		if err != nil {
 			return err
 		}
@@ -107,6 +167,10 @@ func fromFolder(packagePath string) (*PackageFiles, error) {
 	if err != nil {
 		return nil, err
 	}
+	// final check
+	if result.Operator == nil || result.Params == nil {
+		return nil, fmt.Errorf("incomplete operator package in path: %v", packagePath)
+	}
 	return &result, nil
 }
 
@@ -153,7 +217,7 @@ func parseTarPackage(r io.Reader) (*PackageFiles, error) {
 		case tar.TypeReg:
 			bytes, err := ioutil.ReadAll(tr)
 			if err != nil {
-				return nil, errors.Wrapf(err, "while reading file from bundle tarball %s", header.Name)
+				return nil, errors.Wrapf(err, "while reading file from package tarball %s", header.Name)
 			}
 
 			err = parsePackageFile(header.Name, bytes, &result)

pkg/kudoctl/bundle/finder/bundle_finder.go

@@ -3,10 +3,10 @@ package finder
 import (
 	"fmt"
 	"io"
-	"os"
 
 	"github.com/kudobuilder/kudo/pkg/kudoctl/bundle"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/http"
+	"github.com/spf13/afero"
 )
 
 // Finder is a bundle finder and is any implementation which can find/discover a bundle.
@@ -17,6 +17,7 @@ type Finder interface {
 
 // LocalFinder will find local operator bundle: folders or tgz
 type LocalFinder struct {
+	fs afero.Fs
 }
 
 // URLFinder will find an operator bundle from a url
@@ -44,7 +45,7 @@ func New() *Manager {
 func (f *Manager) GetBundle(name string, version string) (bundle.Bundle, error) {
 
 	// if local folder return the bundle
-	if _, err := os.Stat(name); err == nil {
+	if _, err := f.local.fs.Stat(name); err == nil {
 		b, err := f.local.GetBundle(name, version)
 		if err != nil {
 			return nil, err
@@ -89,19 +90,19 @@ func (f *URLFinder) getBundleByURL(url string) (io.Reader, error) {
 // GetBundle provides a bundle for the local folder or tarball provided
 func (f *LocalFinder) GetBundle(name string, version string) (bundle.Bundle, error) {
 	//	make sure file exists
-	_, err := os.Stat(name)
+	_, err := f.fs.Stat(name)
 	if err != nil {
 		return nil, fmt.Errorf("unsupported file system format %v. Expect either a tar.gz file or a folder", name)
 	}
 	// order of discovery
 	// 1. tarball
 	// 2. file based
-	return bundle.NewBundle(name)
+	return bundle.NewBundle(f.fs, name)
 }
 
 // NewLocal creates a finder for local operator bundles
 func NewLocal() *LocalFinder {
-	return &LocalFinder{}
+	return &LocalFinder{fs: afero.NewOsFs()}
 }
 
 // NewURL creates an instance of a URLFinder

pkg/kudoctl/bundle/package_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/go-test/deep"
 	"github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 	"sigs.k8s.io/yaml"
 )
 
@@ -30,10 +31,11 @@ func TestReadFileSystemPackage(t *testing.T) {
 		{"zookeeper", "zookeeper-xn8fg", "testdata/zk", "testdata/zk-crd-golden1"},
 		{"zookeeper", "zookeeper-txhzt", "testdata/zk.tar.gz", "testdata/zk-crd-golden2"},
 	}
+	var fs = afero.NewOsFs()
 
 	for _, tt := range tests {
 		t.Run(fmt.Sprintf("%s-from-%s", tt.name, tt.path), func(t *testing.T) {
-			bundle, err := NewBundle(tt.path)
+			bundle, err := NewBundle(fs, tt.path)
 			if err != nil {
 				t.Fatalf("Found unexpected error: %v", err)
 			}