Repo Index

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/leodido/go-urn v1.1.0 // indirect
+	github.com/magiconair/properties v1.8.0
 	github.com/markbates/inflect v1.0.4 // indirect
 	github.com/masterminds/sprig v2.18.0+incompatible
 	github.com/onsi/ginkgo v1.7.0 // indirect

pkg/bundle/operator.go

@@ -7,6 +7,7 @@ type Operator struct {
 	Name              string                        `json:"name"`
 	Description       string                        `json:"description,omitempty"`
 	Version           string                        `json:"version"`
+	AppVersion        string                        `json:"appVersion,omitempty"`
 	KUDOVersion       string                        `json:"kudoVersion,omitempty"`
 	KubernetesVersion string                        `json:"kubernetesVersion,omitempty"`
 	Maintainers       []v1alpha1.Maintainer         `json:"maintainers,omitempty"`

pkg/kudoctl/bundle/bundle.go

@@ -7,9 +7,10 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
+	"github.com/kudobuilder/kudo/pkg/kudoctl/files"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/afero"
 )
@@ -19,7 +20,10 @@ import (
 
 // Bundle is an abstraction of the collection of files that makes up a package.  It is anything we can retrieve the PackageCRDs from.
 type Bundle interface {
+	// transformed server view
 	GetCRDs() (*PackageCRDs, error)
+	// working with local package files
+	GetPkgFiles() (*PackageFiles, error)
 }
 
 type tarBundle struct {
@@ -68,23 +72,32 @@ func NewBundleFromReader(r io.Reader) Bundle {
 	return tarBundle{r}
 }
 
-func (b tarBundle) GetCRDs() (*PackageCRDs, error) {
+// GetPkgFiles returns the command side package files
+func (b tarBundle) GetPkgFiles() (*PackageFiles, error) {
+	return parseTarPackage(b.reader)
+}
 
-	p, err := parseTarPackage(b.reader)
+// GetCRDs returns the server side CRDs
+func (b tarBundle) GetCRDs() (*PackageCRDs, error) {
+	p, err := b.GetPkgFiles()
 	if err != nil {
 		return nil, errors.Wrap(err, "while extracting package files")
 	}
 	return p.getCRDs()
 }
 
 func (b fileBundle) GetCRDs() (*PackageCRDs, error) {
-	p, err := fromFolder(b.fs, b.path)
+	p, err := b.GetPkgFiles()
 	if err != nil {
 		return nil, errors.Wrap(err, "while reading package from the file system")
 	}
 	return p.getCRDs()
 }
 
+func (b fileBundle) GetPkgFiles() (*PackageFiles, error) {
+	return fromFolder(b.fs, b.path)
+}
+
 // ToTarBundle takes a path to operator files and creates a tgz of those files with the destination and name provided
 func ToTarBundle(fs afero.Fs, path string, destination string, overwrite bool) (string, error) {
 	pkg, err := fromFolder(fs, path)
@@ -93,7 +106,7 @@ func ToTarBundle(fs afero.Fs, path string, destination string, overwrite bool) (
 	}
 
 	name := packageVersionedName(pkg)
-	target, e := getFullPathToTarget(fs, destination, name, overwrite)
+	target, e := files.FullPathToTarget(fs, destination, fmt.Sprintf("%v.tgz", name), overwrite)
 	if e != nil {
 		return "", e
 	}
@@ -111,31 +124,6 @@ func ToTarBundle(fs afero.Fs, path string, destination string, overwrite bool) (
 	return target, err
 }
 
-// getFullPathToTarget takes destination path and file name and provides a clean full path while ensure the file does not exist.
-func getFullPathToTarget(fs afero.Fs, destination string, name string, overwrite bool) (string, error) {
-	if destination == "." {
-		destination = ""
-	}
-	if destination != "" {
-		if strings.Contains(destination, "~") {
-			userHome, _ := os.UserHomeDir()
-			destination = strings.Replace(destination, "~", userHome, 1)
-		}
-		fi, err := fs.Stat(destination)
-		if err != nil || !fi.Mode().IsDir() {
-			return "", fmt.Errorf("destination \"%v\" is not a proper directory", destination)
-		}
-		name = filepath.Join(destination, name)
-	}
-	target := filepath.Clean(fmt.Sprintf("%v.tgz", name))
-	if exists, _ := afero.Exists(fs, target); exists {
-		if !overwrite {
-			return "", fmt.Errorf("target file exists. Remove or --overwrite. File:%v", target)
-		}
-	}
-	return target, nil
-}
-
 // packageVersionedName provides the version name of a package provided a set of PackageFiles.  Ex. "zookeeper-0.1.0"
 func packageVersionedName(pkg *PackageFiles) string {
 	return fmt.Sprintf("%v-%v", pkg.Operator.Name, pkg.Operator.Version)

pkg/kudoctl/bundle/package.go

@@ -2,19 +2,21 @@ package bundle
 
 import (
 	"fmt"
+	"io"
 	"regexp"
 	"strconv"
 	"strings"
 
-	"github.com/kudobuilder/kudo/pkg/util/kudo"
-
-	"k8s.io/apimachinery/pkg/util/rand"
-
 	"github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 	"github.com/kudobuilder/kudo/pkg/bundle"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/files"
+	"github.com/kudobuilder/kudo/pkg/util/kudo"
+
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
 	"sigs.k8s.io/yaml"
 )
 
@@ -41,6 +43,12 @@ type PackageFiles struct {
 	Params    []v1alpha1.Parameter
 }
 
+// PackageFilesDigest is a tuple of data used to return the package files AND the digest of a tarball
+type PackageFilesDigest struct {
+	PkgFiles *PackageFiles
+	Digest   string
+}
+
 func parsePackageFile(filePath string, fileBytes []byte, currentPackage *PackageFiles) error {
 	isOperatorFile := func(name string) bool {
 		return strings.HasSuffix(name, operatorFileName)
@@ -196,3 +204,53 @@ func (p *PackageFiles) getCRDs() (*PackageCRDs, error) {
 		Instance:        instance,
 	}, nil
 }
+
+// GetFilesDigest maps []string of paths to the [] Operators
+func GetFilesDigest(fs afero.Fs, paths []string) []*PackageFilesDigest {
+	return mapPaths(fs, paths, pathToOperator)
+}
+
+// work of map path, swallows errors to return only packages that are valid
+func mapPaths(fs afero.Fs, paths []string, f func(afero.Fs, string) (*PackageFilesDigest, error)) []*PackageFilesDigest {
+	ops := make([]*PackageFilesDigest, 0)
+	for _, path := range paths {
+		op, err := f(fs, path)
+		if err != nil {
+			fmt.Printf("WARNING: operator: %v is invalid", path)
+			continue
+		}
+		ops = append(ops, op)
+	}
+
+	return ops
+}
+
+// pathToOperator takes a single path and returns an operator or error
+func pathToOperator(fs afero.Fs, path string) (*PackageFilesDigest, error) {
+	reader, err := fs.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	digest, err := files.Sha256Sum(reader)
+	if err != nil {
+		return nil, err
+	}
+	// restart reading of file after getting digest
+	reader.Seek(0, io.SeekStart)
+
+	pkg, err := readerToOperator(reader)
+	pfd := &PackageFilesDigest{
+		pkg,
+		digest,
+	}
+	return pfd, err
+}
+
+func readerToOperator(r io.Reader) (*PackageFiles, error) {
+	b := NewBundleFromReader(r)
+	pkg, err := b.GetPkgFiles()
+	if err != nil {
+		return nil, err
+	}
+	return pkg, nil
+}

pkg/kudoctl/bundle/testdata/zk-crd-golden1/operator.golden

@@ -9,5 +9,6 @@ spec:
   maintainers:
   - Alena Varkockova <avarkockova@mesosphere.com>
   - Tom Runyon <runyontr@gmail.com>
+  - Ken Sipe <kensipe@gmail.com>
   kudoVersion: 0.2.0
-  kubernetesVersion: 1.5.1
+  kubernetesVersion: 1.15.0

pkg/kudoctl/bundle/testdata/zk-crd-golden2/operator.golden

@@ -9,5 +9,6 @@ spec:
   maintainers:
   - Alena Varkockova <avarkockova@mesosphere.com>
   - Tom Runyon <runyontr@gmail.com>
+  - Ken Sipe <kensipe@gmail.com>
   kudoVersion: 0.2.0
-  kubernetesVersion: 1.5.1
+  kubernetesVersion: 1.15.0

pkg/kudoctl/bundle/testdata/zk/operator.yaml

@@ -1,10 +1,12 @@
 name: "zookeeper"
 version: "0.1.0"
+appVersion: "3.4.10"
 kudoVersion: 0.2.0
-kubernetesVersion: 1.5.1
+kubernetesVersion: 1.15.0
 maintainers:
 - Alena Varkockova <avarkockova@mesosphere.com>
 - Tom Runyon <runyontr@gmail.com>
+- Ken Sipe <kensipe@gmail.com>
 url: https://zookeeper.apache.org/
 tasks:
   infra:

pkg/kudoctl/cmd/package_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/kudobuilder/kudo/pkg/kudoctl/files"
@@ -22,6 +23,8 @@ var packageCmdArgs = []struct {
 
 func TestTableNewBundleCmd(t *testing.T) {
 	fs := afero.NewMemMapFs()
+	testdir, _ := filepath.Abs("")
+	fs.Mkdir(testdir, 0777)
 	files.CopyOperatorToFs(fs, "../bundle/testdata/zk", "/opt")
 	for _, test := range packageCmdArgs {
 		newCmdBundle := newPackageCmd(fs, os.Stdout)

pkg/kudoctl/cmd/repo.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"io"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+)
+
+const repoDesc = `
+	This command consists of multiple sub-commands to interact with KUDO repositories.
+
+	It can be used to add, remove, list, and index kudo repositories.
+	Example usage:
+		$ kubectl kudo repo add [NAME] [REPO_URL]`
+
+// newRepoCmd for repo commands such as building a repo index
+func newRepoCmd(fs afero.Fs, out io.Writer) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "repo [FLAGS] add|remove|list|index| update [ARGS]",
+		Short:   "Package an official KUDO operator.",
+		Long:    `Add, list, remove, update, and index kudo repositories`,
+		Example: repoDesc,
+	}
+
+	cmd.AddCommand(newRepoIndexCmd(fs, out))
+
+	return cmd
+}

pkg/kudoctl/cmd/repo_index.go

@@ -0,0 +1,82 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/kudobuilder/kudo/pkg/kudoctl/files"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/util/repo"
+
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+)
+
+const repoIndexDesc = `
+Read the provided directory and generate an index file based on the packages found.
+
+This tool is used for creating an 'index.yaml' file for a KUDO package repository. To
+set an absolute URL to the operators, use '--url' flag.
+
+# Create an index file for all KUDO packages in the repo-dir.
+	$ kubectl kudo repo index repo-dir`
+
+type repoIndexCmd struct {
+	path      string
+	url       string
+	overwrite bool
+	out       io.Writer
+	fs        afero.Fs
+}
+
+// newRepoIndexCmd for repo commands such as building a repo index file.
+func newRepoIndexCmd(fs afero.Fs, out io.Writer) *cobra.Command {
+	index := &repoIndexCmd{out: out, fs: fs}
+	cmd := &cobra.Command{
+		Use:   "index [flags] <DIR>",
+		Short: "Generate an index file given a directory containing KUDO packages",
+		Long:  repoIndexDesc,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := validateRepoIndex(args); err != nil {
+				return err
+			}
+			index.path = args[0]
+			if err := index.run(); err != nil {
+				return err
+			}
+			return nil
+		},
+		SilenceUsage: true,
+	}
+
+	f := cmd.Flags()
+	f.StringVar(&index.url, "url", "", "URL of the operator repository")
+	f.BoolVarP(&index.overwrite, "overwrite", "o", false, "Overwrite existing package")
+
+	return cmd
+}
+
+func validateRepoIndex(args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("expecting exactly one argument - directory containing the operators to package")
+	}
+	return nil
+}
+
+// run returns the errors associated with cmd env
+func (ri *repoIndexCmd) run() error {
+	target, err := files.FullPathToTarget(ri.fs, ri.path, "index.yaml", ri.overwrite)
+	if err != nil {
+		return err
+	}
+
+	t := time.Now()
+	i, err := repo.IndexDirectory(ri.fs, ri.path, ri.url, &t)
+	if err != nil {
+		return err
+	}
+
+	i.WriteFile(fs, target)
+	fmt.Fprintf(ri.out, "index %v created.\n", target)
+	return nil
+}

pkg/kudoctl/cmd/root.go

@@ -56,6 +56,7 @@ and serves as an API aggregation layer.
 	cmd.AddCommand(newPackageCmd(fs, cmd.OutOrStdout()))
 	cmd.AddCommand(newGetCmd())
 	cmd.AddCommand(newPlanCmd())
+	cmd.AddCommand(newRepoCmd(fs, cmd.OutOrStdout()))
 	cmd.AddCommand(newTestCmd())
 	cmd.AddCommand(newVersionCmd())
 	cmd.PersistentFlags().String("kubeconfig", os.Getenv("HOME")+"/.kube/config", "Path to your Kubernetes configuration file")