KEP-29: Install dependencies of operators.

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/stretchr/testify v1.5.1
 	github.com/thoas/go-funk v0.6.0
 	github.com/xlab/treeprint v1.0.0
+	github.com/yourbasic/graph v0.0.0-20170921192928-40eb135c0b26
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	golang.org/x/sys v0.0.0-20200408040146-ea54a3c99b9b // indirect

go.sum

@@ -506,6 +506,8 @@ github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSf
 github.com/xlab/treeprint v1.0.0 h1:J0TkWtiuYgtdlrkkrDLISYBQ92M+X5m4LrIIMKrbDTs=
 github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yourbasic/graph v0.0.0-20170921192928-40eb135c0b26 h1:4u7nCRnWizT8R6xOP7cGaq+Ov0oBGkKMsLWZKiwDFas=
+github.com/yourbasic/graph v0.0.0-20170921192928-40eb135c0b26/go.mod h1:Rfzr+sqaDreiCaoQbFCu3sTXxeFq/9kXRuyOoSlGQHE=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=

pkg/kudoctl/cmd/install/install.go

@@ -95,5 +95,6 @@ func installOperator(operatorArgument string, options *Options, fs afero.Fs, set
 		settings.Namespace,
 		*pkg.Resources,
 		options.Parameters,
+		resolver,
 		installOpts)
 }

pkg/kudoctl/packages/install/dependencies.go

@@ -0,0 +1,146 @@
+package install
+
+import (
+	"fmt"
+
+	"github.com/thoas/go-funk"
+	"github.com/yourbasic/graph"
+
+	"github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
+	engtask "github.com/kudobuilder/kudo/pkg/engine/task"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/clog"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/packages"
+	pkgresolver "github.com/kudobuilder/kudo/pkg/kudoctl/packages/resolver"
+)
+
+// dependencyGraph is modeled after 'graph.Mutable' but allows to add vertices.
+type dependencyGraph struct {
+	edges []map[int]struct{}
+}
+
+// AddVertex adds a new vertex to the dependency graph.
+func (g *dependencyGraph) AddVertex() {
+	g.edges = append(g.edges, map[int]struct{}{})
+}
+
+// AddEdge adds an edge from vertex v to w to the dependency graph.
+func (g *dependencyGraph) AddEdge(v, w int) {
+	g.edges[v][w] = struct{}{}
+}
+
+// Order returns the number of vertices of the dependency graph.
+func (g *dependencyGraph) Order() int {
+	return len(g.edges)
+}
+
+func (g *dependencyGraph) Visit(v int, do func(w int, c int64) bool) bool {
+	for w := range g.edges[v] {
+		if do(w, 1) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// gatherDependencies resolved all dependencies of a package.
+// Dependencies are resolved recursively.
+// Cyclic dependencies are detected and result in an error.
+func gatherDependencies(root packages.Resources, resolver pkgresolver.Resolver) ([]packages.Resources, error) {
+	pkgs := []packages.Resources{
+		root,
+	}
+
+	// Each vertex in 'g' matches an index in 'pkgs'.
+	g := dependencyGraph{
+		edges: []map[int]struct{}{{}},
+	}
+
+	if err := dependencyWalk(&pkgs, &g, root, resolver); err != nil {
+		return nil, err
+	}
+
+	// Remove 'root' from the list of dependencies.
+	pkgs = funk.Drop(pkgs, 1).([]packages.Resources) //nolint:errcheck
+
+	return pkgs, nil
+}
+
+func dependencyWalk(
+	pkgs *[]packages.Resources,
+	g *dependencyGraph,
+	parent packages.Resources,
+	resolver pkgresolver.Resolver) error {
+	//nolint:errcheck
+	operatorTasks := funk.Filter(parent.OperatorVersion.Spec.Tasks, func(task v1beta1.Task) bool {
+		return task.Kind == engtask.KudoOperatorTaskKind
+	}).([]v1beta1.Task)
+
+	versionOf := func(pkg packages.Resources) func(packages.Resources) bool {
+		return func(r packages.Resources) bool {
+			return r.Operator.Name == pkg.Operator.Name &&
+				r.OperatorVersion.Spec.AppVersion == pkg.OperatorVersion.Spec.AppVersion &&
+				r.OperatorVersion.Spec.Version == pkg.OperatorVersion.Spec.Version
+		}
+	}
+
+	for _, task := range operatorTasks {
+		pkg, err := resolver.Resolve(
+			task.Spec.Package,
+			task.Spec.AppVersion,
+			task.Spec.OperatorVersion)
+		if err != nil {
+			return fmt.Errorf(
+				"failed to resolve package %s-%s-%s, dependency of package %s-%s-%s: %v",
+				task.Spec.Package,
+				task.Spec.AppVersion,
+				task.Spec.OperatorVersion,
+				parent.Operator.Name,
+				parent.OperatorVersion.Spec.AppVersion,
+				parent.OperatorVersion.Spec.Version,
+				err)
+		}
+
+		parentIndex := funk.IndexOf(*pkgs, funk.Find(*pkgs, versionOf(parent)))
+		if parentIndex == -1 {
+			panic("failed to find parent index in dependency graph")
+		}
+
+		if funk.Find(*pkgs, versionOf(*pkg.Resources)) == nil {
+			clog.Printf(
+				"Adding new dependency %s-%s-%s",
+				pkg.Resources.Operator.Name,
+				pkg.Resources.OperatorVersion.Spec.AppVersion,
+				pkg.Resources.OperatorVersion.Spec.Version)
+
+			*pkgs = append(*pkgs, *pkg.Resources)
+
+			// The number of vertices in 'g' has to match the number of packages
+			// we're tracking.
+			g.AddVertex()
+		}
+
+		pkgIndex := funk.IndexOf(*pkgs, funk.Find(*pkgs, versionOf(*pkg.Resources)))
+		if pkgIndex == -1 {
+			panic("failed to find package index in dependency graph")
+		}
+
+		// This is a directed graph. The edge represents a dependency of
+		// the parent package on the current package.
+		g.AddEdge(parentIndex, pkgIndex)
+
+		if !graph.Acyclic(g) {
+			return fmt.Errorf(
+				"cyclic package dependency found when adding package %s-%s-%s",
+				pkg.Resources.Operator.Name,
+				pkg.Resources.OperatorVersion.Spec.AppVersion,
+				pkg.Resources.OperatorVersion.Spec.Version)
+		}
+
+		if err := dependencyWalk(pkgs, g, *pkg.Resources, resolver); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/kudoctl/packages/install/dependencies_test.go

@@ -0,0 +1,157 @@
+package install
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/thoas/go-funk"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
+	engtask "github.com/kudobuilder/kudo/pkg/engine/task"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/packages"
+)
+
+type nameResolver struct {
+	Pkgs []packages.Package
+}
+
+func (resolver nameResolver) Resolve(
+	name string,
+	appVersion string,
+	operatorVersion string) (*packages.Package, error) {
+	for _, pkg := range resolver.Pkgs {
+		if pkg.Resources.Operator.Name == name {
+			return &pkg, nil
+		}
+	}
+
+	return nil, fmt.Errorf("package not found")
+}
+
+func createPackage(name string, dependencies ...string) packages.Package {
+	p := packages.Package{
+		Resources: &packages.Resources{
+			Operator: &v1beta1.Operator{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: name,
+				},
+			},
+			OperatorVersion: &v1beta1.OperatorVersion{},
+		},
+	}
+
+	for _, dependency := range dependencies {
+		p.Resources.OperatorVersion.Spec.Tasks = append(
+			p.Resources.OperatorVersion.Spec.Tasks,
+			v1beta1.Task{
+				Name: "dependency",
+				Kind: engtask.KudoOperatorTaskKind,
+				Spec: v1beta1.TaskSpec{
+					KudoOperatorTaskSpec: v1beta1.KudoOperatorTaskSpec{
+						Package: dependency,
+					},
+				},
+			})
+	}
+
+	return p
+}
+
+func TestGatherDependencies(t *testing.T) {
+	tests := []struct {
+		name         string
+		pkgs         []packages.Package
+		expectedDeps []string
+		expectedErr  string
+	}{
+		{
+			// A <---> A
+			"trivial circular dependency",
+			[]packages.Package{
+				createPackage("A", "A"),
+			},
+			[]string{},
+			"cyclic package dependency found when adding package A--",
+		},
+		{
+			// A <---> B
+			"circular dependency",
+			[]packages.Package{
+				createPackage("A", "B"),
+				createPackage("B", "A"),
+			},
+			[]string{},
+			"cyclic package dependency found when adding package A--",
+		},
+		{
+			// A ---> (B)
+			"unknown dependency",
+			[]packages.Package{
+				createPackage("A", "B"),
+			},
+			[]string{},
+			"failed to resolve package B--, dependency of package A--: package not found",
+		},
+		{
+			// A ---> B ---> C
+			"simple dependency",
+			[]packages.Package{
+				createPackage("A", "B"),
+				createPackage("B", "C"),
+				createPackage("C"),
+			},
+			[]string{"B", "C"},
+			"",
+		},
+		{
+			//        B -----
+			//        |      \
+			//        |      |
+			//        v      |
+			// A ---> C      |
+			// |      |      |
+			// |      |      |
+			// \      v      v
+			//  ----> D ---> E
+			"complex dependency",
+			[]packages.Package{
+				createPackage("A", "C", "D"),
+				createPackage("B", "C", "E"),
+				createPackage("C", "D"),
+				createPackage("D", "E"),
+				createPackage("E"),
+			},
+			[]string{"C", "D", "E"},
+			"",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+
+		resolver := nameResolver{test.pkgs}
+
+		actual, err := gatherDependencies(*test.pkgs[0].Resources, resolver)
+		if err != nil {
+			if test.expectedErr == "" {
+				t.Errorf("%s: expected no error but got %v", test.name, err)
+			}
+
+			assert.EqualError(t, err, test.expectedErr, test.name)
+		} else {
+			if test.expectedErr != "" {
+				t.Errorf("%s: expected an error but got none", test.name)
+			}
+
+			for _, operatorName := range test.expectedDeps {
+				operatorName := operatorName
+
+				assert.NotNil(t, funk.Find(actual, func(p packages.Resources) bool {
+					return p.Operator.Name == operatorName
+				}), test.name)
+			}
+		}
+	}
+}

pkg/kudoctl/packages/install/package.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/clog"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/packages"
+	"github.com/kudobuilder/kudo/pkg/kudoctl/packages/resolver"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/util/kudo"
 )
 
@@ -23,12 +24,16 @@ type Options struct {
 // Instance name, namespace and operator parameters are applied to the
 // operator package resources. These rendered resources are then created
 // on the Kubernetes cluster.
+// Packages can have dependencies on other packages. In that case,
+// dependent packages are resolved and their Operator and
+// Operatorversion resources created on the Kubernetes cluster.
 func Package(
 	client *kudo.Client,
 	instanceName string,
 	namespace string,
 	resources packages.Resources,
 	parameters map[string]string,
+	resolver resolver.Resolver,
 	options Options) error {
 	clog.V(3).Printf(
 		"Preparing %s/%s:%s for installation",
@@ -54,6 +59,20 @@ func Package(
 		}
 	}
 
+	dependencies, err := gatherDependencies(resources, resolver)
+	if err != nil {
+		return err
+	}
+
+	for _, dependency := range dependencies {
+		dependency.Operator.SetNamespace(namespace)
+		dependency.OperatorVersion.SetNamespace(namespace)
+
+		if err := installOperatorAndOperatorVersion(client, dependency); err != nil {
+			return err
+		}
+	}
+
 	if err := installOperatorAndOperatorVersion(client, resources); err != nil {
 		return err
 	}

pkg/kudoctl/packages/install/package_test.go

@@ -101,7 +101,7 @@ func Test_InstallPackage(t *testing.T) {
 			SkipInstance: tt.skipInstance,
 		}
 
-		err := Package(kc, "", namespace, testResources, tt.installParameters, options)
+		err := Package(kc, "", namespace, testResources, tt.installParameters, nil, options)
 		if tt.err != "" {
 			assert.EqualError(t, err, tt.err)
 		}

pkg/kudoctl/packages/resolver/resolver.go

@@ -10,7 +10,7 @@ import (
 // Resolver will try to resolve a given package name to either local tarball, folder, remote url or
 // an operator in the remote repository.
 type Resolver interface {
-	Resolve(name string, version string) (*packages.Package, error)
+	Resolve(name string, appVersion string, operatorVersion string) (*packages.Package, error)
 }
 
 // PackageResolver is the source of resolver of operator packages.