stacks: multiple namespace support

cmd/crossplane/main.go

@@ -105,7 +105,7 @@ func main() {
 		kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
 
 	case extManageCmd.FullCommand():
-		log := logging.NewLogrLogger(zl.WithName("stack-manager"))
+		log := logging.NewLogrLogger(zl.WithName(stack.LabelValueStackManager))
 		log.Debug("Starting", "sync-period", syncPeriod.String())
 
 		cfg, err := getRestConfig(*extManageTenantKubeconfig)

go.sum

@@ -81,8 +81,10 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -169,6 +171,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
@@ -196,6 +199,7 @@ github.com/gophercloud/gophercloud v0.6.0 h1:Xb2lcqZtml1XjgYZxbeayEemq7ASbeTp09m
 github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -276,6 +280,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -293,6 +298,7 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
@@ -341,6 +347,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

pkg/controller/stacks/install/installjob.go

@@ -28,7 +28,6 @@ import (
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/api/core/v1"
-	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -232,13 +231,12 @@ func (jc *stackInstallJobCompleter) createJobOutputObject(ctx context.Context, o
 		return nil
 	}
 
-	ns := i.GetNamespace()
-
 	// Modify Stack and StackDefinition resources based on StackInstall
 	isStack := isStackObject(obj)
 	isStackDefinition := !isStack && isStackDefinitionObject(obj)
 
 	if isStack || isStackDefinition {
+		ns := i.GetNamespace()
 		name := i.GetName()
 		if obj.GetName() == "" {
 			obj.SetName(name)
@@ -253,6 +251,9 @@ func (jc *stackInstallJobCompleter) createJobOutputObject(ctx context.Context, o
 			saAnnotationSetter(i.GetServiceAccountAnnotations()),
 		}
 
+		labels := stacks.ParentLabels(i)
+		meta.AddLabels(obj, labels)
+
 		// StackDefinition controllers need the name of the StackDefinition
 		// which, by design, matches the StackInstall
 		if isStackDefinition {
@@ -265,33 +266,13 @@ func (jc *stackInstallJobCompleter) createJobOutputObject(ctx context.Context, o
 		}
 	}
 
-	// We want to clean up any installed CRDS when we're deleted. We can't rely
-	// on garbage collection because a namespaced object (StackInstall) can't
-	// own a cluster scoped object (CustomResourceDefinition), so we use labels
-	// instead.
-	labels := stacks.ParentLabels(i)
-
-	// CRDs are labeled with the namespaces of the stacks they are managed by.
-	// This will allow for a single Namespaced stack to be installed in multiple
-	// namespaces, or different stacks (possibly only differing by versions) to
-	// provide the same CRDs without the risk that a single StackInstall removal
-	// will delete a CRD until there are no remaining namespace labels.
-	if isCRDObject(obj) {
-		labelNamespace := fmt.Sprintf(stacks.LabelNamespaceFmt, ns)
-
-		labels[labelNamespace] = "true"
-	}
-
-	meta.AddLabels(obj, labels)
-
 	jc.log.Debug(
 		"creating object from job output",
 		"job", job.Name,
 		"name", obj.GetName(),
 		"namespace", obj.GetNamespace(),
 		"apiVersion", obj.GetAPIVersion(),
 		"kind", obj.GetKind(),
-		"labels", labels,
 	)
 	if err := jc.client.Create(ctx, obj); err != nil && !kerrors.IsAlreadyExists(err) {
 		return errors.Wrapf(err, "failed to create object %s from job output %s", obj.GetName(), job.Name)
@@ -325,16 +306,6 @@ func isStackDefinitionObject(obj stacks.KindlyIdentifier) bool {
 		strings.EqualFold(gvk.Kind, v1alpha1.StackDefinitionKind)
 }
 
-func isCRDObject(obj runtime.Object) bool {
-	if obj == nil {
-		return false
-	}
-	gvk := obj.GetObjectKind().GroupVersionKind()
-
-	return apiextensions.SchemeGroupVersion == gvk.GroupVersion() &&
-		strings.EqualFold(gvk.Kind, "CustomResourceDefinition")
-}
-
 func setupStackDefinitionController(obj *unstructured.Unstructured, modifiers ...stackSpecModifier) error {
 	if len(modifiers) == 0 {
 		return nil

pkg/controller/stacks/install/installjob_test.go

@@ -29,9 +29,11 @@ import (
 	"github.com/pkg/errors"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -366,13 +368,16 @@ var (
 	_ jobCompleter = &stackInstallJobCompleter{}
 )
 
-func nsLabel(ns string) string {
-	return fmt.Sprintf(stacks.LabelNamespaceFmt, ns)
-}
-
 // Job modifiers
 type jobModifier func(*batchv1.Job)
 
+func withJobName(nn types.NamespacedName) jobModifier {
+	return func(j *batchv1.Job) {
+		j.SetNamespace(nn.Namespace)
+		j.SetName(nn.Name)
+	}
+}
+
 func withJobConditions(jobConditionType batchv1.JobConditionType, message string) jobModifier {
 	return func(j *batchv1.Job) {
 		j.Status.Conditions = []batchv1.JobCondition{
@@ -630,14 +635,6 @@ func TestHandleJobCompletion(t *testing.T) {
 			jc: &stackInstallJobCompleter{
 				client: &test.MockClient{
 					MockCreate: func(ctx context.Context, obj runtime.Object, _ ...client.CreateOption) error {
-						if isCRDObject(obj) {
-							if crd, ok := obj.(*unstructured.Unstructured); ok {
-								if labels := crd.GetLabels(); labels == nil || labels["namespace.crossplane.io/"+namespace] != "true" {
-									return errors.New("expected CRD namespace label")
-								}
-							}
-
-						}
 						return nil
 					},
 					MockGet: func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
@@ -824,6 +821,10 @@ func TestCreate(t *testing.T) {
 		ext    *v1alpha1.StackInstall
 	}
 
+	noJobs := func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
+		return kerrors.NewNotFound(schema.GroupResource{Group: "batch", Resource: "Job"}, key.String())
+	}
+
 	tests := []struct {
 		name    string
 		handler *stackInstallHandler
@@ -839,6 +840,7 @@ func TestCreate(t *testing.T) {
 					MockStatusUpdate: func(ctx context.Context, obj runtime.Object, _ ...client.UpdateOption) error { return nil },
 				},
 				hostKube: &test.MockClient{
+					MockGet:    noJobs,
 					MockCreate: func(ctx context.Context, obj runtime.Object, _ ...client.CreateOption) error { return nil },
 				},
 				executorInfo: &stacks.ExecutorInfo{Image: stackPackageImage},
@@ -859,12 +861,45 @@ func TestCreate(t *testing.T) {
 			name: "CreateInstallJobHosted",
 			handler: &stackInstallHandler{
 				kube: &test.MockClient{
+
 					MockPatch: func(_ context.Context, obj runtime.Object, patch client.Patch, _ ...client.PatchOption) error {
 						return nil
 					},
 					MockStatusUpdate: func(ctx context.Context, obj runtime.Object, _ ...client.UpdateOption) error { return nil },
 				},
 				hostKube: &test.MockClient{
+					MockGet:    noJobs,
+					MockCreate: func(ctx context.Context, obj runtime.Object, _ ...client.CreateOption) error { return nil },
+				},
+				hostAwareConfig: &hosted.Config{HostControllerNamespace: hostControllerNamespace},
+				executorInfo:    &stacks.ExecutorInfo{Image: stackPackageImage},
+				ext:             resource(),
+				log:             logging.NewNopLogger(),
+			},
+			want: want{
+				result: requeueOnSuccess,
+				err:    nil,
+				ext: resource(
+					withFinalizers(installFinalizer),
+					withConditions(runtimev1alpha1.Creating(), runtimev1alpha1.ReconcileSuccess()),
+					withInstallJob(&corev1.ObjectReference{Name: fmt.Sprintf("%s.%s", namespace, resourceName), Namespace: hostControllerNamespace}),
+				),
+			},
+		},
+		{
+			name: "ExistingInstallJobHosted",
+			handler: &stackInstallHandler{
+				kube: &test.MockClient{
+					MockPatch: func(_ context.Context, obj runtime.Object, patch client.Patch, _ ...client.PatchOption) error {
+						return nil
+					},
+					MockStatusUpdate: func(ctx context.Context, obj runtime.Object, _ ...client.UpdateOption) error { return nil },
+				},
+				hostKube: &test.MockClient{
+					MockGet: func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
+						*obj.(*batchv1.Job) = *(job(withJobName(types.NamespacedName{Name: fmt.Sprintf("%s.%s", namespace, resourceName), Namespace: hostControllerNamespace})))
+						return nil
+					},
 					MockCreate: func(ctx context.Context, obj runtime.Object, _ ...client.CreateOption) error { return nil },
 				},
 				hostAwareConfig: &hosted.Config{HostControllerNamespace: hostControllerNamespace},
@@ -892,6 +927,7 @@ func TestCreate(t *testing.T) {
 					MockStatusUpdate: func(ctx context.Context, obj runtime.Object, _ ...client.UpdateOption) error { return nil },
 				},
 				hostKube: &test.MockClient{
+					MockGet:    noJobs,
 					MockCreate: func(ctx context.Context, obj runtime.Object, _ ...client.CreateOption) error { return errBoom },
 				},
 				hostAwareConfig: &hosted.Config{HostControllerNamespace: hostControllerNamespace},
@@ -1117,7 +1153,6 @@ func TestCreateJobOutputObject(t *testing.T) {
 				err: errors.Wrapf(errBoom, "failed to create object %s from job output %s", crdName, resourceName),
 				obj: unstructuredObj(crdRaw,
 					withUnstructuredObjLabels(wantedParentLabels),
-					withUnstructuredObjLabels(map[string]string{nsLabel(namespace): "true"}),
 				),
 			},
 		},
@@ -1134,10 +1169,7 @@ func TestCreateJobOutputObject(t *testing.T) {
 			obj:            unstructuredObj(crdRaw),
 			want: want{
 				err: nil,
-				obj: unstructuredObj(crdRaw,
-					withUnstructuredObjLabels(wantedParentLabels),
-					withUnstructuredObjLabels(map[string]string{nsLabel(namespace): "true"}),
-				),
+				obj: unstructuredObj(crdRaw),
 			},
 		},
 		{