[WIP] Wait for cr from pr

changelogs/unreleased/1937-nrb

@@ -0,0 +1 @@
+Wait for CustomResourceDefinitions to be ready before restoring CustomResources

pkg/builder/customresourcedefinition_builder.go

@@ -41,16 +41,51 @@ func ForCustomResourceDefinition(name string) *CustomResourceDefinitionBuilder {
 	}
 }
 
+// Condition adds a CustomResourceDefinitionCondition objects to a CustomResourceDefinitionBuilder.
+func (c *CustomResourceDefinitionBuilder) Condition(cond apiextv1beta1.CustomResourceDefinitionCondition) *CustomResourceDefinitionBuilder {
+	c.object.Status.Conditions = append(c.object.Status.Conditions, cond)
+	return c
+}
+
 // Result returns the built CustomResourceDefinition.
 func (b *CustomResourceDefinitionBuilder) Result() *apiextv1beta1.CustomResourceDefinition {
 	return b.object
 }
 
-// ObjectMeta applies functional options to the Namespace's ObjectMeta.
+// ObjectMeta applies functional options to the CustomResourceDefinition's ObjectMeta.
 func (b *CustomResourceDefinitionBuilder) ObjectMeta(opts ...ObjectMetaOpt) *CustomResourceDefinitionBuilder {
 	for _, opt := range opts {
 		opt(b.object)
 	}
 
 	return b
 }
+
+// CustomResourceDefinitionConditionBuilder builds CustomResourceDefinitionCondition objects.
+type CustomResourceDefinitionConditionBuilder struct {
+	object apiextv1beta1.CustomResourceDefinitionCondition
+}
+
+// ForCustomResourceDefinitionConditionBuilder is the construction for a CustomResourceDefinitionConditionBuilder.
+func ForCustomResourceDefinitionCondition() *CustomResourceDefinitionConditionBuilder {
+	return &CustomResourceDefinitionConditionBuilder{
+		object: apiextv1beta1.CustomResourceDefinitionCondition{},
+	}
+}
+
+// Type sets the Condition's type.
+func (c *CustomResourceDefinitionConditionBuilder) Type(t apiextv1beta1.CustomResourceDefinitionConditionType) *CustomResourceDefinitionConditionBuilder {
+	c.object.Type = t
+	return c
+}
+
+// Status sets the Condition's status.
+func (c *CustomResourceDefinitionConditionBuilder) Status(cs apiextv1beta1.ConditionStatus) *CustomResourceDefinitionConditionBuilder {
+	c.object.Status = cs
+	return c
+}
+
+// Results returns the built CustomResourceDefinitionCondition.
+func (b *CustomResourceDefinitionConditionBuilder) Result() apiextv1beta1.CustomResourceDefinitionCondition {
+	return b.object
+}

pkg/install/install.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/vmware-tanzu/velero/pkg/client"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
 )
 
 // kindToResource translates a Kind (mixed case, singular) to a Resource (lowercase, plural) string.
@@ -56,21 +57,6 @@ type ResourceGroup struct {
 	OtherResources []*unstructured.Unstructured
 }
 
-// crdIsReady checks a CRD to see if it's ready, so that objects may be created from it.
-func crdIsReady(crd *apiextv1beta1.CustomResourceDefinition) bool {
-	var isEstablished, namesAccepted bool
-	for _, cond := range crd.Status.Conditions {
-		if cond.Type == apiextv1beta1.Established {
-			isEstablished = true
-		}
-		if cond.Type == apiextv1beta1.NamesAccepted {
-			namesAccepted = true
-		}
-	}
-
-	return (isEstablished && namesAccepted)
-}
-
 // crdsAreReady polls the API server to see if the BackupStorageLocation and VolumeSnapshotLocation CRDs are ready to create objects.
 func crdsAreReady(factory client.DynamicFactory, crdKinds []string) (bool, error) {
 	gvk := schema.FromAPIVersionAndKind(apiextv1beta1.SchemeGroupVersion.String(), "CustomResourceDefinition")
@@ -108,7 +94,7 @@ func crdsAreReady(factory client.DynamicFactory, crdKinds []string) (bool, error
 		}
 
 		for _, crd := range foundCRDs {
-			if !crdIsReady(crd) {
+			if !kube.IsCRDReady(crd) {
 				return false, nil
 			}
 

pkg/kuberesource/kuberesource.go

@@ -21,12 +21,13 @@ import (
 )
 
 var (
-	ClusterRoleBindings    = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterrolebindings"}
-	ClusterRoles           = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterroles"}
-	Jobs                   = schema.GroupResource{Group: "batch", Resource: "jobs"}
-	Namespaces             = schema.GroupResource{Group: "", Resource: "namespaces"}
-	PersistentVolumeClaims = schema.GroupResource{Group: "", Resource: "persistentvolumeclaims"}
-	PersistentVolumes      = schema.GroupResource{Group: "", Resource: "persistentvolumes"}
-	Pods                   = schema.GroupResource{Group: "", Resource: "pods"}
-	ServiceAccounts        = schema.GroupResource{Group: "", Resource: "serviceaccounts"}
+	ClusterRoleBindings       = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterrolebindings"}
+	ClusterRoles              = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterroles"}
+	CustomResourceDefinitions = schema.GroupResource{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}
+	Jobs                      = schema.GroupResource{Group: "batch", Resource: "jobs"}
+	Namespaces                = schema.GroupResource{Group: "", Resource: "namespaces"}
+	PersistentVolumeClaims    = schema.GroupResource{Group: "", Resource: "persistentvolumeclaims"}
+	PersistentVolumes         = schema.GroupResource{Group: "", Resource: "persistentvolumes"}
+	Pods                      = schema.GroupResource{Group: "", Resource: "pods"}
+	ServiceAccounts           = schema.GroupResource{Group: "", Resource: "serviceaccounts"}
 )

pkg/restore/restore.go

@@ -31,6 +31,7 @@ import (
 	uuid "github.com/satori/go.uuid"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
+	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -653,6 +654,41 @@ func (ctx *context) shouldRestore(name string, pvClient client.Dynamic) (bool, e
 	return shouldRestore, err
 }
 
+// crdAvailable waits for a CRD to be available for use before letting the restore continue.
+func (ctx *context) crdAvailable(name string, crdClient client.Dynamic) (bool, error) {
+	crdLogger := ctx.log.WithField("crdName", name)
+
+	var available bool
+	// Wait 1 minute rather than the standard resource timeout, since each CRD will transition fairly quickly
+	err := wait.PollImmediate(time.Second, time.Minute*1, func() (bool, error) {
+		unstructuredCRD, err := crdClient.Get(name, metav1.GetOptions{})
+		if err != nil {
+			return true, err
+		}
+
+		clusterCRD := new(apiextv1beta1.CustomResourceDefinition)
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredCRD.Object, clusterCRD); err != nil {
+			return true, err
+		}
+
+		available = kube.IsCRDReady(clusterCRD)
+
+		if !available {
+			crdLogger.Debug("CRD not yet ready for use")
+		}
+
+		// If the CRD is not available, keep polling (false, nil)
+		// If the CRD is available, break the poll and return back to caller (true, nil)
+		return available, nil
+	})
+
+	if err == wait.ErrWaitTimeout {
+		crdLogger.Debug("timeout reached waiting for custom resource definition to be ready")
+	}
+
+	return available, err
+}
+
 // restoreResource restores the specified cluster or namespace scoped resource. If namespace is
 // empty we are restoring a cluster level resource, otherwise into the specified namespace.
 func (ctx *context) restoreResource(resource, targetNamespace, originalNamespace string, items []string) (Result, Result) {
@@ -1085,6 +1121,17 @@ func (ctx *context) restoreItem(obj *unstructured.Unstructured, groupResource sc
 		restorePodVolumeBackups(ctx, createdObj, originalNamespace)
 	}
 
+	// Wait for a CRD to be available for instantiating resources
+	// before continuing.
+	if groupResource == kuberesource.CustomResourceDefinitions {
+		available, err := ctx.crdAvailable(name, resourceClient)
+		if err != nil {
+			addToResult(&errs, namespace, errors.Wrapf(err, "error verifying custom resource definition is ready to use"))
+		} else if !available {
+			addToResult(&errs, namespace, fmt.Errorf("CRD %s is not available to use for custom resources.", name))
+		}
+	}
+
 	return warnings, errs
 }
 

pkg/util/kube/utils.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2017 the Velero contributors.
+Copyright 2017, 2019 the Velero contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	corev1api "k8s.io/api/core/v1"
+	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -135,3 +136,18 @@ func GetVolumeDirectory(pod *corev1api.Pod, volumeName string, pvcLister corev1l
 
 	return pvc.Spec.VolumeName, nil
 }
+
+// IsCRDReady checks a CRD to see if it's ready, with both the Established and NamesAccepted conditions.
+func IsCRDReady(crd *apiextv1beta1.CustomResourceDefinition) bool {
+	var isEstablished, namesAccepted bool
+	for _, cond := range crd.Status.Conditions {
+		if cond.Type == apiextv1beta1.Established && cond.Status == apiextv1beta1.ConditionTrue {
+			isEstablished = true
+		}
+		if cond.Type == apiextv1beta1.NamesAccepted && cond.Status == apiextv1beta1.ConditionTrue {
+			namesAccepted = true
+		}
+	}
+
+	return (isEstablished && namesAccepted)
+}

pkg/util/kube/utils_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -197,3 +198,42 @@ func TestGetVolumeDirectorySuccess(t *testing.T) {
 		assert.Equal(t, tc.want, dir)
 	}
 }
+
+func TestIsCRDReady(t *testing.T) {
+	tests := []struct {
+		name string
+		crd  *apiextv1beta1.CustomResourceDefinition
+		want bool
+	}{
+		{
+			name: "CRD is not established & not accepting names - not ready",
+			crd:  builder.ForCustomResourceDefinition("MyCRD").Result(),
+			want: false,
+		},
+		{
+			name: "CRD is established & not accepting names - not ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).Result(),
+			want: false,
+		},
+		{
+			name: "CRD is not established & accepting names - not ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).Result(),
+			want: false,
+		},
+		{
+			name: "CRD is established & accepting names - ready",
+			crd: builder.ForCustomResourceDefinition("MyCRD").
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).
+				Condition(builder.ForCustomResourceDefinitionCondition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).
+				Result(),
+			want: true,
+		},
+	}
+
+	for _, tc := range tests {
+		result := IsCRDReady(tc.crd)
+		assert.Equal(t, tc.want, result)
+	}
+}