Enable OperatorCondition overrides for OperatorUpgradeable

Cluster Admin will be able to override the OperatorUpgradeable
condition.

Add an e2e test for OperatorUpgradeable condition.

Signed-off-by: Vu Dinh <vdinh@redhat.com>

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/onsi/gomega v1.9.0
 	github.com/openshift/api v0.0.0-20200331152225-585af27e34fd
 	github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0
-	github.com/operator-framework/api v0.3.23
+	github.com/operator-framework/api v0.3.25
 	github.com/operator-framework/operator-registry v1.13.6
 	github.com/otiai10/copy v1.2.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -744,8 +744,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/operator-framework/api v0.3.7-0.20200602203552-431198de9fc2/go.mod h1:Xbje9x0SHmh0nihE21kpesB38vk3cyxnE6JdDS8Jo1Q=
-github.com/operator-framework/api v0.3.23 h1:+QiJ0m5SjfV+iMv8uzuGac+PoJFlTaMFIN8eVccUnoY=
-github.com/operator-framework/api v0.3.23/go.mod h1:GVNiB6AQucwdZz3ZFXNv9HtcLOzcFnr6O/QldzKG93g=
+github.com/operator-framework/api v0.3.25 h1:d6WgHCshCffT37okVZeL+IbGlhrsHy57xdfMnopC8rI=
+github.com/operator-framework/api v0.3.25/go.mod h1:GVNiB6AQucwdZz3ZFXNv9HtcLOzcFnr6O/QldzKG93g=
 github.com/operator-framework/operator-registry v1.13.6 h1:h/dIjQQS7uneQNRifrSz7h0xg4Xyjg6C9f6XZofbMPg=
 github.com/operator-framework/operator-registry v1.13.6/go.mod h1:YhnIzOVjRU2ZwZtzt+fjcjW8ujJaSFynBEu7QVKaSdU=
 github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=

pkg/controller/operators/olm/operator.go

@@ -220,6 +220,22 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat
 			return nil, err
 		}
 
+		// Register OperatorCondition QueueInformer
+		opConditionInformer := extInformerFactory.Operators().V1().OperatorConditions()
+		op.lister.OperatorsV1().RegisterOperatorConditionLister(namespace, opConditionInformer.Lister())
+		opConditionQueueInformer, err := queueinformer.NewQueueInformer(
+			ctx,
+			queueinformer.WithLogger(op.logger),
+			queueinformer.WithInformer(opConditionInformer.Informer()),
+			queueinformer.WithSyncer(k8sSyncer),
+		)
+		if err != nil {
+			return nil, err
+		}
+		if err := op.RegisterQueueInformer(opConditionQueueInformer); err != nil {
+			return nil, err
+		}
+
 		subInformer := extInformerFactory.Operators().V1alpha1().Subscriptions()
 		op.lister.OperatorsV1alpha1().RegisterSubscriptionLister(namespace, subInformer.Lister())
 		subQueueInformer, err := queueinformer.NewQueueInformer(
@@ -452,22 +468,6 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat
 		return nil, err
 	}
 
-	// Register OperatorCondition QueueInformer
-	opConditionInformer := extInformerFactory.Operators().V1().OperatorConditions()
-	op.lister.OperatorsV1().RegisterOperatorConditionLister(namespace, opConditionInformer.Lister())
-	opConditionQueueInformer, err := queueinformer.NewQueueInformer(
-		ctx,
-		queueinformer.WithLogger(op.logger),
-		queueinformer.WithInformer(opConditionInformer.Informer()),
-		queueinformer.WithSyncer(k8sSyncer),
-	)
-	if err != nil {
-		return nil, err
-	}
-	if err := op.RegisterQueueInformer(opConditionInformer); err != nil {
-		return nil, err
-	}
-
 	// setup proxy env var injection policies
 	discovery := config.operatorClient.KubernetesInterface().Discovery()
 	proxyAPIExists, err := proxy.IsAPIAvailable(discovery)
@@ -1774,12 +1774,6 @@ func (a *Operator) updateInstallStatus(csv *v1alpha1.ClusterServiceVersion, inst
 		return nil
 	}
 
-	// Only issue warning for OperatorCondition error for now
-	// TODO: Determine a new CSV reason to reflect OperatorCondition failure
-	if condErr != nil {
-		a.logger.Warn(condErr.Error())
-	}
-
 	// installcheck determined we can't progress (e.g. deployment failed to come up in time)
 	if install.IsErrorUnrecoverable(strategyErr) {
 		csv.SetPhaseWithEventIfChanged(v1alpha1.CSVPhaseFailed, v1alpha1.CSVReasonInstallCheckFailed, fmt.Sprintf("install failed: %s", strategyErr), now, a.recorder)
@@ -1824,6 +1818,11 @@ func (a *Operator) updateInstallStatus(csv *v1alpha1.ClusterServiceVersion, inst
 		return strategyErr
 	}
 
+	if condErr != nil {
+		a.logger.WithError(condErr).Debug("operator is not upgradeable")
+		return condErr
+	}
+
 	return nil
 }
 

pkg/controller/operators/olm/operatorconditions.go

@@ -1,13 +1,13 @@
 package olm
 
 import (
-	"context"
 	"fmt"
 
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	meta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	operatorv1 "github.com/operator-framework/api/pkg/operators/v1"
+	"github.com/operator-framework/api/pkg/operators/v1"
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
 )
 
@@ -16,20 +16,33 @@ func (a *Operator) isOperatorUpgradeable(csv *v1alpha1.ClusterServiceVersion) (b
 		return false, fmt.Errorf("CSV is invalid")
 	}
 
-	cond, err := a.lister.OperatorsV1().OperatorConditions(csv.GetNamespace()).Get(context.TODO(), csv.GetName(), metav1.GetOptions{})
+	cond, err := a.lister.OperatorsV1().OperatorConditionLister().OperatorConditions(csv.GetNamespace()).Get(csv.GetName())
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
 			return true, nil
 		}
 		return false, err
 	}
 
+	// Check condition overrides
+	for _, override := range cond.Spec.Overrides {
+		if override.Type == v1.OperatorUpgradeable {
+			if override.Status == metav1.ConditionTrue {
+				return true, nil
+			}
+			return false, fmt.Errorf("The operator is not upgradeable: %s", override.Message)
+		}
+	}
+
+	// If no condition, proceed with normal flow
 	if len(cond.Status.Conditions) < 1 {
-		return false, fmt.Errorf("No operator conditions are available")
+		return true, nil
+	}
+	if c := meta.FindStatusCondition(cond.Status.Conditions, v1.OperatorUpgradeable); c != nil {
+		if c.Status == metav1.ConditionFalse {
+			return false, fmt.Errorf("The operator is not upgradeable: %s", c.Message)
+		}
 	}
-  if upgradeCond := cond.GetCondition(operatorv1.OperatorUpgradeable); upgradeCond.Status == corev1.ConditionFalse {
-    return false, fmt.Errorf("The operator is not upgradeable: %s", upgradeCond.Message)
-  }
 
-  return true, nil
+	return true, nil
 }

pkg/lib/operatorclient/client.go

@@ -47,6 +47,8 @@ type CustomResourceClient interface {
 	DeleteCustomResource(apiGroup, version, namespace, resourceKind, resourceName string) error
 	AtomicModifyCustomResource(apiGroup, version, namespace, resourceKind, resourceName string, f CustomResourceModifier, data interface{}) error
 	ListCustomResource(apiGroup, version, namespace, resourceKind string) (*CustomResourceList, error)
+	UpdateCustomResourceStatus(item *unstructured.Unstructured) error
+	UpdateCustomResourceStatusRaw(apiGroup, version, namespace, resourcePlural, resourceName string, data []byte) error
 }
 
 // APIServiceClient contains methods for manipulating APIServiceBindings.

pkg/lib/operatorclient/customresources.go

@@ -154,6 +154,49 @@ func (c *Client) UpdateCustomResourceRaw(apiGroup, version, namespace, resourceP
 	return nil
 }
 
+// UpdateCustomResourceStatus updates the custom resource's status.
+func (c *Client) UpdateCustomResourceStatus(item *unstructured.Unstructured) error {
+	klog.V(4).Infof("[UPDATE CUSTOM RESOURCE STATUS]: %s:%s", item.GetNamespace(), item.GetName())
+	kind := item.GetKind()
+	name := item.GetName()
+	namespace := item.GetNamespace()
+	apiVersion := item.GetAPIVersion()
+	apiGroup, version, err := parseAPIVersion(apiVersion)
+	if err != nil {
+		return err
+	}
+
+	data, err := json.Marshal(item)
+	if err != nil {
+		return err
+	}
+
+	return c.UpdateCustomResourceStatusRaw(apiGroup, version, namespace, kind, name, data)
+}
+
+// UpdateCustomResourceStatusRaw updates the thirdparty resource with the raw data.
+func (c *Client) UpdateCustomResourceStatusRaw(apiGroup, version, namespace, resourcePlural, resourceName string, data []byte) error {
+	klog.V(4).Infof("[UPDATE CUSTOM RESOURCE STATUS RAW]: %s:%s", namespace, resourceName)
+	var statusCode int
+
+	httpRestClient := c.extInterface.ApiextensionsV1beta1().RESTClient()
+	uri := customResourceStatusURI(apiGroup, version, namespace, resourcePlural, resourceName)
+	klog.V(4).Infof("[PUT]: %s", uri)
+	result := httpRestClient.Put().RequestURI(uri).Body(data).Do(context.TODO())
+
+	if result.Error() != nil {
+		return result.Error()
+	}
+
+	result.StatusCode(&statusCode)
+	klog.V(4).Infof("Updated %s, status: %d", uri, statusCode)
+
+	if statusCode != 200 {
+		return fmt.Errorf("unexpected status code %d, expecting 200", statusCode)
+	}
+	return nil
+}
+
 // CreateOrUpdateCustomeResourceRaw creates the custom resource if it doesn't exist.
 // If the custom resource exists, it updates the existing one.
 func (c *Client) CreateOrUpdateCustomeResourceRaw(apiGroup, version, namespace, resourcePlural, resourceName string, data []byte) error {
@@ -254,6 +297,20 @@ func customResourceURI(apiGroup, version, namespace, resourcePlural, resourceNam
 		strings.ToLower(resourceName))
 }
 
+// customResourceStatusURI returns the URI for the thirdparty resource's status.
+func customResourceStatusURI(apiGroup, version, namespace, resourcePlural, resourceName string) string {
+	if namespace == "" {
+		namespace = metav1.NamespaceDefault
+	}
+
+	return fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s/%s/status",
+		strings.ToLower(apiGroup),
+		strings.ToLower(version),
+		strings.ToLower(namespace),
+		strings.ToLower(resourcePlural),
+		strings.ToLower(resourceName))
+}
+
 // customResourceDefinitionURI returns the URI for the CRD.
 //
 // Example of apiGroup: "tco.coreos.com"

pkg/lib/operatorlister/operatorcondition.go

@@ -18,12 +18,12 @@ type UnionOperatorConditionLister struct {
 }
 
 // List lists all OperatorConditions in the indexer.
-func (uol *UnionOperatorConditionLister) List(selector labels.Selector) (ret []*v1.OperatorConditionOperatorCondition, err error) {
-	uol.csvLock.RLock()
-	defer uol.csvLock.RUnlock()
+func (uol *UnionOperatorConditionLister) List(selector labels.Selector) (ret []*v1.OperatorCondition, err error) {
+	uol.opConditionLock.RLock()
+	defer uol.opConditionLock.RUnlock()
 
 	set := make(map[types.UID]*v1.OperatorCondition)
-	for _, cl := range uol.csvListers {
+	for _, cl := range uol.opConditionListers {
 		csvs, err := cl.List(selector)
 		if err != nil {
 			return nil, err
@@ -43,39 +43,39 @@ func (uol *UnionOperatorConditionLister) List(selector labels.Selector) (ret []*
 
 // OperatorConditions returns an object that can list and get OperatorConditions.
 func (uol *UnionOperatorConditionLister) OperatorConditions(namespace string) listers.OperatorConditionNamespaceLister {
-	uol.csvLock.RLock()
-	defer uol.csvLock.RUnlock()
+	uol.opConditionLock.RLock()
+	defer uol.opConditionLock.RUnlock()
 
 	// Check for specific namespace listers
-	if cl, ok := uol.csvListers[namespace]; ok {
+	if cl, ok := uol.opConditionListers[namespace]; ok {
 		return cl.OperatorConditions(namespace)
 	}
 
 	// Check for any namespace-all listers
-	if cl, ok := uol.csvListers[metav1.NamespaceAll]; ok {
+	if cl, ok := uol.opConditionListers[metav1.NamespaceAll]; ok {
 		return cl.OperatorConditions(namespace)
 	}
 
 	return &NullOperatorConditionNamespaceLister{}
 }
 
 func (uol *UnionOperatorConditionLister) RegisterOperatorConditionLister(namespace string, lister listers.OperatorConditionLister) {
-	uol.csvLock.Lock()
-	defer uol.csvLock.Unlock()
+	uol.opConditionLock.Lock()
+	defer uol.opConditionLock.Unlock()
 
-	if uol.csvListers == nil {
-		uol.csvListers = make(map[string]listers.OperatorConditionLister)
+	if uol.opConditionListers == nil {
+		uol.opConditionListers = make(map[string]listers.OperatorConditionLister)
 	}
 
-	uol.csvListers[namespace] = lister
+	uol.opConditionListers[namespace] = lister
 }
 
 func (l *operatorsV1Lister) RegisterOperatorConditionLister(namespace string, lister listers.OperatorConditionLister) {
-	l.OperatorConditionLister.RegisterOperatorConditionLister(namespace, lister)
+	l.operatorConditionLister.RegisterOperatorConditionLister(namespace, lister)
 }
 
 func (l *operatorsV1Lister) OperatorConditionLister() listers.OperatorConditionLister {
-	return l.OperatorConditionLister
+	return l.operatorConditionLister
 }
 
 // NullOperatorConditionNamespaceLister is an implementation of a null OperatorConditionNamespaceLister. It is