Implement addition CA config maps merge and propagation to Che server (…

…#531)

Implement CA certs sources merge and propagate resulting config map to Che server

Signed-off-by: Mykola Morhun <mmorhun@redhat.com>

pkg/controller/che/che_controller.go

@@ -458,6 +458,18 @@ func (r *ReconcileChe) Reconcile(request reconcile.Request) (reconcile.Result, e
 		}
 	}
 
+	// Make sure that CA certificates from all marked config maps are merged into single config map to be propageted to Che components
+	cm, err := deploy.SyncAdditionalCACertsConfigMapToCluster(instance, deployContext)
+	if err != nil {
+		logrus.Errorf("Error updating additional CA config map: %v", err)
+		return reconcile.Result{}, err
+	}
+	if cm == nil && !tests {
+		// Config map update is in progress
+		// Return and do not force reconcile. When update finishes it will trigger reconcile loop.
+		return reconcile.Result{}, err
+	}
+
 	// Get custom ConfigMap
 	// if it exists, add the data into CustomCheProperties
 	customConfigMap := &corev1.ConfigMap{}
@@ -1034,6 +1046,7 @@ func getServerExposingServiceName(cr *orgv1.CheCluster) string {
 	return deploy.CheServiceName
 }
 
+// isTrustedBundleConfigMap detects whether given config map is the config map with additional CA certificates to be trusted by Che
 func isTrustedBundleConfigMap(mgr manager.Manager, obj handler.MapObject) (bool, reconcile.Request) {
 	checlusters := &orgv1.CheClusterList{}
 	if err := mgr.GetClient().List(context.TODO(), checlusters, &client.ListOptions{}); err != nil {
@@ -1044,8 +1057,22 @@ func isTrustedBundleConfigMap(mgr manager.Manager, obj handler.MapObject) (bool,
 		return false, reconcile.Request{}
 	}
 
+	// Check if config map is the config map from CR
 	if checlusters.Items[0].Spec.Server.ServerTrustStoreConfigMapName != obj.Meta.GetName() {
-		return false, reconcile.Request{}
+		// No, it is not form CR
+		// Check for labels
+
+		// Check for part of Che label
+		if value, exists := obj.Meta.GetLabels()[deploy.PartOfCheLabelKey]; !exists || value != deploy.PartOfCheLabelValue {
+			// Labels do not match
+			return false, reconcile.Request{}
+		}
+
+		// Check for CA bundle label
+		if value, exists := obj.Meta.GetLabels()[deploy.CheCACertsConfigMapLabelKey]; !exists || value != deploy.CheCACertsConfigMapLabelValue {
+			// Labels do not match
+			return false, reconcile.Request{}
+		}
 	}
 
 	return true, reconcile.Request{

pkg/controller/che/proxy.go

@@ -13,10 +13,10 @@ package che
 
 import (
 	"context"
-	"github.com/eclipse/che-operator/pkg/deploy/server"
 
 	orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
 	"github.com/eclipse/che-operator/pkg/deploy"
+	"github.com/eclipse/che-operator/pkg/deploy/server"
 	"github.com/eclipse/che-operator/pkg/util"
 	configv1 "github.com/openshift/api/config/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -36,7 +36,7 @@ func (r *ReconcileChe) getProxyConfiguration(checluster *orgv1.CheCluster) (*dep
 
 		// If proxy configuration exists in CR then cluster wide proxy configuration is ignored
 		// otherwise cluster wide proxy configuration is used and non proxy hosts
-		// are merted with defined ones in CR
+		// are merged with defined ones in CR
 		if proxy.HttpProxy == "" && clusterProxy.Status.HTTPProxy != "" {
 			proxy, err = deploy.ReadClusterWideProxyConfiguration(clusterProxy, proxy.NoProxy)
 			if err != nil {

pkg/deploy/configmap.go

@@ -26,6 +26,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
+// SyncConfigMapToCluster makes sure that given config map spec is actual.
+// It compares config map data and labels.
+// If returned config map is nil then it means that the config map update is in progress and reconcile loop probably should be restarted.
 func SyncConfigMapToCluster(deployContext *DeployContext, specConfigMap *corev1.ConfigMap) (*corev1.ConfigMap, error) {
 	clusterConfigMap, err := GetClusterConfigMap(specConfigMap.Name, specConfigMap.Namespace, deployContext.ClusterAPI.Client)
 	if err != nil {
@@ -38,18 +41,21 @@ func SyncConfigMapToCluster(deployContext *DeployContext, specConfigMap *corev1.
 		return nil, err
 	}
 
-	diff := cmp.Diff(clusterConfigMap.Data, specConfigMap.Data)
-	if len(diff) > 0 {
+	dataDiff := cmp.Diff(clusterConfigMap.Data, specConfigMap.Data)
+	labelsDiff := cmp.Diff(clusterConfigMap.ObjectMeta.Labels, specConfigMap.ObjectMeta.Labels)
+	if len(dataDiff) > 0 || len(labelsDiff) > 0 {
 		logrus.Infof("Updating existing object: %s, name: %s", specConfigMap.Kind, specConfigMap.Name)
-		fmt.Printf("Difference:\n%s", diff)
+		fmt.Printf("Difference:\n%s\n%s", dataDiff, labelsDiff)
 		clusterConfigMap.Data = specConfigMap.Data
+		clusterConfigMap.ObjectMeta.Labels = specConfigMap.ObjectMeta.Labels
 		err := deployContext.ClusterAPI.Client.Update(context.TODO(), clusterConfigMap)
 		return nil, err
 	}
 
 	return clusterConfigMap, nil
 }
 
+// GetSpecConfigMap returns config map spec template
 func GetSpecConfigMap(
 	deployContext *DeployContext,
 	name string,
@@ -79,6 +85,7 @@ func GetSpecConfigMap(
 	return configMap, nil
 }
 
+// GetClusterConfigMap reads config map from cluster
 func GetClusterConfigMap(name string, namespace string, client runtimeClient.Client) (*corev1.ConfigMap, error) {
 	configMap := &corev1.ConfigMap{}
 	namespacedName := types.NamespacedName{

pkg/deploy/defaults.go

@@ -14,11 +14,12 @@ package deploy
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/eclipse/che-operator/pkg/util"
 	"github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
@@ -98,6 +99,9 @@ const (
 	OldDefaultCodeReadyServerImageRepo = "registry.redhat.io/codeready-workspaces/server-rhel8"
 	OldDefaultCodeReadyServerImageTag  = "1.2"
 	OldCrwPluginRegistryUrl            = "https://che-plugin-registry.openshift.io"
+
+	PartOfCheLabelKey   = "app.kubernetes.io/part-of"
+	PartOfCheLabelValue = "che.eclipse.org"
 )
 
 func InitDefaults(defaultsPath string) {

pkg/deploy/identity-provider/deployment_keycloak.go

@@ -18,12 +18,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/eclipse/che-operator/pkg/deploy/server"
-
+	orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
 	"github.com/eclipse/che-operator/pkg/deploy"
 	"github.com/eclipse/che-operator/pkg/deploy/postgres"
-
-	orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
 	"github.com/eclipse/che-operator/pkg/util"
 	"github.com/google/go-cmp/cmp"
 	"github.com/sirupsen/logrus"
@@ -109,7 +106,7 @@ func getSpecKeycloakDeployment(
 		}
 	}
 
-	cmResourceVersions := server.GetTrustStoreConfigMapVersion(deployContext)
+	cmResourceVersions := deploy.GetAdditionalCACertsConfigMapVersion(deployContext)
 	terminationGracePeriodSeconds := int64(30)
 	cheCertSecretVersion := getSecretResourceVersion("self-signed-certificate", deployContext.CheCluster.Namespace, deployContext.ClusterAPI)
 	openshiftApiCertSecretVersion := getSecretResourceVersion("openshift-api-crt", deployContext.CheCluster.Namespace, deployContext.ClusterAPI)
@@ -139,14 +136,12 @@ func getSpecKeycloakDeployment(
 
 	customPublicCertsDir := "/public-certs"
 	customPublicCertsVolumeSource := corev1.VolumeSource{}
-	if deployContext.CheCluster.Spec.Server.ServerTrustStoreConfigMapName != "" {
-		customPublicCertsVolumeSource = corev1.VolumeSource{
-			ConfigMap: &corev1.ConfigMapVolumeSource{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: deployContext.CheCluster.Spec.Server.ServerTrustStoreConfigMapName,
-				},
+	customPublicCertsVolumeSource = corev1.VolumeSource{
+		ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{
+				Name: deploy.CheAllCACertsConfigMapName,
 			},
-		}
+		},
 	}
 	customPublicCertsVolume := corev1.Volume{
 		Name:         "che-public-certs",

pkg/deploy/server/che_configmap.go

@@ -236,7 +236,7 @@ func GetCheConfigMapData(deployContext *deploy.DeployContext) (cheEnv map[string
 		CheServerSecureExposerJwtProxyImage:    deploy.DefaultCheServerSecureExposerJwtProxyImage(deployContext.CheCluster),
 		CheJGroupsKubernetesLabels:             cheLabels,
 		CheMetricsEnabled:                      cheMetrics,
-		CheTrustedCABundlesConfigMap:           deployContext.CheCluster.Spec.Server.ServerTrustStoreConfigMapName,
+		CheTrustedCABundlesConfigMap:           deploy.CheAllCACertsConfigMapName,
 		ServerStrategy:                         ingressStrategy,
 		WorkspaceExposure:                      workspaceExposure,
 		SingleHostGatewayConfigMapLabels:       singleHostGatewayConfigMapLabels,

pkg/deploy/server/configmap_cert.go

@@ -54,14 +54,3 @@ func SyncTrustStoreConfigMapToCluster(deployContext *deploy.DeployContext) (*cor
 
 	return clusterConfigMap, nil
 }
-
-func GetTrustStoreConfigMapVersion(deployContext *deploy.DeployContext) string {
-	if deployContext.CheCluster.Spec.Server.ServerTrustStoreConfigMapName != "" {
-		trustStoreConfigMap, _ := deploy.GetClusterConfigMap(deployContext.CheCluster.Spec.Server.ServerTrustStoreConfigMapName, deployContext.CheCluster.Namespace, deployContext.ClusterAPI.Client)
-		if trustStoreConfigMap != nil {
-			return trustStoreConfigMap.ResourceVersion
-		}
-	}
-
-	return ""
-}

pkg/deploy/server/deployment_che.go

@@ -57,7 +57,7 @@ func getSpecCheDeployment(deployContext *deploy.DeployContext) (*appsv1.Deployme
 	}
 
 	cmResourceVersions := GetCheConfigMapVersion(deployContext)
-	cmResourceVersions += "," + GetTrustStoreConfigMapVersion(deployContext)
+	cmResourceVersions += "," + deploy.GetAdditionalCACertsConfigMapVersion(deployContext)
 
 	terminationGracePeriodSeconds := int64(30)
 	cheFlavor := deploy.DefaultCheFlavor(deployContext.CheCluster)
@@ -69,14 +69,12 @@ func getSpecCheDeployment(deployContext *deploy.DeployContext) (*appsv1.Deployme
 		Value: "",
 	}
 	customPublicCertsVolumeSource := corev1.VolumeSource{}
-	if deployContext.CheCluster.Spec.Server.ServerTrustStoreConfigMapName != "" {
-		customPublicCertsVolumeSource = corev1.VolumeSource{
-			ConfigMap: &corev1.ConfigMapVolumeSource{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: deployContext.CheCluster.Spec.Server.ServerTrustStoreConfigMapName,
-				},
+	customPublicCertsVolumeSource = corev1.VolumeSource{
+		ConfigMap: &corev1.ConfigMapVolumeSource{
+			LocalObjectReference: corev1.LocalObjectReference{
+				Name: deploy.CheAllCACertsConfigMapName,
 			},
-		}
+		},
 	}
 	customPublicCertsVolume := corev1.Volume{
 		Name:         "che-public-certs",

pkg/deploy/tls.go

@@ -18,17 +18,22 @@ import (
 	"encoding/pem"
 	stderrors "errors"
 	"net/http"
+	"reflect"
 	"strings"
 	"time"
 
+	orgv1 "github.com/eclipse/che-operator/pkg/apis/org/v1"
 	"github.com/eclipse/che-operator/pkg/util"
 	routev1 "github.com/openshift/api/route/v1"
 	"github.com/sirupsen/logrus"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
@@ -42,6 +47,21 @@ const (
 	CheTLSJobComponentName                = "che-create-tls-secret-job"
 	CheTLSSelfSignedCertificateSecretName = "self-signed-certificate"
 	DefaultCheTLSSecretName               = "che-tls"
+
+	// CheCACertsConfigMapLabelKey is the label key which marks config map with additional CA certificates
+	CheCACertsConfigMapLabelKey = "app.kubernetes.io/component"
+	// CheCACertsConfigMapLabelKey is the label value which marks config map with additional CA certificates
+	CheCACertsConfigMapLabelValue = "ca-bundle"
+	// CheAllCACertsConfigMapName is the name of config map which contains all additional trusted by Che TLS CA certificates
+	CheAllCACertsConfigMapName = "ca-certs-merged"
+	// CheMergedCAConfigMapRevisionsAnnotationKey is annotation name which holds versions of included config maps in format: cm-name1=ver1,cm-name2=ver2
+	CheMergedCAConfigMapRevisionsAnnotationKey = "che.eclipse.org/included-configmaps"
+
+	// Local constants
+	// labelEqualSign consyant is used as a replacement for '=' symbol in labels because '=' is not allowed there
+	labelEqualSign = "-"
+	// labelCommaSign consyant is used as a replacement for ',' symbol in labels because ',' is not allowed there
+	labelCommaSign = "."
 )
 
 // IsSelfSignedCertificateUsed detects whether endpoints are/should be secured by self-signed certificate.
@@ -460,3 +480,116 @@ func deleteJob(deployContext *DeployContext, job *batchv1.Job) {
 		logrus.Errorf("Error deleting job: '%s', error: %v", CheTLSJobName, err)
 	}
 }
+
+// SyncAdditionalCACertsConfigMapToCluster makes sure that additional CA certs config map is up to date if any
+func SyncAdditionalCACertsConfigMapToCluster(cr *orgv1.CheCluster, deployContext *DeployContext) (*corev1.ConfigMap, error) {
+	// Get all source config maps, if any
+	caConfigMaps, err := getCACertsConfigMaps(deployContext)
+	if err != nil {
+		return nil, err
+	}
+	if len(cr.Spec.Server.ServerTrustStoreConfigMapName) > 0 {
+		crConfigMap := &corev1.ConfigMap{}
+		err := deployContext.ClusterAPI.Client.Get(context.TODO(), types.NamespacedName{Namespace: deployContext.CheCluster.Namespace, Name: cr.Spec.Server.ServerTrustStoreConfigMapName}, crConfigMap)
+		if err != nil {
+			return nil, err
+		}
+		caConfigMaps = append(caConfigMaps, *crConfigMap)
+	}
+
+	mergedCAConfigMap := &corev1.ConfigMap{}
+	err = deployContext.ClusterAPI.Client.Get(context.TODO(), types.NamespacedName{Namespace: deployContext.CheCluster.Namespace, Name: CheAllCACertsConfigMapName}, mergedCAConfigMap)
+	if err == nil {
+		// Merged config map exists. Check if it is up to date.
+		caConfigMapsCurrentRevisions := make(map[string]string)
+		for _, cm := range caConfigMaps {
+			caConfigMapsCurrentRevisions[cm.Name] = cm.ResourceVersion
+		}
+
+		caConfigMapsCachedRevisions := make(map[string]string)
+		if mergedCAConfigMap.ObjectMeta.Annotations != nil {
+			if revisions, exists := mergedCAConfigMap.ObjectMeta.Annotations[CheMergedCAConfigMapRevisionsAnnotationKey]; exists {
+				for _, cmNameRevision := range strings.Split(revisions, labelCommaSign) {
+					nameRevision := strings.Split(cmNameRevision, labelEqualSign)
+					if len(nameRevision) != 2 {
+						// The label value is invalid, recreate merged config map
+						break
+					}
+					caConfigMapsCachedRevisions[nameRevision[0]] = nameRevision[1]
+				}
+			}
+		}
+
+		if reflect.DeepEqual(caConfigMapsCurrentRevisions, caConfigMapsCachedRevisions) {
+			// Existing merged config map is up to date, do nothing
+			return mergedCAConfigMap, nil
+		}
+	} else {
+		if !errors.IsNotFound(err) {
+			return nil, err
+		}
+		// Merged config map doesn't exist. Create it.
+	}
+
+	// Merged config map is out of date or doesn't exist
+	// Merge all config maps into single one to mount inside Che components and workspaces
+	data := make(map[string]string)
+	revisions := ""
+	for _, cm := range caConfigMaps {
+		// Copy data
+		for key, dataRecord := range cm.Data {
+			data[cm.ObjectMeta.Name+"."+key] = dataRecord
+		}
+
+		// Save source config map revision
+		if revisions != "" {
+			revisions += labelCommaSign
+		}
+		revisions += cm.ObjectMeta.Name + labelEqualSign + cm.ObjectMeta.ResourceVersion
+	}
+
+	mergedCAConfigMapSpec, err := GetSpecConfigMap(deployContext, CheAllCACertsConfigMapName, data)
+	if err != nil {
+		return nil, err
+	}
+	mergedCAConfigMapSpec.ObjectMeta.Labels[PartOfCheLabelKey] = PartOfCheLabelValue
+
+	if mergedCAConfigMapSpec.ObjectMeta.Annotations == nil {
+		mergedCAConfigMapSpec.ObjectMeta.Annotations = make(map[string]string)
+	}
+	mergedCAConfigMapSpec.ObjectMeta.Annotations[CheMergedCAConfigMapRevisionsAnnotationKey] = revisions
+
+	logrus.Infof("Updating additional CA certs config map: %s", CheAllCACertsConfigMapName)
+	mergedCAConfigMap, err = SyncConfigMapToCluster(deployContext, mergedCAConfigMapSpec)
+	if err != nil {
+		return nil, err
+	}
+	return mergedCAConfigMap, nil
+}
+
+// getCACertsConfigMaps returns list of config maps with additional CA certificates that should be trusted by Che
+// The selection is based on the specific label
+func getCACertsConfigMaps(deployContext *DeployContext) ([]corev1.ConfigMap, error) {
+	CACertsConfigMapList := &corev1.ConfigMapList{}
+
+	caBundleLabelSelectorRequirement, _ := labels.NewRequirement(CheCACertsConfigMapLabelKey, selection.Equals, []string{CheCACertsConfigMapLabelValue})
+	cheComponetLabelSelectorRequirement, _ := labels.NewRequirement(PartOfCheLabelKey, selection.Equals, []string{PartOfCheLabelValue})
+	listOptions := &client.ListOptions{
+		LabelSelector: labels.NewSelector().Add(*cheComponetLabelSelectorRequirement).Add(*caBundleLabelSelectorRequirement),
+	}
+	if err := deployContext.ClusterAPI.Client.List(context.TODO(), CACertsConfigMapList, listOptions); err != nil {
+		return nil, err
+	}
+
+	return CACertsConfigMapList.Items, nil
+}
+
+// GetAdditionalCACertsConfigMapVersion returns revision of merged additional CA certs config map
+func GetAdditionalCACertsConfigMapVersion(deployContext *DeployContext) string {
+	trustStoreConfigMap, _ := GetClusterConfigMap(CheAllCACertsConfigMapName, deployContext.CheCluster.Namespace, deployContext.ClusterAPI.Client)
+	if trustStoreConfigMap != nil {
+		return trustStoreConfigMap.ResourceVersion
+	}
+
+	return ""
+}