Propagate CA certs bundle on all infrastructures (#18377)

Implement CA bundle propagation on Kubernetes infrastructures family

Signed-off-by: Mykola Morhun <mmorhun@redhat.com>

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties

@@ -45,14 +45,6 @@ che.workspace.http_proxy=
 che.workspace.https_proxy=
 che.workspace.no_proxy=
 
-# When cluster-wide proxy is configured, che-operator creates a special configmap
-# and allows the OpenShift Network Operator to inject the ca-bundle into it.
-# In addition, it adds the `pass:[CHE_TRUSTED__CA__BUNDLES__CONFIGMAP]` key with
-# the name of this configmap in the Che server configmap (and corresponding
-# environment variable). So, its presence can be used to detect if proxy mode is
-# enabled or not. Do not set this property manually unless required for this purpose.
-che.trusted_ca_bundles_configmap=NULL
-
 # By default, when users access a workspace with its URL, the workspace
 # automatically starts (if currently stopped). Set this to `false` to disable this behavior.
 che.workspace.auto_start=true
@@ -250,7 +242,6 @@ che.infra.kubernetes.ingress.domain=
 # the namespace specified by the che.infra.kubernetes.namespace.default will be created and used.
 che.infra.kubernetes.namespace=
 
-
 # Indicates whether Che server is allowed to create namespaces/projects for user
 # workspaces, or they're intended to be created manually by cluster administrator.
 # This property is also used by the OpenShift infra.
@@ -488,6 +479,26 @@ che.infra.kubernetes.tls_cert=NULL
 # - Che Server communicates with Kubernetes API using token from OAuth provider;
 che.infra.kubernetes.runtimes_consistency_check_period_min=-1
 
+
+# Name of cofig map in Che server namespace with additional CA TLS certificates to be propagated into all user's workspaces.
+# If the property is set on OpenShift 4 infrastructure, and che.infra.openshift.trusted_ca.dest_configmap_labels includes
+# config.openshift.io/inject-trusted-cabundle=true label, then cluster CA bundle will be propagated too.
+che.infra.kubernetes.trusted_ca.src_configmap=NULL
+
+# Name of configmap in a workspace namespace with additional CA TLS certificates.
+# Holds the copy of che.infra.kubernetes.trusted_ca.src_configmap but in a workspace namespace.
+# Content of this config map is mounted into all workspace containers including plugin brokers.
+# Do not change the config map name unless it conflicts with the already existing config map.
+che.infra.kubernetes.trusted_ca.dest_configmap=ca-certs
+
+# Configures path on workspace containers where the CA bundle should be mount.
+# Content of config map specified by che.infra.kubernetes.trusted_ca.dest_configmap is mounted.
+che.infra.kubernetes.trusted_ca.mount_path=/public-certs
+
+# Comma separated list of labels to add to the CA certificates config map in user workspace.
+# See che.infra.kubernetes.trusted_ca.dest_configmap property.
+che.infra.kubernetes.trusted_ca.dest_configmap_labels=
+
 ### OpenShift Infra parameters
 
 # Since OpenShift infrastructure reuse Kubernetes infrastructure components
@@ -506,19 +517,10 @@ che.infra.kubernetes.runtimes_consistency_check_period_min=-1
 # the namespace specified by the che.infra.kubernetes.namespace.default will be created and used.
 che.infra.openshift.project=
 
-
-# Configures name of the trust-store config map where the CA bundles are stored in Openshift 4.
-# This map is supposed to be initially created by Che installer (operator or etc) with basically
-# any name, and Che server finds it by specific label (see below) during workspace startup and then
-# creates and mounts same map in the namespace of the workspace. The property defines name of the
-# map in workspace namespace.
-che.infra.openshift.trusted_ca_bundles_config_map=ca-certs
-
-# Label name for config maps which are used for automatic certificate injection in Openshift 4.
-che.infra.openshift.trusted_ca_bundles_config_map_labels=config.openshift.io/inject-trusted-cabundle=true
-
-# Configures path on workspace containers where the CA bundles are mount.
-che.infra.openshift.trusted_ca_bundles_mount_path=/public-certs
+# Comma separated list of labels to add to the CA certificates config map in user workspace.
+# See che.infra.kubernetes.trusted_ca.dest_configmap property.
+# This default value is used for automatic cluster CA bundle injection in Openshift 4.
+che.infra.openshift.trusted_ca.dest_configmap_labels=config.openshift.io/inject-trusted-cabundle=true
 
 # Additional labels to add into every Route created by Che server
 # to allow clear identification.

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che_aliases.properties

@@ -37,3 +37,8 @@ che.infra.docker.certificates_folder=docker.client.certificates_folder
 che.limits.workspace.idle.timeout=che.workspace.agent.dev.inactive_stop_timeout_ms
 
 che.workspace.default_memory_limit_mb=che.workspace.default_memory_mb
+
+che.infra.kubernetes.trusted_ca.src_configmap=che.trusted_ca_bundles_configmap
+che.infra.kubernetes.trusted_ca.dest_configmap=che.infra.openshift.trusted_ca_bundles_config_map
+che.infra.kubernetes.trusted_ca.mount_path=che.infra.openshift.trusted_ca_bundles_mount_path
+che.infra.openshift.trusted_ca.dest_configmap_labels=che.infra.openshift.trusted_ca_bundles_config_map_labels

deploy/kubernetes/helm/che/custom-charts/che-keycloak/templates/deployment.yaml

@@ -109,6 +109,10 @@ spec:
           name: keycloak-data
         - mountPath: /opt/jboss/keycloak/standalone/log
           name: keycloak-log
+{{- if not (eq .Values.global.tls.serverTrustStoreConfigMapName "") }}
+        - mountPath: /public-certs
+          name: che-public-certs
+{{- end }}
       restartPolicy: Always
       securityContext:
         # `fsGroup`, `runAsGroup`, and `runAsUser` must be
@@ -125,4 +129,9 @@ spec:
       - name: keycloak-log
         persistentVolumeClaim:
           claimName: keycloak-log
+{{- if not (eq .Values.global.tls.serverTrustStoreConfigMapName "") }}
+      - name: che-public-certs
+        configMap:
+          name: {{ .Values.global.tls.serverTrustStoreConfigMapName }}
+{{- end }}
 status: {}

deploy/kubernetes/helm/che/custom-charts/che-keycloak/values.yaml

@@ -14,6 +14,9 @@
 global:
   ingress:
     class: "nginx"
+  tls:
+    ## Name of the config-map with public certificates to add to Java trust store of the Keycloak
+    serverTrustStoreConfigMapName: ""
 
 image: quay.io/eclipse/che-keycloak:nightly
 requireAdminPasswordChange: true

deploy/kubernetes/helm/che/templates/configmap.yaml

@@ -49,6 +49,9 @@ data:
 {{- end }}
 {{- if and .Values.global.multiuser .Values.customOidcUsernameClaim }}
   CHE_KEYCLOAK_USERNAME__CLAIM: {{ .Values.customOidcUsernameClaim }}
+{{- end }}
+{{- if not (eq .Values.global.tls.serverTrustStoreConfigMapName "") }}
+  CHE_TRUSTED__CA__BUNDLES__CONFIGMAP: {{ .Values.global.tls.serverTrustStoreConfigMapName }}
 {{- end }}
   # This is only kept for the upgrade purposes where we need to make sure we don't loose this piece of old configuration
   # so that we can find the data of old workspaces

deploy/kubernetes/helm/che/values.yaml

@@ -59,7 +59,8 @@ global:
     useSelfSignedCerts: true
     selfSignedCertSecretName: self-signed-certificate
 
-    ## Name of the config-map with public certificates to add to Java trust store of the Che server.
+    ## Name of the config-map with public certificates to add to Java trust store 
+    ## of the Che server, Keycloak and propagate into user workspaces.
     serverTrustStoreConfigMapName: ""
 
 

dockerfiles/keycloak/kc_realm_user.sh

@@ -33,10 +33,27 @@ if [ $KEYCLOAK_USER ] && [ $KEYCLOAK_PASSWORD ]; then
     /opt/jboss/keycloak/bin/add-user-keycloak.sh --user $KEYCLOAK_USER --password $KEYCLOAK_PASSWORD
 fi
 
+# Handle CA certificates
+KEYSTORE_PATH=/scripts/openshift.jks
+TRUST_STORE_PASSWORD=${TRUSTPASS:-openshift}
+CUSTOM_CERTS_DIR=/public-certs
+
+# Check for additional CA certificates propagated to Keycloak
+if [[ -d $CUSTOM_CERTS_DIR && -n $(find ${CUSTOM_CERTS_DIR} -type f) ]]; then
+    for certfile in ${CUSTOM_CERTS_DIR}/* ; do
+        keytool -importcert -alias CERT_$(basename $certfile) -keystore $KEYSTORE_PATH -file $certfile -storepass $TRUST_STORE_PASSWORD  -noprompt;
+    done
+fi
+
+# Check for self-sighed certificate
 if [ "${CHE_SELF__SIGNED__CERT}" != "" ]; then
     echo "${CHE_SELF__SIGNED__CERT}" > /scripts/openshift.cer
-    keytool -importcert -alias HOSTDOMAIN -keystore /scripts/openshift.jks -file /scripts/openshift.cer -storepass openshift -noprompt
-    keytool -importkeystore -srckeystore $JAVA_HOME/jre/lib/security/cacerts -destkeystore /scripts/openshift.jks -srcstorepass changeit -deststorepass openshift
+    keytool -importcert -alias HOSTDOMAIN -keystore $KEYSTORE_PATH -file /scripts/openshift.cer -storepass $TRUST_STORE_PASSWORD -noprompt
+fi
+
+# Export Java trust store into one that is propagated to Keycloak
+if [ -f "$KEYSTORE_PATH" ]; then
+    keytool -importkeystore -srckeystore $JAVA_HOME/jre/lib/security/cacerts -destkeystore $KEYSTORE_PATH -srcstorepass changeit -deststorepass $TRUST_STORE_PASSWORD
     /opt/jboss/keycloak/bin/jboss-cli.sh --file=/scripts/cli/add_openshift_certificate.cli && rm -rf /opt/jboss/keycloak/standalone/configuration/standalone_xml_history
 fi
 

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/CheServerKubernetesClientFactory.java

@@ -24,7 +24,7 @@
  * This {@link KubernetesClientFactory} ensures that we use `che` ServiceAccount and not related to
  * any workspace. It always provides client with default {@link Config}. It's useful for operations
  * that needs permissions of `che` SA, such as operations inside `che` namespace (like creating a
- * ConfigMaps for Gateway router) or some cluste-wide actions (like labeling the namespaces).
+ * ConfigMaps for Gateway router) or some cluster-wide actions (like labeling the namespaces).
  */
 @Singleton
 public class CheServerKubernetesClientFactory extends KubernetesClientFactory {

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java

@@ -26,6 +26,7 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayRouterProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GitConfigProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ImagePullSecretProvisioner;
+import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesTrustedCAProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.NodeSelectorProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.PodTerminationGracePeriodProvisioner;
@@ -86,6 +87,7 @@ class KubernetesEnvironmentProvisionerImpl
     private final PreviewUrlExposer<KubernetesEnvironment> previewUrlExposer;
     private final VcsSslCertificateProvisioner vcsSslCertificateProvisioner;
     private final GatewayRouterProvisioner gatewayRouterProvisioner;
+    private final KubernetesTrustedCAProvisioner trustedCAProvisioner;
 
     @Inject
     public KubernetesEnvironmentProvisionerImpl(
@@ -111,7 +113,8 @@ public KubernetesEnvironmentProvisionerImpl(
         GitConfigProvisioner gitConfigProvisioner,
         PreviewUrlExposer<KubernetesEnvironment> previewUrlExposer,
         VcsSslCertificateProvisioner vcsSslCertificateProvisioner,
-        GatewayRouterProvisioner gatewayRouterProvisioner) {
+        GatewayRouterProvisioner gatewayRouterProvisioner,
+        KubernetesTrustedCAProvisioner trustedCAProvisioner) {
       this.pvcEnabled = pvcEnabled;
       this.volumesStrategy = volumesStrategy;
       this.uniqueNamesProvisioner = uniqueNamesProvisioner;
@@ -135,6 +138,7 @@ public KubernetesEnvironmentProvisionerImpl(
       this.gitConfigProvisioner = gitConfigProvisioner;
       this.previewUrlExposer = previewUrlExposer;
       this.gatewayRouterProvisioner = gatewayRouterProvisioner;
+      this.trustedCAProvisioner = trustedCAProvisioner;
     }
 
     @Traced
@@ -178,6 +182,7 @@ public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
       vcsSslCertificateProvisioner.provision(k8sEnv, identity);
       gitConfigProvisioner.provision(k8sEnv, identity);
       gatewayRouterProvisioner.provision(k8sEnv, identity);
+      trustedCAProvisioner.provision(k8sEnv, identity);
       LOG.debug("Provisioning Kubernetes environment done for workspace '{}'", workspaceId);
     }
   }

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java

@@ -60,8 +60,10 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiExternalEnvVarProvider;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiInternalEnvVarProvider;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesPreviewUrlCommandProvisioner;
+import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesTrustedCAProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.PreviewUrlCommandProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TlsProvisioner;
+import org.eclipse.che.workspace.infrastructure.kubernetes.provision.TrustedCAProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.server.ServersConverter;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.IngressAnnotationsProvider;
@@ -107,6 +109,8 @@ protected void configure() {
     bind(RuntimeInfrastructure.class).to(KubernetesInfrastructure.class);
     bind(InconsistentRuntimesDetector.class).asEagerSingleton();
 
+    bind(TrustedCAProvisioner.class).to(KubernetesTrustedCAProvisioner.class);
+
     MapBinder<WorkspaceExposureType, TlsProvisioner<KubernetesEnvironment>> tlsProvisioners =
         MapBinder.newMapBinder(
             binder(),

infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/provision/Openshift4TrustedCAProvisioner.java → infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/KubernetesTrustedCAProvisioner.java

@@ -9,7 +9,7 @@
  * Contributors:
  *   Red Hat, Inc. - initial API and implementation
  */
-package org.eclipse.che.workspace.infrastructure.openshift.provision;
+package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
 
 import static com.google.common.base.Strings.isNullOrEmpty;
 
@@ -21,80 +21,100 @@
 import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
 import io.fabric8.kubernetes.api.model.VolumeBuilder;
 import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Singleton;
+import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
 import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
 import org.eclipse.che.commons.annotation.Nullable;
 import org.eclipse.che.workspace.infrastructure.kubernetes.CheServerKubernetesClientFactory;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.CheInstallationLocation;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodData;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodRole;
-import org.eclipse.che.workspace.infrastructure.openshift.project.OpenShiftProject;
+import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace;
+import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory;
 
 /**
- * Checks if config map with CA bundles is configured by specific property, and if it is, creates
- * map for ca bundles in workspace project, allowing Openshift to auto-inject values into it. (see
- * https://docs.openshift.com/container-platform/4.3/networking/configuring-a-custom-pki.html#certificate-injection-using-operators_configuring-a-custom-pki)
+ * Checks if config maps with CA bundles is configured by specific property. If they are, then
+ * creates single config map for all ca bundles in workspace namespace and mounts it into pods.
  */
 @Singleton
-public class Openshift4TrustedCAProvisioner {
+public class KubernetesTrustedCAProvisioner implements TrustedCAProvisioner {
 
-  public static final String CHE_TRUST_STORE_VOLUME = "che-self-signed-certs";
+  public static final String CHE_TRUST_STORE_VOLUME = "che-ca-certs";
 
   private final String certificateMountPath;
   private final boolean trustedStoreInitialized;
   private final String caBundleConfigMap;
   private final String configMapName;
-  private final Map<String, String> configMapLabelKeyValue;
   private final CheServerKubernetesClientFactory cheServerClientFactory;
   private final String installationLocationNamespace;
+  private final KubernetesNamespaceFactory namespaceFactory;
+  private final Map<String, String> configMapLabelKeyValue;
 
   @Inject
-  public Openshift4TrustedCAProvisioner(
-      @Nullable @Named("che.trusted_ca_bundles_configmap") String caBundleConfigMap,
-      @Named("che.infra.openshift.trusted_ca_bundles_config_map") String configMapName,
-      @Named("che.infra.openshift.trusted_ca_bundles_config_map_labels") String configMapLabel,
-      @Named("che.infra.openshift.trusted_ca_bundles_mount_path") String certificateMountPath,
+  public KubernetesTrustedCAProvisioner(
+      @Nullable @Named("che.infra.kubernetes.trusted_ca.src_configmap") String caBundleConfigMap,
+      @Named("che.infra.kubernetes.trusted_ca.dest_configmap") String configMapName,
+      @Named("che.infra.kubernetes.trusted_ca.mount_path") String certificateMountPath,
+      @Nullable @Named("che.infra.kubernetes.trusted_ca.dest_configmap_labels")
+          String configMapLabels,
       CheInstallationLocation cheInstallationLocation,
+      KubernetesNamespaceFactory namespaceFactory,
       CheServerKubernetesClientFactory cheServerClientFactory)
       throws InfrastructureException {
     this.cheServerClientFactory = cheServerClientFactory;
     this.trustedStoreInitialized = !isNullOrEmpty(caBundleConfigMap);
     this.configMapName = configMapName;
     this.caBundleConfigMap = caBundleConfigMap;
     this.certificateMountPath = certificateMountPath;
-    this.configMapLabelKeyValue = Splitter.on(",").withKeyValueSeparator("=").split(configMapLabel);
     this.installationLocationNamespace = cheInstallationLocation.getInstallationLocationNamespace();
+    this.namespaceFactory = namespaceFactory;
+
+    if (configMapLabels != null && !configMapLabels.trim().equals("")) {
+      this.configMapLabelKeyValue =
+          Splitter.on(",").withKeyValueSeparator("=").split(configMapLabels);
+    } else {
+      this.configMapLabelKeyValue = new HashMap<>();
+    }
   }
 
   public boolean isTrustedStoreInitialized() {
     return trustedStoreInitialized;
   }
 
-  public void provision(KubernetesEnvironment k8sEnv, OpenShiftProject project)
+  /**
+   * Propagates additional CA certificates into config map and mounts them into all pods of given
+   * namespace
+   *
+   * @param k8sEnv available objects in the scope
+   * @param runtimeID defines namespace into which config map should be provisioned
+   * @throws InfrastructureException if failed to CRUD a resource
+   */
+  public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity runtimeID)
       throws InfrastructureException {
     if (!trustedStoreInitialized) {
       return;
     }
-    ConfigMap configMap =
+
+    ConfigMap allCaCertsConfigMap =
         cheServerClientFactory
             .create()
             .configMaps()
             .inNamespace(installationLocationNamespace)
             .withName(caBundleConfigMap)
             .get();
-
-    if (configMap == null) {
+    if (allCaCertsConfigMap == null) {
       return;
     }
 
-    Optional<ConfigMap> existing = project.configMaps().get(configMapName);
-
-    if (!existing.isPresent() || !existing.get().getData().equals(configMap.getData())) {
+    KubernetesNamespace namespace = namespaceFactory.getOrCreate(runtimeID);
+    Optional<ConfigMap> existing = namespace.configMaps().get(configMapName);
+    if (existing.isEmpty() || !existing.get().getData().equals(allCaCertsConfigMap.getData())) {
       // create or renew map
       k8sEnv
           .getConfigMaps()
@@ -104,11 +124,11 @@ public void provision(KubernetesEnvironment k8sEnv, OpenShiftProject project)
                   .withMetadata(
                       new ObjectMetaBuilder()
                           .withName(configMapName)
-                          .withAnnotations(configMap.getMetadata().getAnnotations())
+                          .withAnnotations(allCaCertsConfigMap.getMetadata().getAnnotations())
                           .withLabels(configMapLabelKeyValue)
                           .build())
-                  .withApiVersion(configMap.getApiVersion())
-                  .withData(configMap.getData())
+                  .withApiVersion(allCaCertsConfigMap.getApiVersion())
+                  .withData(allCaCertsConfigMap.getData())
                   .build());
     }