diff --git a/pkg/constants/metadata.go b/pkg/constants/metadata.go index 8fb032ed7..21d077c19 100644 --- a/pkg/constants/metadata.go +++ b/pkg/constants/metadata.go @@ -102,4 +102,12 @@ const ( // NamespacedConfigLabelKey is a label applied to configmaps to mark them as a configuration for all DevWorkspaces in // the current namespace. NamespacedConfigLabelKey = "controller.devfile.io/namespaced-config" + + // NamespacePodTolerationsAnnotation is an annotation applied to a namespace to configure pod tolerations for all workspaces + // in that namespace. Value should be json-encoded []corev1.Toleration struct. + NamespacePodTolerationsAnnotation = "controller.devfile.io/pod-tolerations" + + // NamespaceNodeSelectorAnnotation is an annotation applied to a namespace to configure the node selector for all workspaces + // in that namespace. Value should be json-encoded map[string]string + NamespaceNodeSelectorAnnotation = "controller.devfile.io/node-selector" ) diff --git a/pkg/provision/config/config.go b/pkg/provision/config/config.go index e172b30b6..448b0f5ea 100644 --- a/pkg/provision/config/config.go +++ b/pkg/provision/config/config.go @@ -16,12 +16,14 @@ package config import ( + "encoding/json" "fmt" "strings" "github.com/devfile/devworkspace-operator/pkg/provision/sync" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/devfile/devworkspace-operator/pkg/constants" @@ -72,3 +74,32 @@ func ReadNamespacedConfig(namespace string, api sync.ClusterAPI) (*NamespacedCon CommonPVCSize: cm.Data[commonPVCSizeKey], }, nil } + +// GetNamespacePodTolerationsAndNodeSelector gets pod tolerations and the node selector that should be applied to all pods created +// for workspaces in a given namespace. Tolerations and node selector are unmarshalled from json-formatted annotations on the namespace +// itself. Returns an error if annotations are not valid JSON. +func GetNamespacePodTolerationsAndNodeSelector(namespace string, api sync.ClusterAPI) ([]corev1.Toleration, map[string]string, error) { + ns := &corev1.Namespace{} + err := api.Client.Get(api.Ctx, types.NamespacedName{Name: namespace}, ns) + if err != nil { + return nil, nil, err + } + + var podTolerations []corev1.Toleration + podTolerationsAnnot, ok := ns.Annotations[constants.NamespacePodTolerationsAnnotation] + if ok && podTolerationsAnnot != "" { + if err := json.Unmarshal([]byte(podTolerationsAnnot), &podTolerations); err != nil { + return nil, nil, fmt.Errorf("failed to parse %s annotation: %w", constants.NamespacePodTolerationsAnnotation, err) + } + } + + nodeSelector := map[string]string{} + nodeSelectorAnnot, ok := ns.Annotations[constants.NamespaceNodeSelectorAnnotation] + if ok && nodeSelectorAnnot != "" { + if err := json.Unmarshal([]byte(nodeSelectorAnnot), &nodeSelector); err != nil { + return nil, nil, fmt.Errorf("failed to parse %s annotation: %w", constants.NamespaceNodeSelectorAnnotation, err) + } + } + + return podTolerations, nodeSelector, nil +} diff --git a/pkg/provision/storage/asyncstorage/deployment.go b/pkg/provision/storage/asyncstorage/deployment.go index e7fc18174..b5ca5169f 100644 --- a/pkg/provision/storage/asyncstorage/deployment.go +++ b/pkg/provision/storage/asyncstorage/deployment.go @@ -17,6 +17,7 @@ package asyncstorage import ( "github.com/devfile/devworkspace-operator/internal/images" + nsconfig "github.com/devfile/devworkspace-operator/pkg/provision/config" "github.com/devfile/devworkspace-operator/pkg/provision/sync" wsprovision "github.com/devfile/devworkspace-operator/pkg/provision/workspace" appsv1 "k8s.io/api/apps/v1" @@ -27,7 +28,12 @@ import ( ) func SyncWorkspaceSyncDeploymentToCluster(namespace string, sshConfigMap *corev1.ConfigMap, storage *corev1.PersistentVolumeClaim, clusterAPI sync.ClusterAPI) (*appsv1.Deployment, error) { - specDeployment := getWorkspaceSyncDeploymentSpec(namespace, sshConfigMap, storage) + podTolerations, nodeSelector, err := nsconfig.GetNamespacePodTolerationsAndNodeSelector(namespace, clusterAPI) + if err != nil { + return nil, err + } + + specDeployment := getWorkspaceSyncDeploymentSpec(namespace, sshConfigMap, storage, podTolerations, nodeSelector) clusterObj, err := sync.SyncObjectWithCluster(specDeployment, clusterAPI) switch err.(type) { case nil: @@ -47,7 +53,13 @@ func SyncWorkspaceSyncDeploymentToCluster(namespace string, sshConfigMap *corev1 return nil, NotReadyError } -func getWorkspaceSyncDeploymentSpec(namespace string, sshConfigMap *corev1.ConfigMap, storage *corev1.PersistentVolumeClaim) *appsv1.Deployment { +func getWorkspaceSyncDeploymentSpec( + namespace string, + sshConfigMap *corev1.ConfigMap, + storage *corev1.PersistentVolumeClaim, + tolerations []corev1.Toleration, + nodeSelector map[string]string) *appsv1.Deployment { + replicas := int32(1) terminationGracePeriod := int64(1) modeReadOnly := int32(0640) @@ -140,6 +152,15 @@ func getWorkspaceSyncDeploymentSpec(namespace string, sshConfigMap *corev1.Confi }, }, } + + if tolerations != nil && len(tolerations) > 0 { + deployment.Spec.Template.Spec.Tolerations = tolerations + } + + if nodeSelector != nil && len(nodeSelector) > 0 { + deployment.Spec.Template.Spec.NodeSelector = nodeSelector + } + return deployment } diff --git a/pkg/provision/storage/cleanup.go b/pkg/provision/storage/cleanup.go index 9e7c7160e..8606facb9 100644 --- a/pkg/provision/storage/cleanup.go +++ b/pkg/provision/storage/cleanup.go @@ -21,6 +21,7 @@ import ( "time" dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" + nsconfig "github.com/devfile/devworkspace-operator/pkg/provision/config" "github.com/devfile/devworkspace-operator/pkg/provision/sync" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -106,6 +107,7 @@ func getSpecCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.Clus if restrictedAccess, needsRestrictedAccess := workspace.Annotations[constants.DevWorkspaceRestrictedAccessAnnotation]; needsRestrictedAccess { jobLabels[constants.DevWorkspaceRestrictedAccessAnnotation] = restrictedAccess } + job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: common.PVCCleanupJobName(workspaceId), @@ -161,10 +163,20 @@ func getSpecCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.Clus }, } - err := controllerutil.SetControllerReference(workspace, job, clusterAPI.Scheme) + podTolerations, nodeSelector, err := nsconfig.GetNamespacePodTolerationsAndNodeSelector(workspace.Namespace, clusterAPI) if err != nil { return nil, err } + if podTolerations != nil && len(podTolerations) > 0 { + job.Spec.Template.Spec.Tolerations = podTolerations + } + if nodeSelector != nil && len(nodeSelector) > 0 { + job.Spec.Template.Spec.NodeSelector = nodeSelector + } + + if err := controllerutil.SetControllerReference(workspace, job, clusterAPI.Scheme); err != nil { + return nil, err + } return job, nil } diff --git a/pkg/provision/workspace/deployment.go b/pkg/provision/workspace/deployment.go index 71418df0f..49094cc1c 100644 --- a/pkg/provision/workspace/deployment.go +++ b/pkg/provision/workspace/deployment.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + nsconfig "github.com/devfile/devworkspace-operator/pkg/provision/config" "github.com/devfile/devworkspace-operator/pkg/provision/sync" "k8s.io/apimachinery/pkg/fields" @@ -113,9 +114,20 @@ func SyncDeploymentToCluster( envFromSourceAdditions = append(envFromSourceAdditions, automountEnv...) } + podTolerations, nodeSelector, err := nsconfig.GetNamespacePodTolerationsAndNodeSelector(workspace.Namespace, clusterAPI) + if err != nil { + return DeploymentProvisioningStatus{ + ProvisioningStatus{ + Message: "failed to read pod tolerations and node selector from namespace", + Err: err, + FailStartup: true, + }, + } + } + // [design] we have to pass components and routing pod additions separately because we need mountsources from each // component. - specDeployment, err := getSpecDeployment(workspace, podAdditions, envFromSourceAdditions, saName, clusterAPI.Scheme) + specDeployment, err := getSpecDeployment(workspace, podAdditions, envFromSourceAdditions, saName, podTolerations, nodeSelector, clusterAPI.Scheme) if err != nil { return DeploymentProvisioningStatus{ ProvisioningStatus{ @@ -228,6 +240,8 @@ func getSpecDeployment( podAdditionsList []v1alpha1.PodAdditions, envFromSourceAdditions []corev1.EnvFromSource, saName string, + podTolerations []corev1.Toleration, + nodeSelector map[string]string, scheme *runtime.Scheme) (*appsv1.Deployment, error) { replicas := int32(1) terminationGracePeriod := int64(10) @@ -294,6 +308,13 @@ func getSpecDeployment( }, } + if podTolerations != nil && len(podTolerations) > 0 { + deployment.Spec.Template.Spec.Tolerations = podTolerations + } + if nodeSelector != nil && len(nodeSelector) > 0 { + deployment.Spec.Template.Spec.NodeSelector = nodeSelector + } + if needsPVCWorkaround(podAdditions) { // Kubernetes creates directories in a PVC to support subpaths such that only the leaf directory has g+rwx permissions. // This means that mounting the subpath e.g. /plugins will result in the directory being