[installer]: set affinities for pods and jobs

gitpod-io · Nov 24, 2021 · c1deb14 · c1deb14
1 parent 17b1b7f
commit c1deb14
Show file tree

Hide file tree

Showing 18 changed files with 82 additions and 15 deletions.
diff --git a/installer/pkg/components/blobserve/deployment.go b/installer/pkg/components/blobserve/deployment.go
@@ -6,6 +6,7 @@ package blobserve
 
 import (
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry"
 	appsv1 "k8s.io/api/apps/v1"
@@ -69,7 +70,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 					},
 					Spec: corev1.PodSpec{
-						Affinity:           &corev1.Affinity{},
+						Affinity:           common.Affinity(cluster.AffinityLabelWorkspaceServices),
 						ServiceAccountName: Component,
 						EnableServiceLinks: pointer.Bool(false),
 						Volumes: []corev1.Volume{{

diff --git a/installer/pkg/components/content-service/deployment.go b/installer/pkg/components/content-service/deployment.go
@@ -5,6 +5,7 @@
 package content_service
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 
 	v1 "k8s.io/api/apps/v1"
@@ -24,7 +25,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 	}
 
 	podSpec := corev1.PodSpec{
-		Affinity:                      &corev1.Affinity{},
+		Affinity:                      common.Affinity(cluster.AffinityLabelMeta),
 		ServiceAccountName:            Component,
 		EnableServiceLinks:            pointer.Bool(false),
 		DNSPolicy:                     "ClusterFirst",

diff --git a/installer/pkg/components/dashboard/deployment.go b/installer/pkg/components/dashboard/deployment.go
@@ -5,6 +5,7 @@
 package dashboard
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -39,7 +40,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						Labels:    labels,
 					},
 					Spec: corev1.PodSpec{
-						Affinity:                      &corev1.Affinity{},
+						Affinity:                      common.Affinity(cluster.AffinityLabelMeta),
 						ServiceAccountName:            Component,
 						EnableServiceLinks:            pointer.Bool(false),
 						DNSPolicy:                     "ClusterFirst",

diff --git a/installer/pkg/components/database/incluster/helm.go b/installer/pkg/components/database/incluster/helm.go
@@ -5,6 +5,7 @@
 package incluster
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/helm"
 	"github.com/gitpod-io/gitpod/installer/third_party/charts"
@@ -13,6 +14,16 @@ import (
 
 var Helm = common.CompositeHelmFunc(
 	helm.ImportTemplate(charts.MySQL(), helm.TemplateConfig{}, func(cfg *common.RenderContext) (*common.HelmConfig, error) {
+		affinity, err := helm.AffinityYaml(cluster.AffinityLabelMeta)
+		if err != nil {
+			return nil, err
+		}
+
+		primaryAffinityTemplate, err := helm.KeyFileValue("mysql.primary.affinity", affinity)
+		if err != nil {
+			return nil, err
+		}
+
 		return &common.HelmConfig{
 			Enabled: true,
 			Values: &values.Options{
@@ -23,6 +34,10 @@ var Helm = common.CompositeHelmFunc(
 					helm.KeyValue("mysql.initdbScriptsConfigMap", SQLInitScripts),
 					helm.KeyValue("mysql.serviceAccount.name", Component),
 				},
+				// This is too complex to be sent as a string
+				FileValues: []string{
+					primaryAffinityTemplate,
+				},
 			},
 		}, nil
 	}),

diff --git a/installer/pkg/components/database/init/job.go b/installer/pkg/components/database/init/job.go
@@ -8,6 +8,7 @@ package init
 
 import (
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	batchv1 "k8s.io/api/batch/v1"
@@ -32,7 +33,7 @@ func job(ctx *common.RenderContext) ([]runtime.Object, error) {
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: objectMeta,
 				Spec: corev1.PodSpec{
-					Affinity:           &corev1.Affinity{},
+					Affinity:           common.Affinity(cluster.AffinityLabelMeta),
 					RestartPolicy:      corev1.RestartPolicyNever,
 					ServiceAccountName: Component,
 					EnableServiceLinks: pointer.Bool(false),

diff --git a/installer/pkg/components/image-builder-mk3/deployment.go b/installer/pkg/components/image-builder-mk3/deployment.go
@@ -6,6 +6,7 @@ package image_builder_mk3
 
 import (
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry"
@@ -78,7 +79,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 					},
 				},
 				Spec: corev1.PodSpec{
-					Affinity:                      &corev1.Affinity{},
+					Affinity:                      common.Affinity(cluster.AffinityLabelMeta),
 					ServiceAccountName:            Component,
 					EnableServiceLinks:            pointer.Bool(false),
 					DNSPolicy:                     "ClusterFirst",

diff --git a/installer/pkg/components/migrations/job.go b/installer/pkg/components/migrations/job.go
@@ -5,6 +5,7 @@
 package migrations
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -28,7 +29,7 @@ func job(ctx *common.RenderContext) ([]runtime.Object, error) {
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: objectMeta,
 				Spec: corev1.PodSpec{
-					Affinity:           &corev1.Affinity{},
+					Affinity:           common.Affinity(cluster.AffinityLabelMeta),
 					RestartPolicy:      corev1.RestartPolicyNever,
 					ServiceAccountName: Component,
 					EnableServiceLinks: pointer.Bool(false),

diff --git a/installer/pkg/components/minio/incluster/minio.go b/installer/pkg/components/minio/incluster/minio.go
@@ -6,6 +6,7 @@ package incluster
 
 import (
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/helm"
 	"github.com/gitpod-io/gitpod/installer/third_party/charts"
@@ -15,6 +16,16 @@ import (
 var Helm = func(apiPort int32, consolePort int32) common.HelmFunc {
 	return common.CompositeHelmFunc(
 		helm.ImportTemplate(charts.Minio(), helm.TemplateConfig{}, func(cfg *common.RenderContext) (*common.HelmConfig, error) {
+			affinity, err := helm.AffinityYaml(cluster.AffinityLabelMeta)
+			if err != nil {
+				return nil, err
+			}
+
+			affinityTemplate, err := helm.KeyFileValue("minio.affinity", affinity)
+			if err != nil {
+				return nil, err
+			}
+
 			return &common.HelmConfig{
 				Enabled: true,
 				Values: &values.Options{
@@ -24,6 +35,10 @@ var Helm = func(apiPort int32, consolePort int32) common.HelmFunc {
 						helm.KeyValue("minio.service.ports.api", fmt.Sprintf("%d", apiPort)),
 						helm.KeyValue("minio.service.ports.console", fmt.Sprintf("%d", consolePort)),
 					},
+					// This is too complex to be sent as a string
+					FileValues: []string{
+						affinityTemplate,
+					},
 				},
 			}, nil
 		}),

diff --git a/installer/pkg/components/openvsx-proxy/statefulset.go b/installer/pkg/components/openvsx-proxy/statefulset.go
@@ -6,6 +6,7 @@ package openvsx_proxy
 
 import (
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	appsv1 "k8s.io/api/apps/v1"
@@ -50,7 +51,7 @@ func statefulset(ctx *common.RenderContext) ([]runtime.Object, error) {
 					},
 				},
 				Spec: v1.PodSpec{
-					Affinity:                      &v1.Affinity{},
+					Affinity:                      common.Affinity(cluster.AffinityLabelIDE),
 					ServiceAccountName:            Component,
 					EnableServiceLinks:            pointer.Bool(false),
 					DNSPolicy:                     "ClusterFirst",

diff --git a/installer/pkg/components/proxy/deployment.go b/installer/pkg/components/proxy/deployment.go
@@ -6,6 +6,7 @@ package proxy
 
 import (
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 
@@ -115,7 +116,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 					},
 					Spec: corev1.PodSpec{
-						Affinity:                      &corev1.Affinity{},
+						Affinity:                      common.Affinity(cluster.AffinityLabelMeta),
 						PriorityClassName:             common.SystemNodeCritical,
 						ServiceAccountName:            Component,
 						EnableServiceLinks:            pointer.Bool(false),

diff --git a/installer/pkg/components/rabbitmq/helm.go b/installer/pkg/components/rabbitmq/helm.go
@@ -7,6 +7,7 @@ package rabbitmq
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/helm"
 	"github.com/gitpod-io/gitpod/installer/third_party/charts"
@@ -258,6 +259,16 @@ var Helm = common.CompositeHelmFunc(
 			return nil, err
 		}
 
+		affinity, err := helm.AffinityYaml(cluster.AffinityLabelMeta)
+		if err != nil {
+			return nil, err
+		}
+
+		affinityTemplate, err := helm.KeyFileValue("rabbitmq.affinity", affinity)
+		if err != nil {
+			return nil, err
+		}
+
 		return &common.HelmConfig{
 			Enabled: true,
 			Values: &values.Options{
@@ -271,6 +282,7 @@ var Helm = common.CompositeHelmFunc(
 				},
 				// This is too complex to be sent as a string
 				FileValues: []string{
+					affinityTemplate,
 					loadDefinitionFilename,
 					shovelsTemplateFileName,
 				},

diff --git a/installer/pkg/components/registry-facade/daemonset.go b/installer/pkg/components/registry-facade/daemonset.go
@@ -88,8 +88,7 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 					},
 				},
 				Spec: corev1.PodSpec{
-					PriorityClassName: common.SystemNodeCritical,
-					// todo(sje): do we need affinity?
+					PriorityClassName:             common.SystemNodeCritical,
 					Affinity:                      common.Affinity(cluster.AffinityLabelWorkspacesRegular, cluster.AffinityLabelWorkspacesHeadless),
 					ServiceAccountName:            Component,
 					EnableServiceLinks:            pointer.Bool(false),

diff --git a/installer/pkg/components/server/deployment.go b/installer/pkg/components/server/deployment.go
@@ -8,6 +8,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	wsmanager "github.com/gitpod-io/gitpod/installer/pkg/components/ws-manager"
@@ -59,7 +60,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 					},
 					Spec: corev1.PodSpec{
-						Affinity:           &corev1.Affinity{},
+						Affinity:           common.Affinity(cluster.AffinityLabelMeta),
 						PriorityClassName:  common.SystemNodeCritical,
 						ServiceAccountName: Component,
 						EnableServiceLinks: pointer.Bool(false),

diff --git a/installer/pkg/components/ws-manager-bridge/deployment.go b/installer/pkg/components/ws-manager-bridge/deployment.go
@@ -6,6 +6,7 @@ package wsmanagerbridge
 
 import (
 	"fmt"
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	wsmanager "github.com/gitpod-io/gitpod/installer/pkg/components/ws-manager"
 
@@ -63,7 +64,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 					},
 					Spec: corev1.PodSpec{
-						Affinity:                      &corev1.Affinity{},
+						Affinity:                      common.Affinity(cluster.AffinityLabelMeta),
 						ServiceAccountName:            Component,
 						PriorityClassName:             common.SystemNodeCritical,
 						EnableServiceLinks:            pointer.Bool(false),

diff --git a/installer/pkg/components/ws-manager/deployment.go b/installer/pkg/components/ws-manager/deployment.go
@@ -5,6 +5,7 @@
 package wsmanager
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon"
 	appsv1 "k8s.io/api/apps/v1"
@@ -25,7 +26,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 
 	podSpec := corev1.PodSpec{
 		PriorityClassName:  common.SystemNodeCritical,
-		Affinity:           &corev1.Affinity{},
+		Affinity:           common.Affinity(cluster.AffinityLabelWorkspaceServices),
 		EnableServiceLinks: pointer.Bool(false),
 		ServiceAccountName: Component,
 		SecurityContext: &corev1.PodSecurityContext{

diff --git a/installer/pkg/components/ws-proxy/deployment.go b/installer/pkg/components/ws-proxy/deployment.go
@@ -5,6 +5,7 @@
 package wsproxy
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 
 	wsmanager "github.com/gitpod-io/gitpod/installer/pkg/components/ws-manager"
@@ -68,7 +69,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 					},
 					Spec: corev1.PodSpec{
 						PriorityClassName:  common.SystemNodeCritical,
-						Affinity:           &corev1.Affinity{},
+						Affinity:           common.Affinity(cluster.AffinityLabelWorkspaceServices),
 						EnableServiceLinks: pointer.Bool(false),
 						ServiceAccountName: Component,
 						SecurityContext: &corev1.PodSecurityContext{

diff --git a/installer/pkg/components/ws-scheduler/deployment.go b/installer/pkg/components/ws-scheduler/deployment.go
@@ -5,6 +5,7 @@
 package wsscheduler
 
 import (
+	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 
 	wsmanager "github.com/gitpod-io/gitpod/installer/pkg/components/ws-manager"
@@ -47,7 +48,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 				},
 				Spec: corev1.PodSpec{
 					PriorityClassName:  common.SystemNodeCritical,
-					Affinity:           &corev1.Affinity{},
+					Affinity:           common.Affinity(cluster.AffinityLabelWorkspaceServices),
 					EnableServiceLinks: pointer.Bool(false),
 					ServiceAccountName: Component,
 					SecurityContext: &corev1.PodSecurityContext{

diff --git a/installer/pkg/helm/helm.go b/installer/pkg/helm/helm.go
@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"sigs.k8s.io/yaml"
 	"syscall"
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
@@ -108,6 +109,18 @@ func writeCharts(chart *charts.Chart) (string, error) {
 	return dir, nil
 }
 
+// AffinityYaml convert an affinity into a YAML byte array
+func AffinityYaml(orLabels ...string) ([]byte, error) {
+	affinities := common.Affinity(orLabels...)
+
+	marshal, err := yaml.Marshal(affinities)
+	if err != nil {
+		return nil, err
+	}
+
+	return marshal, nil
+}
+
 // ImportTemplate allows for Helm charts to be imported into the installer manifest
 func ImportTemplate(chart *charts.Chart, templateCfg TemplateConfig, pkgConfig PkgConfig) common.HelmFunc {
 	return func(cfg *common.RenderContext) (r []string, err error) {