Scalyr agent sidecar for log shipping

zalando · Dec 20, 2017 · 175fecc · 175fecc
1 parent 04024f9
commit 175fecc
Show file tree

Hide file tree

Showing 4 changed files with 105 additions and 40 deletions.
diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go
@@ -295,6 +295,33 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 	if len(c.Statefulset.Spec.Template.Spec.Containers) != len(statefulSet.Spec.Template.Spec.Containers) {
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's container specification doesn't match the current one")
+	} else {
+		for index, container1 := range c.Statefulset.Spec.Template.Spec.Containers {
+			container2 := statefulSet.Spec.Template.Spec.Containers[index]
+			if container1.Image != container2.Image {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d image doesn't match the current one", index))
+			}
+			if !reflect.DeepEqual(container1.Ports, container2.Ports) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d ports don't match the current one", index))
+			}
+			if !compareResources(&container1.Resources, &container2.Resources) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d resources don't match the current ones", index))
+			}
+			if !reflect.DeepEqual(container1.Env, container2.Env) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d environment doesn't match the current one", index))
+			}
+			if !reflect.DeepEqual(container1.EnvFrom, container2.EnvFrom) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d environment sources don't match the current one", index))
+			}
+			if needsRollUpdate || needsReplace {
+				match = false
+			}
+		}
 	}
 	if len(c.Statefulset.Spec.Template.Spec.Containers) == 0 {
 		c.logger.Warningf("statefulset %q has no container", util.NameFromMeta(c.Statefulset.ObjectMeta))
@@ -355,35 +382,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 		}
 	}
 
-	container1 := c.Statefulset.Spec.Template.Spec.Containers[0]
-	container2 := statefulSet.Spec.Template.Spec.Containers[0]
-	if container1.Image != container2.Image {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container image doesn't match the current one")
-	}
-
-	if !reflect.DeepEqual(container1.Ports, container2.Ports) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container ports don't match the current one")
-	}
-
-	if !compareResources(&container1.Resources, &container2.Resources) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container resources don't match the current ones")
-	}
-	if !reflect.DeepEqual(container1.Env, container2.Env) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container environment doesn't match the current one")
-	}
-	if !reflect.DeepEqual(container1.EnvFrom, container2.EnvFrom) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container environment sources don't match the current one")
-	}
-
-	if needsRollUpdate || needsReplace {
-		match = false
-	}
-
 	return &compareStatefulsetResult{match: match, reasons: reasons, rollingUpdate: needsRollUpdate, replace: needsReplace}
 }
 

diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go
@@ -276,13 +276,16 @@ func (c *Cluster) tolerations(tolerationsSpec *[]v1.Toleration) []v1.Toleration
 	}
 }
 
-func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequirements,
+func (c *Cluster) generatePodTemplate(
+	resourceRequirements *v1.ResourceRequirements,
+	resourceRequirementsScalyrSidecar *v1.ResourceRequirements,
 	tolerationsSpec *[]v1.Toleration,
 	pgParameters *spec.PostgresqlParam,
 	patroniParameters *spec.Patroni,
 	cloneDescription *spec.CloneDescription,
 	dockerImage *string,
-	customPodEnvVars map[string]string) *v1.PodTemplateSpec {
+	customPodEnvVars map[string]string,
+) *v1.PodTemplateSpec {
 	spiloConfiguration := c.generateSpiloJSONConfiguration(pgParameters, patroniParameters)
 
 	envVars := []v1.EnvVar{
@@ -391,6 +394,12 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 	if dockerImage != nil && *dockerImage != "" {
 		containerImage = *dockerImage
 	}
+	volumeMounts := []v1.VolumeMount{
+		{
+			Name:      constants.DataVolumeName,
+			MountPath: constants.PostgresDataMount, //TODO: fetch from manifest
+		},
+	}
 	container := v1.Container{
 		Name:            c.containerName(),
 		Image:           containerImage,
@@ -410,13 +419,8 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 				Protocol:      v1.ProtocolTCP,
 			},
 		},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      constants.DataVolumeName,
-				MountPath: constants.PostgresDataMount, //TODO: fetch from manifesto
-			},
-		},
-		Env: envVars,
+		VolumeMounts: volumeMounts,
+		Env:          envVars,
 		SecurityContext: &v1.SecurityContext{
 			Privileged: &privilegedMode,
 		},
@@ -431,6 +435,29 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 		Tolerations:                   c.tolerations(tolerationsSpec),
 	}
 
+	if c.OpConfig.ScalyrAPIKey != "" {
+		podSpec.Containers = append(
+			podSpec.Containers,
+			v1.Container{
+				Name:            "scalyr-sidecar",
+				Image:           c.OpConfig.ScalyrImage,
+				ImagePullPolicy: v1.PullIfNotPresent,
+				Resources:       *resourceRequirementsScalyrSidecar,
+				VolumeMounts:    volumeMounts,
+				Env: []v1.EnvVar{
+					{
+						Name:  "SCALYR_API_KEY",
+						Value: c.OpConfig.ScalyrAPIKey,
+					},
+					{
+						Name:  "SCALYR_SERVER_HOST",
+						Value: c.Name,
+					},
+				},
+			},
+		)
+	}
+
 	template := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels:    c.labelsSet(),
@@ -445,11 +472,40 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 	return &template
 }
 
+func makeResources(
+	cpuRequest string,
+	memoryRequest string,
+	cpuLimit string,
+	memoryLimit string,
+) spec.Resources {
+	return spec.Resources{
+		ResourceRequest: spec.ResourceDescription{
+			CPU:    cpuRequest,
+			Memory: memoryRequest,
+		},
+		ResourceLimits: spec.ResourceDescription{
+			CPU:    cpuLimit,
+			Memory: memoryLimit,
+		},
+	}
+}
+
 func (c *Cluster) generateStatefulSet(spec *spec.PostgresSpec) (*v1beta1.StatefulSet, error) {
 	resourceRequirements, err := c.resourceRequirements(spec.Resources)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate resource requirements: %v", err)
 	}
+	resourceRequirementsScalyrSidecar, err := c.resourceRequirements(
+		makeResources(
+			c.OpConfig.ScalyrCPURequest,
+			c.OpConfig.ScalyrMemoryRequest,
+			c.OpConfig.ScalyrCPULimit,
+			c.OpConfig.ScalyrMemoryLimit,
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("could not generate Scalyr sidecar resource requirements: %v", err)
+	}
 	var customPodEnvVars map[string]string
 	if c.OpConfig.PodEnvironmentConfigMap != "" {
 		if cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap, metav1.GetOptions{}); err != nil {
@@ -458,7 +514,7 @@ func (c *Cluster) generateStatefulSet(spec *spec.PostgresSpec) (*v1beta1.Statefu
 			customPodEnvVars = cm.Data
 		}
 	}
-	podTemplate := c.generatePodTemplate(resourceRequirements, &spec.Tolerations, &spec.PostgresqlParam, &spec.Patroni, &spec.Clone, &spec.DockerImage, customPodEnvVars)
+	podTemplate := c.generatePodTemplate(resourceRequirements, resourceRequirementsScalyrSidecar, &spec.Tolerations, &spec.PostgresqlParam, &spec.Patroni, &spec.Clone, &spec.DockerImage, customPodEnvVars)
 	volumeClaimTemplate, err := generatePersistentVolumeClaimTemplate(spec.Volume.Size, spec.Volume.StorageClass)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate volume claim template: %v", err)

diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go
@@ -319,7 +319,7 @@ func (c *Cluster) syncSecrets() error {
 			if err2 != nil {
 				return fmt.Errorf("could not get current secret: %v", err2)
 			}
-			c.logger.Debugf("secret %q already exists, fetching it's password", util.NameFromMeta(curSecret.ObjectMeta))
+			c.logger.Debugf("secret %q already exists, fetching its password", util.NameFromMeta(curSecret.ObjectMeta))
 			if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
 				secretUsername = constants.SuperuserKeyName
 				userMap = c.systemUsers

diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
@@ -50,11 +50,22 @@ type Auth struct {
 	ReplicationUsername           string              `name:"replication_username" default:"standby"`
 }
 
+// Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping:
+type Scalyr struct {
+	ScalyrAPIKey        string `name:"scalyr_api_key" default:""`
+	ScalyrImage         string `name:"scalyr_image" default:"registry.opensource.zalan.do/acid/spilo-sidecar-scalyr:cpr-1-18"`
+	ScalyrCPURequest    string `name:"scalyr_cpu_request" default:"100m"`
+	ScalyrMemoryRequest string `name:"scalyr_memory_request" default:"50Mi"`
+	ScalyrCPULimit      string `name:"scalyr_cpu_limit" default:"1"`
+	ScalyrMemoryLimit   string `name:"scalyr_memory_limit" default:"1Gi"`
+}
+
 // Config describes operator config
 type Config struct {
 	CRD
 	Resources
 	Auth
+	Scalyr
 	Namespace                string            `name:"namespace"`
 	EtcdHost                 string            `name:"etcd_host" default:"etcd-client.default.svc.cluster.local:2379"`
 	DockerImage              string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spiloprivate-9.6:1.2-p4"`