Add termination grace period and liveness probes (#65)

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>
thanos-io · Dec 13, 2019 · 13609ff · 13609ff
1 parent 7f91fb0
commit 13609ff
Show file tree

Hide file tree

Showing 12 changed files with 143 additions and 7 deletions.
diff --git a/examples/all/manifests/thanos-bucket-deployment.yaml b/examples/all/manifests/thanos-bucket-deployment.yaml
@@ -27,14 +27,29 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 8080
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-bucket
         ports:
         - containerPort: 8080
           name: http
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: 250m
             memory: 512Mi
           requests:
             cpu: 100m
             memory: 256Mi
+      terminationGracePeriodSeconds: 120
diff --git a/examples/all/manifests/thanos-compactor-statefulSet.yaml b/examples/all/manifests/thanos-compactor-statefulSet.yaml
@@ -32,6 +32,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-compactor
         ports:
         - containerPort: 10902
@@ -41,6 +48,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: 500m
@@ -52,6 +61,7 @@ spec:
         - mountPath: /var/thanos/compactor
           name: thanos-compactor-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
       volumes:
       - emptyDir: {}
         name: thanos-compactor-data
diff --git a/examples/all/manifests/thanos-querier-deployment.yaml b/examples/all/manifests/thanos-querier-deployment.yaml
@@ -26,6 +26,13 @@ spec:
         - --store=dnssrv+_grpc._tcp.prometheus-k8s.monitoring.svc.cluster.local
         - --store=dnssrv+_grpc._tcp.thanos-rule.monitoring.svc.cluster.local
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 9090
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-querier
         ports:
         - containerPort: 10901
@@ -37,10 +44,13 @@ spec:
             path: /-/ready
             port: 9090
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "1"
             memory: 1Gi
           requests:
             cpu: 100m
             memory: 256Mi
+      terminationGracePeriodSeconds: 120
diff --git a/examples/all/manifests/thanos-receive-statefulSet.yaml b/examples/all/manifests/thanos-receive-statefulSet.yaml
@@ -37,6 +37,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-receive
         ports:
         - containerPort: 10901
@@ -50,6 +57,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "1"
@@ -61,6 +70,7 @@ spec:
         - mountPath: /var/thanos/receive
           name: thanos-receive-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
   volumeClaimTemplates:
   - metadata:
       name: thanos-receive-data

diff --git a/examples/all/manifests/thanos-store-statefulSet.yaml b/examples/all/manifests/thanos-store-statefulSet.yaml
@@ -30,6 +30,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-store
         ports:
         - containerPort: 10901
@@ -41,6 +48,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "2"
@@ -52,6 +61,7 @@ spec:
         - mountPath: /var/thanos/store
           name: thanos-store-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
   volumeClaimTemplates:
   - metadata:
       name: thanos-store-data

diff --git a/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet b/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet
@@ -50,12 +50,25 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.mixin.resources.withLimits({ cpu: '250m', memory: '512Mi' }) +
           container.withPorts([
             { name: 'http', containerPort: tb.ports.http },
-          ]);
+          ]) +
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.bucket.service.spec.ports[0].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.bucket.service.spec.ports[0].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         deployment.new(tb.name, 1, c, $.thanos.bucket.deployment.metadata.labels) +
         deployment.mixin.metadata.withNamespace(tb.namespace) +
         deployment.mixin.metadata.withLabels({ 'app.kubernetes.io/name': tb.name }) +
-        deployment.mixin.spec.selector.withMatchLabels(tb.labels),
+        deployment.mixin.spec.selector.withMatchLabels(tb.labels) +
+        deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120),
     },
   },
 }
diff --git a/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet b/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet
@@ -56,12 +56,24 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.withVolumeMounts([
             containerVolumeMount.new('thanos-compactor-data', '/var/thanos/compactor', false),
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         statefulSet.new(tc.name, 1, c, [], $.thanos.compactor.statefulSet.metadata.labels) +
         statefulSet.mixin.metadata.withNamespace(tc.namespace) +
         statefulSet.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.compactor.statefulSet.metadata.name }) +
         statefulSet.mixin.spec.withServiceName($.thanos.compactor.service.metadata.name) +
+        statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) +
         statefulSet.mixin.spec.template.spec.withVolumes([
           volume.fromEmptyDir('thanos-compactor-data'),
         ]) +

diff --git a/jsonnet/kube-thanos/kube-thanos-querier.libsonnet b/jsonnet/kube-thanos/kube-thanos-querier.libsonnet
@@ -46,12 +46,24 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
             { name: 'grpc', containerPort: $.thanos.querier.service.spec.ports[0].port },
             { name: 'http', containerPort: $.thanos.querier.service.spec.ports[1].port },
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         deployment.new(tq.name, tq.replicas, c, $.thanos.querier.deployment.metadata.labels) +
         deployment.mixin.metadata.withNamespace(tq.namespace) +
         deployment.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.querier.deployment.metadata.name }) +
-        deployment.mixin.spec.selector.withMatchLabels($.thanos.querier.deployment.metadata.labels),
+        deployment.mixin.spec.selector.withMatchLabels($.thanos.querier.deployment.metadata.labels) +
+        deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120),
     },
   },
 }
diff --git a/jsonnet/kube-thanos/kube-thanos-receive.libsonnet b/jsonnet/kube-thanos/kube-thanos-receive.libsonnet
@@ -64,13 +64,25 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.withVolumeMounts([
             containerVolumeMount.new(tr.name + '-data', '/var/thanos/receive', false),
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         sts.new(tr.name, tr.replicas, c, [], $.thanos.receive.statefulSet.metadata.labels) +
         sts.mixin.metadata.withNamespace(tr.namespace) +
         sts.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.receive.statefulSet.metadata.name }) +
         sts.mixin.spec.withServiceName($.thanos.receive.service.metadata.name) +
         sts.mixin.spec.selector.withMatchLabels($.thanos.receive.statefulSet.metadata.labels) +
+        sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) +
         sts.mixin.spec.template.spec.withVolumes([
           volume.fromEmptyDir('data'),
         ]) +

diff --git a/jsonnet/kube-thanos/kube-thanos-store.libsonnet b/jsonnet/kube-thanos/kube-thanos-store.libsonnet
@@ -64,13 +64,25 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.withVolumeMounts([
             containerVolumeMount.new(ts.name + '-data', '/var/thanos/store', false),
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         sts.new(ts.name, ts.replicas, c, [], $.thanos.store.statefulSet.metadata.labels) +
         sts.mixin.metadata.withNamespace(ts.namespace) +
         sts.mixin.metadata.withLabels({ 'app.kubernetes.io/name': ts.name }) +
         sts.mixin.spec.withServiceName($.thanos.store.service.metadata.name) +
         sts.mixin.spec.selector.withMatchLabels($.thanos.store.statefulSet.metadata.labels) +
+        sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) +
         sts.mixin.spec.template.spec.withVolumes([
           volume.fromEmptyDir(ts.name + '-data'),
         ]) +

diff --git a/manifests/thanos-querier-deployment.yaml b/manifests/thanos-querier-deployment.yaml
@@ -23,6 +23,13 @@ spec:
         - --http-address=0.0.0.0:9090
         - --store=dnssrv+_grpc._tcp.thanos-store.monitoring.svc.cluster.local
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 9090
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-querier
         ports:
         - containerPort: 10901
@@ -34,10 +41,13 @@ spec:
             path: /-/ready
             port: 9090
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "1"
             memory: 1Gi
           requests:
             cpu: 100m
             memory: 256Mi
+      terminationGracePeriodSeconds: 120
diff --git a/manifests/thanos-store-statefulSet.yaml b/manifests/thanos-store-statefulSet.yaml
@@ -30,6 +30,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-store
         ports:
         - containerPort: 10901
@@ -41,6 +48,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "2"
@@ -52,6 +61,7 @@ spec:
         - mountPath: /var/thanos/store
           name: thanos-store-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
       volumes:
       - emptyDir: {}
         name: thanos-store-data