jsonnet: Update use of latest kube-thanos and define app labels

openshift · Apr 6, 2020 · d01a0e6 · d01a0e6
1 parent b0ef73b
commit d01a0e6
Show file tree

Hide file tree

Showing 12 changed files with 112 additions and 59 deletions.
diff --git a/assets/thanos-querier/cluster-role-binding.yaml b/assets/thanos-querier/cluster-role-binding.yaml
@@ -1,6 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier
 roleRef:
   apiGroup: rbac.authorization.k8s.io

diff --git a/assets/thanos-querier/cluster-role.yaml b/assets/thanos-querier/cluster-role.yaml
@@ -1,6 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier
 rules:
 - apiGroups:

diff --git a/assets/thanos-querier/deployment.yaml b/assets/thanos-querier/deployment.yaml
@@ -2,32 +2,54 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    app.kubernetes.io/name: thanos-querier
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier
   namespace: openshift-monitoring
 spec:
   replicas: 2
   selector:
     matchLabels:
-      app.kubernetes.io/name: thanos-querier
+      app.kubernetes.io/component: query-layer
+      app.kubernetes.io/instance: thanos-querier
+      app.kubernetes.io/name: thanos-query
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: thanos-querier
+        app.kubernetes.io/component: query-layer
+        app.kubernetes.io/instance: thanos-querier
+        app.kubernetes.io/name: thanos-query
+        app.kubernetes.io/version: 0.11.0
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-query
+              namespaces:
+              - openshift-monitoring
+              topologyKey: kubernetes.io/hostname
+            weight: 100
       containers:
       - args:
         - query
-        - --query.replica-label=prometheus_replica
-        - --query.replica-label=thanos_ruler_replica
         - --grpc-address=127.0.0.1:10901
         - --http-address=127.0.0.1:9090
+        - --query.replica-label=prometheus_replica
+        - --query.replica-label=thanos_ruler_replica
+        - --store=dnssrv+_grpc._tcp.prometheus-operated.openshift-monitoring.svc.cluster.local
         - --grpc-client-tls-secure
         - --grpc-client-tls-cert=/etc/tls/grpc/client.crt
         - --grpc-client-tls-key=/etc/tls/grpc/client.key
         - --grpc-client-tls-ca=/etc/tls/grpc/ca.crt
         - --grpc-client-server-name=prometheus-grpc
-        - --store=dnssrv+_grpc._tcp.prometheus-operated.openshift-monitoring.svc.cluster.local
         image: quay.io/openshift/origin-thanos:latest
         livenessProbe:
           exec:
@@ -37,7 +59,7 @@ spec:
             - if [ -x "$(command -v curl)" ]; then curl http://localhost:9090/-/healthy;
               elif [ -x "$(command -v wget)" ]; then wget --quiet --tries=1 --spider
               http://localhost:9090/-/healthy; else exit 1; fi
-        name: thanos-querier
+        name: thanos-query
         ports:
         - containerPort: 9090
           name: http
@@ -53,6 +75,7 @@ spec:
           requests:
             cpu: 5m
             memory: 12Mi
+        terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
         - mountPath: /etc/tls/grpc
           name: secret-grpc-tls
@@ -132,7 +155,6 @@ spec:
             memory: 20Mi
         terminationMessagePolicy: FallbackToLogsOnError
       priorityClassName: system-cluster-critical
-      securityContext: {}
       serviceAccountName: thanos-querier
       terminationGracePeriodSeconds: 120
       volumes:

diff --git a/assets/thanos-querier/grpc-tls-secret.yaml b/assets/thanos-querier/grpc-tls-secret.yaml
@@ -3,7 +3,10 @@ data: {}
 kind: Secret
 metadata:
   labels:
-    k8s-app: thanos-querier
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier-grpc-tls
   namespace: openshift-monitoring
 type: Opaque
diff --git a/assets/thanos-querier/kube-rbac-proxy-secret.yaml b/assets/thanos-querier/kube-rbac-proxy-secret.yaml
@@ -4,7 +4,10 @@ data:
 kind: Secret
 metadata:
   labels:
-    k8s-app: thanos
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier-kube-rbac-proxy
   namespace: openshift-monitoring
 type: Opaque
diff --git a/assets/thanos-querier/oauth-cookie-secret.yaml b/assets/thanos-querier/oauth-cookie-secret.yaml
@@ -3,7 +3,10 @@ data: {}
 kind: Secret
 metadata:
   labels:
-    k8s-app: thanos-querier
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier-oauth-cookie
   namespace: openshift-monitoring
 type: Opaque
diff --git a/assets/thanos-querier/oauth-htpasswd-secret.yaml b/assets/thanos-querier/oauth-htpasswd-secret.yaml
@@ -3,7 +3,10 @@ data: {}
 kind: Secret
 metadata:
   labels:
-    k8s-app: thanos-querier
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier-oauth-htpasswd
   namespace: openshift-monitoring
 type: Opaque
diff --git a/assets/thanos-querier/route.yaml b/assets/thanos-querier/route.yaml
@@ -1,6 +1,11 @@
 apiVersion: v1
 kind: Route
 metadata:
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier
   namespace: openshift-monitoring
 spec:

diff --git a/assets/thanos-querier/service-account.yaml b/assets/thanos-querier/service-account.yaml
@@ -3,5 +3,10 @@ kind: ServiceAccount
 metadata:
   annotations:
     serviceaccounts.openshift.io/oauth-redirectreference.thanos-querier: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"thanos-querier"}}'
+  labels:
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier
   namespace: openshift-monitoring
diff --git a/assets/thanos-querier/service.yaml b/assets/thanos-querier/service.yaml
@@ -4,7 +4,10 @@ metadata:
   annotations:
     service.alpha.openshift.io/serving-cert-secret-name: thanos-querier-tls
   labels:
-    app.kubernetes.io/name: thanos-querier
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/version: 0.11.0
   name: thanos-querier
   namespace: openshift-monitoring
 spec:
@@ -16,5 +19,7 @@ spec:
     port: 9092
     targetPort: tenancy
   selector:
-    app.kubernetes.io/name: thanos-querier
+    app.kubernetes.io/component: query-layer
+    app.kubernetes.io/instance: thanos-querier
+    app.kubernetes.io/name: thanos-query
   type: ClusterIP
diff --git a/jsonnet/main.jsonnet b/jsonnet/main.jsonnet
@@ -5,19 +5,6 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
            (import 'kube-prometheus/kube-prometheus-anti-affinity.libsonnet') +
            (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') +
            (import 'kube-prometheus/kube-prometheus-thanos-sidecar.libsonnet') +
-           (import 'kube-thanos/kube-thanos-querier.libsonnet') +
-           (import 'kube-thanos/kube-thanos-ruler.libsonnet') +
-           (import 'kube-thanos/kube-thanos-store.libsonnet') +
-           {
-             thanos+:: {
-               ruler+: {
-                 objectStorageConfig: {
-                   name: 'storename',
-                   key: 'storekey',
-                 },
-               },
-             },
-           } +
            (import 'openshift-state-metrics/openshift-state-metrics.libsonnet') +
            {
              prometheusK8s+:: $.prometheus {

diff --git a/jsonnet/thanos-querier.jsonnet b/jsonnet/thanos-querier.jsonnet
@@ -29,20 +29,32 @@ local authorizationRole =
   local config = super._config,
 
   thanos+:: {
-    image:: config.imageRepos.openshiftThanos + ':' + config.versions.openshiftThanos,
 
-    querier+: {
+    querier+: (import 'kube-thanos/kube-thanos-query.libsonnet') {
+      local tq = self,
+
+      config+:: {
+        name: 'thanos-querier',
+        namespace: config.namespace,
+        image: config.imageRepos.openshiftThanos + ':' + config.versions.openshiftThanos,
+        version: '0.11.0',
+        replicas: 2,
+        replicaLabels: ['prometheus_replica', 'thanos_ruler_replica'],
+        stores: ['dnssrv+_grpc._tcp.prometheus-operated.openshift-monitoring.svc.cluster.local'],
+      },
+
       trustedCaBundle:
         configmap.new('thanos-querier-trusted-ca-bundle', { 'ca-bundle.crt': '' }) +
-        configmap.mixin.metadata.withNamespace($._config.namespace) +
+        configmap.mixin.metadata.withNamespace(tq.config.namespace) +
         configmap.mixin.metadata.withLabels({ 'config.openshift.io/inject-trusted-cabundle': 'true' }),
 
       route: {
         apiVersion: 'v1',
         kind: 'Route',
         metadata: {
           name: 'thanos-querier',
-          namespace: $._config.namespace,
+          namespace: tq.config.namespace,
+          labels: tq.config.commonLabels,
         },
         spec: {
           to: {
@@ -62,41 +74,43 @@ local authorizationRole =
       clusterRole:
         clusterRole.new() +
         clusterRole.mixin.metadata.withName('thanos-querier') +
+        clusterRole.mixin.metadata.withLabels(tq.config.commonLabels) +
         clusterRole.withRules([authenticationRole, authorizationRole]),
 
       clusterRoleBinding:
         local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
 
         clusterRoleBinding.new() +
         clusterRoleBinding.mixin.metadata.withName('thanos-querier') +
+        clusterRoleBinding.mixin.metadata.withLabels(tq.config.commonLabels) +
         clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
         clusterRoleBinding.mixin.roleRef.withName('thanos-querier') +
         clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
         clusterRoleBinding.withSubjects([{
           kind: 'ServiceAccount',
           name: 'thanos-querier',
-          namespace: $._config.namespace,
+          namespace: tq.config.namespace,
         }]),
 
       grpcTlsSecret:
         secret.new('thanos-querier-grpc-tls', {}) +
-        secret.mixin.metadata.withNamespace($._config.namespace) +
-        secret.mixin.metadata.withLabels({ 'k8s-app': 'thanos-querier' }),
+        secret.mixin.metadata.withNamespace(tq.config.namespace) +
+        secret.mixin.metadata.withLabels(tq.config.commonLabels),
 
       // holds the secret which is used encrypt/decrypt cookies
       // issued by the oauth proxy.
       oauthCookieSecret:
         secret.new('thanos-querier-oauth-cookie', {}) +
-        secret.mixin.metadata.withNamespace($._config.namespace) +
-        secret.mixin.metadata.withLabels({ 'k8s-app': 'thanos-querier' }),
+        secret.mixin.metadata.withNamespace(tq.config.namespace) +
+        secret.mixin.metadata.withLabels(tq.config.commonLabels),
 
       // holds the htpasswd configuration
       // which includes a static secret used to authenticate/authorize
       // requests originating from grafana.
       oauthHtpasswdSecret:
         secret.new('thanos-querier-oauth-htpasswd', {}) +
-        secret.mixin.metadata.withNamespace($._config.namespace) +
-        secret.mixin.metadata.withLabels({ 'k8s-app': 'thanos-querier' }),
+        secret.mixin.metadata.withNamespace(tq.config.namespace) +
+        secret.mixin.metadata.withLabels(tq.config.commonLabels),
 
       // holds the kube-rbac-proxy configuration as a secret.
       // It configures to template the request in flight
@@ -124,14 +138,15 @@ local authorizationRole =
         };
 
         secret.new('thanos-querier-kube-rbac-proxy', config) +
-        secret.mixin.metadata.withNamespace($._config.namespace) +
-        secret.mixin.metadata.withLabels({ 'k8s-app': 'thanos' }),
+        secret.mixin.metadata.withNamespace(tq.config.namespace) +
+        secret.mixin.metadata.withLabels(tq.config.commonLabels),
 
       serviceAccount:
         local serviceAccount = k.core.v1.serviceAccount;
 
         serviceAccount.new('thanos-querier') +
-        serviceAccount.mixin.metadata.withNamespace($._config.namespace) +
+        serviceAccount.mixin.metadata.withNamespace(tq.config.namespace) +
+        serviceAccount.mixin.metadata.withLabels(tq.config.commonLabels) +
 
         // The ServiceAccount needs this annotation, to signify the identity
         // provider, that when a users it doing the oauth flow through the
@@ -148,7 +163,7 @@ local authorizationRole =
         service.mixin.metadata.withAnnotations({
           'service.alpha.openshift.io/serving-cert-secret-name': 'thanos-querier-tls',
         }) +
-        service.mixin.metadata.withNamespace(config.namespace) +
+        service.mixin.metadata.withLabels(tq.config.commonLabels) +
         // The ClusterIP is explicitly set, as it signifies the
         // cluster-monitoring-operator, that when reconciling this service the
         // cluster IP needs to be retained.
@@ -159,22 +174,17 @@ local authorizationRole =
         ]),
 
       deployment+:
-        deployment.mixin.metadata.withNamespace(config.namespace) +
         {
           spec+: {
-            replicas: 2,
             template+: {
               spec+: {
-                // TODO (the library adds some affinity rules here, we have to dynamically dependency inject the namespace here)
-                affinity+:: {},
-                volumes: [
+                volumes+: [
                   volume.fromSecret('secret-thanos-querier-tls', 'thanos-querier-tls'),
                   volume.fromSecret('secret-thanos-querier-oauth-cookie', 'thanos-querier-oauth-cookie'),
                   volume.fromSecret('secret-thanos-querier-oauth-htpasswd', 'thanos-querier-oauth-htpasswd'),
                   volume.fromSecret('secret-thanos-querier-kube-rbac-proxy', 'thanos-querier-kube-rbac-proxy'),
                 ],
                 serviceAccountName: 'thanos-querier',
-                securityContext: {},
                 priorityClassName: 'system-cluster-critical',
                 containers: [
                   super.containers[0] {
@@ -190,21 +200,18 @@ local authorizationRole =
                         command: ['sh', '-c', 'if [ -x "$(command -v curl)" ]; then curl http://localhost:9090/-/healthy; elif [ -x "$(command -v wget)" ]; then wget --quiet --tries=1 --spider http://localhost:9090/-/healthy; else exit 1; fi'],
                       },
                     },
-                    args: [
-                      'query',
-                      '--query.replica-label=prometheus_replica',
-                      '--query.replica-label=thanos_ruler_replica',
-                      '--grpc-address=127.0.0.1:10901',
-                      '--http-address=127.0.0.1:9090',
+                    args: std.map(
+                      function(a)
+                        if std.startsWith(a, '--grpc-address=') then '--grpc-address=127.0.0.1:10901'
+                        else if std.startsWith(a, '--http-address=') then '--http-address=127.0.0.1:9090'
+                        else a,
+                      super.args
+                    ) + [
                       '--grpc-client-tls-secure',
                       '--grpc-client-tls-cert=/etc/tls/grpc/client.crt',
                       '--grpc-client-tls-key=/etc/tls/grpc/client.key',
                       '--grpc-client-tls-ca=/etc/tls/grpc/ca.crt',
                       '--grpc-client-server-name=prometheus-grpc',
-                      '--store=dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [
-                        'prometheus-operated',
-                        'openshift-monitoring',
-                      ],
                     ],
                     resources: {
                       requests: {
@@ -218,7 +225,7 @@ local authorizationRole =
                         name: 'http',
                       },
                     ],
-                    volumeMounts: [
+                    volumeMounts+: [
                       {
                         mountPath: '/etc/tls/grpc',
                         name: 'secret-grpc-tls',