diff --git a/build.sh b/build.sh index 8dc7a7cb6..c5e604f0f 100755 --- a/build.sh +++ b/build.sh @@ -23,6 +23,7 @@ jsonnet -J vendor environments/openshift/telemeter-prometheus-ams.jsonnet | gojs jsonnet -J vendor environments/openshift/telemeter.jsonnet | gojsontoyaml >environments/openshift/manifests/telemeter-template.yaml jsonnet -J vendor environments/openshift/thanos.jsonnet | gojsontoyaml >environments/openshift/manifests/thanos-template.yaml jsonnet -J vendor environments/openshift/jaeger.jsonnet | gojsontoyaml >environments/openshift/manifests/jaeger-template.yaml +jsonnet -J vendor environments/openshift/observatorium-api.jsonnet | gojsontoyaml >environments/openshift/manifests/observatorium-api-template.yaml find environments/openshift/manifests -type f ! -name '*.yaml' -delete # Make sure to start with a clean 'servicemonitors' dir diff --git a/environments/kubernetes/main.jsonnet b/environments/kubernetes/main.jsonnet index 65f03ab64..636fc21bc 100644 --- a/environments/kubernetes/main.jsonnet +++ b/environments/kubernetes/main.jsonnet @@ -1,8 +1,10 @@ local app = (import 'kube-thanos.libsonnet') + (import 'telemeter.libsonnet') + + (import 'observatorium.libsonnet') + (import 'jaeger.libsonnet'); +{ ['observatorium-api-' + name]: app.observatorium.api[name] for name in std.objectFields(app.observatorium.api) } + { ['thanos-querier-' + name]: app.thanos.querier[name] for name in std.objectFields(app.thanos.querier) } + { ['thanos-receive-' + name]: app.thanos.receive[name] for name in std.objectFields(app.thanos.receive) } + { ['thanos-compactor-' + name]: app.thanos.compactor[name] for name in std.objectFields(app.thanos.compactor) } + diff --git a/environments/kubernetes/manifests/observatorium-api-deployment.yaml b/environments/kubernetes/manifests/observatorium-api-deployment.yaml new file mode 100644 index 000000000..ee4e37ec5 --- /dev/null +++ b/environments/kubernetes/manifests/observatorium-api-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: observatorium-api + name: observatorium-api + namespace: observatorium +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: observatorium-api + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + app.kubernetes.io/name: observatorium-api + spec: + containers: + - args: + - --web.listen=0.0.0.0:8080 + - --metrics.ui.endpoint=http://thanos-querier.observatorium.svc.cluster.local:9090 + - --metrics.query.endpoint=http://observatorium-cache.observatorium.svc.cluster.local:9090/api/v1/query + - --metrics.write.endpoint=http://thanos-receive.observatorium.svc.cluster.local:19291/api/v1/receive + - --log.level=warn + image: quay.io/observatorium/observatorium:latest + livenessProbe: + failureThreshold: 4 + httpGet: + path: /-/healthy + port: 8080 + scheme: HTTP + periodSeconds: 30 + name: observatorium-api + ports: + - containerPort: 8080 + name: http + readinessProbe: + failureThreshold: 3 + httpGet: + path: /-/ready + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + limits: + cpu: "2" + memory: 1Gi + requests: + cpu: "1" + memory: 256Mi diff --git a/environments/kubernetes/manifests/observatorium-api-service.yaml b/environments/kubernetes/manifests/observatorium-api-service.yaml new file mode 100644 index 000000000..ca6d13391 --- /dev/null +++ b/environments/kubernetes/manifests/observatorium-api-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: observatorium-api +spec: + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + app.kubernetes.io/name: observatorium-api diff --git a/environments/kubernetes/observatorium.libsonnet b/environments/kubernetes/observatorium.libsonnet new file mode 100644 index 000000000..1940f3b13 --- /dev/null +++ b/environments/kubernetes/observatorium.libsonnet @@ -0,0 +1,42 @@ +local kt = (import 'kube-thanos.libsonnet'); + +(import 'observatorium/observatorium-api.libsonnet') { + observatorium+:: { + namespace:: 'observatorium', + api+: { + replicas:: 3, + deployment+: { + spec+: { + template+: { + spec+: { + containers: [ + super.containers[0] + { + args: [ + '--web.listen=0.0.0.0:8080', + '--metrics.ui.endpoint=http://%s.%s.svc.cluster.local:%d' % [ + kt.thanos.querier.service.metadata.name, + kt.thanos.querier.service.metadata.namespace, + kt.thanos.querier.service.spec.ports[1].port, + ], + '--metrics.query.endpoint=http://%s.%s.svc.cluster.local:%d/api/v1/query' % [ + kt.thanos.querierCache.service.metadata.name, + kt.thanos.querierCache.service.metadata.namespace, + kt.thanos.querierCache.service.spec.ports[0].port, + ], + '--metrics.write.endpoint=http://%s.%s.svc.cluster.local:%d/api/v1/receive' % [ + kt.thanos.receive.service.metadata.name, + kt.thanos.receive.service.metadata.namespace, + kt.thanos.receive.service.spec.ports[2].port, + ], + '--log.level=warn', + ], + }, + ], + }, + }, + }, + }, + }, + }, +} diff --git a/environments/openshift/manifests/observatorium-api-template.yaml b/environments/openshift/manifests/observatorium-api-template.yaml new file mode 100644 index 000000000..d9be620db --- /dev/null +++ b/environments/openshift/manifests/observatorium-api-template.yaml @@ -0,0 +1,147 @@ +apiVersion: v1 +kind: Template +metadata: + name: observatorium-api +objects: +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: observatorium-api + name: observatorium-api + namespace: ${NAMESPACE} + spec: + replicas: ${OBSERVATORIUM_API_REPLICAS} + selector: + matchLabels: + app.kubernetes.io/name: observatorium-api + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + app.kubernetes.io/name: observatorium-api + spec: + containers: + - args: + - --web.listen=0.0.0.0:8080 + - --metrics.ui.endpoint=http://thanos-querier.observatorium.svc.cluster.local:9090 + - --metrics.query.endpoint=http://observatorium-cache.observatorium.svc.cluster.local:9090/api/v1/query + - --metrics.write.endpoint=http://thanos-receive.observatorium.svc.cluster.local:19291/api/v1/receive + - --log.level=warn + image: ${OBSERVATORIUM_API_IMAGE}:${OBSERVATORIUM_API_IMAGE_TAG} + livenessProbe: + failureThreshold: 4 + httpGet: + path: /-/healthy + port: 8080 + scheme: HTTP + periodSeconds: 30 + name: observatorium-api + ports: + - containerPort: 8080 + name: http + readinessProbe: + failureThreshold: 3 + httpGet: + path: /-/ready + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + resources: + limits: + cpu: ${OBSERVATORIUM_API_CPU_LIMIT} + memory: ${OBSERVATORIUM_API_MEMORY_LIMIT} + requests: + cpu: ${OBSERVATORIUM_API_CPU_REQUEST} + memory: ${OBSERVATORIUM_API_MEMORY_REQUEST} + - args: + - -provider=openshift + - -https-address=:8081 + - -http-address= + - -email-domain=* + - -upstream=http://localhost:8080 + - -openshift-service-account=prometheus-telemeter + - '-openshift-sar={"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", + "namespace": "${NAMESPACE}"}' + - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", + "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}' + - -tls-cert=/etc/tls/private/tls.crt + - -tls-key=/etc/tls/private/tls.key + - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token + - -cookie-secret-file=/etc/proxy/secrets/session_secret + - -openshift-ca=/etc/pki/tls/cert.pem + - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - -skip-auth-regex=^/metrics + image: ${PROXY_IMAGE}:${PROXY_IMAGE_TAG} + name: proxy + ports: + - containerPort: 8081 + name: https + volumeMounts: + - mountPath: /etc/tls/private + name: secret-api-tls + readOnly: false + - mountPath: /etc/proxy/secrets + name: secret-api-proxy + readOnly: false + serviceAccount: prometheus-telemeter + serviceAccountName: prometheus-telemeter + volumes: + - name: secret-api-tls + secret: + secretName: api-tls + - name: secret-api-proxy + secret: + secretName: api-proxy +- apiVersion: v1 + data: + session_secret: "" + kind: Secret + metadata: + labels: + app.kubernetes.io/name: observatorium-api + name: observatorium-proxy + namespace: ${NAMESPACE} + type: Opaque +- apiVersion: v1 + kind: Service + metadata: + annotations: + service.alpha.openshift.io/serving-cert-secret-name: observatorium-tls + name: observatorium-api + namespace: ${NAMESPACE} + spec: + ports: + - name: http + port: 8080 + targetPort: 8080 + - name: https + port: 8081 + targetPort: https + selector: + app.kubernetes.io/name: observatorium-api +parameters: +- name: NAMESPACE + value: telemeter +- name: OBSERVATORIUM_API_IMAGE + value: quay.io/observatorium/observatorium +- name: OBSERVATORIUM_API_IMAGE_TAG + value: master-2020-01-14-d076eab +- name: PROXY_IMAGE + value: openshift/oauth-proxy +- name: PROXY_IMAGE_TAG + value: v1.1.0 +- name: OBSERVATORIUM_API_REPLICAS + value: "3" +- name: OBSERVATORIUM_API_CPU_REQUEST + value: 100m +- name: OBSERVATORIUM_API_CPU_LIMIT + value: "1" +- name: OBSERVATORIUM_API_MEMORY_REQUEST + value: 256Mi +- name: OBSERVATORIUM_API_MEMORY_LIMIT + value: 1Gi diff --git a/environments/openshift/observatorium-api.jsonnet b/environments/openshift/observatorium-api.jsonnet new file mode 100644 index 000000000..8124c7186 --- /dev/null +++ b/environments/openshift/observatorium-api.jsonnet @@ -0,0 +1,42 @@ +local list = import 'telemeter/lib/list.libsonnet'; + +// This entire file takes what we have for Kubernetes and +// generates an OpenShift specific Template on top of that. + +local app = + (import 'observatorium.libsonnet') + { + apiVersion: 'v1', + kind: 'Template', + metadata: { + name: 'observatorium', + }, + objects: [ + $.observatorium.api[name] + for name in std.objectFields($.observatorium.api) + ], + parameters: [ + { name: 'NAMESPACE', value: 'telemeter' }, + { name: 'OBSERVATORIUM_API_IMAGE', value: 'quay.io/observatorium/observatorium' }, + { name: 'OBSERVATORIUM_API_IMAGE_TAG', value: 'master-2020-01-14-d076eab' }, + { name: 'PROXY_IMAGE', value: 'openshift/oauth-proxy' }, + { name: 'PROXY_IMAGE_TAG', value: 'v1.1.0' }, + { name: 'OBSERVATORIUM_API_REPLICAS', value: '3' }, + { name: 'OBSERVATORIUM_API_CPU_REQUEST', value: '100m' }, + { name: 'OBSERVATORIUM_API_CPU_LIMIT', value: '1' }, + { name: 'OBSERVATORIUM_API_MEMORY_REQUEST', value: '256Mi' }, + { name: 'OBSERVATORIUM_API_MEMORY_LIMIT', value: '1Gi' }, + ], + template: + list.asList('observatorium-api', {}, []) + { + objects: $.objects, + parameters: $.parameters, + }, + } + { + template+: { + parameters: + std.filter(function(param) !(param.name == 'NAMESPACE' && param.value == 'observatorium'), super.parameters), + }, + }; + +// Output only the template +app.template diff --git a/environments/openshift/observatorium.libsonnet b/environments/openshift/observatorium.libsonnet new file mode 100644 index 000000000..1705a9e46 --- /dev/null +++ b/environments/openshift/observatorium.libsonnet @@ -0,0 +1,107 @@ +local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local list = import 'telemeter/lib/list.libsonnet'; + +local service = k.core.v1.service; +local configmap = k.core.v1.configMap; +local secret = k.core.v1.secret; +local deployment = k.apps.v1.deployment; + +(import '../kubernetes/observatorium.libsonnet') + +{ + observatorium+:: { + local namespace = '${NAMESPACE}', + namespace:: namespace, + + proxyImage:: '${PROXY_IMAGE}:${PROXY_IMAGE_TAG}', + proxyConfig+:: { + sessionSecret: '', + }, + + api+: { + image:: '${OBSERVATORIUM_API_IMAGE}:${OBSERVATORIUM_API_IMAGE_TAG}', + + // The proxy secret is there to encrypt session created by the oauth proxy. + proxySecret: + secret.new('observatorium-proxy', { + session_secret: std.base64($.observatorium.proxyConfig.sessionSecret), + }) + + secret.mixin.metadata.withNamespace(namespace) + + secret.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.observatorium.api.name }), + + service+: + service.mixin.metadata.withNamespace(namespace) + + service.mixin.metadata.withAnnotations({ + 'service.alpha.openshift.io/serving-cert-secret-name': 'observatorium-tls', + }) + { + spec+: { + ports+: [ + service.mixin.spec.portsType.newNamed('https', 8081, 'https'), + ], + }, + }, + local volume = deployment.mixin.spec.template.spec.volumesType, + local container = deployment.mixin.spec.template.spec.containersType, + local volumeMount = container.volumeMountsType, + deployment+: + { + spec+: { + template+: { + spec+: { + containers: [ + if c.name == 'observatorium-api' then c { + resources: { + requests: { + cpu: '${OBSERVATORIUM_API_CPU_REQUEST}', + memory: '${OBSERVATORIUM_API_MEMORY_REQUEST}', + }, + limits: { + cpu: '${OBSERVATORIUM_API_CPU_LIMIT}', + memory: '${OBSERVATORIUM_API_MEMORY_LIMIT}', + }, + }, + } else c + for c in super.containers + ] + [ + container.new('proxy', $.observatorium.proxyImage) + + container.withArgs([ + '-provider=openshift', + '-https-address=:%d' % $.observatorium.api.service.spec.ports[1].port, + '-http-address=', + '-email-domain=*', + '-upstream=http://localhost:%d' % $.observatorium.api.service.spec.ports[0].port, + '-openshift-service-account=prometheus-telemeter', + '-openshift-sar={"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}', + '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", "name": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}', + '-tls-cert=/etc/tls/private/tls.crt', + '-tls-key=/etc/tls/private/tls.key', + '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token', + '-cookie-secret-file=/etc/proxy/secrets/session_secret', + '-openshift-ca=/etc/pki/tls/cert.pem', + '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + '-skip-auth-regex=^/metrics', + ]) + + container.withPorts([ + { name: 'https', containerPort: $.observatorium.api.service.spec.ports[1].port }, + ]) + + container.withVolumeMounts( + [ + volumeMount.new('secret-api-tls', '/etc/tls/private'), + volumeMount.new('secret-api-proxy', '/etc/proxy/secrets'), + ] + ), + ], + }, + }, + }, + } + + deployment.mixin.metadata.withNamespace(namespace) + + deployment.mixin.spec.withReplicas('${OBSERVATORIUM_API_REPLICAS}') + + deployment.mixin.spec.template.spec.withServiceAccount('prometheus-telemeter') + + deployment.mixin.spec.template.spec.withServiceAccountName('prometheus-telemeter') + + deployment.mixin.spec.template.spec.withVolumes([ + volume.fromSecret('secret-api-tls', 'api-tls'), + volume.fromSecret('secret-api-proxy', 'api-proxy'), + ]), + }, + }, +} diff --git a/environments/sre/servicemonitors.jsonnet b/environments/sre/servicemonitors.jsonnet index af7f96e14..871554343 100644 --- a/environments/sre/servicemonitors.jsonnet +++ b/environments/sre/servicemonitors.jsonnet @@ -142,6 +142,27 @@ local sm = }, }, }, + } + (import '../openshift/observatorium.libsonnet') + { + observatorium+:: { + api+: { + serviceMonitor+: { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + metadata: { + name: 'observatorium-api', + labels: { prometheus: 'app-sre' }, + }, + spec+: { + selector+: { + matchLabels: $.observatorium.api.labels, + }, + endpoints: [ + { port: $.observatorium.api.service.spec.ports[0].name }, + ], + }, + }, + }, + }, }; { @@ -159,6 +180,9 @@ local sm = } for tenant in tenants } { + 'observatorium-api.servicemonitor': sm.observatorium.api.serviceMonitor { + spec+: { namespaceSelector+: { matchNames: [$.namespace] } }, + }, 'observatorium-prometheus-ams.servicemonitor': prom.prometheusAms.serviceMonitor { metadata: { name: prom.prometheusAms.serviceMonitor.metadata.name, labels: { prometheus: 'app-sre' } }, spec+: { namespaceSelector+: { matchNames: [$.namespace] } }, diff --git a/environments/sre/servicemonitors/observatorium-api.servicemonitor.yaml b/environments/sre/servicemonitors/observatorium-api.servicemonitor.yaml new file mode 100644 index 000000000..0247074ea --- /dev/null +++ b/environments/sre/servicemonitors/observatorium-api.servicemonitor.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + prometheus: app-sre + name: observatorium-api +spec: + endpoints: + - port: http + namespaceSelector: + matchNames: + - '{{namespace}}' + selector: + matchLabels: + app.kubernetes.io/name: observatorium-api diff --git a/jsonnetfile.json b/jsonnetfile.json index ade7bf0bb..84f4f8993 100644 --- a/jsonnetfile.json +++ b/jsonnetfile.json @@ -89,6 +89,16 @@ } }, "version": "master" + }, + { + "name": "observatorium", + "source": { + "git": { + "remote": "https://github.com/observatorium/observatorium", + "subdir": "jsonnet/lib" + } + }, + "version": "master" } ] } diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json index 016410933..c290d7fb8 100644 --- a/jsonnetfile.lock.json +++ b/jsonnetfile.lock.json @@ -55,6 +55,17 @@ "version": "0c90793e469a6822c4f0c7de57ccf5ab0449f676", "sum": "ASNsquhtTSMENd7OFq2IOtGQlrNQ9apnfzAJRMEFflU=" }, + { + "name": "observatorium", + "source": { + "git": { + "remote": "https://github.com/observatorium/observatorium", + "subdir": "jsonnet/lib" + } + }, + "version": "d076eab78dfb3c2fd95101fb8a27149177770c8d", + "sum": "10pjPejBy1oGkIdqigomiPv9PCFtuZBoAPbR/3bKKvc=" + }, { "name": "prometheus-operator", "source": {