Skip to content

Commit

Permalink
pkg/tasks/prometheus: add trusted CA bundle
Browse files Browse the repository at this point in the history
  • Loading branch information
@s-urbaniak
s-urbaniak committed Nov 21, 2019
1 parent 31cee2f commit 42c8622
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 26 deletions.
8 changes: 8 additions & 0 deletions jsonnet/prometheus.jsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ local alertmanagerRole =

{
prometheusK8s+:: {
trustedCaBundle:
configmap.new('prometheus-trusted-ca-bundle', { 'ca-bundle.crt': '' }) +
configmap.mixin.metadata.withNamespace($._config.namespace) +
configmap.mixin.metadata.withLabels({ 'config.openshift.io/inject-trusted-cabundle': 'true' }),

grpcTlsSecret:
secret.new('prometheus-k8s-grpc-tls', {}) +
secret.mixin.metadata.withNamespace($._config.namespace) +
Expand Down Expand Up @@ -425,6 +430,9 @@ local alertmanagerRole =
},
],
},
{
name: 'prometheus',
},
],
},
},
Expand Down
60 changes: 53 additions & 7 deletions pkg/manifests/manifests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ var (
PrometheusK8sServingCertsCABundle = "assets/prometheus-k8s/serving-certs-ca-bundle.yaml"
PrometheusK8sKubeletServingCABundle = "assets/prometheus-k8s/kubelet-serving-ca-bundle.yaml"
PrometheusK8sGrpcTLSSecret = "assets/prometheus-k8s/grpc-tls-secret.yaml"
PrometheusK8sTrustedCABundle = "assets/prometheus-k8s/trusted-ca-bundle.yaml"

PrometheusUserWorkloadServingCertsCABundle = "assets/prometheus-user-workload/serving-certs-ca-bundle.yaml"
PrometheusUserWorkloadServiceAccount = "assets/prometheus-user-workload/service-account.yaml"
Expand Down Expand Up @@ -1057,7 +1058,26 @@ func (f *Factory) SharingConfig(promHost, amHost, grafanaHost, thanosHost *url.U
}
}

func (f *Factory) PrometheusK8s(host string, grpcTLS *v1.Secret) (*monv1.Prometheus, error) {
func (f *Factory) PrometheusK8sTrustedCABundle() (*v1.ConfigMap, error) {
cm, err := f.NewConfigMap(MustAssetReader(PrometheusK8sTrustedCABundle))
if err != nil {
return nil, err
}

return cm, nil
}

const (
// These constants refer to indices of prometheus-k8s containers.
// They need to be in sync with jsonnet/prometheus.jsonnet
K8S_CONTAINER_OAUTH_PROXY = 0
K8S_CONTAINER_KUBE_RBAC_PROXY = 1
K8S_CONTAINER_PROM_LABEL_PROXY = 2
K8S_CONTAINER_THANOS_SIDECAR = 3
K8S_CONTAINER_PROMETHEUS = 4
)

func (f *Factory) PrometheusK8s(host string, grpcTLS *v1.Secret, trustedCABundleCM *v1.ConfigMap) (*monv1.Prometheus, error) {
p, err := f.NewPrometheus(MustAssetReader(PrometheusK8s))
if err != nil {
return nil, err
Expand Down Expand Up @@ -1111,17 +1131,18 @@ func (f *Factory) PrometheusK8s(host string, grpcTLS *v1.Secret) (*monv1.Prometh
p.Spec.Thanos.Image = &f.config.Images.Thanos
}

p.Spec.Containers[0].Image = f.config.Images.OauthProxy
p.Spec.Containers[1].Image = f.config.Images.KubeRbacProxy
p.Spec.Containers[2].Image = f.config.Images.PromLabelProxy
p.Spec.Containers[K8S_CONTAINER_OAUTH_PROXY].Image = f.config.Images.OauthProxy
p.Spec.Containers[K8S_CONTAINER_KUBE_RBAC_PROXY].Image = f.config.Images.KubeRbacProxy
p.Spec.Containers[K8S_CONTAINER_PROM_LABEL_PROXY].Image = f.config.Images.PromLabelProxy

p.Spec.Alerting.Alertmanagers[0].Namespace = f.namespace
p.Spec.Alerting.Alertmanagers[0].TLSConfig.ServerName = fmt.Sprintf("alertmanager-main.%s.svc", f.namespace)
p.Namespace = f.namespace

setEnv := func(name, value string) {
for i := range p.Spec.Containers[0].Env {
if p.Spec.Containers[0].Env[i].Name == name {
p.Spec.Containers[0].Env[i].Value = value
for i := range p.Spec.Containers[K8S_CONTAINER_OAUTH_PROXY].Env {
if p.Spec.Containers[K8S_CONTAINER_OAUTH_PROXY].Env[i].Name == name {
p.Spec.Containers[K8S_CONTAINER_OAUTH_PROXY].Env[i].Value = value
break
}
}
Expand All @@ -1145,6 +1166,31 @@ func (f *Factory) PrometheusK8s(host string, grpcTLS *v1.Secret) (*monv1.Prometh
},
})

if trustedCABundleCM != nil {
volumeName := "prometheus-trusted-ca-bundle"
volumePath := "/etc/pki/ca-trust/extracted/pem/"
volume := trustedCABundleVolume(trustedCABundleCM.Name, volumeName)
volume.VolumeSource.ConfigMap.Items = append(volume.VolumeSource.ConfigMap.Items, v1.KeyToPath{
Key: "ca-bundle.crt",
Path: "tls-ca-bundle.pem",
})
p.Spec.Volumes = append(p.Spec.Volumes, volume)

// we only need the trusted CA bundle in:
// 1. Prometheus, because users might want to configure external remote write.
// 2. In OAuth proxy, as that communicates externally when executing the OAuth handshake.

p.Spec.Containers[K8S_CONTAINER_OAUTH_PROXY].VolumeMounts = append(
p.Spec.Containers[K8S_CONTAINER_OAUTH_PROXY].VolumeMounts,
trustedCABundleVolumeMount(volumeName, volumePath),
)

p.Spec.Containers[K8S_CONTAINER_PROMETHEUS].VolumeMounts = append(
p.Spec.Containers[K8S_CONTAINER_PROMETHEUS].VolumeMounts,
trustedCABundleVolumeMount(volumeName, volumePath),
)
}

return p, nil
}

Expand Down
19 changes: 14 additions & 5 deletions pkg/manifests/manifests_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,12 @@ func TestUnconfiguredManifests(t *testing.T) {
t.Fatal(err)
}

_, err = f.PrometheusK8s("prometheus-k8s.openshift-monitoring.svc", &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
_, err = f.PrometheusK8sTrustedCABundle()
if err != nil {
t.Fatal(err)
}

_, err = f.PrometheusK8s("prometheus-k8s.openshift-monitoring.svc", &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, nil)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -734,7 +739,11 @@ ingress:
})

f := NewFactory("openshift-monitoring", "openshift-user-workload-monitoring", c)
p, err := f.PrometheusK8s("prometheus-k8s.openshift-monitoring.svc", &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
p, err := f.PrometheusK8s(
"prometheus-k8s.openshift-monitoring.svc",
&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
)
if err != nil {
t.Fatal(err)
}
Expand All @@ -747,15 +756,15 @@ ingress:
t.Fatal("Prometheus image is not configured correctly")
}

if p.Spec.Containers[0].Image != "docker.io/openshift/origin-oauth-proxy:latest" {
if p.Spec.Containers[K8S_CONTAINER_OAUTH_PROXY].Image != "docker.io/openshift/origin-oauth-proxy:latest" {
t.Fatal("oauth-proxy image is not configured correctly")
}

if p.Spec.Containers[1].Image != "docker.io/openshift/origin-kube-rbac-proxy:latest" {
if p.Spec.Containers[K8S_CONTAINER_KUBE_RBAC_PROXY].Image != "docker.io/openshift/origin-kube-rbac-proxy:latest" {
t.Fatal("kube-rbac-proxy image is not configured correctly")
}

if p.Spec.Containers[2].Image != "docker.io/openshift/origin-prom-label-proxy:latest" {
if p.Spec.Containers[K8S_CONTAINER_PROM_LABEL_PROXY].Image != "docker.io/openshift/origin-prom-label-proxy:latest" {
t.Fatal("prom-label-proxy image is not configured correctly")
}

Expand Down
57 changes: 43 additions & 14 deletions pkg/tasks/prometheus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,23 +269,52 @@ func (t *PrometheusTask) Run() error {
if err != nil {
return errors.Wrap(err, "error creating Prometheus Client GRPC TLS secret")
}
{
// Create trusted CA bundle ConfigMap.
trustedCA, err := t.factory.PrometheusK8sTrustedCABundle()
if err != nil {
return errors.Wrap(err, "initializing Prometheus CA bundle ConfigMap failed")
}

klog.V(4).Info("initializing Prometheus object")
p, err := t.factory.PrometheusK8s(host, s)
if err != nil {
return errors.Wrap(err, "initializing Prometheus object failed")
}
trustedCA, err = t.client.CreateIfNotExistConfigMap(trustedCA)
if err != nil {
return errors.Wrap(err, " creating Promehteus CA bundle ConfigMap failed")
}
// In the case when there is no data but the ConfigMap is there, we just continue.
// We will catch this on the next loop.
trustedCA = t.factory.HashTrustedCA(trustedCA, "prometheus")
if trustedCA != nil {
err = t.client.CreateOrUpdateConfigMap(trustedCA)
if err != nil {
return errors.Wrap(err, "reconciling Prometheus CA bundle ConfigMap failed")
}

err = t.client.DeleteHashedConfigMap(
string(trustedCA.Labels["monitoring.openshift.io/hash"]),
"prometheus",
)
if err != nil {
return errors.Wrap(err, "deleting old Prometheus configmaps failed")
}
}

klog.V(4).Info("reconciling Prometheus object")
err = t.client.CreateOrUpdatePrometheus(p)
if err != nil {
return errors.Wrap(err, "reconciling Prometheus object failed")
}
klog.V(4).Info("initializing Prometheus object")
p, err := t.factory.PrometheusK8s(host, s, trustedCA)
if err != nil {
return errors.Wrap(err, "initializing Prometheus object failed")
}

klog.V(4).Info("waiting for Prometheus object changes")
err = t.client.WaitForPrometheus(p)
if err != nil {
return errors.Wrap(err, "waiting for Prometheus object changes failed")
klog.V(4).Info("reconciling Prometheus object")
err = t.client.CreateOrUpdatePrometheus(p)
if err != nil {
return errors.Wrap(err, "reconciling Prometheus object failed")
}

klog.V(4).Info("waiting for Prometheus object changes")
err = t.client.WaitForPrometheus(p)
if err != nil {
return errors.Wrap(err, "waiting for Prometheus object changes failed")
}
}

smk, err := t.factory.PrometheusK8sKubeletServiceMonitor()
Expand Down

0 comments on commit 42c8622

Please sign in to comment.