Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
*: Enable multi-namespace monitoring
  • Loading branch information
brancz committed Oct 31, 2018
1 parent 1cff58e commit e26c0d5
Show file tree
Hide file tree
Showing 9 changed files with 61 additions and 35 deletions.
10 changes: 2 additions & 8 deletions assets/prometheus-k8s/prometheus.yaml
Expand Up @@ -98,12 +98,6 @@ spec:
- kube-rbac-proxy
securityContext: {}
serviceAccountName: prometheus-k8s
serviceMonitorNamespaceSelector:
matchExpressions:
- key: openshift.io/cluster-monitoring
operator: Exists
serviceMonitorSelector:
matchExpressions:
- key: k8s-app
operator: Exists
serviceMonitorNamespaceSelector: {}
serviceMonitorSelector: {}
version: v2.4.2
3 changes: 2 additions & 1 deletion cmd/operator/main.go
Expand Up @@ -79,6 +79,7 @@ func (t *tags) Type() string {
func Main() int {
flagset := flag.CommandLine
namespace := flagset.String("namespace", "openshift-monitoring", "Namespace to deploy and manage cluster monitoring stack in.")
namespaceSelector := flagset.String("namespace-selector", "openshift.io/cluster-monitoring=true", "Selector for namespaces to monitor.")
configMapName := flagset.String("configmap", "cluster-monitoring-config", "ConfigMap name to configure the cluster monitoring stack.")
kubeconfigPath := flagset.String("kubeconfig", "", "The path to the kubeconfig to connect to the apiserver with.")
apiserver := flagset.String("apiserver", "", "The address of the apiserver to talk to.")
Expand Down Expand Up @@ -113,7 +114,7 @@ func Main() int {
return 1
}

o, err := cmo.New(config, *namespace, *configMapName, tags.asMap())
o, err := cmo.New(config, *namespace, *namespaceSelector, *configMapName, tags.asMap())
if err != nil {
fmt.Fprint(os.Stderr, err)
return 1
Expand Down
4 changes: 2 additions & 2 deletions jsonnet/prometheus.jsonnet
Expand Up @@ -300,8 +300,8 @@ local namespacesRole = policyRule.new() +
'kube-rbac-proxy',
],
configMaps: ['prometheus-serving-certs-ca-bundle'],
serviceMonitorSelector: selector.withMatchExpressions({ key: 'k8s-app', operator: 'Exists' }),
serviceMonitorNamespaceSelector: selector.withMatchExpressions({ key: 'openshift.io/cluster-monitoring', operator: 'Exists' }),
serviceMonitorSelector: {},
serviceMonitorNamespaceSelector: {},
listenLocal: true,
containers: [
{
Expand Down
33 changes: 25 additions & 8 deletions pkg/client/client.go
Expand Up @@ -51,16 +51,17 @@ const (
)

type Client struct {
namespace string
appVersionName string
kclient kubernetes.Interface
ossclient openshiftsecurityclientset.Interface
osrclient openshiftrouteclientset.Interface
mclient monitoring.Interface
eclient apiextensionsclient.Interface
namespace string
namespaceSelector string
appVersionName string
kclient kubernetes.Interface
ossclient openshiftsecurityclientset.Interface
osrclient openshiftrouteclientset.Interface
mclient monitoring.Interface
eclient apiextensionsclient.Interface
}

func New(cfg *rest.Config, namespace string, appVersionName string) (*Client, error) {
func New(cfg *rest.Config, namespace string, namespaceSelector string, appVersionName string) (*Client, error) {
mclient, err := monitoring.NewForConfig(
&monv1.DefaultCrdKinds,
monv1.Group,
Expand Down Expand Up @@ -176,6 +177,22 @@ func (c *Client) CreateRouteIfNotExists(r *routev1.Route) error {
return nil
}

func (c *Client) NamespaceToMonitor() ([]string, error) {
namespaces, err := c.kclient.CoreV1().Namespaces().List(metav1.ListOptions{
LabelSelector: c.namespaceSelector,
})
if err != nil {
return nil, errors.Wrap(err, "listing namespaces failed")
}

namespaceNames := make([]string, len(namespaces.Items))
for i, namespace := range namespaces.Items {
namespaceNames[i] = namespace.Name
}

return namespaceNames, nil
}

func (c *Client) CreateOrUpdatePrometheus(p *monv1.Prometheus) error {
pclient := c.mclient.MonitoringV1().Prometheuses(p.GetNamespace())
_, err := pclient.Get(p.GetName(), metav1.GetOptions{})
Expand Down
4 changes: 2 additions & 2 deletions pkg/manifests/bindata.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions pkg/manifests/manifests.go
Expand Up @@ -817,7 +817,7 @@ func (f *Factory) PrometheusOperatorServiceAccount() (*v1.ServiceAccount, error)
return s, nil
}

func (f *Factory) PrometheusOperatorDeployment() (*appsv1.Deployment, error) {
func (f *Factory) PrometheusOperatorDeployment(namespaces []string) (*appsv1.Deployment, error) {
d, err := f.NewDeployment(MustAssetReader(PrometheusOperatorDeployment))
if err != nil {
return nil, err
Expand All @@ -840,7 +840,7 @@ func (f *Factory) PrometheusOperatorDeployment() (*appsv1.Deployment, error) {
args := d.Spec.Template.Spec.Containers[0].Args
for i := range args {
if strings.HasPrefix(args[i], PrometheusOperatorNamespaceFlag) {
args[i] = PrometheusOperatorNamespaceFlag + f.namespace
args[i] = PrometheusOperatorNamespaceFlag + strings.Join(namespaces, ",")
}

if strings.HasPrefix(args[i], PrometheusConfigReloaderFlag) && f.config.PrometheusOperatorConfig.PrometheusConfigReloader != "" {
Expand Down
24 changes: 16 additions & 8 deletions pkg/manifests/manifests_test.go
Expand Up @@ -183,7 +183,7 @@ func TestUnconfiguredManifests(t *testing.T) {
t.Fatal(err)
}

_, err = f.PrometheusOperatorDeployment()
_, err = f.PrometheusOperatorDeployment([]string{"default", "openshift-monitoring"})
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -448,7 +448,7 @@ func TestPrometheusOperatorConfiguration(t *testing.T) {
}

f := NewFactory("openshift-monitoring", c)
d, err := f.PrometheusOperatorDeployment()
d, err := f.PrometheusOperatorDeployment([]string{"default", "openshift-monitoring"})
if err != nil {
t.Fatal(err)
}
Expand All @@ -467,13 +467,17 @@ func TestPrometheusOperatorConfiguration(t *testing.T) {

configReloaderFound := false
prometheusReloaderFound := false
namespacesFound := false
for i := range d.Spec.Template.Spec.Containers[0].Args {
if strings.HasPrefix(d.Spec.Template.Spec.Containers[0].Args[i], PrometheusConfigReloaderFlag+"quay.io/test/prometheus-config-reloader") {
prometheusReloaderFound = true
}
if strings.HasPrefix(d.Spec.Template.Spec.Containers[0].Args[i], ConfigReloaderImageFlag+"quay.io/test/configmap-reload") {
configReloaderFound = true
}
if strings.HasPrefix(d.Spec.Template.Spec.Containers[0].Args[i], PrometheusOperatorNamespaceFlag+"default,openshift-monitoring") {
namespacesFound = true
}
}

if !configReloaderFound {
Expand All @@ -483,6 +487,10 @@ func TestPrometheusOperatorConfiguration(t *testing.T) {
if !prometheusReloaderFound {
t.Fatal("Configuring the Prometheus Reloader base image failed")
}

if !namespacesFound {
t.Fatal("Configuring the namespaces to watch failed")
}
}

func TestPrometheusK8sConfiguration(t *testing.T) {
Expand Down Expand Up @@ -698,9 +706,9 @@ kubeRbacProxy:

func TestPrometheusEtcdRulesFiltered(t *testing.T) {
enabled := false
cfg := NewDefaultConfig()
cfg.EtcdConfig.Enabled = &enabled
f := NewFactory("openshift-monitoring", cfg)
c := NewDefaultConfig()
c.EtcdConfig.Enabled = &enabled
f := NewFactory("openshift-monitoring", c)

r, err := f.PrometheusK8sRules()
if err != nil {
Expand Down Expand Up @@ -740,9 +748,9 @@ func TestPrometheusEtcdRules(t *testing.T) {

func TestEtcdGrafanaDashboardFiltered(t *testing.T) {
enabled := false
cfg := NewDefaultConfig()
cfg.EtcdConfig.Enabled = &enabled
f := NewFactory("openshift-monitoring", cfg)
c := NewDefaultConfig()
c.EtcdConfig.Enabled = &enabled
f := NewFactory("openshift-monitoring", c)

cms, err := f.GrafanaDashboardDefinitions()
if err != nil {
Expand Down
7 changes: 4 additions & 3 deletions pkg/operator/operator.go
Expand Up @@ -45,7 +45,8 @@ const (
)

type Operator struct {
namespace string
namespace string

configMapName string
tagOverrides map[string]string

Expand All @@ -60,8 +61,8 @@ type Operator struct {
reconcileErrors prometheus.Counter
}

func New(config *rest.Config, namespace, configMapName string, tagOverrides map[string]string) (*Operator, error) {
c, err := client.New(config, namespace, configMapName)
func New(config *rest.Config, namespace, namespaceSelector, configMapName string, tagOverrides map[string]string) (*Operator, error) {
c, err := client.New(config, namespace, namespaceSelector, configMapName)
if err != nil {
return nil, err
}
Expand Down
7 changes: 6 additions & 1 deletion pkg/tasks/prometheusoperator.go
Expand Up @@ -73,7 +73,12 @@ func (t *PrometheusOperatorTask) Run() error {
return errors.Wrap(err, "reconciling Prometheus Operator Service failed")
}

d, err := t.factory.PrometheusOperatorDeployment()
namespaces, err := t.client.NamespaceToMonitor()
if err != nil {
return errors.Wrap(err, "listing namespaces to monitor failed")
}

d, err := t.factory.PrometheusOperatorDeployment(namespaces)
if err != nil {
return errors.Wrap(err, "initializing Prometheus Operator Deployment failed")
}
Expand Down

0 comments on commit e26c0d5

Please sign in to comment.