Skip to content

Commit

Permalink
feat: add metrics sleep_workload_total also for cronjobs
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidebianchi
davidebianchi committed May 1, 2022
1 parent 91afbf5 commit 78f301a
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 11 deletions.
20 changes: 19 additions & 1 deletion controllers/sleepinfo/cronjobs/cronjobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,18 @@ import (
"strings"

kubegreenv1alpha1 "github.com/kube-green/kube-green/api/v1alpha1"
"github.com/kube-green/kube-green/controllers/sleepinfo/metrics"
"github.com/kube-green/kube-green/controllers/sleepinfo/resource"
"github.com/prometheus/client_golang/prometheus"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const resourceType = "cronjob"

var (
ErrFetchingCronJobs = errors.New("error fetching cronjobs")
)
Expand All @@ -26,14 +30,18 @@ type cronjobs struct {
data []unstructured.Unstructured
OriginalSuspendStatus OriginalSuspendStatus
areToSuspend bool
metricsClient metrics.Metrics
namespace string
}

func NewResource(ctx context.Context, res resource.ResourceClient, namespace string, originalSuspendStatus map[string]bool) (cronjobs, error) {
func NewResource(ctx context.Context, res resource.ResourceClient, namespace string, originalSuspendStatus map[string]bool, metricsClient metrics.Metrics) (cronjobs, error) {
d := cronjobs{
ResourceClient: res,
OriginalSuspendStatus: originalSuspendStatus,
areToSuspend: res.SleepInfo.IsCronjobsToSuspend(),
data: []unstructured.Unstructured{},
metricsClient: metricsClient,
namespace: namespace,
}
if !d.areToSuspend {
return d, nil
Expand All @@ -54,6 +62,8 @@ func getSuspendStatus(cronjob unstructured.Unstructured) (bool, bool, error) {
}

func (c cronjobs) Sleep(ctx context.Context) error {
numberOfCronjobSleeped := float64(0)

for _, cronjob := range c.data {
cronjobSuspended, found, err := getSuspendStatus(cronjob)
if err != nil {
Expand All @@ -62,6 +72,8 @@ func (c cronjobs) Sleep(ctx context.Context) error {
if found && cronjobSuspended {
continue
}
numberOfCronjobSleeped++

newCronJob := cronjob.DeepCopy()
if err = unstructured.SetNestedField(newCronJob.Object, true, "spec", "suspend"); err != nil {
return err
Expand All @@ -71,6 +83,12 @@ func (c cronjobs) Sleep(ctx context.Context) error {
return err
}
}

c.metricsClient.SleepWorkloadTotal.With(prometheus.Labels{
"resource_type": resourceType,
"namespace": c.namespace,
}).Add(numberOfCronjobSleeped)

return nil
}

Expand Down
88 changes: 86 additions & 2 deletions controllers/sleepinfo/cronjobs/cronjobs_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
package cronjobs

import (
"bytes"
"context"
"fmt"
"strconv"
"testing"

"github.com/kube-green/kube-green/api/v1alpha1"
"github.com/kube-green/kube-green/controllers/internal/testutil"
"github.com/kube-green/kube-green/controllers/sleepinfo/metrics"
"github.com/kube-green/kube-green/controllers/sleepinfo/resource"

promTestutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand All @@ -20,6 +23,10 @@ import (
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

func getMetrics() metrics.Metrics {
return metrics.SetupMetricsOrDie("test_prefix")
}

func TestCronJobs(t *testing.T) {
testLogger := zap.New(zap.UseDevMode(true))

Expand Down Expand Up @@ -54,12 +61,23 @@ func TestCronJobs(t *testing.T) {
},
}

getNewResourceDisabled := func(t *testing.T, client client.Client, originalSuspendedCronJob map[string]bool) cronjobs {
c, err := NewResource(context.Background(), resource.ResourceClient{
Client: client,
Log: testLogger,
SleepInfo: &v1alpha1.SleepInfo{},
}, namespace, originalSuspendedCronJob, getMetrics())
require.NoError(t, err)

return c
}

getNewResource := func(t *testing.T, client client.Client, originalSuspendedCronJob map[string]bool) cronjobs {
c, err := NewResource(context.Background(), resource.ResourceClient{
Client: client,
Log: testLogger,
SleepInfo: sleepInfo,
}, namespace, originalSuspendedCronJob)
}, namespace, originalSuspendedCronJob, getMetrics())
require.NoError(t, err)

return c
Expand Down Expand Up @@ -144,7 +162,7 @@ func TestCronJobs(t *testing.T) {
SleepInfo: test.sleepInfo,
}

resource, err := NewResource(context.Background(), r, namespace, map[string]bool{})
resource, err := NewResource(context.Background(), r, namespace, map[string]bool{}, getMetrics())
if test.throws {
require.EqualError(t, err, fmt.Sprintf("%s: error during list", ErrFetchingCronJobs))
} else {
Expand Down Expand Up @@ -318,6 +336,72 @@ func TestCronJobs(t *testing.T) {
}, suspendedStatus)
})
})

t.Run("metrics", func(t *testing.T) {
t.Run("SleepWorkloadTotal", func(t *testing.T) {
t.Run("cronjob to suspend", func(t *testing.T) {
ctx := context.Background()
fakeClient := testutil.PossiblyErroringFakeCtrlRuntimeClient{
Client: getFakeClient().
WithRuntimeObjects(&cronJob1, &cronJob2, &suspendedCronJobs, &cronJobSuspendSetToFalseNotEmpty).
Build(),
}
c := getNewResource(t, fakeClient, nil)
m := c.metricsClient
require.NoError(t, c.Sleep(ctx))

require.Equal(t, 1, promTestutil.CollectAndCount(m.SleepWorkloadTotal))
expected := bytes.NewBufferString(fmt.Sprintf(`
# HELP test_prefix_sleep_workload_total Total number of workload stopped by the controller
# TYPE test_prefix_sleep_workload_total counter
test_prefix_sleep_workload_total{namespace="%s",resource_type="cronjob"} 3
`, namespace))
require.NoError(t, promTestutil.CollectAndCompare(m.SleepWorkloadTotal, expected))

require.NoError(t, c.fetch(ctx, namespace))
require.NoError(t, c.WakeUp(ctx))

require.Equal(t, 1, promTestutil.CollectAndCount(m.SleepWorkloadTotal))
expected = bytes.NewBufferString(fmt.Sprintf(`
# HELP test_prefix_sleep_workload_total Total number of workload stopped by the controller
# TYPE test_prefix_sleep_workload_total counter
test_prefix_sleep_workload_total{namespace="%s",resource_type="cronjob"} 3
`, namespace))
require.NoError(t, promTestutil.CollectAndCompare(m.SleepWorkloadTotal, expected))
})

t.Run("cronjob not to suspend", func(t *testing.T) {
ctx := context.Background()
fakeClient := testutil.PossiblyErroringFakeCtrlRuntimeClient{
Client: getFakeClient().
WithRuntimeObjects(&cronJob1, &cronJob2, &suspendedCronJobs, &cronJobSuspendSetToFalseNotEmpty).
Build(),
}
c := getNewResourceDisabled(t, fakeClient, nil)
m := c.metricsClient
require.NoError(t, c.Sleep(ctx))

require.Equal(t, 1, promTestutil.CollectAndCount(m.SleepWorkloadTotal))
expected := bytes.NewBufferString(fmt.Sprintf(`
# HELP test_prefix_sleep_workload_total Total number of workload stopped by the controller
# TYPE test_prefix_sleep_workload_total counter
test_prefix_sleep_workload_total{namespace="%s",resource_type="cronjob"} 0
`, namespace))
require.NoError(t, promTestutil.CollectAndCompare(m.SleepWorkloadTotal, expected))

require.NoError(t, c.fetch(ctx, namespace))
require.NoError(t, c.WakeUp(ctx))

require.Equal(t, 1, promTestutil.CollectAndCount(m.SleepWorkloadTotal))
expected = bytes.NewBufferString(fmt.Sprintf(`
# HELP test_prefix_sleep_workload_total Total number of workload stopped by the controller
# TYPE test_prefix_sleep_workload_total counter
test_prefix_sleep_workload_total{namespace="%s",resource_type="cronjob"} 0
`, namespace))
require.NoError(t, promTestutil.CollectAndCompare(m.SleepWorkloadTotal, expected))
})
})
})
}

func suspendAndUpdateResourceVersion(t *testing.T, cronJob unstructured.Unstructured) unstructured.Unstructured {
Expand Down
5 changes: 0 additions & 5 deletions controllers/sleepinfo/resource/testutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ type ResourceMock struct {
MockSleep func(context.Context) error
MockWakeUp func(context.Context) error
MockOriginalInfoToSave func() ([]byte, error)
MockType string
}

func (r ResourceMock) HasResource() bool {
Expand Down Expand Up @@ -35,10 +34,6 @@ func (r ResourceMock) GetOriginalInfoToSave() ([]byte, error) {
return r.MockOriginalInfoToSave()
}

func (r ResourceMock) GetType() string {
return r.MockType
}

func GetResourceMock(mock ResourceMock) Resource {
return mock
}
2 changes: 1 addition & 1 deletion controllers/sleepinfo/resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func NewResources(ctx context.Context, resourceClient resource.ResourceClient, n
resourceClient.Log.Error(err, "fails to init deployments")
return Resources{}, err
}
cronJobResource, err := cronjobs.NewResource(ctx, resourceClient, namespace, sleepInfoData.OriginalCronJobStatus)
cronJobResource, err := cronjobs.NewResource(ctx, resourceClient, namespace, sleepInfoData.OriginalCronJobStatus, metricsClient)
if err != nil {
resourceClient.Log.Error(err, "fails to init cronjobs")
return Resources{}, err
Expand Down
4 changes: 2 additions & 2 deletions controllers/sleepinfo/sleepinfo_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -923,7 +923,7 @@ func assertCorrectSleepOperation(assert AssertOperation) {
By("metrics correctly collected - quantitatively", func() {
metrics := sleepInfoReconciler.Metrics

Expect(promTestutil.CollectAndCount(metrics.SleepWorkloadTotal)).To(Equal(1))
Expect(promTestutil.CollectAndCount(metrics.SleepWorkloadTotal)).To(Equal(2))
Expect(promTestutil.CollectAndCount(metrics.ActualSleepReplicas)).To(Equal(1))
Expect(promTestutil.CollectAndCount(metrics.SleepInfoInfo)).To(Equal(0))
Expect(promTestutil.CollectAndCount(metrics.SleepDurationSeconds)).To(Equal(0))
Expand Down Expand Up @@ -991,7 +991,7 @@ func assertCorrectWakeUpOperation(assert AssertOperation) {
By("metrics correctly collected - quantitatively", func() {
metrics := sleepInfoReconciler.Metrics

Expect(promTestutil.CollectAndCount(metrics.SleepWorkloadTotal)).To(Equal(1))
Expect(promTestutil.CollectAndCount(metrics.SleepWorkloadTotal)).To(Equal(2))
Expect(promTestutil.CollectAndCount(metrics.ActualSleepReplicas)).To(Equal(1))
Expect(promTestutil.CollectAndCount(metrics.SleepInfoInfo)).To(Equal(0))
Expect(promTestutil.CollectAndCount(metrics.SleepDurationSeconds)).To(Equal(0))
Expand Down

0 comments on commit 78f301a

Please sign in to comment.