Skip to content

Commit

Permalink
Add unit tests
Browse files Browse the repository at this point in the history
  • Loading branch information
zugao committed Oct 5, 2022
1 parent 09eea72 commit ee73039
Show file tree
Hide file tree
Showing 3 changed files with 219 additions and 14 deletions.
4 changes: 2 additions & 2 deletions pkg/database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ var (
type AggregatedBucket struct {
Organization string
// Storage in bytes
StorageUsed int64
StorageUsed float64
}

// Database holds raw url of the postgresql database with the opened connection
Expand Down Expand Up @@ -173,7 +173,7 @@ func (d *Database) getBestMatch(ctx context.Context) error {

var quantity float64
if query.Unit == defaultUnit {
quantity = float64(d.aggregatedBucket.StorageUsed) / 1000 / 1000 / 1000
quantity = d.aggregatedBucket.StorageUsed / 1000 / 1000 / 1000
} else {
return fmt.Errorf("unknown query unit %s", query.Unit)
}
Expand Down
32 changes: 20 additions & 12 deletions pkg/sos/objectstorage.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ func (o *ObjectStorage) Execute(ctx context.Context) error {
return p.RunWithContext(ctx)
}

// getBucketUsage gets bucket usage from Exoscale and matches them with the bucket from the cluster
// If there are no buckets in Exoscale, the API will return an empty slice
func (o *ObjectStorage) getBucketUsage(ctx context.Context) error {
log := ctrl.LoggerFrom(ctx)
log.Info("Fetching bucket usage from Exoscale")
Expand All @@ -65,7 +67,12 @@ func (o *ObjectStorage) getBucketUsage(ctx context.Context) error {
return err
}

aggregatedBuckets := *getAggregatedBuckets(ctx, resp.JSON200.SosBucketsUsage, &o.bucketDetails)
aggregatedBuckets := getAggregatedBuckets(ctx, *resp.JSON200.SosBucketsUsage, o.bucketDetails)
if len(aggregatedBuckets) == 0 {
log.Info("There are no bucket usage to be saved in the database")
return nil
}

o.aggregatedBuckets = aggregatedBuckets
return nil
}
Expand All @@ -81,7 +88,7 @@ func (o *ObjectStorage) fetchManagedBuckets(ctx context.Context) error {
if err != nil {
return fmt.Errorf("cannot list buckets: %w", err)
}
o.bucketDetails = *addOrgAndNamespaceToBucket(ctx, &buckets)
o.bucketDetails = addOrgAndNamespaceToBucket(ctx, buckets)
return nil
}

Expand Down Expand Up @@ -124,20 +131,20 @@ func (o *ObjectStorage) getBillingDate(_ context.Context) error {
return nil
}

func getAggregatedBuckets(ctx context.Context, sosBucketsUsage *[]oapi.SosBucketUsage, bucketDetails *[]BucketDetail) *map[string]db.AggregatedBucket {
func getAggregatedBuckets(ctx context.Context, sosBucketsUsage []oapi.SosBucketUsage, bucketDetails []BucketDetail) map[string]db.AggregatedBucket {
log := ctrl.LoggerFrom(ctx)
log.Info("Aggregating buckets by namespace")
aggregatedBuckets := make(map[string]db.AggregatedBucket)
for _, bucketDetail := range *bucketDetails {
for _, bucketDetail := range bucketDetails {
log.V(1).Info("Checking bucket", "bucket", bucketDetail.BucketName)
storageUsed := int64(0)
storageUsed := float64(0)
usageBucketExists := false

// Match managed kubernetes resource bucket to the exoscale usage bucket to retrieve the amount of storage used
for _, bucketUsage := range *sosBucketsUsage {
for _, bucketUsage := range sosBucketsUsage {
if bucketDetail.BucketName == *bucketUsage.Name {
log.V(1).Info("Found exoscale bucket usage", "bucket", bucketUsage.Name, "bucket size", bucketUsage.Name)
storageUsed = *bucketUsage.Size
storageUsed = float64(*bucketUsage.Size)
usageBucketExists = true
break
}
Expand All @@ -149,7 +156,8 @@ func getAggregatedBuckets(ctx context.Context, sosBucketsUsage *[]oapi.SosBucket
if exists {
log.V(1).Info("AggregatedBucket exists, adding its used storage to the current AggregatedBucket",
"bucket", bucketDetail.BucketName, "storage to be added", storageUsed)
aggregatedBucket.StorageUsed += storageUsed
aggregatedBucket.StorageUsed = aggregatedBucket.StorageUsed + storageUsed
aggregatedBuckets[bucketDetail.Namespace] = aggregatedBucket
} else {
log.V(1).Info("Adding new AggregatedBucket object", "bucket", bucketDetail.BucketName, "bucket storage", storageUsed)
aggregatedBuckets[bucketDetail.Namespace] = db.AggregatedBucket{
Expand All @@ -161,14 +169,14 @@ func getAggregatedBuckets(ctx context.Context, sosBucketsUsage *[]oapi.SosBucket
log.Info("Could not find any bucket on exoscale", "bucket", bucketDetail.BucketName)
}
}
return &aggregatedBuckets
return aggregatedBuckets
}

func addOrgAndNamespaceToBucket(ctx context.Context, buckets *exoscalev1.BucketList) *[]BucketDetail {
func addOrgAndNamespaceToBucket(ctx context.Context, buckets exoscalev1.BucketList) []BucketDetail {
log := ctrl.LoggerFrom(ctx)
log.V(1).Info("Gathering more information from buckets")

var bucketDetails []BucketDetail
bucketDetails := make([]BucketDetail, 0, 10)
for _, bucket := range buckets.Items {
bucketDetail := BucketDetail{
BucketName: bucket.Spec.ForProvider.BucketName,
Expand Down Expand Up @@ -197,5 +205,5 @@ func addOrgAndNamespaceToBucket(ctx context.Context, buckets *exoscalev1.BucketL
"organization", bucketDetail.Organization)
bucketDetails = append(bucketDetails, bucketDetail)
}
return &bucketDetails
return bucketDetails
}
197 changes: 197 additions & 0 deletions pkg/sos/objectstorage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
package sos

import (
"context"
"github.com/exoscale/egoscale/v2/oapi"
"github.com/stretchr/testify/assert"
db "github.com/vshn/exoscale-metrics-collector/pkg/database"
exoscalev1 "github.com/vshn/provider-exoscale/apis/exoscale/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"testing"
"time"
)

func TestObjectStorage_GetBillingDate(t *testing.T) {
t.Run("GivenContext_WhenGetBillingDate_ThenReturnYesterdayDate", func(t *testing.T) {
// Given
ctx := context.Background()
utc, _ := time.LoadLocation("UTC")
now := time.Now().In(utc)
expected := time.Date(now.Year(), now.Month(), time.Now().Day()-1, 6, 0, 0, 0, now.Location())

//When
o := ObjectStorage{}
err := o.getBillingDate(ctx)

// Then
assert.NoError(t, err)
assert.Equal(t, o.billingDate, expected)
})
}

func TestObjectStorage_GetAggregatedBuckets(t *testing.T) {
tests := map[string]struct {
givenSosBucketsUsage []oapi.SosBucketUsage
givenBucketDetails []BucketDetail
expectedAggregatedBuckets map[string]db.AggregatedBucket
}{
"GivenSosBucketsUsageAndBuckets_WhenMatch_ThenExpectAggregatedBucketObjects": {
givenSosBucketsUsage: []oapi.SosBucketUsage{
createSosBucketUsage("bucket-test-1", 1),
createSosBucketUsage("bucket-test-2", 4),
createSosBucketUsage("bucket-test-3", 9),
createSosBucketUsage("bucket-test-4", 0),
createSosBucketUsage("bucket-test-5", 5),
},
givenBucketDetails: []BucketDetail{
createBucketDetail("bucket-test-1", "default", "orgA"),
createBucketDetail("bucket-test-2", "alpha", "orgB"),
createBucketDetail("bucket-test-3", "alpha", "orgB"),
createBucketDetail("bucket-test-4", "omega", "orgC"),
createBucketDetail("no-metrics-bucket", "beta", "orgD"),
},
expectedAggregatedBuckets: map[string]db.AggregatedBucket{
"default": createAggregatedBucket("orgA", 1),
"alpha": createAggregatedBucket("orgB", 13),
"omega": createAggregatedBucket("orgC", 0),
},
},
"GivenSosBucketsUsageAndBuckets_WhenMatch_ThenExpectNoAggregatedBucketObjects": {
givenSosBucketsUsage: []oapi.SosBucketUsage{
createSosBucketUsage("bucket-test-1", 1),
createSosBucketUsage("bucket-test-2", 4),
},
givenBucketDetails: []BucketDetail{
createBucketDetail("bucket-test-3", "default", "orgA"),
createBucketDetail("bucket-test-4", "alpha", "orgB"),
createBucketDetail("bucket-test-5", "alpha", "orgB"),
},
expectedAggregatedBuckets: map[string]db.AggregatedBucket{},
},
"GivenSosBucketsUsageAndBuckets_WhenSosBucketsUsageEmpty_ThenExpectNoAggregatedBucketObjects": {
givenSosBucketsUsage: []oapi.SosBucketUsage{
createSosBucketUsage("bucket-test-1", 1),
createSosBucketUsage("bucket-test-2", 4),
},
givenBucketDetails: []BucketDetail{},
expectedAggregatedBuckets: map[string]db.AggregatedBucket{},
},
"GivenSosBucketsUsageAndBuckets_WhenNoBuckets_ThenExpectNoAggregatedBucketObjects": {
givenSosBucketsUsage: []oapi.SosBucketUsage{},
givenBucketDetails: []BucketDetail{
createBucketDetail("bucket-test-3", "default", "orgA"),
createBucketDetail("bucket-test-4", "alpha", "orgB"),
},
expectedAggregatedBuckets: map[string]db.AggregatedBucket{},
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
// Given
ctx := context.Background()

// When
aggregatedBuckets := getAggregatedBuckets(ctx, tc.givenSosBucketsUsage, tc.givenBucketDetails)

// Then
assert.Equal(t, aggregatedBuckets, tc.expectedAggregatedBuckets)
})
}
}

func TestObjectStorage_AadOrgAndNamespaceToBucket(t *testing.T) {
tests := map[string]struct {
givenBucketList exoscalev1.BucketList
expectedBucketDetails []BucketDetail
}{
"GivenBucketListFromExoscale_WhenOrgAndNamespaces_ThenExpectBucketDetailObjects": {
givenBucketList: exoscalev1.BucketList{
Items: []exoscalev1.Bucket{
createBucket("bucket-1", "alpha", "orgA"),
createBucket("bucket-2", "beta", "orgB"),
createBucket("bucket-3", "alpha", "orgA"),
createBucket("bucket-4", "omega", "orgB"),
createBucket("bucket-5", "theta", "orgC"),
},
},
expectedBucketDetails: []BucketDetail{
createBucketDetail("bucket-1", "alpha", "orgA"),
createBucketDetail("bucket-2", "beta", "orgB"),
createBucketDetail("bucket-3", "alpha", "orgA"),
createBucketDetail("bucket-4", "omega", "orgB"),
createBucketDetail("bucket-5", "theta", "orgC"),
},
},
"GivenBucketListFromExoscale_WhenNoOrgOrNamespaces_ThenExpectNoBucketDetailObjects": {
givenBucketList: exoscalev1.BucketList{
Items: []exoscalev1.Bucket{
createBucket("bucket-1", "", "orgA"),
createBucket("bucket-2", "beta", ""),
createBucket("bucket-3", "", ""),
},
},
expectedBucketDetails: []BucketDetail{},
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
// Given
ctx := context.Background()

// When
bucketDetails := addOrgAndNamespaceToBucket(ctx, tc.givenBucketList)

// Then
assert.Equal(t, tc.expectedBucketDetails, bucketDetails)
})
}
}

func createBucket(name, namespace, organization string) exoscalev1.Bucket {
labels := make(map[string]string)
if namespace != "" {
labels[namespaceLabel] = namespace
}
if organization != "" {
labels[organizationLabel] = organization
}
return exoscalev1.Bucket{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: labels,
},
Spec: exoscalev1.BucketSpec{
ForProvider: exoscalev1.BucketParameters{
BucketName: name,
},
},
}
}

func createAggregatedBucket(organization string, size float64) db.AggregatedBucket {
return db.AggregatedBucket{
Organization: organization,
StorageUsed: size,
}
}

func createBucketDetail(bucketName, namespace, organization string) BucketDetail {
return BucketDetail{
Organization: organization,
BucketName: bucketName,
Namespace: namespace,
}
}

func createSosBucketUsage(bucketName string, size int) oapi.SosBucketUsage {
date := time.Now()
actualSize := int64(size)
zone := oapi.ZoneName("ch-gva-2")
return oapi.SosBucketUsage{
CreatedAt: &date,
Name: &bucketName,
Size: &actualSize,
ZoneName: &zone,
}
}

0 comments on commit ee73039

Please sign in to comment.