Skip to content

Commit

Permalink
Add unit tests and fix bugs
Browse files Browse the repository at this point in the history
  • Loading branch information
Krishnan Parthasarathi authored and Krishnan Parthasarathi committed Jan 12, 2024
1 parent 955cd93 commit b995f95
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 24 deletions.
36 changes: 21 additions & 15 deletions cmd/data-usage-cache.go
@@ -1,4 +1,4 @@
// Copyright (c) 2015-2021 MinIO, Inc.
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
Expand Down Expand Up @@ -775,6 +775,23 @@ func (h *sizeHistogram) add(size int64) {
}
}

// mergeV1 is used to migrate data usage cache from sizeHistogramV1 to
// sizeHistogram
func (h *sizeHistogram) mergeV1(v sizeHistogramV1) {
var oidx, nidx int
for oidx < len(v) {
intOld, intNew := ObjectsHistogramIntervalsV1[oidx], ObjectsHistogramIntervals[nidx]
// skip intervals that aren't common to both histograms
if intOld.start != intNew.start || intOld.end != intNew.end {
nidx++
continue
}
h[nidx] += v[oidx]
oidx++
nidx++
}
}

// toMap returns the map to a map[string]uint64.
func (h *sizeHistogram) toMap() map[string]uint64 {
res := make(map[string]uint64, dataUsageBucketLen)
Expand All @@ -785,11 +802,11 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
case humanize.KiByte == szInt.start && szInt.end == humanize.MiByte-1:
// spl interval: [1024B, 1MiB)
res[szInt.name] = splCount
case humanize.KiByte <= szInt.start || szInt.end <= humanize.MiByte-1:
case humanize.KiByte <= szInt.start && szInt.end <= humanize.MiByte-1:
// intervals that fall within the spl interval above, not exported; they
// appear earlier in this array of intervals, see
// ObjectsHistogramIntervals
splCount++
splCount += count
default:
res[szInt.name] = count
}
Expand Down Expand Up @@ -1330,18 +1347,7 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
for k, v := range dold.Cache {
var szHist sizeHistogram
var oidx, nidx int
for _, count := range v.ObjSizes {
intOld, intNew := ObjectsHistogramIntervalsV1[oidx], ObjectsHistogramIntervalsV1[nidx]
// skip intervals that aren't common to both histograms
if intOld.start != intNew.start || intOld.end != intNew.end {
nidx++
continue
}
szHist[nidx] = count
oidx++
nidx++
}
szHist.mergeV1(v.ObjSizes)
d.Cache[k] = dataUsageEntry{
Children: v.Children,
Size: v.Size,
Expand Down
95 changes: 95 additions & 0 deletions cmd/data-usage-cache_test.go
@@ -0,0 +1,95 @@
// Copyright (c) 2015-2023 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
"fmt"
"testing"

"github.com/dustin/go-humanize"
)

func TestSizeHistogramToMap(t *testing.T) {
tests := []struct {
sizes []int64
want map[string]uint64
}{
{
sizes: []int64{100, 1000, 72_000, 100_000},
want: map[string]uint64{
"LESS_THAN_1024_B": 2,
"BETWEEN_1024B_AND_1_MB": 2,
},
},
{
sizes: []int64{100, 1000, 2000, 100_000, 13 * humanize.MiByte},
want: map[string]uint64{
"LESS_THAN_1024_B": 2,
"BETWEEN_1024B_AND_1_MB": 2,
"BETWEEN_10_MB_AND_64_MB": 1,
},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("Test-%d", i), func(t *testing.T) {
var h sizeHistogram
for _, sz := range test.sizes {
h.add(sz)
}
got := h.toMap()
exp := test.want
// what is in exp is in got
for k := range exp {
if exp[k] != got[k] {
t.Fatalf("interval %s: Expected %d values but got %d values\n", k, exp[k], got[k])
}
}
// what is absent in exp is absent in got too
for k := range got {
if _, ok := exp[k]; !ok && got[k] > 0 {
t.Fatalf("Unexpected interval: %s has value %d\n", k, got[k])
}
}
})
}
}

func TestMigrateSizeHistogramFromV1(t *testing.T) {
tests := []struct {
v sizeHistogramV1
want sizeHistogram
}{
{
v: sizeHistogramV1{0: 10, 1: 20, 2: 3},
want: sizeHistogram{0: 10, 5: 20, 6: 3},
},
{
v: sizeHistogramV1{0: 10, 1: 20, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7},
want: sizeHistogram{0: 10, 5: 20, 6: 3, 7: 4, 8: 5, 9: 6, 10: 7},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
var got sizeHistogram
got.mergeV1(test.v)
if got != test.want {
t.Fatalf("Expected %v but got %v", test.want, got)
}
})
}
}
17 changes: 8 additions & 9 deletions cmd/data-usage_test.go
Expand Up @@ -82,7 +82,7 @@ func TestDataUsageUpdate(t *testing.T) {
size: 1322310,
flatten: true,
objs: 8,
oSizes: sizeHistogram{0: 2, 1: 6},
oSizes: sizeHistogram{0: 2, 1: 3, 2: 2, 4: 1},
},
{
path: "/",
Expand All @@ -94,7 +94,7 @@ func TestDataUsageUpdate(t *testing.T) {
path: "/dir1",
size: 1302010,
objs: 5,
oSizes: sizeHistogram{0: 1, 1: 4},
oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1},
},
{
path: "/dir1/dira",
Expand Down Expand Up @@ -167,7 +167,6 @@ func TestDataUsageUpdate(t *testing.T) {
size: 200,
},
}

createUsageTestFiles(t, base, bucket, files)
err = os.RemoveAll(filepath.Join(base, bucket, "dir1/dira/dirasub/dcfile"))
if err != nil {
Expand All @@ -194,14 +193,14 @@ func TestDataUsageUpdate(t *testing.T) {
size: 363515,
flatten: true,
objs: 14,
oSizes: sizeHistogram{0: 7, 1: 7},
oSizes: sizeHistogram{0: 7, 1: 5, 2: 2},
},
{
path: "/dir1",
size: 342210,
objs: 7,
flatten: false,
oSizes: sizeHistogram{0: 2, 1: 5},
oSizes: sizeHistogram{0: 2, 1: 3, 2: 2},
},
{
path: "/newfolder",
Expand Down Expand Up @@ -308,7 +307,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
path: "flat",
size: 1322310 + expectSize,
objs: 8 + expectSize,
oSizes: sizeHistogram{0: 2 + expectSize, 1: 6},
oSizes: sizeHistogram{0: 2 + expectSize, 1: 3, 2: 2, 4: 1},
},
{
path: "bucket/",
Expand All @@ -321,7 +320,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
path: "bucket/dir1",
size: 1302010,
objs: 5,
oSizes: sizeHistogram{0: 1, 1: 4},
oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1},
},
{
// Gets compacted at this level...
Expand Down Expand Up @@ -436,13 +435,13 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
path: "flat",
size: 363515 + expectSize,
objs: 14 + expectSize,
oSizes: sizeHistogram{0: 7 + expectSize, 1: 7},
oSizes: sizeHistogram{0: 7 + expectSize, 1: 5, 2: 2},
},
{
path: "bucket/dir1",
size: 342210,
objs: 7,
oSizes: sizeHistogram{0: 2, 1: 5},
oSizes: sizeHistogram{0: 2, 1: 3, 2: 2},
},
{
path: "bucket/",
Expand Down

0 comments on commit b995f95

Please sign in to comment.