Skip to content

Commit

Permalink
add test that tests gaps in chunks
Browse files Browse the repository at this point in the history
  • Loading branch information
@replay
replay committed Aug 12, 2021
1 parent 1e9ba49 commit d478766
Showing 1 changed file with 137 additions and 67 deletions.
204 changes: 137 additions & 67 deletions pkg/querier/block_test.go
View file
Expand Up @@ -116,91 +116,129 @@ func TestBlockQuerierSeriesSet(t *testing.T) {
// It would be possible to split this test into smaller parts, but I prefer to keep
// it as is, to also test transitions between series.

bss := &blockQuerierSeriesSet{
series: []*storepb.Series{
// first, with one chunk.
{
Labels: mkZLabels("__name__", "first", "a", "a"),
Chunks: []storepb.AggrChunk{
createAggrChunkWithSineSamples(now, now.Add(100*time.Second), 3*time.Millisecond), // ceil(100 / 0.003) samples (= 33334)
bss := func() *blockQuerierSeriesSet {
return &blockQuerierSeriesSet{
series: []*storepb.Series{
// first, with one chunk.
{
Labels: mkZLabels("__name__", "first", "a", "a"),
Chunks: []storepb.AggrChunk{
createAggrChunkWithSineSamples(now, now.Add(100*time.Second), 3*time.Millisecond), // ceil(100 / 0.003) samples (= 33334)
},
},
},

// continuation of previous series. Must have exact same labels.
{
Labels: mkZLabels("__name__", "first", "a", "a"),
Chunks: []storepb.AggrChunk{
createAggrChunkWithSineSamples(now.Add(100*time.Second), now.Add(200*time.Second), 3*time.Millisecond), // ceil(100 / 0.003) samples more, 66668 in total
// continuation of previous series. Must have exact same labels.
{
Labels: mkZLabels("__name__", "first", "a", "a"),
Chunks: []storepb.AggrChunk{
createAggrChunkWithSineSamples(now.Add(100*time.Second), now.Add(200*time.Second), 3*time.Millisecond), // ceil(100 / 0.003) samples more, 66668 in total
},
},
},

// second, with multiple chunks
{
Labels: mkZLabels("__name__", "second"),
Chunks: []storepb.AggrChunk{
// unordered chunks
createAggrChunkWithSineSamples(now.Add(400*time.Second), now.Add(600*time.Second), 5*time.Millisecond), // 200 / 0.005 (= 40000 samples, = 120000 in total)
createAggrChunkWithSineSamples(now.Add(200*time.Second), now.Add(400*time.Second), 5*time.Millisecond), // 200 / 0.005 (= 40000 samples)
createAggrChunkWithSineSamples(now, now.Add(200*time.Second), 5*time.Millisecond), // 200 / 0.005 (= 40000 samples)
// second, with multiple chunks
{
Labels: mkZLabels("__name__", "second"),
Chunks: []storepb.AggrChunk{
// unordered chunks
createAggrChunkWithSineSamples(now.Add(400*time.Second), now.Add(600*time.Second), 5*time.Millisecond), // 200 / 0.005 (= 40000 samples, = 120000 in total)
createAggrChunkWithSineSamples(now.Add(200*time.Second), now.Add(400*time.Second), 5*time.Millisecond), // 200 / 0.005 (= 40000 samples)
createAggrChunkWithSineSamples(now, now.Add(200*time.Second), 5*time.Millisecond), // 200 / 0.005 (= 40000 samples)
},
},
},

// overlapping
{
Labels: mkZLabels("__name__", "overlapping"),
Chunks: []storepb.AggrChunk{
createAggrChunkWithSineSamples(now, now.Add(10*time.Second), 5*time.Millisecond), // 10 / 0.005 = 2000 samples
// overlapping
{
Labels: mkZLabels("__name__", "overlapping"),
Chunks: []storepb.AggrChunk{
createAggrChunkWithSineSamples(now, now.Add(10*time.Second), 5*time.Millisecond), // 10 / 0.005 = 2000 samples
},
},
},
{
Labels: mkZLabels("__name__", "overlapping"),
Chunks: []storepb.AggrChunk{
// 10 / 0.005 = 2000 samples, but first 1000 are overlapping with previous series, so this chunk only contributes 1000
createAggrChunkWithSineSamples(now.Add(5*time.Second), now.Add(15*time.Second), 5*time.Millisecond),
{
Labels: mkZLabels("__name__", "overlapping"),
Chunks: []storepb.AggrChunk{
// 10 / 0.005 = 2000 samples, but first 1000 are overlapping with previous series, so this chunk only contributes 1000
createAggrChunkWithSineSamples(now.Add(5*time.Second), now.Add(15*time.Second), 5*time.Millisecond),
},
},
},

// overlapping 2. Chunks here come in wrong order.
{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// entire range overlaps with the next chunk, so this chunks contributes 0 samples (it will be sorted as second)
createAggrChunkWithSineSamples(now.Add(3*time.Second), now.Add(7*time.Second), 5*time.Millisecond),
// overlapping 2. Chunks here come in wrong order.
{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// entire range overlaps with the next chunk, so this chunks contributes 0 samples (it will be sorted as second)
createAggrChunkWithSineSamples(now.Add(3*time.Second), now.Add(7*time.Second), 5*time.Millisecond),
},
},
},
{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// this chunk has completely overlaps previous chunk. Since its minTime is lower, it will be sorted as first.
createAggrChunkWithSineSamples(now, now.Add(10*time.Second), 5*time.Millisecond), // 10 / 0.005 = 2000 samples
{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// this chunk has completely overlaps previous chunk. Since its minTime is lower, it will be sorted as first.
createAggrChunkWithSineSamples(now, now.Add(10*time.Second), 5*time.Millisecond), // 10 / 0.005 = 2000 samples
},
},
},
{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// no samples
createAggrChunkWithSineSamples(now, now, 5*time.Millisecond),
{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// no samples
createAggrChunkWithSineSamples(now, now, 5*time.Millisecond),
},
},
},

{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// 2000 samples more (10 / 0.005)
createAggrChunkWithSineSamples(now.Add(20*time.Second), now.Add(30*time.Second), 5*time.Millisecond),
{
Labels: mkZLabels("__name__", "overlapping2"),
Chunks: []storepb.AggrChunk{
// 2000 samples more (10 / 0.005)
createAggrChunkWithSineSamples(now.Add(20*time.Second), now.Add(30*time.Second), 5*time.Millisecond),
},
},

// overlapping 3. Chunk1 has a gap which is filled in Chunk2.
{
Labels: mkZLabels("__name__", "overlapping3"),
Chunks: []storepb.AggrChunk{
// chunk with 10 second time range and a 2 second gap in the middle
createAggrChunkWithSineSamples(
now,
now.Add(10*time.Second),
5*time.Millisecond,
[2]time.Time{now.Add(4 * time.Second), now.Add(6 * time.Second)}, // add gap of 2 seconds
),
},
},
{
Labels: mkZLabels("__name__", "overlapping3"),
Chunks: []storepb.AggrChunk{
// chunk which fills in the gap of the above chunk
createAggrChunkWithSineSamples(now.Add(3*time.Second), now.Add(7*time.Second), 5*time.Millisecond),
},
},
},
},
}
}

verifyNextSeries(t, bss, labels.FromStrings("__name__", "first", "a", "a"), 66668)
verifyNextSeries(t, bss, labels.FromStrings("__name__", "second"), 120000)
verifyNextSeries(t, bss, labels.FromStrings("__name__", "overlapping"), 3000)
verifyNextSeries(t, bss, labels.FromStrings("__name__", "overlapping2"), 4000)
require.False(t, bss.Next())
ss := bss()
verifyNextSeriesViaNext(t, ss, labels.FromStrings("__name__", "first", "a", "a"), 66668)
verifyNextSeriesViaNext(t, ss, labels.FromStrings("__name__", "second"), 120000)
verifyNextSeriesViaNext(t, ss, labels.FromStrings("__name__", "overlapping"), 3000)
verifyNextSeriesViaNext(t, ss, labels.FromStrings("__name__", "overlapping2"), 4000)
verifyNextSeriesViaNext(t, ss, labels.FromStrings("__name__", "overlapping3"), 2000)
require.False(t, ss.Next())

type timeRange struct {
minT time.Time
maxT time.Time
step time.Duration
}

ss = bss()
verifyNextSeriesViaSeek(t, ss, labels.FromStrings("__name__", "first", "a", "a"), timeRange{now, now.Add(100 * time.Second), 3 * time.Millisecond})
verifyNextSeriesViaSeek(t, ss, labels.FromStrings("__name__", "second"), timeRange{now, now.Add(600 * time.Second), 5 * time.Millisecond})
verifyNextSeriesViaSeek(t, ss, labels.FromStrings("__name__", "overlapping"), timeRange{now, now.Add(15 * time.Second), 5 * time.Millisecond})
verifyNextSeriesViaSeek(t, ss, labels.FromStrings("__name__", "overlapping2"), timeRange{now, now.Add(10 * time.Second), 5 * time.Millisecond}, timeRange{now.Add(20 * time.Second), now.Add(30 * time.Second), 5 * time.Millisecond})
verifyNextSeriesViaSeek(t, ss, labels.FromStrings("__name__", "overlapping3"), timeRange{now, now.Add(10 * time.Second), 5 * time.Millisecond})
}

func verifyNextSeries(t *testing.T, ss storage.SeriesSet, labels labels.Labels, samples int) {
func verifyNextSeriesViaNext(t *testing.T, ss storage.SeriesSet, labels labels.Labels, samples int) {
require.True(t, ss.Next())

s := ss.At()
Expand All @@ -219,14 +257,46 @@ func verifyNextSeries(t *testing.T, ss storage.SeriesSet, labels labels.Labels,
require.Equal(t, samples, count)
}

func createAggrChunkWithSineSamples(minTime, maxTime time.Time, step time.Duration) storepb.AggrChunk {
// verifyNextSeriesViaSeek verifies a series by consuming it via the iterator's Seek() method.
// The last parameter "ranges" is a slice of arrays where each array consists of <minT,maxT,step>.
func verifyNextSeriesViaSeek(t *testing.T, ss storage.SeriesSet, labels labels.Labels, ranges ...struct {
minT time.Time
maxT time.Time
step time.Duration
}) {
require.True(t, ss.Next())

s := ss.At()
require.Equal(t, labels, s.Labels())

it := s.Iterator()
for _, r := range ranges {
for wantTs := r.minT.Unix() * 1000; wantTs < r.maxT.Unix()*1000; wantTs += r.step.Milliseconds() {
require.True(t, it.Seek(wantTs))
gotTs, v := it.At()
require.Equal(t, wantTs, gotTs)
require.Equal(t, math.Sin(float64(wantTs)), v)
}
}
}

func createAggrChunkWithSineSamples(minTime, maxTime time.Time, step time.Duration, gaps ...[2]time.Time) storepb.AggrChunk {
var samples []promql.Point

minT := minTime.Unix() * 1000
maxT := maxTime.Unix() * 1000
stepMillis := step.Milliseconds()

sampleTimes:
for t := minT; t < maxT; t += stepMillis {
// If gaps are defined then we want to check whether the current "t" is within one of them,
// if so then we want to skip the current "t".
for _, gap := range gaps {
if t >= gap[0].Unix()*1000 && t < gap[1].Unix()*1000 {
continue sampleTimes
}
}

samples = append(samples, promql.Point{T: t, V: math.Sin(float64(t))})
}

Expand Down

0 comments on commit d478766

Please sign in to comment.