Skip to content

Commit

Permalink
fix: GROUP BY queries with offset that crosses a DST boundary fail. (#…
Browse files Browse the repository at this point in the history 
…25082) (#25087) (#25089)

This is actually the second fix for
#20238
for when the time zone falls back in autumn.

closes #25078

(cherry picked from commit d60741b)

closes #25080

(cherry picked from commit 0c77b4c)

closes #25081
  • Loading branch information
@davidby-influx
davidby-influx committed Jun 25, 2024
1 parent 852a03d commit 3e756c6
Show file tree
Hide file tree
Showing 2 changed files with 169 additions and 1 deletion.
5 changes: 4 additions & 1 deletion influxql/query/iterator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,10 @@ func (opt IteratorOptions) Window(t int64) (start, end int64) {
end = t + dt
}

// As above, the offset has to happen before the time zone calculation.
// This is another fix for https://github.com/influxdata/influxdb/issues/20238
// that was missed the first time.
end += int64(opt.Interval.Offset)
// Retrieve the zone offset for the end time.
if opt.Location != nil {
_, endOffset := opt.Zone(end)
Expand All @@ -863,7 +867,6 @@ func (opt IteratorOptions) Window(t int64) (start, end int64) {
}
}
}
end += int64(opt.Interval.Offset)
return
}

Expand Down
165 changes: 165 additions & 0 deletions influxql/query/iterator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/pkg/deep"
"github.com/influxdata/influxdb/v2/pkg/testing/assert"
"github.com/influxdata/influxql"
)

Expand Down Expand Up @@ -880,6 +881,170 @@ func TestGroupByIterator_DST(t *testing.T) {
}
}

// A count() GROUP BY query with an offset that caused an interval
// to cross a daylight savings change back to standard time dropped
// rows in a grouped count() expression.

func TestGroupByIterator_DST_End(t *testing.T) {
// const RFC822 string = "02 Jan 06 15:04 MST"
inputIter := &IntegerIterator{
Points: []query.IntegerPoint{
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T00:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T01:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T02:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T03:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T04:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T05:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T06:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T07:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T08:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T09:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T10:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T11:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T12:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T13:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T14:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T15:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T16:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T17:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T18:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T19:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T20:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T21:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T22:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-02T23:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T00:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T01:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T02:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T03:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T04:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T05:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T06:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T07:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T08:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T09:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T10:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T11:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T12:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T13:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T14:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T15:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T16:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T17:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T18:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T19:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T20:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T21:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T22:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-03T23:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T00:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T01:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T02:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T03:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T04:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T05:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T06:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T07:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T08:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T09:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T10:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T11:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T12:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T13:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T14:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T15:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T16:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T17:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T18:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T19:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T20:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T21:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T22:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-04T23:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T00:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T01:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T02:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T03:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T04:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T05:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T06:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T07:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T08:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T09:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T10:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T11:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T12:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T13:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T14:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T15:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T16:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T17:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T18:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T19:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T20:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T21:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T22:00:01Z").UnixNano(), Value: 1},
{Name: "a", Tags: ParseTags("t=a"), Time: mustParseTime("2023-11-05T23:00:01Z").UnixNano(), Value: 1},
},
}
const location = "America/Los_Angeles"
loc, err := time.LoadLocation(location)
if err != nil {
t.Fatalf("Cannot find timezone for %s: %s", location, err)
}
opt := query.IteratorOptions{
StartTime: mustParseTime("2023-11-02T00:00:00Z").UnixNano(),
EndTime: mustParseTime("2023-11-06T00:00:00Z").UnixNano(),
Ascending: true,

Ordered: true,
StripName: false,
Fill: influxql.NoFill,
FillValue: nil,
Dedupe: false,
Interval: query.Interval{
Duration: 24 * time.Hour,
Offset: 12 * time.Hour,
},
Expr: MustParseExpr("count(Value)"),
Location: loc,
}

groupByIter, err := query.NewCallIterator(inputIter, opt)
if err != nil {
t.Fatalf("Cannot create Count and Group By iterator: %s", err)
} else {
groupByIter = query.NewFillIterator(groupByIter, MustParseExpr("count(Value)"), opt)
}

if all, err := (Iterators{groupByIter}).ReadAll(); err != nil {
t.Fatalf("unexpected error: %s", err)
} else {
results := [][]query.IntegerPoint{
{query.IntegerPoint{Name: "a", Aggregated: 19, Time: mustParseTime("2023-11-01T12:00:00-07:00").UnixNano(), Value: 19}},
{query.IntegerPoint{Name: "a", Aggregated: 24, Time: mustParseTime("2023-11-02T12:00:00-07:00").UnixNano(), Value: 24}},
{query.IntegerPoint{Name: "a", Aggregated: 24, Time: mustParseTime("2023-11-03T12:00:00-07:00").UnixNano(), Value: 24}},
// The extra time from falling back means more than 24 hours counted
{query.IntegerPoint{Name: "a", Aggregated: 25, Time: mustParseTime("2023-11-04T12:00:00-07:00").UnixNano(), Value: 25}},
{query.IntegerPoint{Name: "a", Aggregated: 04, Time: mustParseTime("2023-11-05T12:00:00-08:00").UnixNano(), Value: 4}},
}

for i, a := range all {
for j, p := range a {
switch ip := p.(type) {
case *query.IntegerPoint:
assert.Equal(t, results[i][j].Time, ip.Time,
"Time mismatch at i=%d j=%d: expect %v, got %v", i, j, time.Unix(0, results[i][j].Time), time.Unix(0, ip.Time))
assert.Equal(t, results[i][j].Value, ip.Value,
"Value mismatch at i=%d j=%d: expect %d, got %d", i, j, results[i][j].Value, ip.Value)
assert.Equal(t, results[i][j].Aggregated, ip.Aggregated,
"Aggregated mismatch at i=%d j=%d: expect %d, got %d", i, j, results[i][j].Aggregated, ip.Aggregated)
}
}
}
}
}

func TestFillIterator_DST(t *testing.T) {
for _, tt := range []struct {
name string
Expand Down

0 comments on commit 3e756c6

Please sign in to comment.