Skip to content

Commit 951d5db

Browse files
profclemsStorj Robot
profclems
authored and
Storj Robot
committed
storagenode: fix hour_interval for first day defaulted to 24h
Previously because of the use of a LAG to calculate the hour_interval the first record, which is usually the first day of the month usually, doesn’t have a previous record and always assumes the at_rest_total is for 24 hours. Resolves #5390 Change-Id: Id532f8b38fe9df61432e62655318ff119a733d13
1 parent 93fad70 commit 951d5db

File tree

2 files changed

+83
-53
lines changed

2 files changed

+83
-53
lines changed

‎storagenode/storagenodedb/storageusage.go‎

Lines changed: 77 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -54,21 +54,28 @@ func (db *storageUsageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID
5454

5555
// hour_interval = current row interval_end_time - previous row interval_end_time
5656
// Rows with 0-hour difference are assumed to be 24 hours.
57-
query := `SELECT satellite_id,
58-
at_rest_total,
57+
query := `SELECT su1.satellite_id,
58+
su1.at_rest_total,
5959
COALESCE(
60-
(
61-
CAST(strftime('%s', interval_end_time) AS NUMERIC)
62-
-
63-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
64-
) / 3600,
65-
24
60+
(
61+
CAST(strftime('%s', su1.interval_end_time) AS NUMERIC)
62+
-
63+
CAST(strftime('%s', (
64+
SELECT interval_end_time
65+
FROM storage_usage
66+
WHERE satellite_id = su1.satellite_id
67+
AND timestamp < su1.timestamp
68+
ORDER BY timestamp DESC
69+
LIMIT 1
70+
)) AS NUMERIC)
71+
) / 3600,
72+
24
6673
) AS hour_interval,
67-
timestamp
68-
FROM storage_usage
69-
WHERE satellite_id = ?
70-
AND ? <= timestamp AND timestamp <= ?
71-
ORDER BY timestamp`
74+
su1.timestamp
75+
FROM storage_usage su1
76+
WHERE su1.satellite_id = ?
77+
AND ? <= su1.timestamp AND su1.timestamp <= ?
78+
ORDER BY su1.timestamp ASC`
7279

7380
rows, err := db.QueryContext(ctx, query, satelliteID, from.UTC(), to.UTC())
7481
if err != nil {
@@ -106,22 +113,30 @@ func (db *storageUsageDB) GetDailyTotal(ctx context.Context, from, to time.Time)
106113

107114
// hour_interval = current row interval_end_time - previous row interval_end_time
108115
// Rows with 0-hour difference are assumed to be 24 hours.
109-
query := `SELECT SUM(usages.at_rest_total), SUM(usages.hour_interval), usages.timestamp
116+
query := `SELECT SUM(su3.at_rest_total), SUM(su3.hour_interval), su3.timestamp
110117
FROM (
111-
SELECT at_rest_total, timestamp,
112-
COALESCE(
113-
(
114-
CAST(strftime('%s', interval_end_time) AS NUMERIC)
115-
-
116-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
117-
) / 3600,
118-
24
119-
) AS hour_interval
120-
FROM storage_usage
121-
WHERE ? <= timestamp AND timestamp <= ?
122-
) as usages
123-
GROUP BY usages.timestamp
124-
ORDER BY usages.timestamp`
118+
SELECT su1.at_rest_total,
119+
COALESCE(
120+
(
121+
CAST(strftime('%s', su1.interval_end_time) AS NUMERIC)
122+
-
123+
CAST(strftime('%s', (
124+
SELECT interval_end_time
125+
FROM storage_usage su2
126+
WHERE su2.satellite_id = su1.satellite_id
127+
AND su2.timestamp < su1.timestamp
128+
ORDER BY su2.timestamp DESC
129+
LIMIT 1
130+
)) AS NUMERIC)
131+
) / 3600,
132+
24
133+
) AS hour_interval,
134+
su1.timestamp
135+
FROM storage_usage su1
136+
WHERE ? <= su1.timestamp AND su1.timestamp <= ?
137+
) as su3
138+
GROUP BY su3.timestamp
139+
ORDER BY su3.timestamp ASC`
125140

126141
rows, err := db.QueryContext(ctx, query, from.UTC(), to.UTC())
127142
if err != nil {
@@ -157,23 +172,30 @@ func (db *storageUsageDB) Summary(ctx context.Context, from, to time.Time) (_, _
157172
defer mon.Task()(&ctx, from, to)(&err)
158173
var summary, averageUsageInBytes sql.NullFloat64
159174

160-
query := `SELECT SUM(usages.at_rest_total), AVG(usages.at_rest_total_bytes)
175+
query := `SELECT SUM(su3.at_rest_total), AVG(su3.at_rest_total_bytes)
161176
FROM (
162177
SELECT
163178
at_rest_total,
164179
at_rest_total / (
165180
COALESCE(
166-
(
167-
CAST(strftime('%s', interval_end_time) AS NUMERIC)
168-
-
169-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
170-
) / 3600,
171-
24
172-
)
181+
(
182+
CAST(strftime('%s', su1.interval_end_time) AS NUMERIC)
183+
-
184+
CAST(strftime('%s', (
185+
SELECT interval_end_time
186+
FROM storage_usage su2
187+
WHERE su2.satellite_id = su1.satellite_id
188+
AND su2.timestamp < su1.timestamp
189+
ORDER BY su2.timestamp DESC
190+
LIMIT 1
191+
)) AS NUMERIC)
192+
) / 3600,
193+
24
194+
)
173195
) AS at_rest_total_bytes
174-
FROM storage_usage
196+
FROM storage_usage su1
175197
WHERE ? <= timestamp AND timestamp <= ?
176-
) as usages`
198+
) as su3`
177199

178200
err = db.QueryRowContext(ctx, query, from.UTC(), to.UTC()).Scan(&summary, &averageUsageInBytes)
179201
return summary.Float64, averageUsageInBytes.Float64, err
@@ -184,24 +206,31 @@ func (db *storageUsageDB) SatelliteSummary(ctx context.Context, satelliteID stor
184206
defer mon.Task()(&ctx, satelliteID, from, to)(&err)
185207
var summary, averageUsageInBytes sql.NullFloat64
186208

187-
query := `SELECT SUM(usages.at_rest_total), AVG(usages.at_rest_total_bytes)
209+
query := `SELECT SUM(su3.at_rest_total), AVG(su3.at_rest_total_bytes)
188210
FROM (
189211
SELECT
190212
at_rest_total,
191213
at_rest_total / (
192214
COALESCE(
193-
(
194-
CAST(strftime('%s', interval_end_time) AS NUMERIC)
195-
-
196-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
197-
) / 3600,
198-
24
199-
)
215+
(
216+
CAST(strftime('%s', su1.interval_end_time) AS NUMERIC)
217+
-
218+
CAST(strftime('%s', (
219+
SELECT interval_end_time
220+
FROM storage_usage su2
221+
WHERE su2.satellite_id = su1.satellite_id
222+
AND su2.timestamp < su1.timestamp
223+
ORDER BY su2.timestamp DESC
224+
LIMIT 1
225+
)) AS NUMERIC)
226+
) / 3600,
227+
24
228+
)
200229
) AS at_rest_total_bytes
201-
FROM storage_usage
230+
FROM storage_usage su1
202231
WHERE satellite_id = ?
203232
AND ? <= timestamp AND timestamp <= ?
204-
) as usages`
233+
) as su3`
205234

206235
err = db.QueryRowContext(ctx, query, satelliteID, from.UTC(), to.UTC()).Scan(&summary, &averageUsageInBytes)
207236
return summary.Float64, averageUsageInBytes.Float64, err

‎storagenode/storageusage/storageusage_test.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ func TestStorageUsage(t *testing.T) {
8181
})
8282

8383
t.Run("get daily", func(t *testing.T) {
84-
res, err := storageUsageDB.GetDaily(ctx, satelliteID, time.Time{}, now)
84+
from := now.AddDate(0, 0, -20)
85+
res, err := storageUsageDB.GetDaily(ctx, satelliteID, from, now)
8586
assert.NoError(t, err)
8687
assert.NotNil(t, res)
8788

@@ -143,17 +144,17 @@ func TestEmptyStorageUsage(t *testing.T) {
143144
})
144145

145146
t.Run("summary satellite", func(t *testing.T) {
146-
summ, hourInterval, err := storageUsageDB.SatelliteSummary(ctx, storj.NodeID{}, time.Time{}, now)
147+
summ, averageUsageInBytes, err := storageUsageDB.SatelliteSummary(ctx, storj.NodeID{}, time.Time{}, now)
147148
assert.NoError(t, err)
148149
assert.Equal(t, emptySummary, summ)
149-
assert.Equal(t, zeroHourInterval, hourInterval)
150+
assert.Equal(t, zeroHourInterval, averageUsageInBytes)
150151
})
151152

152153
t.Run("summary", func(t *testing.T) {
153-
summ, hourInterval, err := storageUsageDB.Summary(ctx, time.Time{}, now)
154+
summ, averageUsageInBytes, err := storageUsageDB.Summary(ctx, time.Time{}, now)
154155
assert.NoError(t, err)
155156
assert.Equal(t, emptySummary, summ)
156-
assert.Equal(t, zeroHourInterval, hourInterval)
157+
assert.Equal(t, zeroHourInterval, averageUsageInBytes)
157158
})
158159
})
159160
}

0 commit comments

Comments
 (0)