Skip to content

Commit

Permalink
{storagenode,web/multinode}: fix storage usage db/cache retrieval que…
Browse files Browse the repository at this point in the history 
…ries

The query changes we did while fixing the usage graph led to wrong
payout calculations directly linked to disk space.

This change:

- avoids converting from Bh to B directly in the query
- returns the at_rest_total in the original bytes*hour value
- returns at_rest_total_bytes as the calculated disk spaced used in bytes
- uses the at_rest_total_bytes only for the disk space graph
- return summary_bytes as the average disk space used within the specified date
- updates the disk space graph header to "average disk space used this month"

The total disk used in the month is also displayed in B not B*day

Resolves #5355

Change-Id: I2cfefb0fe711f9c59de2adb547c4ab50b05c7cbb
  • Loading branch information
@profclems
profclems committed Dec 9, 2022
1 parent 3cf7ebf commit 7461ffe
Show file tree
Hide file tree
Showing 24 changed files with 397 additions and 358 deletions.
375 changes: 188 additions & 187 deletions private/multinodepb/multinode.pb.go
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions private/multinodepb/multinode.proto
View file
Expand Up @@ -47,7 +47,7 @@ message StorageUsageRequest {
message StorageUsageResponse {
repeated StorageUsage storage_usage = 1;
double summary = 2;
double summary_bytes = 3;
double average_usage_bytes = 3;
}

message StorageUsageSatelliteRequest {
Expand All @@ -60,7 +60,7 @@ message StorageUsageSatelliteRequest {
message StorageUsageSatelliteResponse {
repeated StorageUsage storage_usage = 1;
double summary = 2;
double summary_bytes = 3;
double average_usage_bytes = 3;
}

service Bandwidth {
Expand Down
40 changes: 22 additions & 18 deletions storagenode/console/service.go
View file
Expand Up @@ -256,6 +256,7 @@ type Satellite struct {
StorageDaily []storageusage.Stamp `json:"storageDaily"`
BandwidthDaily []bandwidth.UsageRollup `json:"bandwidthDaily"`
StorageSummary float64 `json:"storageSummary"`
AverageUsageBytes float64 `json:"averageUsageBytes"`
BandwidthSummary int64 `json:"bandwidthSummary"`
EgressSummary int64 `json:"egressSummary"`
IngressSummary int64 `json:"ingressSummary"`
Expand Down Expand Up @@ -296,7 +297,7 @@ func (s *Service) GetSatelliteData(ctx context.Context, satelliteID storj.NodeID
return nil, SNOServiceErr.Wrap(err)
}

storageSummary, err := s.storageUsageDB.SatelliteSummary(ctx, satelliteID, from, to)
storageSummary, averageUsageInBytes, err := s.storageUsageDB.SatelliteSummary(ctx, satelliteID, from, to)
if err != nil {
return nil, SNOServiceErr.Wrap(err)
}
Expand Down Expand Up @@ -334,6 +335,7 @@ func (s *Service) GetSatelliteData(ctx context.Context, satelliteID storj.NodeID
StorageDaily: storageDaily,
BandwidthDaily: bandwidthDaily,
StorageSummary: storageSummary,
AverageUsageBytes: averageUsageInBytes,
BandwidthSummary: bandwidthSummary.Total(),
CurrentStorageUsed: currentStorageUsed,
EgressSummary: egressSummary.Total(),
Expand All @@ -352,14 +354,15 @@ func (s *Service) GetSatelliteData(ctx context.Context, satelliteID storj.NodeID

// Satellites represents consolidated data across all satellites.
type Satellites struct {
StorageDaily []storageusage.Stamp `json:"storageDaily"`
BandwidthDaily []bandwidth.UsageRollup `json:"bandwidthDaily"`
StorageSummary float64 `json:"storageSummary"`
BandwidthSummary int64 `json:"bandwidthSummary"`
EgressSummary int64 `json:"egressSummary"`
IngressSummary int64 `json:"ingressSummary"`
EarliestJoinedAt time.Time `json:"earliestJoinedAt"`
Audits []Audits `json:"audits"`
StorageDaily []storageusage.Stamp `json:"storageDaily"`
BandwidthDaily []bandwidth.UsageRollup `json:"bandwidthDaily"`
StorageSummary float64 `json:"storageSummary"`
AverageUsageBytes float64 `json:"averageUsageBytes"`
BandwidthSummary int64 `json:"bandwidthSummary"`
EgressSummary int64 `json:"egressSummary"`
IngressSummary int64 `json:"ingressSummary"`
EarliestJoinedAt time.Time `json:"earliestJoinedAt"`
Audits []Audits `json:"audits"`
}

// Audits represents audit, suspension and online scores of SNO across all satellites.
Expand Down Expand Up @@ -403,7 +406,7 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
return nil, SNOServiceErr.Wrap(err)
}

storageSummary, err := s.storageUsageDB.Summary(ctx, from, to)
storageSummary, averageUsageInBytes, err := s.storageUsageDB.Summary(ctx, from, to)
if err != nil {
return nil, SNOServiceErr.Wrap(err)
}
Expand Down Expand Up @@ -436,14 +439,15 @@ func (s *Service) GetAllSatellitesData(ctx context.Context) (_ *Satellites, err
}

return &Satellites{
StorageDaily: storageDaily,
BandwidthDaily: bandwidthDaily,
StorageSummary: storageSummary,
BandwidthSummary: bandwidthSummary.Total(),
EgressSummary: egressSummary.Total(),
IngressSummary: ingressSummary.Total(),
EarliestJoinedAt: joinedAt,
Audits: audits,
StorageDaily: storageDaily,
BandwidthDaily: bandwidthDaily,
StorageSummary: storageSummary,
AverageUsageBytes: averageUsageInBytes,
BandwidthSummary: bandwidthSummary.Total(),
EgressSummary: egressSummary.Total(),
IngressSummary: ingressSummary.Total(),
EarliestJoinedAt: joinedAt,
Audits: audits,
}, nil
}

Expand Down
24 changes: 14 additions & 10 deletions storagenode/multinode/storage.go
View file
Expand Up @@ -85,22 +85,24 @@ func (storage *StorageEndpoint) Usage(ctx context.Context, req *multinodepb.Stor
if err != nil {
return nil, rpcstatus.Wrap(rpcstatus.Internal, err)
}
summary, err := storage.usage.Summary(ctx, from, to)
summary, averageUsageInBytes, err := storage.usage.Summary(ctx, from, to)
if err != nil {
return nil, rpcstatus.Wrap(rpcstatus.Internal, err)
}

var usage []*multinodepb.StorageUsage
for _, stamp := range stamps {
usage = append(usage, &multinodepb.StorageUsage{
AtRestTotal: stamp.AtRestTotal,
IntervalStart: stamp.IntervalStart,
AtRestTotal: stamp.AtRestTotal,
AtRestTotalBytes: stamp.AtRestTotalBytes,
IntervalStart: stamp.IntervalStart,
})
}

return &multinodepb.StorageUsageResponse{
StorageUsage: usage,
Summary: summary,
StorageUsage: usage,
Summary: summary,
AverageUsageBytes: averageUsageInBytes,
}, nil
}

Expand Down Expand Up @@ -129,21 +131,23 @@ func (storage *StorageEndpoint) UsageSatellite(ctx context.Context, req *multino
if err != nil {
return nil, rpcstatus.Wrap(rpcstatus.Internal, err)
}
summary, err := storage.usage.SatelliteSummary(ctx, req.SatelliteId, from, to)
summary, averageUsageInBytes, err := storage.usage.SatelliteSummary(ctx, req.SatelliteId, from, to)
if err != nil {
return nil, rpcstatus.Wrap(rpcstatus.Internal, err)
}

var usage []*multinodepb.StorageUsage
for _, stamp := range stamps {
usage = append(usage, &multinodepb.StorageUsage{
AtRestTotal: stamp.AtRestTotal,
IntervalStart: stamp.IntervalStart,
AtRestTotal: stamp.AtRestTotal,
AtRestTotalBytes: stamp.AtRestTotalBytes,
IntervalStart: stamp.IntervalStart,
})
}

return &multinodepb.StorageUsageSatelliteResponse{
StorageUsage: usage,
Summary: summary,
StorageUsage: usage,
Summary: summary,
AverageUsageBytes: averageUsageInBytes,
}, nil
}
4 changes: 2 additions & 2 deletions storagenode/payouts/estimatedpayouts/service.go
View file
Expand Up @@ -150,8 +150,8 @@ func (s *Service) estimationUsagePeriod(ctx context.Context, period time.Time, j
for j := 0; j < len(storageDaily); j++ {
payout.DiskSpace += storageDaily[j].AtRestTotal
}
// dividing by 30 to show tbm instead of tb.
payout.DiskSpace /= 30
// dividing by 720 to show tbm instead of tbh.
payout.DiskSpace /= 720
payout.SetDiskSpacePayout(priceModel.DiskSpace)
payout.SetHeldAmount()
payout.SetPayout()
Expand Down
17 changes: 13 additions & 4 deletions storagenode/storagenodedb/storagenodedbtest/common.go
View file
Expand Up @@ -19,14 +19,23 @@ func MakeStorageUsageStamps(satellites []storj.NodeID, days int, endDate time.Ti

startDate := time.Date(endDate.Year(), endDate.Month(), endDate.Day()-days, 0, 0, 0, 0, endDate.Location())
for _, satellite := range satellites {
previousStampIntervalEndTime := startDate
for i := 0; i < days; i++ {
h := testrand.Intn(24)
intervalEndTime := startDate.Add(time.Hour * 24 * time.Duration(i)).Add(time.Hour * time.Duration(h))
atRestTotalBytes := math.Round(testrand.Float64n(100))
intervalInHours := float64(24)
if i > 0 {
intervalInHours = intervalEndTime.Sub(previousStampIntervalEndTime).Hours()
}
previousStampIntervalEndTime = intervalEndTime
stamp := storageusage.Stamp{
SatelliteID: satellite,
AtRestTotal: math.Round(testrand.Float64n(1000)),
IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()),
IntervalEndTime: intervalEndTime,
SatelliteID: satellite,
AtRestTotalBytes: atRestTotalBytes,
AtRestTotal: atRestTotalBytes * intervalInHours,
IntervalInHours: intervalInHours,
IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()),
IntervalEndTime: intervalEndTime,
}
stamps = append(stamps, stamp)
}
Expand Down

1 comment on commit 7461ffe

@storjrobot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This commit has been mentioned on Storj Community Forum (official). There might be relevant details there:

https://forum.storj.io/t/current-month-earnings-in-node-v1-67-1/20319/42

Please sign in to comment.