Skip to content

Commit

Permalink
satellite/{console, web}: add limitations for usage report endpoint
Browse files Browse the repository at this point in the history 
Added 13 month date range limit for usage report endpoint.
Also, added user ID rate limiter.

Issue:
#6513

Change-Id: Id3a2ca87362f9e5bc611eac42b0587c31e6c1d1f
  • Loading branch information
@VitaliiShpital
VitaliiShpital authored and Storj Robot committed Jan 10, 2024
1 parent 60402b5 commit 14def22
Show file tree
Hide file tree
Showing 14 changed files with 112 additions and 60 deletions.
2 changes: 2 additions & 0 deletions satellite/console/consoleweb/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package consoleweb

import (
"encoding/json"
"time"

"storj.io/common/memory"
"storj.io/storj/satellite/console"
Expand Down Expand Up @@ -56,6 +57,7 @@ type FrontendConfig struct {
LimitIncreaseRequestEnabled bool `json:"limitIncreaseRequestEnabled"`
SignupActivationCodeEnabled bool `json:"signupActivationCodeEnabled"`
NewSignupFlowEnabled bool `json:"newSignupFlowEnabled"`
AllowedUsageReportDateRange time.Duration `json:"allowedUsageReportDateRange"`
}

// Satellites is a configuration value that contains a list of satellite names and addresses.
Expand Down
18 changes: 13 additions & 5 deletions satellite/console/consoleweb/consoleapi/usagelimits.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,17 @@ var (

// UsageLimits is an api controller that exposes all usage and limits related functionality.
type UsageLimits struct {
log *zap.Logger
service *console.Service
log *zap.Logger
service *console.Service
allowedReportDateRange time.Duration
}

// NewUsageLimits is a constructor for api usage and limits controller.
func NewUsageLimits(log *zap.Logger, service *console.Service) *UsageLimits {
func NewUsageLimits(log *zap.Logger, service *console.Service, allowedReportDateRange time.Duration) *UsageLimits {
return &UsageLimits{
log: log,
service: service,
log: log,
service: service,
allowedReportDateRange: allowedReportDateRange,
}
}

Expand Down Expand Up @@ -126,6 +128,12 @@ func (ul *UsageLimits) UsageReport(w http.ResponseWriter, r *http.Request) {
since := time.Unix(sinceStamp, 0).UTC()
before := time.Unix(beforeStamp, 0).UTC()

duration := before.Sub(since)
if duration > ul.allowedReportDateRange {
ul.serveJSONError(ctx, w, http.StatusForbidden, errs.New("date range must be less than %v", ul.allowedReportDateRange))
return
}

var projectID uuid.UUID

idParam := r.URL.Query().Get("projectID")
Expand Down
8 changes: 7 additions & 1 deletion satellite/console/consoleweb/consoleapi/usagelimits_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ func Test_TotalUsageReport(t *testing.T) {
inAnHour = now.Add(1 * time.Hour)
since = fmt.Sprintf("%d", now.Unix())
before = fmt.Sprintf("%d", inAnHour.Unix())
notAllowedBefore = fmt.Sprintf("%d", now.Add(satelliteSys.Config.Console.AllowedUsageReportDateRange+1*time.Second).Unix())
expectedCSVValue = fmt.Sprintf("%f", float64(0))
)

Expand All @@ -201,6 +202,11 @@ func Test_TotalUsageReport(t *testing.T) {
user, err := satelliteSys.AddUser(ctx, newUser, 3)
require.NoError(t, err)

endpoint := fmt.Sprintf("projects/usage-report?since=%s&before=%s&projectID=", since, notAllowedBefore)
_, status, err := doRequestWithAuth(ctx, t, satelliteSys, user, http.MethodGet, endpoint, nil)
require.NoError(t, err)
require.Equal(t, http.StatusForbidden, status)

project1, err := satelliteSys.AddProject(ctx, user.ID, "testProject1")
require.NoError(t, err)

Expand Down Expand Up @@ -235,7 +241,7 @@ func Test_TotalUsageReport(t *testing.T) {
err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, inFiveMinutes, bucketTallies)
require.NoError(t, err)

endpoint := fmt.Sprintf("projects/usage-report?since=%s&before=%s&projectID=", since, before)
endpoint = fmt.Sprintf("projects/usage-report?since=%s&before=%s&projectID=", since, before)
body, status, err := doRequestWithAuth(ctx, t, satelliteSys, user, http.MethodGet, endpoint, nil)
require.NoError(t, err)
require.Equal(t, http.StatusOK, status)
Expand Down
70 changes: 36 additions & 34 deletions satellite/console/consoleweb/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,38 +71,39 @@ type Config struct {
AuthTokenSecret string `help:"secret used to sign auth tokens" releaseDefault:"" devDefault:"my-suppa-secret-key"`
AuthCookieDomain string `help:"optional domain for cookies to use" default:""`

ContactInfoURL string `help:"url link to contacts page" default:"https://forum.storj.io"`
LetUsKnowURL string `help:"url link to let us know page" default:"https://storjlabs.atlassian.net/servicedesk/customer/portals"`
SEO string `help:"used to communicate with web crawlers and other web robots" default:"User-agent: *\nDisallow: \nDisallow: /cgi-bin/"`
SatelliteName string `help:"used to display at web satellite console" default:"Storj"`
SatelliteOperator string `help:"name of organization which set up satellite" default:"Storj Labs" `
TermsAndConditionsURL string `help:"url link to terms and conditions page" default:"https://www.storj.io/terms-of-service/"`
AccountActivationRedirectURL string `help:"url link for account activation redirect" default:""`
PartneredSatellites Satellites `help:"names and addresses of partnered satellites in JSON list format" default:"[{\"name\":\"US1\",\"address\":\"https://us1.storj.io\"},{\"name\":\"EU1\",\"address\":\"https://eu1.storj.io\"},{\"name\":\"AP1\",\"address\":\"https://ap1.storj.io\"}]"`
GeneralRequestURL string `help:"url link to general request page" default:"https://supportdcs.storj.io/hc/en-us/requests/new?ticket_form_id=360000379291"`
ProjectLimitsIncreaseRequestURL string `help:"url link to project limit increase request page" default:"https://supportdcs.storj.io/hc/en-us/requests/new?ticket_form_id=360000683212"`
GatewayCredentialsRequestURL string `help:"url link for gateway credentials requests" default:"https://auth.storjsatelliteshare.io" devDefault:"http://localhost:8000"`
IsBetaSatellite bool `help:"indicates if satellite is in beta" default:"false"`
BetaSatelliteFeedbackURL string `help:"url link for for beta satellite feedback" default:""`
BetaSatelliteSupportURL string `help:"url link for for beta satellite support" default:""`
DocumentationURL string `help:"url link to documentation" default:"https://docs.storj.io/"`
CouponCodeBillingUIEnabled bool `help:"indicates if user is allowed to add coupon codes to account from billing" default:"false"`
CouponCodeSignupUIEnabled bool `help:"indicates if user is allowed to add coupon codes to account from signup" default:"false"`
FileBrowserFlowDisabled bool `help:"indicates if file browser flow is disabled" default:"false"`
LinksharingURL string `help:"url link for linksharing requests within the application" default:"https://link.storjsatelliteshare.io" devDefault:"http://localhost:8001"`
PublicLinksharingURL string `help:"url link for linksharing requests for external sharing" default:"https://link.storjshare.io" devDefault:"http://localhost:8001"`
PathwayOverviewEnabled bool `help:"indicates if the overview onboarding step should render with pathways" default:"true"`
LimitsAreaEnabled bool `help:"indicates whether limit card section of the UI is enabled" default:"true"`
GeneratedAPIEnabled bool `help:"indicates if generated console api should be used" default:"false"`
OptionalSignupSuccessURL string `help:"optional url to external registration success page" default:""`
HomepageURL string `help:"url link to storj.io homepage" default:"https://www.storj.io"`
NativeTokenPaymentsEnabled bool `help:"indicates if storj native token payments system is enabled" default:"false"`
PricingPackagesEnabled bool `help:"whether to allow purchasing pricing packages" default:"false" devDefault:"true"`
GalleryViewEnabled bool `help:"whether to show new gallery view" default:"true"`
UseVuetifyProject bool `help:"whether to use vuetify POC project" default:"false"`
VuetifyHost string `help:"the subdomain the vuetify POC project should be hosted on" default:""`
ObjectBrowserPaginationEnabled bool `help:"whether to use object browser pagination" default:"false"`
LimitIncreaseRequestEnabled bool `help:"whether to allow request limit increases directly from the UI" default:"false"`
ContactInfoURL string `help:"url link to contacts page" default:"https://forum.storj.io"`
LetUsKnowURL string `help:"url link to let us know page" default:"https://storjlabs.atlassian.net/servicedesk/customer/portals"`
SEO string `help:"used to communicate with web crawlers and other web robots" default:"User-agent: *\nDisallow: \nDisallow: /cgi-bin/"`
SatelliteName string `help:"used to display at web satellite console" default:"Storj"`
SatelliteOperator string `help:"name of organization which set up satellite" default:"Storj Labs" `
TermsAndConditionsURL string `help:"url link to terms and conditions page" default:"https://www.storj.io/terms-of-service/"`
AccountActivationRedirectURL string `help:"url link for account activation redirect" default:""`
PartneredSatellites Satellites `help:"names and addresses of partnered satellites in JSON list format" default:"[{\"name\":\"US1\",\"address\":\"https://us1.storj.io\"},{\"name\":\"EU1\",\"address\":\"https://eu1.storj.io\"},{\"name\":\"AP1\",\"address\":\"https://ap1.storj.io\"}]"`
GeneralRequestURL string `help:"url link to general request page" default:"https://supportdcs.storj.io/hc/en-us/requests/new?ticket_form_id=360000379291"`
ProjectLimitsIncreaseRequestURL string `help:"url link to project limit increase request page" default:"https://supportdcs.storj.io/hc/en-us/requests/new?ticket_form_id=360000683212"`
GatewayCredentialsRequestURL string `help:"url link for gateway credentials requests" default:"https://auth.storjsatelliteshare.io" devDefault:"http://localhost:8000"`
IsBetaSatellite bool `help:"indicates if satellite is in beta" default:"false"`
BetaSatelliteFeedbackURL string `help:"url link for for beta satellite feedback" default:""`
BetaSatelliteSupportURL string `help:"url link for for beta satellite support" default:""`
DocumentationURL string `help:"url link to documentation" default:"https://docs.storj.io/"`
CouponCodeBillingUIEnabled bool `help:"indicates if user is allowed to add coupon codes to account from billing" default:"false"`
CouponCodeSignupUIEnabled bool `help:"indicates if user is allowed to add coupon codes to account from signup" default:"false"`
FileBrowserFlowDisabled bool `help:"indicates if file browser flow is disabled" default:"false"`
LinksharingURL string `help:"url link for linksharing requests within the application" default:"https://link.storjsatelliteshare.io" devDefault:"http://localhost:8001"`
PublicLinksharingURL string `help:"url link for linksharing requests for external sharing" default:"https://link.storjshare.io" devDefault:"http://localhost:8001"`
PathwayOverviewEnabled bool `help:"indicates if the overview onboarding step should render with pathways" default:"true"`
LimitsAreaEnabled bool `help:"indicates whether limit card section of the UI is enabled" default:"true"`
GeneratedAPIEnabled bool `help:"indicates if generated console api should be used" default:"false"`
OptionalSignupSuccessURL string `help:"optional url to external registration success page" default:""`
HomepageURL string `help:"url link to storj.io homepage" default:"https://www.storj.io"`
NativeTokenPaymentsEnabled bool `help:"indicates if storj native token payments system is enabled" default:"false"`
PricingPackagesEnabled bool `help:"whether to allow purchasing pricing packages" default:"false" devDefault:"true"`
GalleryViewEnabled bool `help:"whether to show new gallery view" default:"true"`
UseVuetifyProject bool `help:"whether to use vuetify POC project" default:"false"`
VuetifyHost string `help:"the subdomain the vuetify POC project should be hosted on" default:""`
ObjectBrowserPaginationEnabled bool `help:"whether to use object browser pagination" default:"false"`
LimitIncreaseRequestEnabled bool `help:"whether to allow request limit increases directly from the UI" default:"false"`
AllowedUsageReportDateRange time.Duration `help:"allowed usage report request date range" default:"9360h"`

OauthCodeExpiry time.Duration `help:"how long oauth authorization codes are issued for" default:"10m"`
OauthAccessTokenExpiry time.Duration `help:"how long oauth access tokens are issued for" default:"24h"`
Expand Down Expand Up @@ -287,11 +288,11 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, oidc
projectsRouter.Handle("/invitations", http.HandlerFunc(projectsController.GetUserInvitations)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/invitations/{id}/respond", http.HandlerFunc(projectsController.RespondToInvitation)).Methods(http.MethodPost, http.MethodOptions)

usageLimitsController := consoleapi.NewUsageLimits(logger, service)
usageLimitsController := consoleapi.NewUsageLimits(logger, service, server.config.AllowedUsageReportDateRange)
projectsRouter.Handle("/{id}/usage-limits", http.HandlerFunc(usageLimitsController.ProjectUsageLimits)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/usage-limits", http.HandlerFunc(usageLimitsController.TotalUsageLimits)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/{id}/daily-usage", http.HandlerFunc(usageLimitsController.DailyUsage)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/usage-report", http.HandlerFunc(usageLimitsController.UsageReport)).Methods(http.MethodGet, http.MethodOptions)
projectsRouter.Handle("/usage-report", server.userIDRateLimiter.Limit(http.HandlerFunc(usageLimitsController.UsageReport))).Methods(http.MethodGet, http.MethodOptions)

authController := consoleapi.NewAuth(logger, service, accountFreezeService, mailService, server.cookieAuth, server.analytics, config.SatelliteName, server.config.ExternalAddress, config.LetUsKnowURL, config.TermsAndConditionsURL, config.ContactInfoURL, config.GeneralRequestURL, config.SignupActivationCodeEnabled)
authRouter := router.PathPrefix("/api/v0/auth").Subrouter()
Expand Down Expand Up @@ -774,6 +775,7 @@ func (server *Server) frontendConfigHandler(w http.ResponseWriter, r *http.Reque
LimitIncreaseRequestEnabled: server.config.LimitIncreaseRequestEnabled,
SignupActivationCodeEnabled: server.config.SignupActivationCodeEnabled,
NewSignupFlowEnabled: server.config.NewSignupFlowEnabled,
AllowedUsageReportDateRange: server.config.AllowedUsageReportDateRange,
}

err := json.NewEncoder(w).Encode(&cfg)
Expand Down
3 changes: 3 additions & 0 deletions satellite/satellite-config.yaml.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ compensation.withheld-percents: 75,75,75,50,50,50,25,25,25,0,0,0,0,0,0
# server address of the http api gateway and frontend app
# console.address: :10100

# allowed usage report request date range
# console.allowed-usage-report-date-range: 9360h0m0s

# default duration for AS OF SYSTEM TIME
# console.as-of-system-time-duration: -5m0s

Expand Down
6 changes: 2 additions & 4 deletions web/satellite/src/api/projects.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,8 @@ export class ProjectsHttpApi implements ProjectsApi {
*
* @throws Error
*/
public getTotalUsageReportLink(start: Date, end: Date, projectID: string): string {
const since = Time.toUnixTimestamp(start).toString();
const before = Time.toUnixTimestamp(end).toString();
return `${this.ROOT_PATH}/usage-report?since=${since}&before=${before}&projectID=${projectID}`;
public getTotalUsageReportLink(start: number, end: number, projectID: string): string {
return `${this.ROOT_PATH}/usage-report?since=${start.toString()}&before=${end.toString()}&projectID=${projectID}`;
}

/**
Expand Down
10 changes: 7 additions & 3 deletions web/satellite/src/components/account/billing/billingTabs/Overview.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,13 @@ function balanceClicked(): void {
* Handles download usage report click logic.
*/
function downloadUsageReport(): void {
const link = projectsStore.getUsageReportLink();
Download.fileByLink(link);
notify.success('Usage report download started successfully.');
try {
const link = projectsStore.getUsageReportLink();
Download.fileByLink(link);
notify.success('Usage report download started successfully.');
} catch (error) {
notify.notifyError(error, AnalyticsErrorEventSource.BILLING_OVERVIEW_TAB);
}
}
/**
Expand Down
11 changes: 8 additions & 3 deletions web/satellite/src/components/account/billing/billingTabs/UsageAndChargesItem.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ import { useBillingStore } from '@/store/modules/billingStore';
import { useProjectsStore } from '@/store/modules/projectsStore';
import { Download } from '@/utils/download';
import { useNotify } from '@/utils/hooks';
import { AnalyticsErrorEventSource } from '@/utils/constants/analyticsEventNames';
import VButton from '@/components/common/VButton.vue';
Expand Down Expand Up @@ -137,9 +138,13 @@ const projectCharges = computed((): ProjectCharges => {
* Handles download usage report click logic.
*/
function downloadUsageReport(): void {
const link = projectsStore.getUsageReportLink(props.projectId);
Download.fileByLink(link);
notify.success('Usage report download started successfully.');
try {
const link = projectsStore.getUsageReportLink(props.projectId);
Download.fileByLink(link);
notify.success('Usage report download started successfully.');
} catch (error) {
notify.notifyError(error, AnalyticsErrorEventSource.BILLING_OVERVIEW_TAB);
}
}
/**
Expand Down
12 changes: 11 additions & 1 deletion web/satellite/src/store/modules/projectsStore.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import {
import { ProjectsHttpApi } from '@/api/projects';
import { DEFAULT_PAGE_LIMIT } from '@/types/pagination';
import { hexToBase64 } from '@/utils/strings';
import { Duration, Time } from '@/utils/time';
import { useConfigStore } from '@/store/modules/configStore';

const DEFAULT_PROJECT = new Project('', '', '', '', '', true, 0);
const DEFAULT_INVITATION = new ProjectInvitation('', '', '', '', new Date());
Expand Down Expand Up @@ -53,8 +55,16 @@ export const useProjectsStore = defineStore('projects', () => {
const now = new Date();
const endUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), now.getUTCHours(), now.getUTCMinutes()));
const startUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1, 0, 0));
const since = Time.toUnixTimestamp(startUTC);
const before = Time.toUnixTimestamp(endUTC);

return api.getTotalUsageReportLink(startUTC, endUTC, projectID);
const allowedDuration = new Duration(useConfigStore().state.config.allowedUsageReportDateRange);
const duration = before - since; // milliseconds
if (duration > allowedDuration.fullMilliseconds) {
throw new Error(`Date range must be less than ${allowedDuration.shortString}`);
}

return api.getTotalUsageReportLink(since, before, projectID);
}

async function getProjects(): Promise<Project[]> {
Expand Down
1 change: 1 addition & 0 deletions web/satellite/src/types/config.gen.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ export class FrontendConfig {
limitIncreaseRequestEnabled: boolean;
signupActivationCodeEnabled: boolean;
newSignupFlowEnabled: boolean;
allowedUsageReportDateRange: number;
}

export class MultiCaptchaConfig {
Expand Down
2 changes: 1 addition & 1 deletion web/satellite/src/types/projects.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ export interface ProjectsApi {
*
* @throws Error
*/
getTotalUsageReportLink(start: Date, end: Date, projectID: string): string
getTotalUsageReportLink(start: number, end: number, projectID: string): string

/**
* Get project daily usage by specific date range.
Expand Down
4 changes: 4 additions & 0 deletions web/satellite/src/utils/time.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ export class Duration {
return Math.floor((this.nanoseconds / 1000000) / 1000);
}

get fullMilliseconds(): number {
return Math.floor(this.nanoseconds / 1000000);
}

/**
* shortString represents this duration in the appropriate unit.
* */
Expand Down

1 comment on commit 14def22

@storjrobot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This commit has been mentioned on Storj Community Forum (official). There might be relevant details there:

https://forum.storj.io/t/why-is-the-date-range-restricted-at-390-days/25675/7

Please sign in to comment.