Skip to content

Commit

Permalink
Add support for setting bandwidth limit (#3430)
Browse files Browse the repository at this point in the history
  • Loading branch information
kerneltime committed Nov 4, 2020
1 parent 470ae1e commit 40e328e
Show file tree
Hide file tree
Showing 5 changed files with 183 additions and 20 deletions.
56 changes: 40 additions & 16 deletions cmd/admin-bucket-remote-add.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"strconv"
"strings"

humanize "github.com/dustin/go-humanize"
"github.com/fatih/color"
"github.com/minio/cli"
json "github.com/minio/mc/pkg/colorjson"
Expand Down Expand Up @@ -50,6 +51,10 @@ var adminBucketRemoteAddFlags = []cli.Flag{
Name: "label",
Usage: "set a label to identify this target (optional)",
},
cli.StringFlag{
Name: "bandwidth",
Usage: "Set bandwidth limit in bits per second (K,B,G,T for metric and Ki,Bi,Gi,Ti for IEC units)",
},
}
var adminBucketRemoteAddCmd = cli.Command{
Name: "add",
Expand All @@ -61,7 +66,7 @@ var adminBucketRemoteAddCmd = cli.Command{
{{.HelpName}} - {{.Usage}}
USAGE:
{{.HelpName}} TARGET http(s)://ACCESSKEY:SECRETKEY@DEST_URL/DEST_BUCKET [--path | --region | --label] --service
{{.HelpName}} TARGET http(s)://ACCESSKEY:SECRETKEY@DEST_URL/DEST_BUCKET [--path | --region | --label| --bandwidth] --service
TARGET:
Also called as alias/sourcebucketname
Expand All @@ -82,12 +87,14 @@ FLAGS:
{{range .VisibleFlags}}{{.}}
{{end}}
EXAMPLES:
1. Set a new remote replication target 'replicabucket' in region "us-west-1" on https://minio2:9000 for bucket 'srcbucket' on MinIO server.
{{.DisableHistory}}
{{.Prompt}} {{.HelpName}} myminio/srcbucket \
https://foobar:foo12345@minio2:9000/replicabucket \
1. Set a new remote replication target 'targetbucket' in region "us-west-1" on https://minio.siteb.example.com for bucket 'sourcebucket'.
{{.Prompt}} {{.HelpName}} sitea/sourcebucket \
https://foobar:foo12345@minio.siteb.example.com/targetbucket \
--service "replication" --region "us-west-1" --label "hdd-tier"
{{.EnableHistory}}
2. Set a new remote replication target 'targetbucket' in region "us-west-1" on https://minio.siteb.example.com for bucket 'sourcebucket' with bandwidth set to 2 gigabits (2*10^9) per second.
{{.Prompt}} {{.HelpName}} sitea/sourcebucket \
https://foobar:foo12345@minio.siteb.example.com/targetbucket \
--service "replication" --region "us-west-1 --bandwidth "2G"
`,
}

Expand Down Expand Up @@ -117,6 +124,7 @@ type RemoteMessage struct {
Region string `json:"region,omitempty"`
ServiceType string `json:"service"`
TargetLabel string `json:"TargetLabel"`
Bandwidth int64 `json:"bandwidth"`
}

func (r RemoteMessage) String() string {
Expand Down Expand Up @@ -192,23 +200,39 @@ func fetchRemoteTarget(cli *cli.Context) (sourceBucket string, bktTarget *madmin
if !madmin.ServiceType(serviceType).IsValid() {
fatalIf(errInvalidArgument().Trace(serviceType), "Invalid service type. Valid option is `[replication]`.")
}

bandwidthStr := cli.String("bandwidth")
bandwidth, err := getBandwidthInBytes(bandwidthStr)
if err != nil {
fatalIf(errInvalidArgument().Trace(bandwidthStr), "Invalid bandwidth number")
}
console.SetColor(cred, color.New(color.FgYellow, color.Italic))
creds := &auth.Credentials{AccessKey: accessKey, SecretKey: secretKey}
bktTarget = &madmin.BucketTarget{
TargetBucket: TargetBucket,
Secure: secure,
Credentials: creds,
Endpoint: host,
Path: path,
API: "s3v4",
Type: madmin.ServiceType(serviceType),
Region: cli.String("region"),
Label: strings.ToUpper(cli.String("label")),
TargetBucket: TargetBucket,
Secure: secure,
Credentials: creds,
Endpoint: host,
Path: path,
API: "s3v4",
Type: madmin.ServiceType(serviceType),
Region: cli.String("region"),
BandwidthLimit: int64(bandwidth),
Label: strings.ToUpper(cli.String("label")),
}
return sourceBucket, bktTarget
}

func getBandwidthInBytes(bandwidthStr string) (bandwidth uint64, err error) {
if bandwidthStr != "" {
bandwidth, err = humanize.ParseBytes(bandwidthStr)
if err != nil {
return
}
}
bandwidth = bandwidth / 8
return
}

// mainAdminBucketRemoteAdd is the handle for "mc admin bucket remote set" command.
func mainAdminBucketRemoteAdd(ctx *cli.Context) error {
checkAdminBucketRemoteAddSyntax(ctx)
Expand Down
114 changes: 114 additions & 0 deletions cmd/admin-bucket-remote-add_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package cmd

import (
"testing"
)

func TestGetBandwidthInBytes(t *testing.T) {
type args struct {
bandwidthStr string
}
f1 := 999.123457 * 1024 * 1024 / 8
f2 := 10.123456790 * 1024 * 1024 * 1024 / 8
f3 := 10000.123456790 * 1024 * 1024 * 1024 / 8
f4 := (0.001*1024*1024*1024 + 1) / 8 // round up
tests := []struct {
name string
args args
want uint64
}{
{
name: "1MegaByte",
args: args{
bandwidthStr: "1Mi",
},
want: 1024 * 1024 / 8,
},
{
name: "1MegaBit",
args: args{
bandwidthStr: "1M",
},
want: 1000000 / 8,
},
{
name: "1GigaBit",
args: args{
bandwidthStr: "1G",
},
want: 1000000000 / 8,
},
{
name: "1GigaByte",
args: args{
bandwidthStr: "1Gi",
},
want: 1024 * 1024 * 1024 / 8,
},
{
name: "FractionalMegaBits",
args: args{
bandwidthStr: "999.123456789123456789M",
},
want: 999123457 / 8,
},
{
name: "FractionalGigaBits",
args: args{
bandwidthStr: "10.123456789123456789123456G",
},
want: 10123456789 / 8,
},
{
name: "FractionalBigGigaBits",
args: args{
bandwidthStr: "10000.123456789123456789123456G",
},
want: 10000123456789 / 8,
},
{
name: "FractionalMegaBytes",
args: args{
bandwidthStr: "999.123456789123456789Mi",
},
want: uint64(f1),
},
{
name: "FractionalGigaBytes",
args: args{
bandwidthStr: "10.123456789123456789123456Gi",
},
want: uint64(f2),
},
{
name: "FractionalBigGigaBytes",
args: args{
bandwidthStr: "10000.123456789123456789123456Gi",
},
want: uint64(f3),
},
{
name: "SmallGiga",
args: args{
bandwidthStr: "0.001Gi",
},
want: uint64(f4),
},
{
name: "LargeK",
args: args{
bandwidthStr: "1024Ki",
},
want: 1024 * 1024 / 8,
},
}
t.Parallel()
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
if got, err := getBandwidthInBytes(tt.args.bandwidthStr); err != nil || got != tt.want {
t.Errorf("getBandwidthInBytes() = %v, want %v", got, tt.want)
}
})
}
}
18 changes: 16 additions & 2 deletions docs/minio-admin-complete-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -872,14 +872,28 @@ COMMANDS:
rm remove configured remote target
```
*Example: Add a new replication target `targetbucket` in region `us-west-1` on `https://minio2:9000` for bucket `srcbucket` on MinIO server. `foobar` and `foo12345` are credentials to target endpoint.
*

*Example: Add a new replication target `targetbucket` in region `us-west-1` on `https://minio2:9000` for bucket `srcbucket` on MinIO server. `foobar` and `foo12345` are credentials to target endpoint.*

```
mc admin bucket remote add myminio/srcbucket https://foobar:foobar12345@minio2:9000/targetbucket --service "replication" --region "us-west-1"
ARN = `arn:minio:replication:us-west-1:1f8712ba-e38f-4429-bcb1-a7bb5aa97447:targetbucket`
```

*Example: Add a new replication target `targetbucket` in region `us-west-1` on `https://minio2:9000` for bucket `srcbucket` on MinIO server. `foobar` and `foo12345` are credentials to target endpoint. The max bandwidth is metric 2G (2*10^9)*

```
mc admin bucket remote add myminio/srcbucket https://foobar:foobar12345@minio2:9000/targetbucket --service "replication" --region "us-west-1" --bandwidth "2G"
ARN = `arn:minio:replication:us-west-1:1f8712ba-e38f-4429-bcb1-a7bb5aa97447:targetbucket`
```

*Example: Add a new replication target `targetbucket` in region `us-west-1` on `https://minio2:9000` for bucket `srcbucket` on MinIO server. `foobar` and `foo12345` are credentials to target endpoint. The max bandwidth is IEC 2Gi (2*2^30)*

```
mc admin bucket remote add myminio/srcbucket https://foobar:foobar12345@minio2:9000/targetbucket --service "replication" --region "us-west-1" --bandwidth "2Gi"
ARN = `arn:minio:replication:us-west-1:1f8712ba-e38f-4429-bcb1-a7bb5aa97447:targetbucket`
```

*Example: Get remote target for replication on bucket 'srcbucket' in MinIO.*

```
Expand Down
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dustin/go-humanize v1.0.0
github.com/fatih/color v1.7.0
github.com/go-ini/ini v1.62.0 // indirect
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/klauspost/compress v1.10.3
github.com/mattn/go-colorable v0.1.7 // indirect
Expand All @@ -24,8 +25,9 @@ require (
github.com/posener/complete v1.2.3
github.com/rjeczalik/notify v0.9.2
github.com/rs/xid v1.2.1
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
golang.org/x/net v0.0.0-20200904194848-62affa334b73
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb
golang.org/x/sys v0.0.0-20201013132646-2da7054afaeb // indirect
golang.org/x/text v0.3.3
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU=
github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
Expand Down Expand Up @@ -492,6 +494,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee h1:4yd7jl+vXjalO5ztz6Vc1VADv+S/80LGJmyl1ROJ2AI=
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
Expand Down Expand Up @@ -522,6 +526,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -558,6 +564,9 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORK
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200915084602-288bc346aa39 h1:356XA7ITklAU2//sYkjFeco+dH1bCRD8XCJ9FIEsvo4=
golang.org/x/sys v0.0.0-20200915084602-288bc346aa39/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201013132646-2da7054afaeb h1:HS9IzC4UFbpMBLQUDSQcU+ViVT1vdFCQVjdPVpTlZrs=
golang.org/x/sys v0.0.0-20201013132646-2da7054afaeb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
Expand Down

0 comments on commit 40e328e

Please sign in to comment.