diff --git a/Gopkg.lock b/Gopkg.lock index 308cce99f3..23668d5750 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -612,6 +612,14 @@ pruneopts = "UT" revision = "22422dad46e14561a0854ad42497a75af9b61909" +[[projects]] + digest = "1:190ff84d9b2ed6589088f178cba8edb4b8ecb334df4572421fb016be1ac20463" + name = "github.com/juju/ratelimit" + packages = ["."] + pruneopts = "UT" + revision = "59fac5042749a5afb9af70e813da1dd5474f0167" + version = "1.0.1" + [[projects]] digest = "1:ae5f4d0779a45e2cb3075d8b3ece6c623e171407f4aac83521392ff06d188871" name = "github.com/kevinburke/ssh_config" @@ -1845,6 +1853,7 @@ "github.com/docker/go-connections/tlsconfig", "github.com/foomo/htpasswd", "github.com/juju/errors", + "github.com/juju/ratelimit", "github.com/mitchellh/go-homedir", "github.com/otiai10/copy", "github.com/pkg/errors", diff --git a/Gopkg.toml b/Gopkg.toml index 8f2089405c..a7433dd97b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,3 +1,7 @@ +[[constraint]] + name = "github.com/juju/ratelimit" + version = "v1.0.1" + [[constraint]] name = "github.com/juju/errors" revision = "22422dad46e14561a0854ad42497a75af9b61909" diff --git a/docs/docs/configuration/config.yaml.md b/docs/docs/configuration/config.yaml.md index 82c1bfcf5a..311cf47001 100644 --- a/docs/docs/configuration/config.yaml.md +++ b/docs/docs/configuration/config.yaml.md @@ -89,9 +89,15 @@ To comfortably sync code to a DevSpace, the DevSpace CLI allows to configure rea - `excludePaths` *string array* paths to exclude files/folders from sync in .gitignore syntax - `downloadExcludePaths` *string array* paths to exclude files/folders from download in .gitignore syntax - `uploadExcludePaths` *string array* paths to exclude files/folders from upload in .gitignore syntax +- `bandwidthLimits` *BandwidthLimits* the bandwidth limits to use for the syncpath In the example above, the entire code within the project would be synchronized with the folder `/app` inside the DevSpace, with the exception of the `node_modules/` folder. +### devspace.sync[].bandwidthLimits +Bandwidth limits for the sync path: +- `upload` *string* kilobytes per second as upper limit to use for uploading files (e.g. 100 means 100 KByte per seconds) +- `download` *string* kilobytes per second as upper limit to use for downloading files (e.g. 100 means 100 KByte per seconds) + ## images This section of the config defines a map of images that can be used in the helm chart that is deployed during `devspace up`. @@ -233,6 +239,12 @@ devSpace: # Exclude node_modules from up and download excludePaths: - node_modules/ + # Bandwidth limits for this sync path in Kbyte/s + bandwidthLimits: + # limit download speed to 100 Kbyte/s + download: 100 + # limit upload speed to 1024 Kbyte/s + upload: 1024 # A map of images that should be build during devspace up images: default: diff --git a/pkg/devspace/config/v1/devspace.go b/pkg/devspace/config/v1/devspace.go index 96f230a040..b434f73b2d 100644 --- a/pkg/devspace/config/v1/devspace.go +++ b/pkg/devspace/config/v1/devspace.go @@ -44,4 +44,11 @@ type SyncConfig struct { ExcludePaths *[]string `yaml:"excludePaths"` DownloadExcludePaths *[]string `yaml:"downloadExcludePaths"` UploadExcludePaths *[]string `yaml:"uploadExcludePaths"` + BandwidthLimits *BandwidthLimits `yaml:"bandwidthLimits,omitempty"` +} + +// BandwidthLimits defines the struct for specifying the sync bandwidth limits +type BandwidthLimits struct { + Download *int64 `yaml:"download,omitempty"` + Upload *int64 `yaml:"upload,omitempty"` } diff --git a/pkg/devspace/services/sync.go b/pkg/devspace/services/sync.go index 5957285946..ca8e1cf966 100644 --- a/pkg/devspace/services/sync.go +++ b/pkg/devspace/services/sync.go @@ -108,6 +108,16 @@ func StartSync(client *kubernetes.Clientset, verboseSync bool, log log.Logger) ( syncConfig.UploadExcludePaths = *syncPath.UploadExcludePaths } + if syncPath.BandwidthLimits != nil { + if syncPath.BandwidthLimits.Download != nil { + syncConfig.DownstreamLimit = *syncPath.BandwidthLimits.Download * 1024 + } + + if syncPath.BandwidthLimits.Upload != nil { + syncConfig.UpstreamLimit = *syncPath.BandwidthLimits.Upload * 1024 + } + } + err = syncConfig.Start() if err != nil { log.Fatalf("Sync error: %s", err.Error()) diff --git a/pkg/devspace/sync/downstream.go b/pkg/devspace/sync/downstream.go index ba16e6b557..652052b3c7 100644 --- a/pkg/devspace/sync/downstream.go +++ b/pkg/devspace/sync/downstream.go @@ -14,6 +14,7 @@ import ( "time" "github.com/juju/errors" + "github.com/juju/ratelimit" "github.com/covexo/devspace/pkg/devspace/kubectl" ) @@ -438,8 +439,14 @@ func (d *downstream) downloadArchive(tarSize int64) (string, error) { defer tempFile.Close() + // Apply rate limit if specified + var downloadReader io.Reader = d.stdoutPipe + if d.config.DownstreamLimit > 0 { + downloadReader = ratelimit.Reader(d.stdoutPipe, ratelimit.NewBucketWithRate(float64(d.config.DownstreamLimit), d.config.DownstreamLimit)) + } + // Write From stdout to temp file - bytesRead, err := io.CopyN(tempFile, d.stdoutPipe, tarSize) + bytesRead, err := io.CopyN(tempFile, downloadReader, tarSize) if err != nil { return "", errors.Trace(err) } diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 2b26315147..ffb6013db8 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -39,6 +39,8 @@ type SyncConfig struct { ExcludePaths []string DownloadExcludePaths []string UploadExcludePaths []string + UpstreamLimit int64 + DownstreamLimit int64 Verbose bool fileIndex *fileIndex diff --git a/pkg/devspace/sync/upstream.go b/pkg/devspace/sync/upstream.go index d6d04b6765..4e0cba6791 100644 --- a/pkg/devspace/sync/upstream.go +++ b/pkg/devspace/sync/upstream.go @@ -10,6 +10,7 @@ import ( "time" "github.com/juju/errors" + "github.com/juju/ratelimit" "github.com/covexo/devspace/pkg/devspace/kubectl" "github.com/rjeczalik/notify" @@ -329,8 +330,14 @@ func (u *upstream) uploadArchive(file *os.File, fileSize string, writtenFiles ma return errors.Trace(err) } + // Apply rate limit if specified + var uploadWriter io.Writer = u.stdinPipe + if u.config.UpstreamLimit > 0 { + uploadWriter = ratelimit.Writer(u.stdinPipe, ratelimit.NewBucketWithRate(float64(u.config.UpstreamLimit), u.config.UpstreamLimit)) + } + // Send file through stdin to remote - _, err = io.Copy(u.stdinPipe, file) + _, err = io.Copy(uploadWriter, file) if err != nil { return errors.Trace(err) }