push: customizable upload chunk size

README.md

@@ -546,6 +546,12 @@ default count of 20:
 drive push -retry-count 4 a/bc/def terms
 ```
 
+* You can also specify the upload chunk size to be used to push each file, by using flag
+`-upload-chunk-size` whose value is in bytes. If you don't specify this flag, by default
+the internal Google APIs use a value of 8MiB from constant `googleapi.DefaultUploadChunkSize`.
+Please note that your value has to be a multiple of and atleast the minimum  upload chunksize
+of 256KiB from constant `googleapi.MinUploadChunkSize`. See https://godoc.org/google.golang.org/api/googleapi#pkg-constants
+
 ### End to End Encryption
 
 See [Issue #543](https://github.com/odeke-em/drive/issues/543)

cmd/drive/main.go

@@ -818,8 +818,9 @@ type pushCmd struct {
 	ExponentialBackoffRetryCount *int    `json:"retry-count"`
 	EncryptionPassword           *string `json:"encryption-password"`
 
-	Files       *bool `json:"files"`
-	Directories *bool `json:"directories"`
+	Files           *bool `json:"files"`
+	Directories     *bool `json:"directories"`
+	UploadChunkSize *int  `json:"upload-chunk-size"`
 }
 
 func (cmd *pushCmd) Flags(fs *flag.FlagSet) *flag.FlagSet {
@@ -847,6 +848,7 @@ func (cmd *pushCmd) Flags(fs *flag.FlagSet) *flag.FlagSet {
 	cmd.EncryptionPassword = fs.String(drive.CLIEncryptionPassword, "", drive.DescEncryptionPassword)
 	cmd.Files = fs.Bool(drive.CLIOptionFiles, false, "push only files")
 	cmd.Directories = fs.Bool(drive.CLIOptionDirectories, false, "push only directories")
+	cmd.UploadChunkSize = fs.Int(drive.CLIOptionUploadChunkSize, 0, "specifies the size of each data chunk to be uploaded. Only set it if you want a custom chunk size. Otherwise the default value of googleapi.DefaultUploadChunkSize ie 8MiB will be used. However it must be at least googleapi.MinUploadChunkSize ie 256KiB. See https://godoc.org/google.golang.org/api/googleapi#pkg-constants")
 
 	return fs
 }
@@ -1064,6 +1066,7 @@ func (pCmd *pushCmd) createPushOptions(absEntryPath string, definedFlags map[str
 		Destination:                  *cmd.Destination,
 		Encrypter:                    encryptFn,
 		ExponentialBackoffRetryCount: retryCount,
+		UploadChunkSize:              *cmd.UploadChunkSize,
 	}
 
 	return opts, nil

src/commands.go

@@ -108,6 +108,11 @@ type Options struct {
 	// clickable files where applicable.
 	// See issue #697.
 	AllowURLLinkedFiles bool
+
+	// Chunksize is the size per block of data uploaded.
+	// If not set, the default value from googleapi.DefaultUploadChunkSize
+	// is used instead.
+	UploadChunkSize int
 }
 
 func (opts *Options) CryptoEnabled() bool {

src/help.go

@@ -254,6 +254,8 @@ const (
 	CLIOptionDesktopLinks       = "desktop-links"
 	CLIOptionKeepParent         = "keep-parent"
 
+	CLIOptionUploadChunkSize = "upload-chunk-size"
+
 	CLIOptionExportsDumpToSameDirectory = "same-exports-dir"
 )
 

src/push.go

@@ -206,14 +206,15 @@ func (g *Commands) PushPiped() error {
 		}
 
 		args := &upsertOpt{
-			parentId:       parent.Id,
-			fsAbsPath:      relToRootPath,
-			src:            fauxSrc,
-			dest:           rem,
-			mask:           g.opts.TypeMask,
-			nonStatable:    true,
-			ignoreChecksum: g.opts.IgnoreChecksum,
-			retryCount:     g.opts.ExponentialBackoffRetryCount,
+			uploadChunkSize: g.opts.UploadChunkSize,
+			parentId:        parent.Id,
+			fsAbsPath:       relToRootPath,
+			src:             fauxSrc,
+			dest:            rem,
+			mask:            g.opts.TypeMask,
+			nonStatable:     true,
+			ignoreChecksum:  g.opts.IgnoreChecksum,
+			retryCount:      g.opts.ExponentialBackoffRetryCount,
 		}
 
 		rem, _, rErr := g.rem.upsertByComparison(os.Stdin, args)
@@ -372,14 +373,15 @@ func (g *Commands) remoteMod(change *Change) (err error) {
 	}
 
 	args := &upsertOpt{
-		parentId:       parent.Id,
-		fsAbsPath:      absPath,
-		src:            change.Src,
-		dest:           change.Dest,
-		mask:           g.opts.TypeMask,
-		ignoreChecksum: g.opts.IgnoreChecksum,
-		debug:          g.opts.Verbose && g.opts.canPreview(),
-		retryCount:     g.opts.ExponentialBackoffRetryCount,
+		uploadChunkSize: g.opts.UploadChunkSize,
+		parentId:        parent.Id,
+		fsAbsPath:       absPath,
+		src:             change.Src,
+		dest:            change.Dest,
+		mask:            g.opts.TypeMask,
+		ignoreChecksum:  g.opts.IgnoreChecksum,
+		debug:           g.opts.Verbose && g.opts.canPreview(),
+		retryCount:      g.opts.ExponentialBackoffRetryCount,
 	}
 
 	coercedMimeKey, ok := g.coercedMimeKey()
@@ -520,10 +522,11 @@ func (g *Commands) remoteMkdirAll(d string) (*File, error) {
 	}
 
 	args := upsertOpt{
-		parentId:   parent.Id,
-		src:        remoteFile,
-		debug:      g.opts.Verbose && g.opts.canPreview(),
-		retryCount: g.opts.ExponentialBackoffRetryCount,
+		uploadChunkSize: g.opts.UploadChunkSize,
+		parentId:        parent.Id,
+		src:             remoteFile,
+		debug:           g.opts.Verbose && g.opts.canPreview(),
+		retryCount:      g.opts.ExponentialBackoffRetryCount,
 	}
 
 	cur, curErr := g.rem.UpsertByComparison(&args)

src/remote.go

@@ -33,6 +33,7 @@ import (
 
 	expb "github.com/odeke-em/exponential-backoff"
 	drive "google.golang.org/api/drive/v2"
+	"google.golang.org/api/googleapi"
 )
 
 const (
@@ -660,17 +661,18 @@ func indexContent(mask int) bool {
 }
 
 type upsertOpt struct {
-	debug          bool
-	parentId       string
-	fsAbsPath      string
-	relToRootPath  string
-	src            *File
-	dest           *File
-	mask           int
-	ignoreChecksum bool
-	mimeKey        string
-	nonStatable    bool
-	retryCount     int
+	debug           bool
+	parentId        string
+	fsAbsPath       string
+	relToRootPath   string
+	src             *File
+	dest            *File
+	mask            int
+	ignoreChecksum  bool
+	mimeKey         string
+	nonStatable     bool
+	retryCount      int
+	uploadChunkSize int
 }
 
 func togglePropertiesInsertCall(req *drive.FilesInsertCall, mask int) *drive.FilesInsertCall {
@@ -756,11 +758,16 @@ func (r *Remote) upsertByComparison(body io.Reader, args *upsertOpt) (f *File, m
 	// Ensure that the ModifiedDate is retrieved from local
 	uploaded.ModifiedDate = toUTCString(args.src.ModTime)
 
+	var mediaOptions []googleapi.MediaOption
+	if args.uploadChunkSize > 0 {
+		mediaOptions = append(mediaOptions, googleapi.ChunkSize(args.uploadChunkSize))
+	}
+
 	if args.src.Id == "" {
 		req := r.service.Files.Insert(uploaded)
 
 		if !args.src.IsDir && body != nil {
-			req = req.Media(body)
+			req = req.Media(body, mediaOptions...)
 			mediaInserted = true
 		}
 
@@ -782,7 +789,7 @@ func (r *Remote) upsertByComparison(body io.Reader, args *upsertOpt) (f *File, m
 	req.SetModifiedDate(true)
 
 	if args.shouldUploadBody() {
-		req = req.Media(body)
+		req = req.Media(body, mediaOptions...)
 		mediaInserted = true
 	}