perf(push): increase files transfer efficiency (#32)

.golangci.yaml

@@ -68,7 +68,6 @@ linters:
   enable:
     - bodyclose
     - deadcode
-    - depguard
     - dogsled
     - dupl
     - errcheck
@@ -118,6 +117,7 @@ linters:
   # - wsl
   # - gochecknoinits
   # - gocritic
+  # - depguard
 
 issues:
   # Excluding configuration per-path, per-linter, per-text and per-source

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.13.0]
+
+### Added
+
+Add flag `-z, --zip` for subcommand `push`.
+
+### Changed
+
+Improve the files transfer efficiency of the subcommand `push`.
+The subcommand `push` no longer uses zip compression by default. If you want to continue using zip compression, you can add the `-z` flag to the command line.
+
 ## [1.12.0]
 
 ### Changed

README.md

@@ -6,8 +6,6 @@
 [![Version](https://img.shields.io/github/v/release/windvalley/gossh?include_prereleases)](https://github.com/windvalley/gossh/releases)
 [![LICENSE](https://img.shields.io/github/license/windvalley/gossh)](LICENSE) <br>
 ![Page Views](https://views.whatilearened.today/views/github/windvalley/gossh.svg)
-[![Traffic Clones Total](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.sre.im%2Fv1%2Fgithub%2Ftraffic%2Fclones%2Ftotal%3Fgit_user%3Dwindvalley%26git_repo%3Dgossh%26type%3Dcount%26label%3Dclones-total)](https://github.com/windvalley/traffic-clones-api)
-[![Traffic Clones Uniques](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.sre.im%2Fv1%2Fgithub%2Ftraffic%2Fclones%2Ftotal%3Fgit_user%3Dwindvalley%26git_repo%3Dgossh%26type%3Duniques%26label%3Dclones-uniques)](https://github.com/windvalley/traffic-clones-api)
 [![Release Download Total](https://img.shields.io/github/downloads/windvalley/gossh/total)](https://github.com/windvalley/gossh/releases)
 
 Gossh is a high-performance and high-concurrency ssh tool written in Go.

internal/cmd/push.go

@@ -34,13 +34,15 @@ import (
 
 	"github.com/windvalley/gossh/internal/pkg/configflags"
 	"github.com/windvalley/gossh/internal/pkg/sshtask"
+	"github.com/windvalley/gossh/pkg/log"
 	"github.com/windvalley/gossh/pkg/util"
 )
 
 var (
 	files          []string
 	fileDstPath    string
 	allowOverwrite bool
+	enableZip      bool
 )
 
 // pushCmd represents the push command
@@ -50,12 +52,15 @@ var pushCmd = &cobra.Command{
 	Long: `
 Copy local files and dirs to target hosts.`,
 	Example: `
-  # Copy a local file or dir to /tmp/ of the target hosts by default.
+  Copy a local file or dir to /tmp/ of the target hosts by default.
   $ gossh push host[1-2] -f /path/foo -k
 
-  # Copy local files and dirs to /home/user/ of the target hosts. 
+  Copy local files and dirs to /home/user/ of the target hosts. 
   $ gossh push host[1-2] -f /path/foo.txt,/path/bar/ -d /home/user -k
 
+  Enable zip files feature (zip first, then push).
+  $ gossh push host[1-2] -f /path/foo.txt,/path/bar/ -d /home/user -k -z
+
   Find more examples at: https://github.com/windvalley/gossh/blob/main/docs/push.md`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		if errs := configflags.Config.Validate(); len(errs) != 0 {
@@ -76,34 +81,49 @@ Copy local files and dirs to target hosts.`,
 
 		var zipFiles []string
 
-		workDir, err := os.Getwd()
-		if err != nil {
-			util.CheckErr(err)
-		}
+		if enableZip {
+			log.Debugf("enabled zip files feature")
 
-		for _, f := range files {
-			fileName := filepath.Base(f)
-			zipName := "." + fileName + "." + fmt.Sprintf("%d", time.Now().UnixMicro())
-			zipFile := path.Join(workDir, zipName)
+			zipTimeStart := time.Now()
 
-			if err := util.Zip(strings.TrimSuffix(f, string(os.PathSeparator)), zipFile); err != nil {
+			workDir, err := os.Getwd()
+			if err != nil {
 				util.CheckErr(err)
 			}
 
-			zipFiles = append(zipFiles, zipFile)
-		}
+			for _, f := range files {
+				fileName := filepath.Base(f)
+				zipName := "." + fileName + "." + fmt.Sprintf("%d", time.Now().UnixMicro())
+				zipFile := path.Join(workDir, zipName)
 
-		defer func() {
-			for _, f := range zipFiles {
-				if err := os.Remove(f); err != nil {
-					fmt.Printf("Warning: %v\n", err)
+				if err := util.Zip(strings.TrimSuffix(f, string(os.PathSeparator)), zipFile); err != nil {
+					util.CheckErr(err)
+				}
+
+				stat, err := os.Stat(zipFile)
+				if err != nil {
+					util.CheckErr(err)
 				}
+				//nolint:gomnd
+				log.Debugf("zip file '%s' size: %d MB", zipFile, stat.Size()/1024/1024)
+
+				zipFiles = append(zipFiles, zipFile)
 			}
-		}()
+
+			defer func() {
+				for _, f := range zipFiles {
+					if err := os.Remove(f); err != nil {
+						fmt.Printf("Warning: %v\n", err)
+					}
+				}
+			}()
+
+			log.Debugf("zip files cost %v", time.Since(zipTimeStart))
+		}
 
 		task.SetTargetHosts(args)
 		task.SetPushfiles(files, zipFiles)
-		task.SetPushOptions(fileDstPath, allowOverwrite)
+		task.SetPushOptions(fileDstPath, allowOverwrite, enableZip)
 
 		task.Start()
 
@@ -128,6 +148,14 @@ func init() {
 		"allow overwrite files/dirs if they already exist on target hosts",
 	)
 
+	pushCmd.Flags().BoolVarP(
+		&enableZip,
+		"zip",
+		"z",
+		false,
+		"enable zip files ('unzip' must be installed on target hosts)",
+	)
+
 	pushCmd.SetHelpFunc(func(command *cobra.Command, strings []string) {
 		util.CobraMarkHiddenGlobalFlags(
 			command,

internal/pkg/sshtask/sshtask.go

@@ -114,6 +114,7 @@ type Task struct {
 	tmpDir         string
 	remove         bool
 	allowOverwrite bool
+	enableZip      bool
 
 	taskOutput   chan taskResult
 	detailOutput chan detailResult
@@ -204,9 +205,10 @@ func (t *Task) SetScriptOptions(destPath string, remove, allowOverwrite bool) {
 }
 
 // SetPushOptions ...
-func (t *Task) SetPushOptions(destPath string, allowOverwrite bool) {
+func (t *Task) SetPushOptions(destPath string, allowOverwrite, enableZip bool) {
 	t.dstDir = destPath
 	t.allowOverwrite = allowOverwrite
+	t.enableZip = enableZip
 }
 
 // SetFetchOptions ...
@@ -227,16 +229,17 @@ func (t *Task) RunSSH(host *batchssh.Host) (string, error) {
 	case ScriptTask:
 		return t.sshClient.ExecuteScript(host, t.scriptFile, t.dstDir, lang, runAs, sudo, t.remove, t.allowOverwrite)
 	case PushTask:
-		return t.sshClient.PushFiles(host, t.pushFiles.files, t.pushFiles.zipFiles, t.dstDir, t.allowOverwrite)
+		return t.sshClient.PushFiles(host, t.pushFiles.files, t.pushFiles.zipFiles, t.dstDir, t.allowOverwrite, t.enableZip)
 	case FetchTask:
 		return t.sshClient.FetchFiles(host, t.fetchFiles, t.dstDir, t.tmpDir, sudo, runAs)
 	default:
 		return "", fmt.Errorf("unknown task type: %v", t.taskType)
 	}
 }
 
-//nolint:gocyclo
 // BatchRun ...
+//
+//nolint:gocyclo
 func (t *Task) BatchRun() {
 	timeNow := time.Now()