Support providing chunk size in multi part upload (#1169)

jfrog · Apr 24, 2024 · dac0f92 · dac0f92
1 parent 5ef50ba
commit dac0f92
Show file tree

Hide file tree

Showing 8 changed files with 16 additions and 55 deletions.
diff --git a/artifactory/commands/generic/upload.go b/artifactory/commands/generic/upload.go
@@ -201,6 +201,7 @@ func getUploadParams(f *spec.File, configuration *utils.UploadConfiguration, bui
 	uploadParams.MinChecksumDeploy = configuration.MinChecksumDeploySize
 	uploadParams.MinSplitSize = configuration.MinSplitSizeMB * rtServicesUtils.SizeMiB
 	uploadParams.SplitCount = configuration.SplitCount
+	uploadParams.ChunkSize = configuration.ChunkSizeMB * rtServicesUtils.SizeMiB
 	uploadParams.AddVcsProps = addVcsProps
 	uploadParams.BuildProps = buildProps
 	uploadParams.Archive = f.Archive

diff --git a/artifactory/commands/transferfiles/utils.go b/artifactory/commands/transferfiles/utils.go
@@ -115,7 +115,7 @@ func (clcm *ChunksLifeCycleManager) StoreStaleChunks(stateManager *state.Transfe
 			for _, file := range uploadedChunkData.ChunkFiles {
 				var sizeStr string
 				if file.Size > 0 {
-					sizeStr = " (" + utils.ConvertIntToStorageSizeString(file.Size) + ")"
+					sizeStr = " (" + serviceUtils.ConvertIntToStorageSizeString(file.Size) + ")"
 				}
 				staleNodeChunk.Files = append(staleNodeChunk.Files, path.Join(file.Repo, file.Path, file.Name)+sizeStr)
 			}

diff --git a/artifactory/utils/storageinfo.go b/artifactory/utils/storageinfo.go
@@ -3,7 +3,6 @@ package utils
 import (
 	"context"
 	"errors"
-	"fmt"
 	"strconv"
 	"strings"
 	"time"
@@ -179,24 +178,3 @@ func convertStorageSizeStringToBytes(sizeStr string) (int64, error) {
 	}
 	return int64(sizeInBytes), nil
 }
-
-func ConvertIntToStorageSizeString(num int64) string {
-	if num > utils.SizeTiB {
-		newNum := float64(num) / float64(utils.SizeTiB)
-		stringNum := fmt.Sprintf("%.1f", newNum)
-		return stringNum + "TB"
-	}
-	if num > utils.SizeGiB {
-		newNum := float64(num) / float64(utils.SizeGiB)
-		stringNum := fmt.Sprintf("%.1f", newNum)
-		return stringNum + "GB"
-	}
-	if num > utils.SizeMiB {
-		newNum := float64(num) / float64(utils.SizeMiB)
-		stringNum := fmt.Sprintf("%.1f", newNum)
-		return stringNum + "MB"
-	}
-	newNum := float64(num) / float64(utils.SizeKib)
-	stringNum := fmt.Sprintf("%.1f", newNum)
-	return stringNum + "KB"
-}
diff --git a/artifactory/utils/storageinfo_test.go b/artifactory/utils/storageinfo_test.go
@@ -159,21 +159,3 @@ func getStorageInfoResponse(t *testing.T, w http.ResponseWriter, r *http.Request
 		assert.NoError(t, err)
 	}
 }
-
-func TestConvertIntToStorageSizeString(t *testing.T) {
-	tests := []struct {
-		num    int
-		output string
-	}{
-		{12546, "12.3KB"},
-		{148576, "145.1KB"},
-		{2587985, "2.5MB"},
-		{12896547, "12.3MB"},
-		{12896547785, "12.0GB"},
-		{5248965785422365, "4773.9TB"},
-	}
-
-	for _, test := range tests {
-		assert.Equal(t, test.output, ConvertIntToStorageSizeString(int64(test.num)))
-	}
-}
diff --git a/artifactory/utils/upload.go b/artifactory/utils/upload.go
@@ -22,6 +22,7 @@ type UploadConfiguration struct {
 	ExplodeArchive        bool
 	SplitCount            int
 	MinSplitSizeMB        int64
+	ChunkSizeMB           int64
 }
 
 func GetMinChecksumDeploySize() (int64, error) {

diff --git a/go.mod b/go.mod
@@ -96,7 +96,7 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )
 
-// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240408071430-62ee0279ac58
+replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240424133643-5bf715f66eac
 
 // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go dev
 

diff --git a/go.sum b/go.sum
@@ -87,8 +87,8 @@ github.com/jfrog/build-info-go v1.9.26 h1:1Ddc6+Ecvhc+UMnKhRVG1jGM6fYNwA49207azT
 github.com/jfrog/build-info-go v1.9.26/go.mod h1:8T7/ajM9aGshvgpwCtXwIFpyF/R6CEn4W+/FLryNXWw=
 github.com/jfrog/gofrog v1.7.1 h1:ME1Meg4hukAT/7X6HUQCVSe4DNjMZACCP8aCY37EW/w=
 github.com/jfrog/gofrog v1.7.1/go.mod h1:X7bjfWoQDN0Z4FQGbE91j3gbPP7Urwzm4Z8tkvrlbRI=
-github.com/jfrog/jfrog-client-go v1.40.1 h1:ISSSV7/IUS8R+QCPfH2lVKLburbv2Xn07fvNyDc17rI=
-github.com/jfrog/jfrog-client-go v1.40.1/go.mod h1:FprEW0Sqhj6ZSFTFk9NCni+ovFAYMA3zCBmNX4hGXgQ=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20240424133643-5bf715f66eac h1:e/QfkFN/Qtkn1rBg/p7JCOMFLmE2EgfPBx57m2yIF1k=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20240424133643-5bf715f66eac/go.mod h1:FprEW0Sqhj6ZSFTFk9NCni+ovFAYMA3zCBmNX4hGXgQ=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=

diff --git a/utils/progressbar/progressbarmng.go b/utils/progressbar/progressbarmng.go
@@ -1,24 +1,23 @@
 package progressbar
 
 import (
-	golangLog "log"
-	"math"
-	"os"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
 	"github.com/gookit/color"
-	artifactoryutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	corelog "github.com/jfrog/jfrog-cli-core/v2/utils/log"
+	servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
 	"github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	"github.com/vbauerster/mpb/v7"
 	"github.com/vbauerster/mpb/v7/decor"
 	"golang.org/x/term"
+	golangLog "log"
+	"math"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
 )
 
 const (
@@ -98,8 +97,8 @@ func (bm *ProgressBarMng) newDoubleValueProgressBar(getVal func() (firstNumerato
 				if err != nil {
 					log.Error(err)
 				}
-				s1 := artifactoryutils.ConvertIntToStorageSizeString(*firstNumerator)
-				s2 := artifactoryutils.ConvertIntToStorageSizeString(*firstDenominator)
+				s1 := servicesUtils.ConvertIntToStorageSizeString(*firstNumerator)
+				s2 := servicesUtils.ConvertIntToStorageSizeString(*firstDenominator)
 				return color.Green.Render(s1 + "/" + s2)
 			}), decor.Name(" "+secondValueLine+": "), decor.Any(func(statistics decor.Statistics) string {
 				_, _, secondNumerator, secondDenominator, err := getVal()