From 797d5fa8059ff15c9213405ef1a1d4b781075380 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Wed, 5 Nov 2025 10:59:38 +0530 Subject: [PATCH 01/15] implemented multipart upload --- internal/commands/scan.go | 23 +- internal/params/binds.go | 4 + internal/params/envs.go | 4 + internal/params/keys.go | 4 + internal/wrappers/feature-flags.go | 1 + internal/wrappers/uploads-http.go | 413 +++++++++++++++++++++++++++++ internal/wrappers/uploads.go | 1 + 7 files changed, 449 insertions(+), 1 deletion(-) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 9dc28ff85..738470a6d 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -129,6 +129,8 @@ const ( sbomScanTypeErrMsg = "The --sbom-only flag can only be used when the scan type is sca" BranchPrimaryPrefix = "--branch-primary=" OverridePolicyManagement = "override-policy-management" + maxSizeGB = 5 // 5 GB + maxSizeBytes = maxSizeGB * 1024 * 1024 * 1024 // 5 GB in bytes ) var ( @@ -2082,7 +2084,26 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip var zipFilePathErr error // Send a request to uploads service var preSignedURL *string - preSignedURL, zipFilePathErr = uploadsWrapper.UploadFile(zipFilePath, featureFlagsWrapper) + + // calculate file size and compare with 5GB limit + fileInfo, err := os.Stat(zipFilePath) + if err != nil { + return "", zipFilePath, errors.Wrapf(err, "Failed to stat %s", zipFilePath) + } + logger.PrintIfVerbose(fmt.Sprintf("Zip size before upload: %.2fMB\n", float64(fileInfo.Size())/mbBytes)) + + // check for INCREASE_FILE_UPLOAD_LIMIT feature flag + flagResponse, _ := wrappers.GetSpecificFeatureFlag(featureFlagsWrapper, wrappers.IncreaseFileUploadLimit) + + if flagResponse.Status && fileInfo.Size() > maxSizeBytes { + // File size >5GB, proceed with multipart upload + logger.PrintIfVerbose("File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts...") + preSignedURL, zipFilePathErr = uploadsWrapper.UploadFileInMultipart(zipFilePath, featureFlagsWrapper) + } else { + // File size is within <=5GB, proceed with upload in single part + logger.PrintIfVerbose("File size is within the limit and uploading file in a single part...") + preSignedURL, zipFilePathErr = uploadsWrapper.UploadFile(zipFilePath, featureFlagsWrapper) + } if zipFilePathErr != nil { if unzip || !userProvidedZip { return "", zipFilePath, errors.Wrapf(zipFilePathErr, "%s: Failed to upload sources file\n", failedCreating) diff --git a/internal/params/binds.go b/internal/params/binds.go index f90d15623..065a78edf 100644 --- a/internal/params/binds.go +++ b/internal/params/binds.go @@ -80,4 +80,8 @@ var EnvVarsBinds = []struct { {RiskManagementPathKey, RiskManagementPathEnv, "api/risk-management/projects/%s/results?scanID=%s"}, {ConfigFilePathKey, ConfigFilePathEnv, ""}, {RealtimeScannerPathKey, RealtimeScannerPathEnv, "api/realtime-scanner"}, + {StartMultiPartUploadPathKey, StartMultiPartUploadPathEnv, "api/uploads/start-multipart-upload"}, + {MultipartPresignedPathKey, MultipartPresignedPathEnv, "api/uploads/multipart-presigned"}, + {CompleteMultiPartUploadPathKey, CompleteMultipartUploadPathEnv, "api/uploads/complete-multipart-upload"}, + {MultipartFileSize, MultipartFileSizeEnv, "2"}, } diff --git a/internal/params/envs.go b/internal/params/envs.go index 19dc0f3c7..9698eb699 100644 --- a/internal/params/envs.go +++ b/internal/params/envs.go @@ -80,4 +80,8 @@ const ( RiskManagementPathEnv = "CX_RISK_MANAGEMENT_PATH" ConfigFilePathEnv = "CX_CONFIG_FILE_PATH" RealtimeScannerPathEnv = "CX_REALTIME_SCANNER_PATH" + StartMultiPartUploadPathEnv = "CX_START_MULTIPART_UPLOAD_PATH" + MultipartPresignedPathEnv = "CX_MULTIPART_PRESIGNED_URL_PATH" + CompleteMultipartUploadPathEnv = "CX_COMPLETE_MULTIPART_UPLOAD_PATH" + MultipartFileSizeEnv = "MULTIPART_FILE_SIZE" ) diff --git a/internal/params/keys.go b/internal/params/keys.go index 839b13e53..8a2732ad8 100644 --- a/internal/params/keys.go +++ b/internal/params/keys.go @@ -79,4 +79,8 @@ var ( RiskManagementPathKey = strings.ToLower(RiskManagementPathEnv) ConfigFilePathKey = strings.ToLower(ConfigFilePathEnv) RealtimeScannerPathKey = strings.ToLower(RealtimeScannerPathEnv) + StartMultiPartUploadPathKey = strings.ToLower(StartMultiPartUploadPathEnv) + MultipartPresignedPathKey = strings.ToLower(MultipartPresignedPathEnv) + CompleteMultiPartUploadPathKey = strings.ToLower(CompleteMultipartUploadPathEnv) + MultipartFileSize = strings.ToLower(MultipartFileSizeEnv) ) diff --git a/internal/wrappers/feature-flags.go b/internal/wrappers/feature-flags.go index 8f92b8cef..6762e860f 100644 --- a/internal/wrappers/feature-flags.go +++ b/internal/wrappers/feature-flags.go @@ -17,6 +17,7 @@ const OssRealtimeEnabled = "OSS_REALTIME_ENABLED" const ScsLicensingV2Enabled = "SSCS_NEW_LICENSING_ENABLED" const DirectAssociationEnabled = "DIRECT_APP_ASSOCIATION_ENABLED" const maxRetries = 3 +const IncreaseFileUploadLimit = "INCREASE_FILE_UPLOAD_LIMIT" var DefaultFFLoad bool = false diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index e2b2808dd..914a5f9e2 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -1,13 +1,18 @@ package wrappers import ( + "bytes" "encoding/json" "fmt" + "io" "net/http" "net/url" "os" + "strconv" + "time" errorConstants "github.com/checkmarx/ast-cli/internal/constants/errors" + "github.com/checkmarx/ast-cli/internal/logger" commonParams "github.com/checkmarx/ast-cli/internal/params" "github.com/pkg/errors" "github.com/spf13/viper" @@ -21,6 +26,34 @@ type UploadsHTTPWrapper struct { path string } +type StartMultipartUploadResponse struct { + ObjectName string `json:"objectName"` + UploadID string `json:"UploadID"` +} +type StartMultipartUploadRequest struct { + FileSize int64 `json:"fileSize"` +} +type MultipartPresignedURl struct { + ObjectName string `json:"objectName"` + UploadID string `json:"UploadID"` + PartNumber int `json:"partNumber"` +} + +type CompleteMultipartUpload struct { + UploadID string `json:"UploadID"` + ObjectName string `json:"objectName"` + PartList []Part `json:"partList"` +} + +type Part struct { + ETag string `json:"eTag"` + PartNumber int `json:"partNumber"` +} + +type UploadModelMultipart struct { + PresignedURL string `json:"presignedURL"` +} + func (u *UploadsHTTPWrapper) UploadFile(sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (*string, error) { preSignedURL, err := u.getPresignedURLForUploading() if err != nil { @@ -130,3 +163,383 @@ func NewUploadsHTTPWrapper(path string) UploadsWrapper { path: path, } } + +func (u *UploadsHTTPWrapper) UploadFileInMultipart(sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (*string, error) { + // calculate file size and compare with 5GB limit + fileInfo, err := os.Stat(sourcesFile) + if err != nil { + return nil, errors.Wrapf(err, "Failed to stat - %s", sourcesFile) + } + + startMultipartUploadRequest := StartMultipartUploadRequest{} + startMultipartUploadRequest.FileSize = fileInfo.Size() + startMultipartUploadResponse, err := startMultipartUpload(startMultipartUploadRequest) + if err != nil { + return nil, err + } + partList, err := SplitZipBySizeGB(sourcesFile) + if err != nil { + return nil, errors.Errorf("Failed to split ZIP file for multipart upload - %s", err.Error()) + } + + // ensure temporary parts are removed when this function returns + defer cleanUpTempParts(partList) + + for i, part := range partList { + logger.PrintfIfVerbose("Part%d created at: %s", i+1, part) + } + + completeMultipartUpload := &CompleteMultipartUpload{ + UploadID: startMultipartUploadResponse.UploadID, + ObjectName: startMultipartUploadResponse.ObjectName, + } + + var presignedURLPart1 string + + for i, partPath := range partList { + partNumber := i + 1 + + // Generate presigned URL + presignedURL, err := getPresignedURLForMultipartUploading(startMultipartUploadResponse, partNumber) + if err != nil { + return nil, fmt.Errorf("failed to get presigned URL for part%d - %s", partNumber, err.Error()) + } + + if partNumber == 1 { + presignedURLPart1 = presignedURL + } + // Upload part + etag, err := uploadPart(presignedURL, partPath, featureFlagsWrapper) + if err != nil { + return nil, fmt.Errorf("failed to upload part%d - %s", partNumber, err.Error()) + } + + // Append part info + completeMultipartUpload.PartList = append(completeMultipartUpload.PartList, Part{ + ETag: etag, + PartNumber: partNumber, + }) + } + + // call the complete multipart upload API + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + path := viper.GetString(commonParams.CompleteMultipartUploadPathEnv) + jsonBytes, err := json.Marshal(completeMultipartUpload) + resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(resp.Body) + defer func() { + _ = resp.Body.Close() + }() + switch resp.StatusCode { + case http.StatusNoContent: + return &presignedURLPart1, nil + case http.StatusUnauthorized: + return nil, errors.New(errorConstants.StatusUnauthorized) + default: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return nil, errors.Errorf("Parsing error model failed - %s", err.Error()) + } + return nil, errors.Errorf("%d - %s", errorModel.Code, errorModel.Message) + } +} + +func startMultipartUpload(startMultipartUploadRequest StartMultipartUploadRequest) (StartMultipartUploadResponse, error) { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + path := viper.GetString(commonParams.StartMultiPartUploadPathEnv) + jsonBytes, err := json.Marshal(startMultipartUploadRequest) + resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return StartMultipartUploadResponse{}, err + } + decoder := json.NewDecoder(resp.Body) + defer func() { + _ = resp.Body.Close() + }() + switch resp.StatusCode { + case http.StatusOK: + startMultipartUpload := StartMultipartUploadResponse{} + err = decoder.Decode(&startMultipartUpload) + if err != nil { + return StartMultipartUploadResponse{}, errors.Errorf("failed to start the multipart upload - %s ", err.Error()) + } + return startMultipartUpload, nil + case http.StatusBadRequest: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return StartMultipartUploadResponse{}, errors.Errorf("failed to start the multipart upload - %s ", err.Error()) + } + return StartMultipartUploadResponse{}, errors.Errorf(errorModel.Message) + case http.StatusUnauthorized: + return StartMultipartUploadResponse{}, errors.New(errorConstants.StatusUnauthorized) + default: + return StartMultipartUploadResponse{}, errors.Errorf("response status code %d", resp.StatusCode) + } +} + +func getPresignedURLForMultipartUploading(response StartMultipartUploadResponse, partNumber int) (string, error) { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + path := viper.GetString(commonParams.MultipartPresignedPathEnv) + + multipartPresignedURl := MultipartPresignedURl{ + ObjectName: response.ObjectName, + UploadID: response.UploadID, + PartNumber: partNumber, + } + jsonBytes, err := json.Marshal(multipartPresignedURl) + if err != nil { + return "", errors.Errorf("Failed to marshal multipart upload presigned URL request body - %s", err.Error()) + } + + resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return "", errors.Errorf("Invoking HTTP request to get pre-signed URL failed - %s", err.Error()) + } + + defer func() { + _ = resp.Body.Close() + }() + + decoder := json.NewDecoder(resp.Body) + + switch resp.StatusCode { + case http.StatusBadRequest: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return "", errors.Errorf("Parsing error model failed - %s", err.Error()) + } + return "", errors.Errorf("%d - %s", errorModel.Code, errorModel.Message) + + case http.StatusOK: + model := UploadModelMultipart{} + err = decoder.Decode(&model) + if err != nil { + return "", errors.Errorf("Parsing upload model failed - %s", err.Error()) + } + return model.PresignedURL, nil + + default: + return "", errors.Errorf("response status code %d", resp.StatusCode) + } +} + +func uploadPart(preSignedURL string, sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (string, error) { + if preSignedURL == "" { + return "", errors.New("preSignedURL is empty or nil") + } + + file, err := os.Open(sourcesFile) + if err != nil { + return "", errors.Errorf("Failed to open file for multipart upload %s - %s", sourcesFile, err.Error()) + } + // Close the file later + defer func() { + _ = file.Close() + }() + + accessToken, err := GetAccessToken() + if err != nil { + return "", err + } + + stat, err := file.Stat() + if err != nil { + return "", errors.Errorf("Failed to stat file %s - %s", sourcesFile, err.Error()) + } + flagResponse, _ := GetSpecificFeatureFlag(featureFlagsWrapper, MinioEnabled) + useAccessToken := flagResponse.Status + resp, err := SendHTTPRequestByFullURLContentLength(http.MethodPut, preSignedURL, file, stat.Size(), useAccessToken, NoTimeout, accessToken, true) + if err != nil { + return "", errors.Errorf("Invoking HTTP request to upload file failed - %s", err.Error()) + } + + defer func() { + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusUnauthorized: + return "", errors.Errorf(errorConstants.StatusUnauthorized) + case http.StatusOK: + return resp.Header.Get("Etag"), nil + case http.StatusBadRequest: + body, err := io.ReadAll(resp.Body) + defer func() { + _ = resp.Body.Close() + }() + if err != nil { + return "", errors.Errorf("Reading response body failed - %s", err.Error()) + } + return "", errors.Errorf("Bad request while uploading part - %s", string(body)) + default: + return "", errors.Errorf("Failed to upload part of multipart - %d", resp.StatusCode) + } +} + +func SplitZipBySizeGB(zipFilePath string) ([]string, error) { + + // Get part size in GB from config if config is not provided, default to 2 GB + partChunkSizeStr := viper.GetString(commonParams.MultipartFileSize) + partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) + if err != nil { + return nil, fmt.Errorf("invalid part size value: %w", err) + } + // Truncate to integer + truncatedSize := int64(partChunkSizeFloat) + if truncatedSize < 1 || truncatedSize > 5 { + // Enforce part size to be between 1 GB and 5 GB. + // If the configured part size is outside this range, default to 2 GB. + logger.PrintIfVerbose(fmt.Sprintf("Configured part size %d GB is outside the allowed range (1 – 5 GB). Defaulting to 2 GB.", truncatedSize)) + truncatedSize = 2 // default to 2 GB if out of range + } + partSizeGB := float64(truncatedSize) + + logger.PrintIfVerbose("Splitting zip file into parts of size: " + fmt.Sprintf("%.0f", partSizeGB) + " GB") + + if partSizeGB <= 0 { + return nil, fmt.Errorf("part size must be greater than 0 GB") + } + + const bytesPerGB = 1024 * 1024 * 1024 + partSizeBytes := int64(partSizeGB * float64(bytesPerGB)) + + f, err := os.Open(zipFilePath) + if err != nil { + return nil, fmt.Errorf("open input - %w", err) + } + defer func() { + if err := f.Close(); err != nil { + logger.PrintfIfVerbose("warning: failed to close input file - %v", err) + } + }() + + stat, err := f.Stat() + if err != nil { + return nil, fmt.Errorf("stat input - %w", err) + } + if stat.Size() == 0 { + return nil, fmt.Errorf("input file is empty") + } + + totalSize := stat.Size() + numParts := int(totalSize / partSizeBytes) + if totalSize%partSizeBytes != 0 { + numParts++ + } + + partSizes := make([]int64, numParts) + for i := 0; i < numParts; i++ { + remaining := totalSize - int64(i)*partSizeBytes + if remaining >= partSizeBytes { + partSizes[i] = partSizeBytes + } else { + partSizes[i] = remaining + } + } + + partNames := make([]string, numParts) + for i := 0; i < numParts; i++ { + partFile, err := os.CreateTemp("", fmt.Sprintf("cx-part%d-*", i+1)) + if err != nil { + for j := 0; j < i; j++ { + if partNames[j] != "" { + if err := os.Remove(partNames[j]); err != nil && !os.IsNotExist(err) { + logger.PrintfIfVerbose("warning: failed to remove part%d - %v", j+1, err) + } + } + } + return nil, fmt.Errorf("create part%d - %w", i+1, err) + } + + offset := int64(0) + for j := 0; j < i; j++ { + offset += partSizes[j] + } + if _, err := f.Seek(offset, io.SeekStart); err != nil { + err := partFile.Close() + if err != nil { + return nil, err + } + err = os.Remove(partFile.Name()) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("seek to part%d - %w", i+1, err) + } + + if _, err := io.CopyN(partFile, f, partSizes[i]); err != nil && err != io.EOF { + err := partFile.Close() + if err != nil { + return nil, err + } + err = os.Remove(partFile.Name()) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("copy to part%d - %w", i+1, err) + } + + if err := partFile.Sync(); err != nil { + logger.PrintfIfVerbose("warning: failed to sync part%d - %v", i+1, err) + } + if err := partFile.Close(); err != nil { + logger.PrintfIfVerbose("warning: failed to close part%d - %v", i+1, err) + } + + partNames[i] = partFile.Name() + } + + return partNames, nil +} + +// cleanUpTempParts removes the temporary part files created during multipart upload. +func cleanUpTempParts(partList []string) { + cleanupMaxRetries := 3 + for i, partPath := range partList { + if partPath != "" { + logger.PrintIfVerbose(fmt.Sprintf("Cleaning up temporary part%d - %s", i+1, partPath)) + tries := cleanupMaxRetries + for attempt := 1; tries > 0; attempt++ { + removeErr := os.Remove(partPath) + if removeErr != nil { + if os.IsNotExist(removeErr) { + logger.PrintIfVerbose(fmt.Sprintf("Temporary part%d already removed - %s", i+1, partPath)) + break + } + logger.PrintIfVerbose(fmt.Sprintf( + "Failed to remove temporary part%d - Attempt %d/%d - %v", + i+1, + attempt, + cleanupMaxRetries, + removeErr, + )) + tries-- + Wait(attempt) + } else { + logger.PrintIfVerbose(fmt.Sprintf("Removed temporary part%d", i+1)) + break + } + } + if tries == 0 { + logger.PrintIfVerbose(fmt.Sprintf("Failed to remove temporary part%d - %s", i+1, partPath)) + } + } else { + logger.PrintIfVerbose(fmt.Sprintf("No temporary part%d to clean", i+1)) + } + } +} + +// Wait implements exponential backoff wait strategy +func Wait(attempt int) { + cleanupRetryWaitSeconds := 15 + // Calculate exponential backoff delay + waitDuration := time.Duration(cleanupRetryWaitSeconds * (1 << (attempt - 1))) // 2^(attempt-1) + logger.PrintIfVerbose(fmt.Sprintf("Waiting %d seconds before retrying...", waitDuration)) + time.Sleep(waitDuration * time.Second) +} diff --git a/internal/wrappers/uploads.go b/internal/wrappers/uploads.go index e2089c781..f778b47bd 100644 --- a/internal/wrappers/uploads.go +++ b/internal/wrappers/uploads.go @@ -2,4 +2,5 @@ package wrappers type UploadsWrapper interface { UploadFile(sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (*string, error) + UploadFileInMultipart(path string, wrapper FeatureFlagsWrapper) (*string, error) } From 17180dff03a47b6aa2c1e81f57144261323aede0 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Wed, 5 Nov 2025 12:12:55 +0530 Subject: [PATCH 02/15] Added wrapper method in mock --- internal/wrappers/mock/uploads-mock.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/wrappers/mock/uploads-mock.go b/internal/wrappers/mock/uploads-mock.go index 59fc41bbb..b8684eef6 100644 --- a/internal/wrappers/mock/uploads-mock.go +++ b/internal/wrappers/mock/uploads-mock.go @@ -11,6 +11,15 @@ import ( type UploadsMockWrapper struct { } +func (u *UploadsMockWrapper) UploadFileInMultipart(path string, wrapper wrappers.FeatureFlagsWrapper) (*string, error) { + fmt.Println("Called Create in UploadsMockWrapper") + if path == "failureCase.zip" { + return nil, errors.New("error from UploadFileInMultipart") + } + url := "/path/to/nowhere" + return &url, nil +} + func (u *UploadsMockWrapper) UploadFile(filePath string, featureFlagsWrapper wrappers.FeatureFlagsWrapper) (*string, error) { fmt.Println("Called Create in UploadsMockWrapper") if filePath == "failureCase.zip" { From 61b8879bc96e12dc37790426d1af4c5e4ff59f19 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Wed, 5 Nov 2025 14:38:41 +0530 Subject: [PATCH 03/15] fix unit test cases and lint issue --- internal/commands/scan_test.go | 43 ++++++++-- internal/wrappers/mock/uploads-mock.go | 11 +-- internal/wrappers/uploads-http.go | 110 +++++++++++++------------ 3 files changed, 101 insertions(+), 63 deletions(-) diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index 3062da8ba..130adadec 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -2972,15 +2972,31 @@ func TestResubmitConfig_ProjectDoesNotExist_ReturnedEmptyConfig(t *testing.T) { func TestUploadZip_whenUserProvideZip_shouldReturnEmptyZipFilePathInSuccessCase(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "test.zip", false, true, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") } func TestUploadZip_whenUserProvideZip_shouldReturnEmptyZipFilePathInFailureCase(t *testing.T) { + // Create a temporary zip file + dir := t.TempDir() + zipPathTemp := filepath.Join(dir, "failureCase.zip") + + // Create the zip file + zipFile, err := os.Create(zipPathTemp) + if err != nil { + t.Fatalf("Failed to create zip file: %v", err) + } + defer func(zipFile *os.File) { + err := zipFile.Close() + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + }(zipFile) + uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "failureCase.zip", false, true, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, true, featureFlagsWrapper) assert.Assert(t, err != nil) assert.Assert(t, strings.Contains(err.Error(), "error from UploadFile"), err.Error()) assert.Equal(t, zipPath, "") @@ -2989,18 +3005,33 @@ func TestUploadZip_whenUserProvideZip_shouldReturnEmptyZipFilePathInFailureCase( func TestUploadZip_whenUserNotProvideZip_shouldReturnZipFilePathInSuccessCase(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "test.zip", false, false, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, false, featureFlagsWrapper) assert.NilError(t, err) - assert.Equal(t, zipPath, "test.zip") + assert.Equal(t, zipPath, "data/sources.zip") } func TestUploadZip_whenUserNotProvideZip_shouldReturnZipFilePathInFailureCase(t *testing.T) { + // Create a temporary zip file + dir := t.TempDir() + zipPathTemp := filepath.Join(dir, "failureCase.zip") + + // Create the zip file + zipFile, err := os.Create(zipPathTemp) + if err != nil { + t.Fatalf("Failed to create zip file: %v", err) + } + defer func(zipFile *os.File) { + err := zipFile.Close() + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + }(zipFile) uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "failureCase.zip", false, false, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, false, featureFlagsWrapper) assert.Assert(t, err != nil) assert.Assert(t, strings.Contains(err.Error(), "error from UploadFile"), err.Error()) - assert.Equal(t, zipPath, "failureCase.zip") + assert.Equal(t, zipPath, zipPathTemp) } func TestAddSastScan_ScanFlags(t *testing.T) { diff --git a/internal/wrappers/mock/uploads-mock.go b/internal/wrappers/mock/uploads-mock.go index b8684eef6..99ff2948c 100644 --- a/internal/wrappers/mock/uploads-mock.go +++ b/internal/wrappers/mock/uploads-mock.go @@ -2,6 +2,7 @@ package mock import ( "fmt" + "strings" "github.com/pkg/errors" @@ -11,18 +12,18 @@ import ( type UploadsMockWrapper struct { } -func (u *UploadsMockWrapper) UploadFileInMultipart(path string, wrapper wrappers.FeatureFlagsWrapper) (*string, error) { - fmt.Println("Called Create in UploadsMockWrapper") - if path == "failureCase.zip" { +func (u *UploadsMockWrapper) UploadFileInMultipart(filePath string, wrapper wrappers.FeatureFlagsWrapper) (*string, error) { + fmt.Println("UploadFileInMultipart called Create in UploadsMockWrapper") + if filePath == "failureCase2.zip" { return nil, errors.New("error from UploadFileInMultipart") } - url := "/path/to/nowhere" + url := "/path/to/largeZipFile" return &url, nil } func (u *UploadsMockWrapper) UploadFile(filePath string, featureFlagsWrapper wrappers.FeatureFlagsWrapper) (*string, error) { fmt.Println("Called Create in UploadsMockWrapper") - if filePath == "failureCase.zip" { + if strings.Contains(filePath, "failureCase.zip") { return nil, errors.New("error from UploadFile") } url := "/path/to/nowhere" diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index 914a5f9e2..386a2c7dd 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -33,7 +33,7 @@ type StartMultipartUploadResponse struct { type StartMultipartUploadRequest struct { FileSize int64 `json:"fileSize"` } -type MultipartPresignedURl struct { +type MultipartPresignedURL struct { ObjectName string `json:"objectName"` UploadID string `json:"UploadID"` PartNumber int `json:"partNumber"` @@ -225,6 +225,9 @@ func (u *UploadsHTTPWrapper) UploadFileInMultipart(sourcesFile string, featureFl clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) path := viper.GetString(commonParams.CompleteMultipartUploadPathEnv) jsonBytes, err := json.Marshal(completeMultipartUpload) + if err != nil { + return nil, errors.Errorf("Failed to marshal complete multipart upload request body - %s", err.Error()) + } resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) if err != nil { return nil, err @@ -252,6 +255,9 @@ func startMultipartUpload(startMultipartUploadRequest StartMultipartUploadReques clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) path := viper.GetString(commonParams.StartMultiPartUploadPathEnv) jsonBytes, err := json.Marshal(startMultipartUploadRequest) + if err != nil { + return StartMultipartUploadResponse{}, errors.Errorf("Failed to marshal start multipart upload request body - %s", err.Error()) + } resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) if err != nil { return StartMultipartUploadResponse{}, err @@ -286,12 +292,12 @@ func getPresignedURLForMultipartUploading(response StartMultipartUploadResponse, clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) path := viper.GetString(commonParams.MultipartPresignedPathEnv) - multipartPresignedURl := MultipartPresignedURl{ + multipartPresignedURL := MultipartPresignedURL{ ObjectName: response.ObjectName, UploadID: response.UploadID, PartNumber: partNumber, } - jsonBytes, err := json.Marshal(multipartPresignedURl) + jsonBytes, err := json.Marshal(multipartPresignedURL) if err != nil { return "", errors.Errorf("Failed to marshal multipart upload presigned URL request body - %s", err.Error()) } @@ -329,7 +335,7 @@ func getPresignedURLForMultipartUploading(response StartMultipartUploadResponse, } } -func uploadPart(preSignedURL string, sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (string, error) { +func uploadPart(preSignedURL, sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (string, error) { if preSignedURL == "" { return "", errors.New("preSignedURL is empty or nil") } @@ -383,41 +389,16 @@ func uploadPart(preSignedURL string, sourcesFile string, featureFlagsWrapper Fea } func SplitZipBySizeGB(zipFilePath string) ([]string, error) { - - // Get part size in GB from config if config is not provided, default to 2 GB - partChunkSizeStr := viper.GetString(commonParams.MultipartFileSize) - partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) + partSizeBytes, err := getPartSizeBytes() if err != nil { - return nil, fmt.Errorf("invalid part size value: %w", err) - } - // Truncate to integer - truncatedSize := int64(partChunkSizeFloat) - if truncatedSize < 1 || truncatedSize > 5 { - // Enforce part size to be between 1 GB and 5 GB. - // If the configured part size is outside this range, default to 2 GB. - logger.PrintIfVerbose(fmt.Sprintf("Configured part size %d GB is outside the allowed range (1 – 5 GB). Defaulting to 2 GB.", truncatedSize)) - truncatedSize = 2 // default to 2 GB if out of range - } - partSizeGB := float64(truncatedSize) - - logger.PrintIfVerbose("Splitting zip file into parts of size: " + fmt.Sprintf("%.0f", partSizeGB) + " GB") - - if partSizeGB <= 0 { - return nil, fmt.Errorf("part size must be greater than 0 GB") + return nil, err } - const bytesPerGB = 1024 * 1024 * 1024 - partSizeBytes := int64(partSizeGB * float64(bytesPerGB)) - f, err := os.Open(zipFilePath) if err != nil { return nil, fmt.Errorf("open input - %w", err) } - defer func() { - if err := f.Close(); err != nil { - logger.PrintfIfVerbose("warning: failed to close input file - %v", err) - } - }() + defer closeFileVerbose(f) stat, err := f.Stat() if err != nil { @@ -427,12 +408,40 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { return nil, fmt.Errorf("input file is empty") } - totalSize := stat.Size() + partSizes := calculatePartSizes(stat.Size(), partSizeBytes) + partNames, err := createParts(f, partSizes) + if err != nil { + cleanUpTempParts(partNames) + return nil, err + } + + return partNames, nil +} + +func getPartSizeBytes() (int64, error) { + // Get part size in GB from config if config is not provided, default to 2 GB + partChunkSizeStr := viper.GetString(commonParams.MultipartFileSize) + partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) + if err != nil { + return 0, fmt.Errorf("invalid part size value: %w", err) + } + // Truncate to integer + truncatedSize := int64(partChunkSizeFloat) + if truncatedSize < 1 || truncatedSize > 5 { + // Enforce part size to be between 1 GB and 5 GB. + // If the configured part size is outside this range, default to 2 GB. + logger.PrintIfVerbose(fmt.Sprintf("Configured part size %d GB is outside the allowed range (1 – 5 GB). Defaulting to 2 GB.", truncatedSize)) + truncatedSize = 2 + } + const bytesPerGB = 1024 * 1024 * 1024 + return int64(float64(truncatedSize) * float64(bytesPerGB)), nil +} + +func calculatePartSizes(totalSize, partSizeBytes int64) []int64 { numParts := int(totalSize / partSizeBytes) if totalSize%partSizeBytes != 0 { numParts++ } - partSizes := make([]int64, numParts) for i := 0; i < numParts; i++ { remaining := totalSize - int64(i)*partSizeBytes @@ -442,21 +451,16 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { partSizes[i] = remaining } } + return partSizes +} - partNames := make([]string, numParts) - for i := 0; i < numParts; i++ { +func createParts(f *os.File, partSizes []int64) ([]string, error) { + partNames := make([]string, len(partSizes)) + for i, size := range partSizes { partFile, err := os.CreateTemp("", fmt.Sprintf("cx-part%d-*", i+1)) if err != nil { - for j := 0; j < i; j++ { - if partNames[j] != "" { - if err := os.Remove(partNames[j]); err != nil && !os.IsNotExist(err) { - logger.PrintfIfVerbose("warning: failed to remove part%d - %v", j+1, err) - } - } - } - return nil, fmt.Errorf("create part%d - %w", i+1, err) + return partNames, fmt.Errorf("create part%d - %w", i+1, err) } - offset := int64(0) for j := 0; j < i; j++ { offset += partSizes[j] @@ -470,10 +474,9 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { if err != nil { return nil, err } - return nil, fmt.Errorf("seek to part%d - %w", i+1, err) + return partNames, fmt.Errorf("seek to part%d - %w", i+1, err) } - - if _, err := io.CopyN(partFile, f, partSizes[i]); err != nil && err != io.EOF { + if _, err := io.CopyN(partFile, f, size); err != nil && err != io.EOF { err := partFile.Close() if err != nil { return nil, err @@ -482,22 +485,25 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { if err != nil { return nil, err } - return nil, fmt.Errorf("copy to part%d - %w", i+1, err) + return partNames, fmt.Errorf("copy to part%d - %w", i+1, err) } - if err := partFile.Sync(); err != nil { logger.PrintfIfVerbose("warning: failed to sync part%d - %v", i+1, err) } if err := partFile.Close(); err != nil { logger.PrintfIfVerbose("warning: failed to close part%d - %v", i+1, err) } - partNames[i] = partFile.Name() } - return partNames, nil } +func closeFileVerbose(f *os.File) { + if err := f.Close(); err != nil { + logger.PrintfIfVerbose("warning: failed to close input file - %v", err) + } +} + // cleanUpTempParts removes the temporary part files created during multipart upload. func cleanUpTempParts(partList []string) { cleanupMaxRetries := 3 From bf44fedd772ec1a5847f598d83cbe6d5b5259c5a Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 6 Nov 2025 18:18:51 +0530 Subject: [PATCH 04/15] Add test cases --- internal/commands/scan.go | 5 +- internal/commands/scan_test.go | 103 +++++++++++++++++++++++++ internal/wrappers/mock/uploads-mock.go | 6 +- internal/wrappers/uploads-http.go | 1 + test/integration/scan_test.go | 19 +++++ 5 files changed, 128 insertions(+), 6 deletions(-) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 738470a6d..d60f96e4c 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -129,8 +129,6 @@ const ( sbomScanTypeErrMsg = "The --sbom-only flag can only be used when the scan type is sca" BranchPrimaryPrefix = "--branch-primary=" OverridePolicyManagement = "override-policy-management" - maxSizeGB = 5 // 5 GB - maxSizeBytes = maxSizeGB * 1024 * 1024 * 1024 // 5 GB in bytes ) var ( @@ -155,6 +153,7 @@ var ( aditionalParameters []string kicsErrorCodes = []string{"60", "50", "40", "30", "20"} containerResolver wrappers.ContainerResolverWrapper + MaxSizeBytes int64 = 5 * 1024 * 1024 * 1024 // 5 GB in bytes ) func NewScanCommand( @@ -2095,7 +2094,7 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip // check for INCREASE_FILE_UPLOAD_LIMIT feature flag flagResponse, _ := wrappers.GetSpecificFeatureFlag(featureFlagsWrapper, wrappers.IncreaseFileUploadLimit) - if flagResponse.Status && fileInfo.Size() > maxSizeBytes { + if flagResponse.Status && fileInfo.Size() > MaxSizeBytes { // File size >5GB, proceed with multipart upload logger.PrintIfVerbose("File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts...") preSignedURL, zipFilePathErr = uploadsWrapper.UploadFileInMultipart(zipFilePath, featureFlagsWrapper) diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index 130adadec..9bd18b6e6 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -4280,3 +4280,106 @@ func TestEnforceLocalResolutionForTarFiles_Integration(t *testing.T) { }) } } + +func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB(t *testing.T) { + fileInfo, err := os.Stat("data/sources.zip") + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + MaxSizeBytes = fileInfo.Size() - 1 + defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test + + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "multiPart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_Exceeds_5GB(t *testing.T) { + fileInfo, err := os.Stat("data/sources.zip") + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + MaxSizeBytes = fileInfo.Size() - 1 + defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test + + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "singlePart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_LessThan_5GB(t *testing.T) { + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "singlePart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_LessThan_5GB(t *testing.T) { + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "singlePart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB_Error(t *testing.T) { + // Create a temporary zip file + dir := t.TempDir() + zipPathTemp := filepath.Join(dir, "failureCaseLarge.zip") + + // Create the zip file + zipFile, err := os.Create(zipPathTemp) + if err != nil { + t.Fatalf("Failed to create zip file: %v", err) + } + defer func(zipFile *os.File) { + err := zipFile.Close() + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + }(zipFile) + + // Seek to 5KB + 1 byte + _, err = zipFile.Seek(5*1024+1, 0) // 5121 bytes + if err != nil { + panic("Failed to seek in zip file: " + err.Error()) + } + + // Write a single byte to allocate space + _, err = zipFile.Write([]byte{0}) + if err != nil { + panic("Failed to write to zip file: " + err.Error()) + } + + fileInfo, err := os.Stat(zipPathTemp) + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + MaxSizeBytes = fileInfo.Size() - 1 + defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // + + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, true, featureFlagsWrapper) + assert.Assert(t, err != nil) + assert.Assert(t, strings.Contains(err.Error(), "error from UploadFileInMultipart"), err.Error()) + assert.Equal(t, zipPath, "") +} diff --git a/internal/wrappers/mock/uploads-mock.go b/internal/wrappers/mock/uploads-mock.go index 99ff2948c..e4d029d1d 100644 --- a/internal/wrappers/mock/uploads-mock.go +++ b/internal/wrappers/mock/uploads-mock.go @@ -14,10 +14,10 @@ type UploadsMockWrapper struct { func (u *UploadsMockWrapper) UploadFileInMultipart(filePath string, wrapper wrappers.FeatureFlagsWrapper) (*string, error) { fmt.Println("UploadFileInMultipart called Create in UploadsMockWrapper") - if filePath == "failureCase2.zip" { + if strings.Contains(filePath, "failureCaseLarge.zip") { return nil, errors.New("error from UploadFileInMultipart") } - url := "/path/to/largeZipFile" + url := "multiPart/path/to/nowhere" return &url, nil } @@ -26,6 +26,6 @@ func (u *UploadsMockWrapper) UploadFile(filePath string, featureFlagsWrapper wra if strings.Contains(filePath, "failureCase.zip") { return nil, errors.New("error from UploadFile") } - url := "/path/to/nowhere" + url := "singlePart/path/to/nowhere" return &url, nil } diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index 386a2c7dd..10aca9182 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -433,6 +433,7 @@ func getPartSizeBytes() (int64, error) { logger.PrintIfVerbose(fmt.Sprintf("Configured part size %d GB is outside the allowed range (1 – 5 GB). Defaulting to 2 GB.", truncatedSize)) truncatedSize = 2 } + logger.PrintIfVerbose("Splitting zip file into parts of size: " + fmt.Sprintf("%.0f", float64(truncatedSize)) + " GB") const bytesPerGB = 1024 * 1024 * 1024 return int64(float64(truncatedSize) * float64(bytesPerGB)), nil } diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 4010effb3..36435fbcc 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -2726,3 +2726,22 @@ func TestCreateScanWithNewProjectName_Assign_Groups(t *testing.T) { assert.NilError(t, err, "Groups should be assigned to newly created projects") } +func TestCreateScan_AsMultipartUpload_Success(t *testing.T) { + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + commands.MaxSizeBytes = 10240 // 10KB less than actual file size + defer func() { commands.MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test + args := []string{ + "scan", "create", + flag(params.ProjectName), getProjectNameForScanTests(), + flag(params.SourcesFlag), "data/insecure.zip", + flag(params.BranchFlag), "dummy_branch", + flag(params.DebugFlag), + } + var buf bytes.Buffer + log.SetOutput(&buf) + + err, _ := executeCommand(t, args...) + assert.NilError(t, err) + log.SetOutput(os.Stderr) + assert.Assert(t, strings.Contains(buf.String(), "File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts"), "uploading file in multiple parts") +} From 0a7fe96d0ffad6bc0d29f2a9e4e761ac35730e29 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 6 Nov 2025 19:40:45 +0530 Subject: [PATCH 05/15] fix unit test case --- internal/commands/scan_test.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index 9bd18b6e6..6805a4a62 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -4282,12 +4282,8 @@ func TestEnforceLocalResolutionForTarFiles_Integration(t *testing.T) { } func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB(t *testing.T) { - fileInfo, err := os.Stat("data/sources.zip") - if err != nil { - t.Fatalf("Failed to close zip file: %v", err) - } // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size - MaxSizeBytes = fileInfo.Size() - 1 + MaxSizeBytes = 1 defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test uploadWrapper := mock.UploadsMockWrapper{} @@ -4366,13 +4362,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB_Error(t *tes panic("Failed to write to zip file: " + err.Error()) } - fileInfo, err := os.Stat(zipPathTemp) - if err != nil { - t.Fatalf("Failed to close zip file: %v", err) - } - // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size - MaxSizeBytes = fileInfo.Size() - 1 + MaxSizeBytes = 1 defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // uploadWrapper := mock.UploadsMockWrapper{} From 296908d2b81f0d38db809e7c1862502959fa40e3 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 6 Nov 2025 20:41:53 +0530 Subject: [PATCH 06/15] debug unit test case --- internal/commands/scan.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index d60f96e4c..0a10930fd 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -2094,6 +2094,8 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip // check for INCREASE_FILE_UPLOAD_LIMIT feature flag flagResponse, _ := wrappers.GetSpecificFeatureFlag(featureFlagsWrapper, wrappers.IncreaseFileUploadLimit) + fmt.Println("flagResponse.Status: ", flagResponse.Status) + fmt.Println("fileInfo.Size(): ", fileInfo.Size()) if flagResponse.Status && fileInfo.Size() > MaxSizeBytes { // File size >5GB, proceed with multipart upload logger.PrintIfVerbose("File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts...") From 2fbc7e208e2a5c9d2279329da2038e1bdd0daa5b Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 6 Nov 2025 20:52:26 +0530 Subject: [PATCH 07/15] debug unit test case --- internal/commands/scan.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 0a10930fd..fe76e0cb3 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -2096,6 +2096,7 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip fmt.Println("flagResponse.Status: ", flagResponse.Status) fmt.Println("fileInfo.Size(): ", fileInfo.Size()) + fmt.Println("MaxSizeBytes: ", MaxSizeBytes) if flagResponse.Status && fileInfo.Size() > MaxSizeBytes { // File size >5GB, proceed with multipart upload logger.PrintIfVerbose("File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts...") From 2fa3eb23f2bcc695c66c984ca7fbf6f8a2bcdec0 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 6 Nov 2025 21:47:45 +0530 Subject: [PATCH 08/15] unit test case --- internal/commands/scan_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index 6805a4a62..f230c07ab 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -4285,10 +4285,9 @@ func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB(t *testing.T // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size MaxSizeBytes = 1 defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test - uploadWrapper := mock.UploadsMockWrapper{} - mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4305,8 +4304,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_Exceeds_5GB(t *testing. defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test uploadWrapper := mock.UploadsMockWrapper{} - mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4315,8 +4314,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_Exceeds_5GB(t *testing. func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_LessThan_5GB(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} - mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4325,8 +4324,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_LessThan_5GB(t *testing. func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_LessThan_5GB(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} - mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4367,8 +4366,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB_Error(t *tes defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // uploadWrapper := mock.UploadsMockWrapper{} - mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, true, featureFlagsWrapper) assert.Assert(t, err != nil) assert.Assert(t, strings.Contains(err.Error(), "error from UploadFileInMultipart"), err.Error()) From 271137b9215487e1f8ade4ac6f40d63dc66d5328 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Thu, 6 Nov 2025 22:49:18 +0530 Subject: [PATCH 09/15] unit test case --- internal/commands/scan.go | 2 +- internal/commands/scan_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index fe76e0cb3..d04fee589 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -2092,7 +2092,7 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip logger.PrintIfVerbose(fmt.Sprintf("Zip size before upload: %.2fMB\n", float64(fileInfo.Size())/mbBytes)) // check for INCREASE_FILE_UPLOAD_LIMIT feature flag - flagResponse, _ := wrappers.GetSpecificFeatureFlag(featureFlagsWrapper, wrappers.IncreaseFileUploadLimit) + flagResponse, _ := featureFlagsWrapper.GetSpecificFlag(wrappers.IncreaseFileUploadLimit) fmt.Println("flagResponse.Status: ", flagResponse.Status) fmt.Println("fileInfo.Size(): ", fileInfo.Size()) diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index f230c07ab..b6cede85c 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -4286,8 +4286,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB(t *testing.T MaxSizeBytes = 1 defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test uploadWrapper := mock.UploadsMockWrapper{} - featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4304,8 +4304,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_Exceeds_5GB(t *testing. defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test uploadWrapper := mock.UploadsMockWrapper{} - featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4314,8 +4314,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_Exceeds_5GB(t *testing. func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_LessThan_5GB(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} - featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4324,8 +4324,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_LessThan_5GB(t *testing. func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_LessThan_5GB(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} - featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") @@ -4366,8 +4366,8 @@ func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB_Error(t *tes defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // uploadWrapper := mock.UploadsMockWrapper{} - featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, true, featureFlagsWrapper) assert.Assert(t, err != nil) assert.Assert(t, strings.Contains(err.Error(), "error from UploadFileInMultipart"), err.Error()) From 2b9649263891b41c65f4c38bae61cb313d28d1dd Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Fri, 7 Nov 2025 09:05:47 +0530 Subject: [PATCH 10/15] Remove print msg --- internal/commands/scan.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index d04fee589..0119f4a5b 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -2093,10 +2093,6 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip // check for INCREASE_FILE_UPLOAD_LIMIT feature flag flagResponse, _ := featureFlagsWrapper.GetSpecificFlag(wrappers.IncreaseFileUploadLimit) - - fmt.Println("flagResponse.Status: ", flagResponse.Status) - fmt.Println("fileInfo.Size(): ", fileInfo.Size()) - fmt.Println("MaxSizeBytes: ", MaxSizeBytes) if flagResponse.Status && fileInfo.Size() > MaxSizeBytes { // File size >5GB, proceed with multipart upload logger.PrintIfVerbose("File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts...") From 4a4dd38360c01e1a269c19a570bbcae3d8019c63 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Fri, 7 Nov 2025 14:30:46 +0530 Subject: [PATCH 11/15] handle multipart_file_size value is empty --- internal/wrappers/uploads-http.go | 4 +++- test/integration/scan_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index 10aca9182..9e46e1bc4 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -422,8 +422,10 @@ func getPartSizeBytes() (int64, error) { // Get part size in GB from config if config is not provided, default to 2 GB partChunkSizeStr := viper.GetString(commonParams.MultipartFileSize) partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) + // If parsing fails or value is empty, default to 2 GB if err != nil { - return 0, fmt.Errorf("invalid part size value: %w", err) + logger.PrintIfVerbose(fmt.Sprintf("Configured part size '%s' is invalid or empty. Defaulting to 2 GB.", partChunkSizeStr)) + partChunkSizeFloat = 2 } // Truncate to integer truncatedSize := int64(partChunkSizeFloat) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 36435fbcc..3f8405fe9 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -2743,5 +2743,5 @@ func TestCreateScan_AsMultipartUpload_Success(t *testing.T) { err, _ := executeCommand(t, args...) assert.NilError(t, err) log.SetOutput(os.Stderr) - assert.Assert(t, strings.Contains(buf.String(), "File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts"), "uploading file in multiple parts") + assert.Assert(t, strings.Contains(buf.String(), "File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts"), "Test for uploading file in multiple parts failed.") } From 0bf16c29f36ff6111bb7a9b3b04b3b77fb0fd691 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Fri, 7 Nov 2025 14:35:42 +0530 Subject: [PATCH 12/15] rename the MultipartFileSizeKey --- internal/params/binds.go | 2 +- internal/params/keys.go | 2 +- internal/wrappers/uploads-http.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/params/binds.go b/internal/params/binds.go index 065a78edf..4dd138e19 100644 --- a/internal/params/binds.go +++ b/internal/params/binds.go @@ -83,5 +83,5 @@ var EnvVarsBinds = []struct { {StartMultiPartUploadPathKey, StartMultiPartUploadPathEnv, "api/uploads/start-multipart-upload"}, {MultipartPresignedPathKey, MultipartPresignedPathEnv, "api/uploads/multipart-presigned"}, {CompleteMultiPartUploadPathKey, CompleteMultipartUploadPathEnv, "api/uploads/complete-multipart-upload"}, - {MultipartFileSize, MultipartFileSizeEnv, "2"}, + {MultipartFileSizeKey, MultipartFileSizeEnv, "2"}, } diff --git a/internal/params/keys.go b/internal/params/keys.go index 8a2732ad8..adabc95a5 100644 --- a/internal/params/keys.go +++ b/internal/params/keys.go @@ -82,5 +82,5 @@ var ( StartMultiPartUploadPathKey = strings.ToLower(StartMultiPartUploadPathEnv) MultipartPresignedPathKey = strings.ToLower(MultipartPresignedPathEnv) CompleteMultiPartUploadPathKey = strings.ToLower(CompleteMultipartUploadPathEnv) - MultipartFileSize = strings.ToLower(MultipartFileSizeEnv) + MultipartFileSizeKey = strings.ToLower(MultipartFileSizeEnv) ) diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index 9e46e1bc4..cf775a991 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -420,7 +420,7 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { func getPartSizeBytes() (int64, error) { // Get part size in GB from config if config is not provided, default to 2 GB - partChunkSizeStr := viper.GetString(commonParams.MultipartFileSize) + partChunkSizeStr := viper.GetString(commonParams.MultipartFileSizeKey) partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) // If parsing fails or value is empty, default to 2 GB if err != nil { From 1af1c1b1fa9aed74cf3bc87e774c07a7dfc58fa0 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Fri, 7 Nov 2025 14:53:11 +0530 Subject: [PATCH 13/15] fix lint issue --- internal/wrappers/uploads-http.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index cf775a991..a94489776 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -389,11 +389,7 @@ func uploadPart(preSignedURL, sourcesFile string, featureFlagsWrapper FeatureFla } func SplitZipBySizeGB(zipFilePath string) ([]string, error) { - partSizeBytes, err := getPartSizeBytes() - if err != nil { - return nil, err - } - + partSizeBytes := getPartSizeBytes() f, err := os.Open(zipFilePath) if err != nil { return nil, fmt.Errorf("open input - %w", err) @@ -418,7 +414,7 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { return partNames, nil } -func getPartSizeBytes() (int64, error) { +func getPartSizeBytes() int64 { // Get part size in GB from config if config is not provided, default to 2 GB partChunkSizeStr := viper.GetString(commonParams.MultipartFileSizeKey) partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) @@ -437,7 +433,7 @@ func getPartSizeBytes() (int64, error) { } logger.PrintIfVerbose("Splitting zip file into parts of size: " + fmt.Sprintf("%.0f", float64(truncatedSize)) + " GB") const bytesPerGB = 1024 * 1024 * 1024 - return int64(float64(truncatedSize) * float64(bytesPerGB)), nil + return int64(float64(truncatedSize) * float64(bytesPerGB)) } func calculatePartSizes(totalSize, partSizeBytes int64) []int64 { From 6874256ba7f32abace2fcd2ea11ba0a5117d6af5 Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Mon, 17 Nov 2025 17:56:36 +0530 Subject: [PATCH 14/15] Made code chnages according to review comments --- internal/commands/scan.go | 9 +- internal/wrappers/uploads-http.go | 135 ++++++++++++++---------------- test/integration/scan_test.go | 2 +- 3 files changed, 67 insertions(+), 79 deletions(-) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 0119f4a5b..e17b2bd54 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -2084,22 +2084,17 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip // Send a request to uploads service var preSignedURL *string - // calculate file size and compare with 5GB limit fileInfo, err := os.Stat(zipFilePath) if err != nil { - return "", zipFilePath, errors.Wrapf(err, "Failed to stat %s", zipFilePath) + return "", zipFilePath, errors.Wrapf(err, "Failed to check the size - %s", zipFilePath) } logger.PrintIfVerbose(fmt.Sprintf("Zip size before upload: %.2fMB\n", float64(fileInfo.Size())/mbBytes)) - // check for INCREASE_FILE_UPLOAD_LIMIT feature flag flagResponse, _ := featureFlagsWrapper.GetSpecificFlag(wrappers.IncreaseFileUploadLimit) if flagResponse.Status && fileInfo.Size() > MaxSizeBytes { - // File size >5GB, proceed with multipart upload - logger.PrintIfVerbose("File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts...") + logger.PrintIfVerbose("Uploading source code in multiple parts.") preSignedURL, zipFilePathErr = uploadsWrapper.UploadFileInMultipart(zipFilePath, featureFlagsWrapper) } else { - // File size is within <=5GB, proceed with upload in single part - logger.PrintIfVerbose("File size is within the limit and uploading file in a single part...") preSignedURL, zipFilePathErr = uploadsWrapper.UploadFile(zipFilePath, featureFlagsWrapper) } if zipFilePathErr != nil { diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index a94489776..fa17d0308 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -39,7 +39,7 @@ type MultipartPresignedURL struct { PartNumber int `json:"partNumber"` } -type CompleteMultipartUpload struct { +type CompleteMultipartUploadRequest struct { UploadID string `json:"UploadID"` ObjectName string `json:"objectName"` PartList []Part `json:"partList"` @@ -165,11 +165,7 @@ func NewUploadsHTTPWrapper(path string) UploadsWrapper { } func (u *UploadsHTTPWrapper) UploadFileInMultipart(sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (*string, error) { - // calculate file size and compare with 5GB limit - fileInfo, err := os.Stat(sourcesFile) - if err != nil { - return nil, errors.Wrapf(err, "Failed to stat - %s", sourcesFile) - } + fileInfo, _ := os.Stat(sourcesFile) startMultipartUploadRequest := StartMultipartUploadRequest{} startMultipartUploadRequest.FileSize = fileInfo.Size() @@ -182,14 +178,13 @@ func (u *UploadsHTTPWrapper) UploadFileInMultipart(sourcesFile string, featureFl return nil, errors.Errorf("Failed to split ZIP file for multipart upload - %s", err.Error()) } - // ensure temporary parts are removed when this function returns defer cleanUpTempParts(partList) for i, part := range partList { logger.PrintfIfVerbose("Part%d created at: %s", i+1, part) } - completeMultipartUpload := &CompleteMultipartUpload{ + completeMultipartUploadRequest := &CompleteMultipartUploadRequest{ UploadID: startMultipartUploadResponse.UploadID, ObjectName: startMultipartUploadResponse.ObjectName, } @@ -199,56 +194,31 @@ func (u *UploadsHTTPWrapper) UploadFileInMultipart(sourcesFile string, featureFl for i, partPath := range partList { partNumber := i + 1 - // Generate presigned URL presignedURL, err := getPresignedURLForMultipartUploading(startMultipartUploadResponse, partNumber) if err != nil { - return nil, fmt.Errorf("failed to get presigned URL for part%d - %s", partNumber, err.Error()) + return nil, errors.Errorf("Failed to get presigned URL for part%d - %s", partNumber, err.Error()) } if partNumber == 1 { presignedURLPart1 = presignedURL } - // Upload part + etag, err := uploadPart(presignedURL, partPath, featureFlagsWrapper) if err != nil { - return nil, fmt.Errorf("failed to upload part%d - %s", partNumber, err.Error()) + return nil, errors.Errorf("Failed to upload part%d - %s", partNumber, err.Error()) } - // Append part info - completeMultipartUpload.PartList = append(completeMultipartUpload.PartList, Part{ + completeMultipartUploadRequest.PartList = append(completeMultipartUploadRequest.PartList, Part{ ETag: etag, PartNumber: partNumber, }) } - // call the complete multipart upload API - clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - path := viper.GetString(commonParams.CompleteMultipartUploadPathEnv) - jsonBytes, err := json.Marshal(completeMultipartUpload) + err = completeMultipartUpload(*completeMultipartUploadRequest) if err != nil { - return nil, errors.Errorf("Failed to marshal complete multipart upload request body - %s", err.Error()) - } - resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) - if err != nil { - return nil, err - } - decoder := json.NewDecoder(resp.Body) - defer func() { - _ = resp.Body.Close() - }() - switch resp.StatusCode { - case http.StatusNoContent: - return &presignedURLPart1, nil - case http.StatusUnauthorized: - return nil, errors.New(errorConstants.StatusUnauthorized) - default: - errorModel := ErrorModel{} - err = decoder.Decode(&errorModel) - if err != nil { - return nil, errors.Errorf("Parsing error model failed - %s", err.Error()) - } - return nil, errors.Errorf("%d - %s", errorModel.Code, errorModel.Message) + return nil, errors.Errorf("Failed to complete multipart upload - %s", err.Error()) } + return &presignedURLPart1, nil } func startMultipartUpload(startMultipartUploadRequest StartMultipartUploadRequest) (StartMultipartUploadResponse, error) { @@ -256,7 +226,7 @@ func startMultipartUpload(startMultipartUploadRequest StartMultipartUploadReques path := viper.GetString(commonParams.StartMultiPartUploadPathEnv) jsonBytes, err := json.Marshal(startMultipartUploadRequest) if err != nil { - return StartMultipartUploadResponse{}, errors.Errorf("Failed to marshal start multipart upload request body - %s", err.Error()) + return StartMultipartUploadResponse{}, err } resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) if err != nil { @@ -271,20 +241,20 @@ func startMultipartUpload(startMultipartUploadRequest StartMultipartUploadReques startMultipartUpload := StartMultipartUploadResponse{} err = decoder.Decode(&startMultipartUpload) if err != nil { - return StartMultipartUploadResponse{}, errors.Errorf("failed to start the multipart upload - %s ", err.Error()) + return StartMultipartUploadResponse{}, err } return startMultipartUpload, nil case http.StatusBadRequest: errorModel := ErrorModel{} err = decoder.Decode(&errorModel) if err != nil { - return StartMultipartUploadResponse{}, errors.Errorf("failed to start the multipart upload - %s ", err.Error()) + return StartMultipartUploadResponse{}, err } return StartMultipartUploadResponse{}, errors.Errorf(errorModel.Message) case http.StatusUnauthorized: return StartMultipartUploadResponse{}, errors.New(errorConstants.StatusUnauthorized) default: - return StartMultipartUploadResponse{}, errors.Errorf("response status code %d", resp.StatusCode) + return StartMultipartUploadResponse{}, errors.Errorf("Response status code %d", resp.StatusCode) } } @@ -299,14 +269,13 @@ func getPresignedURLForMultipartUploading(response StartMultipartUploadResponse, } jsonBytes, err := json.Marshal(multipartPresignedURL) if err != nil { - return "", errors.Errorf("Failed to marshal multipart upload presigned URL request body - %s", err.Error()) + return "", err } resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) if err != nil { - return "", errors.Errorf("Invoking HTTP request to get pre-signed URL failed - %s", err.Error()) + return "", err } - defer func() { _ = resp.Body.Close() }() @@ -318,7 +287,7 @@ func getPresignedURLForMultipartUploading(response StartMultipartUploadResponse, errorModel := ErrorModel{} err = decoder.Decode(&errorModel) if err != nil { - return "", errors.Errorf("Parsing error model failed - %s", err.Error()) + return "", err } return "", errors.Errorf("%d - %s", errorModel.Code, errorModel.Message) @@ -326,25 +295,24 @@ func getPresignedURLForMultipartUploading(response StartMultipartUploadResponse, model := UploadModelMultipart{} err = decoder.Decode(&model) if err != nil { - return "", errors.Errorf("Parsing upload model failed - %s", err.Error()) + return "", err } return model.PresignedURL, nil default: - return "", errors.Errorf("response status code %d", resp.StatusCode) + return "", errors.Errorf("Response status code %d", resp.StatusCode) } } func uploadPart(preSignedURL, sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (string, error) { if preSignedURL == "" { - return "", errors.New("preSignedURL is empty or nil") + return "", errors.New("PreSignedURL is empty or nil") } file, err := os.Open(sourcesFile) if err != nil { - return "", errors.Errorf("Failed to open file for multipart upload %s - %s", sourcesFile, err.Error()) + return "", err } - // Close the file later defer func() { _ = file.Close() }() @@ -356,13 +324,13 @@ func uploadPart(preSignedURL, sourcesFile string, featureFlagsWrapper FeatureFla stat, err := file.Stat() if err != nil { - return "", errors.Errorf("Failed to stat file %s - %s", sourcesFile, err.Error()) + return "", err } flagResponse, _ := GetSpecificFeatureFlag(featureFlagsWrapper, MinioEnabled) useAccessToken := flagResponse.Status resp, err := SendHTTPRequestByFullURLContentLength(http.MethodPut, preSignedURL, file, stat.Size(), useAccessToken, NoTimeout, accessToken, true) if err != nil { - return "", errors.Errorf("Invoking HTTP request to upload file failed - %s", err.Error()) + return "", err } defer func() { @@ -380,11 +348,41 @@ func uploadPart(preSignedURL, sourcesFile string, featureFlagsWrapper FeatureFla _ = resp.Body.Close() }() if err != nil { - return "", errors.Errorf("Reading response body failed - %s", err.Error()) + return "", err } return "", errors.Errorf("Bad request while uploading part - %s", string(body)) default: - return "", errors.Errorf("Failed to upload part of multipart - %d", resp.StatusCode) + return "", errors.Errorf("Response status code %d", resp.StatusCode) + } +} + +func completeMultipartUpload(completeMultipartUploadRequest CompleteMultipartUploadRequest) error { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + path := viper.GetString(commonParams.CompleteMultipartUploadPathEnv) + jsonBytes, err := json.Marshal(completeMultipartUploadRequest) + if err != nil { + return err + } + resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return err + } + decoder := json.NewDecoder(resp.Body) + defer func() { + _ = resp.Body.Close() + }() + switch resp.StatusCode { + case http.StatusNoContent: + return nil + case http.StatusUnauthorized: + return errors.New(errorConstants.StatusUnauthorized) + default: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return err + } + return errors.Errorf("%d - %s", errorModel.Code, errorModel.Message) } } @@ -392,16 +390,16 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { partSizeBytes := getPartSizeBytes() f, err := os.Open(zipFilePath) if err != nil { - return nil, fmt.Errorf("open input - %w", err) + return nil, err } defer closeFileVerbose(f) stat, err := f.Stat() if err != nil { - return nil, fmt.Errorf("stat input - %w", err) + return nil, err } if stat.Size() == 0 { - return nil, fmt.Errorf("input file is empty") + return nil, err } partSizes := calculatePartSizes(stat.Size(), partSizeBytes) @@ -415,19 +413,14 @@ func SplitZipBySizeGB(zipFilePath string) ([]string, error) { } func getPartSizeBytes() int64 { - // Get part size in GB from config if config is not provided, default to 2 GB partChunkSizeStr := viper.GetString(commonParams.MultipartFileSizeKey) partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) - // If parsing fails or value is empty, default to 2 GB if err != nil { logger.PrintIfVerbose(fmt.Sprintf("Configured part size '%s' is invalid or empty. Defaulting to 2 GB.", partChunkSizeStr)) partChunkSizeFloat = 2 } - // Truncate to integer truncatedSize := int64(partChunkSizeFloat) if truncatedSize < 1 || truncatedSize > 5 { - // Enforce part size to be between 1 GB and 5 GB. - // If the configured part size is outside this range, default to 2 GB. logger.PrintIfVerbose(fmt.Sprintf("Configured part size %d GB is outside the allowed range (1 – 5 GB). Defaulting to 2 GB.", truncatedSize)) truncatedSize = 2 } @@ -458,7 +451,7 @@ func createParts(f *os.File, partSizes []int64) ([]string, error) { for i, size := range partSizes { partFile, err := os.CreateTemp("", fmt.Sprintf("cx-part%d-*", i+1)) if err != nil { - return partNames, fmt.Errorf("create part%d - %w", i+1, err) + return partNames, err } offset := int64(0) for j := 0; j < i; j++ { @@ -473,7 +466,7 @@ func createParts(f *os.File, partSizes []int64) ([]string, error) { if err != nil { return nil, err } - return partNames, fmt.Errorf("seek to part%d - %w", i+1, err) + return partNames, err } if _, err := io.CopyN(partFile, f, size); err != nil && err != io.EOF { err := partFile.Close() @@ -484,13 +477,13 @@ func createParts(f *os.File, partSizes []int64) ([]string, error) { if err != nil { return nil, err } - return partNames, fmt.Errorf("copy to part%d - %w", i+1, err) + return partNames, err } if err := partFile.Sync(); err != nil { - logger.PrintfIfVerbose("warning: failed to sync part%d - %v", i+1, err) + return partNames, err } if err := partFile.Close(); err != nil { - logger.PrintfIfVerbose("warning: failed to close part%d - %v", i+1, err) + return partNames, err } partNames[i] = partFile.Name() } @@ -499,7 +492,7 @@ func createParts(f *os.File, partSizes []int64) ([]string, error) { func closeFileVerbose(f *os.File) { if err := f.Close(); err != nil { - logger.PrintfIfVerbose("warning: failed to close input file - %v", err) + logger.PrintfIfVerbose("Warning: failed to close input file - %v", err) } } diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 3f8405fe9..6e0128832 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -2743,5 +2743,5 @@ func TestCreateScan_AsMultipartUpload_Success(t *testing.T) { err, _ := executeCommand(t, args...) assert.NilError(t, err) log.SetOutput(os.Stderr) - assert.Assert(t, strings.Contains(buf.String(), "File size >5GB and INCREASE_FILE_UPLOAD_LIMIT flag is enabled,hence uploading file in multiple parts"), "Test for uploading file in multiple parts failed.") + assert.Assert(t, strings.Contains(buf.String(), "Uploading source code in multiple parts"), "Test for uploading file in multiple parts failed.") } From 03c11896a4539917010104d99edae82a406b483b Mon Sep 17 00:00:00 2001 From: Sumit Morchhale Date: Wed, 19 Nov 2025 13:53:22 +0530 Subject: [PATCH 15/15] remove extra comments --- internal/wrappers/uploads-http.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index fa17d0308..775f1b4d2 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -496,7 +496,6 @@ func closeFileVerbose(f *os.File) { } } -// cleanUpTempParts removes the temporary part files created during multipart upload. func cleanUpTempParts(partList []string) { cleanupMaxRetries := 3 for i, partPath := range partList {