diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 9dc28ff85..e17b2bd54 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -153,6 +153,7 @@ var ( aditionalParameters []string kicsErrorCodes = []string{"60", "50", "40", "30", "20"} containerResolver wrappers.ContainerResolverWrapper + MaxSizeBytes int64 = 5 * 1024 * 1024 * 1024 // 5 GB in bytes ) func NewScanCommand( @@ -2082,7 +2083,20 @@ func uploadZip(uploadsWrapper wrappers.UploadsWrapper, zipFilePath string, unzip var zipFilePathErr error // Send a request to uploads service var preSignedURL *string - preSignedURL, zipFilePathErr = uploadsWrapper.UploadFile(zipFilePath, featureFlagsWrapper) + + fileInfo, err := os.Stat(zipFilePath) + if err != nil { + return "", zipFilePath, errors.Wrapf(err, "Failed to check the size - %s", zipFilePath) + } + logger.PrintIfVerbose(fmt.Sprintf("Zip size before upload: %.2fMB\n", float64(fileInfo.Size())/mbBytes)) + + flagResponse, _ := featureFlagsWrapper.GetSpecificFlag(wrappers.IncreaseFileUploadLimit) + if flagResponse.Status && fileInfo.Size() > MaxSizeBytes { + logger.PrintIfVerbose("Uploading source code in multiple parts.") + preSignedURL, zipFilePathErr = uploadsWrapper.UploadFileInMultipart(zipFilePath, featureFlagsWrapper) + } else { + preSignedURL, zipFilePathErr = uploadsWrapper.UploadFile(zipFilePath, featureFlagsWrapper) + } if zipFilePathErr != nil { if unzip || !userProvidedZip { return "", zipFilePath, errors.Wrapf(zipFilePathErr, "%s: Failed to upload sources file\n", failedCreating) diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index 3062da8ba..b6cede85c 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -2972,15 +2972,31 @@ func TestResubmitConfig_ProjectDoesNotExist_ReturnedEmptyConfig(t *testing.T) { func TestUploadZip_whenUserProvideZip_shouldReturnEmptyZipFilePathInSuccessCase(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "test.zip", false, true, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) assert.NilError(t, err) assert.Equal(t, zipPath, "") } func TestUploadZip_whenUserProvideZip_shouldReturnEmptyZipFilePathInFailureCase(t *testing.T) { + // Create a temporary zip file + dir := t.TempDir() + zipPathTemp := filepath.Join(dir, "failureCase.zip") + + // Create the zip file + zipFile, err := os.Create(zipPathTemp) + if err != nil { + t.Fatalf("Failed to create zip file: %v", err) + } + defer func(zipFile *os.File) { + err := zipFile.Close() + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + }(zipFile) + uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "failureCase.zip", false, true, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, true, featureFlagsWrapper) assert.Assert(t, err != nil) assert.Assert(t, strings.Contains(err.Error(), "error from UploadFile"), err.Error()) assert.Equal(t, zipPath, "") @@ -2989,18 +3005,33 @@ func TestUploadZip_whenUserProvideZip_shouldReturnEmptyZipFilePathInFailureCase( func TestUploadZip_whenUserNotProvideZip_shouldReturnZipFilePathInSuccessCase(t *testing.T) { uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "test.zip", false, false, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, false, featureFlagsWrapper) assert.NilError(t, err) - assert.Equal(t, zipPath, "test.zip") + assert.Equal(t, zipPath, "data/sources.zip") } func TestUploadZip_whenUserNotProvideZip_shouldReturnZipFilePathInFailureCase(t *testing.T) { + // Create a temporary zip file + dir := t.TempDir() + zipPathTemp := filepath.Join(dir, "failureCase.zip") + + // Create the zip file + zipFile, err := os.Create(zipPathTemp) + if err != nil { + t.Fatalf("Failed to create zip file: %v", err) + } + defer func(zipFile *os.File) { + err := zipFile.Close() + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + }(zipFile) uploadWrapper := mock.UploadsMockWrapper{} featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} - _, zipPath, err := uploadZip(&uploadWrapper, "failureCase.zip", false, false, featureFlagsWrapper) + _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, false, featureFlagsWrapper) assert.Assert(t, err != nil) assert.Assert(t, strings.Contains(err.Error(), "error from UploadFile"), err.Error()) - assert.Equal(t, zipPath, "failureCase.zip") + assert.Equal(t, zipPath, zipPathTemp) } func TestAddSastScan_ScanFlags(t *testing.T) { @@ -4249,3 +4280,96 @@ func TestEnforceLocalResolutionForTarFiles_Integration(t *testing.T) { }) } } + +func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB(t *testing.T) { + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + MaxSizeBytes = 1 + defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "multiPart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_Exceeds_5GB(t *testing.T) { + fileInfo, err := os.Stat("data/sources.zip") + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + MaxSizeBytes = fileInfo.Size() - 1 + defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test + + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "singlePart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_LessThan_5GB(t *testing.T) { + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "singlePart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Disable_ZIP_LessThan_5GB(t *testing.T) { + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: false} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + url, zipPath, err := uploadZip(&uploadWrapper, "data/sources.zip", false, true, featureFlagsWrapper) + assert.NilError(t, err) + assert.Equal(t, zipPath, "") + assert.Equal(t, url, "singlePart/path/to/nowhere") +} + +func TestUploadZip_AsMultipartUpload_when_FF_Enable_ZIP_Exceeds_5GB_Error(t *testing.T) { + // Create a temporary zip file + dir := t.TempDir() + zipPathTemp := filepath.Join(dir, "failureCaseLarge.zip") + + // Create the zip file + zipFile, err := os.Create(zipPathTemp) + if err != nil { + t.Fatalf("Failed to create zip file: %v", err) + } + defer func(zipFile *os.File) { + err := zipFile.Close() + if err != nil { + t.Fatalf("Failed to close zip file: %v", err) + } + }(zipFile) + + // Seek to 5KB + 1 byte + _, err = zipFile.Seek(5*1024+1, 0) // 5121 bytes + if err != nil { + panic("Failed to seek in zip file: " + err.Error()) + } + + // Write a single byte to allocate space + _, err = zipFile.Write([]byte{0}) + if err != nil { + panic("Failed to write to zip file: " + err.Error()) + } + + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + MaxSizeBytes = 1 + defer func() { MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // + + uploadWrapper := mock.UploadsMockWrapper{} + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.IncreaseFileUploadLimit, Status: true} + featureFlagsWrapper := &mock.FeatureFlagsMockWrapper{} + _, zipPath, err := uploadZip(&uploadWrapper, zipPathTemp, false, true, featureFlagsWrapper) + assert.Assert(t, err != nil) + assert.Assert(t, strings.Contains(err.Error(), "error from UploadFileInMultipart"), err.Error()) + assert.Equal(t, zipPath, "") +} diff --git a/internal/params/binds.go b/internal/params/binds.go index f90d15623..4dd138e19 100644 --- a/internal/params/binds.go +++ b/internal/params/binds.go @@ -80,4 +80,8 @@ var EnvVarsBinds = []struct { {RiskManagementPathKey, RiskManagementPathEnv, "api/risk-management/projects/%s/results?scanID=%s"}, {ConfigFilePathKey, ConfigFilePathEnv, ""}, {RealtimeScannerPathKey, RealtimeScannerPathEnv, "api/realtime-scanner"}, + {StartMultiPartUploadPathKey, StartMultiPartUploadPathEnv, "api/uploads/start-multipart-upload"}, + {MultipartPresignedPathKey, MultipartPresignedPathEnv, "api/uploads/multipart-presigned"}, + {CompleteMultiPartUploadPathKey, CompleteMultipartUploadPathEnv, "api/uploads/complete-multipart-upload"}, + {MultipartFileSizeKey, MultipartFileSizeEnv, "2"}, } diff --git a/internal/params/envs.go b/internal/params/envs.go index 19dc0f3c7..9698eb699 100644 --- a/internal/params/envs.go +++ b/internal/params/envs.go @@ -80,4 +80,8 @@ const ( RiskManagementPathEnv = "CX_RISK_MANAGEMENT_PATH" ConfigFilePathEnv = "CX_CONFIG_FILE_PATH" RealtimeScannerPathEnv = "CX_REALTIME_SCANNER_PATH" + StartMultiPartUploadPathEnv = "CX_START_MULTIPART_UPLOAD_PATH" + MultipartPresignedPathEnv = "CX_MULTIPART_PRESIGNED_URL_PATH" + CompleteMultipartUploadPathEnv = "CX_COMPLETE_MULTIPART_UPLOAD_PATH" + MultipartFileSizeEnv = "MULTIPART_FILE_SIZE" ) diff --git a/internal/params/keys.go b/internal/params/keys.go index 839b13e53..adabc95a5 100644 --- a/internal/params/keys.go +++ b/internal/params/keys.go @@ -79,4 +79,8 @@ var ( RiskManagementPathKey = strings.ToLower(RiskManagementPathEnv) ConfigFilePathKey = strings.ToLower(ConfigFilePathEnv) RealtimeScannerPathKey = strings.ToLower(RealtimeScannerPathEnv) + StartMultiPartUploadPathKey = strings.ToLower(StartMultiPartUploadPathEnv) + MultipartPresignedPathKey = strings.ToLower(MultipartPresignedPathEnv) + CompleteMultiPartUploadPathKey = strings.ToLower(CompleteMultipartUploadPathEnv) + MultipartFileSizeKey = strings.ToLower(MultipartFileSizeEnv) ) diff --git a/internal/wrappers/feature-flags.go b/internal/wrappers/feature-flags.go index 8f92b8cef..6762e860f 100644 --- a/internal/wrappers/feature-flags.go +++ b/internal/wrappers/feature-flags.go @@ -17,6 +17,7 @@ const OssRealtimeEnabled = "OSS_REALTIME_ENABLED" const ScsLicensingV2Enabled = "SSCS_NEW_LICENSING_ENABLED" const DirectAssociationEnabled = "DIRECT_APP_ASSOCIATION_ENABLED" const maxRetries = 3 +const IncreaseFileUploadLimit = "INCREASE_FILE_UPLOAD_LIMIT" var DefaultFFLoad bool = false diff --git a/internal/wrappers/mock/uploads-mock.go b/internal/wrappers/mock/uploads-mock.go index 59fc41bbb..e4d029d1d 100644 --- a/internal/wrappers/mock/uploads-mock.go +++ b/internal/wrappers/mock/uploads-mock.go @@ -2,6 +2,7 @@ package mock import ( "fmt" + "strings" "github.com/pkg/errors" @@ -11,11 +12,20 @@ import ( type UploadsMockWrapper struct { } +func (u *UploadsMockWrapper) UploadFileInMultipart(filePath string, wrapper wrappers.FeatureFlagsWrapper) (*string, error) { + fmt.Println("UploadFileInMultipart called Create in UploadsMockWrapper") + if strings.Contains(filePath, "failureCaseLarge.zip") { + return nil, errors.New("error from UploadFileInMultipart") + } + url := "multiPart/path/to/nowhere" + return &url, nil +} + func (u *UploadsMockWrapper) UploadFile(filePath string, featureFlagsWrapper wrappers.FeatureFlagsWrapper) (*string, error) { fmt.Println("Called Create in UploadsMockWrapper") - if filePath == "failureCase.zip" { + if strings.Contains(filePath, "failureCase.zip") { return nil, errors.New("error from UploadFile") } - url := "/path/to/nowhere" + url := "singlePart/path/to/nowhere" return &url, nil } diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index e2b2808dd..775f1b4d2 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -1,13 +1,18 @@ package wrappers import ( + "bytes" "encoding/json" "fmt" + "io" "net/http" "net/url" "os" + "strconv" + "time" errorConstants "github.com/checkmarx/ast-cli/internal/constants/errors" + "github.com/checkmarx/ast-cli/internal/logger" commonParams "github.com/checkmarx/ast-cli/internal/params" "github.com/pkg/errors" "github.com/spf13/viper" @@ -21,6 +26,34 @@ type UploadsHTTPWrapper struct { path string } +type StartMultipartUploadResponse struct { + ObjectName string `json:"objectName"` + UploadID string `json:"UploadID"` +} +type StartMultipartUploadRequest struct { + FileSize int64 `json:"fileSize"` +} +type MultipartPresignedURL struct { + ObjectName string `json:"objectName"` + UploadID string `json:"UploadID"` + PartNumber int `json:"partNumber"` +} + +type CompleteMultipartUploadRequest struct { + UploadID string `json:"UploadID"` + ObjectName string `json:"objectName"` + PartList []Part `json:"partList"` +} + +type Part struct { + ETag string `json:"eTag"` + PartNumber int `json:"partNumber"` +} + +type UploadModelMultipart struct { + PresignedURL string `json:"presignedURL"` +} + func (u *UploadsHTTPWrapper) UploadFile(sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (*string, error) { preSignedURL, err := u.getPresignedURLForUploading() if err != nil { @@ -130,3 +163,380 @@ func NewUploadsHTTPWrapper(path string) UploadsWrapper { path: path, } } + +func (u *UploadsHTTPWrapper) UploadFileInMultipart(sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (*string, error) { + fileInfo, _ := os.Stat(sourcesFile) + + startMultipartUploadRequest := StartMultipartUploadRequest{} + startMultipartUploadRequest.FileSize = fileInfo.Size() + startMultipartUploadResponse, err := startMultipartUpload(startMultipartUploadRequest) + if err != nil { + return nil, err + } + partList, err := SplitZipBySizeGB(sourcesFile) + if err != nil { + return nil, errors.Errorf("Failed to split ZIP file for multipart upload - %s", err.Error()) + } + + defer cleanUpTempParts(partList) + + for i, part := range partList { + logger.PrintfIfVerbose("Part%d created at: %s", i+1, part) + } + + completeMultipartUploadRequest := &CompleteMultipartUploadRequest{ + UploadID: startMultipartUploadResponse.UploadID, + ObjectName: startMultipartUploadResponse.ObjectName, + } + + var presignedURLPart1 string + + for i, partPath := range partList { + partNumber := i + 1 + + presignedURL, err := getPresignedURLForMultipartUploading(startMultipartUploadResponse, partNumber) + if err != nil { + return nil, errors.Errorf("Failed to get presigned URL for part%d - %s", partNumber, err.Error()) + } + + if partNumber == 1 { + presignedURLPart1 = presignedURL + } + + etag, err := uploadPart(presignedURL, partPath, featureFlagsWrapper) + if err != nil { + return nil, errors.Errorf("Failed to upload part%d - %s", partNumber, err.Error()) + } + + completeMultipartUploadRequest.PartList = append(completeMultipartUploadRequest.PartList, Part{ + ETag: etag, + PartNumber: partNumber, + }) + } + + err = completeMultipartUpload(*completeMultipartUploadRequest) + if err != nil { + return nil, errors.Errorf("Failed to complete multipart upload - %s", err.Error()) + } + return &presignedURLPart1, nil +} + +func startMultipartUpload(startMultipartUploadRequest StartMultipartUploadRequest) (StartMultipartUploadResponse, error) { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + path := viper.GetString(commonParams.StartMultiPartUploadPathEnv) + jsonBytes, err := json.Marshal(startMultipartUploadRequest) + if err != nil { + return StartMultipartUploadResponse{}, err + } + resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return StartMultipartUploadResponse{}, err + } + decoder := json.NewDecoder(resp.Body) + defer func() { + _ = resp.Body.Close() + }() + switch resp.StatusCode { + case http.StatusOK: + startMultipartUpload := StartMultipartUploadResponse{} + err = decoder.Decode(&startMultipartUpload) + if err != nil { + return StartMultipartUploadResponse{}, err + } + return startMultipartUpload, nil + case http.StatusBadRequest: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return StartMultipartUploadResponse{}, err + } + return StartMultipartUploadResponse{}, errors.Errorf(errorModel.Message) + case http.StatusUnauthorized: + return StartMultipartUploadResponse{}, errors.New(errorConstants.StatusUnauthorized) + default: + return StartMultipartUploadResponse{}, errors.Errorf("Response status code %d", resp.StatusCode) + } +} + +func getPresignedURLForMultipartUploading(response StartMultipartUploadResponse, partNumber int) (string, error) { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + path := viper.GetString(commonParams.MultipartPresignedPathEnv) + + multipartPresignedURL := MultipartPresignedURL{ + ObjectName: response.ObjectName, + UploadID: response.UploadID, + PartNumber: partNumber, + } + jsonBytes, err := json.Marshal(multipartPresignedURL) + if err != nil { + return "", err + } + + resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return "", err + } + defer func() { + _ = resp.Body.Close() + }() + + decoder := json.NewDecoder(resp.Body) + + switch resp.StatusCode { + case http.StatusBadRequest: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return "", err + } + return "", errors.Errorf("%d - %s", errorModel.Code, errorModel.Message) + + case http.StatusOK: + model := UploadModelMultipart{} + err = decoder.Decode(&model) + if err != nil { + return "", err + } + return model.PresignedURL, nil + + default: + return "", errors.Errorf("Response status code %d", resp.StatusCode) + } +} + +func uploadPart(preSignedURL, sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (string, error) { + if preSignedURL == "" { + return "", errors.New("PreSignedURL is empty or nil") + } + + file, err := os.Open(sourcesFile) + if err != nil { + return "", err + } + defer func() { + _ = file.Close() + }() + + accessToken, err := GetAccessToken() + if err != nil { + return "", err + } + + stat, err := file.Stat() + if err != nil { + return "", err + } + flagResponse, _ := GetSpecificFeatureFlag(featureFlagsWrapper, MinioEnabled) + useAccessToken := flagResponse.Status + resp, err := SendHTTPRequestByFullURLContentLength(http.MethodPut, preSignedURL, file, stat.Size(), useAccessToken, NoTimeout, accessToken, true) + if err != nil { + return "", err + } + + defer func() { + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusUnauthorized: + return "", errors.Errorf(errorConstants.StatusUnauthorized) + case http.StatusOK: + return resp.Header.Get("Etag"), nil + case http.StatusBadRequest: + body, err := io.ReadAll(resp.Body) + defer func() { + _ = resp.Body.Close() + }() + if err != nil { + return "", err + } + return "", errors.Errorf("Bad request while uploading part - %s", string(body)) + default: + return "", errors.Errorf("Response status code %d", resp.StatusCode) + } +} + +func completeMultipartUpload(completeMultipartUploadRequest CompleteMultipartUploadRequest) error { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + path := viper.GetString(commonParams.CompleteMultipartUploadPathEnv) + jsonBytes, err := json.Marshal(completeMultipartUploadRequest) + if err != nil { + return err + } + resp, err := SendHTTPRequest(http.MethodPost, path, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return err + } + decoder := json.NewDecoder(resp.Body) + defer func() { + _ = resp.Body.Close() + }() + switch resp.StatusCode { + case http.StatusNoContent: + return nil + case http.StatusUnauthorized: + return errors.New(errorConstants.StatusUnauthorized) + default: + errorModel := ErrorModel{} + err = decoder.Decode(&errorModel) + if err != nil { + return err + } + return errors.Errorf("%d - %s", errorModel.Code, errorModel.Message) + } +} + +func SplitZipBySizeGB(zipFilePath string) ([]string, error) { + partSizeBytes := getPartSizeBytes() + f, err := os.Open(zipFilePath) + if err != nil { + return nil, err + } + defer closeFileVerbose(f) + + stat, err := f.Stat() + if err != nil { + return nil, err + } + if stat.Size() == 0 { + return nil, err + } + + partSizes := calculatePartSizes(stat.Size(), partSizeBytes) + partNames, err := createParts(f, partSizes) + if err != nil { + cleanUpTempParts(partNames) + return nil, err + } + + return partNames, nil +} + +func getPartSizeBytes() int64 { + partChunkSizeStr := viper.GetString(commonParams.MultipartFileSizeKey) + partChunkSizeFloat, err := strconv.ParseFloat(partChunkSizeStr, 64) + if err != nil { + logger.PrintIfVerbose(fmt.Sprintf("Configured part size '%s' is invalid or empty. Defaulting to 2 GB.", partChunkSizeStr)) + partChunkSizeFloat = 2 + } + truncatedSize := int64(partChunkSizeFloat) + if truncatedSize < 1 || truncatedSize > 5 { + logger.PrintIfVerbose(fmt.Sprintf("Configured part size %d GB is outside the allowed range (1 – 5 GB). Defaulting to 2 GB.", truncatedSize)) + truncatedSize = 2 + } + logger.PrintIfVerbose("Splitting zip file into parts of size: " + fmt.Sprintf("%.0f", float64(truncatedSize)) + " GB") + const bytesPerGB = 1024 * 1024 * 1024 + return int64(float64(truncatedSize) * float64(bytesPerGB)) +} + +func calculatePartSizes(totalSize, partSizeBytes int64) []int64 { + numParts := int(totalSize / partSizeBytes) + if totalSize%partSizeBytes != 0 { + numParts++ + } + partSizes := make([]int64, numParts) + for i := 0; i < numParts; i++ { + remaining := totalSize - int64(i)*partSizeBytes + if remaining >= partSizeBytes { + partSizes[i] = partSizeBytes + } else { + partSizes[i] = remaining + } + } + return partSizes +} + +func createParts(f *os.File, partSizes []int64) ([]string, error) { + partNames := make([]string, len(partSizes)) + for i, size := range partSizes { + partFile, err := os.CreateTemp("", fmt.Sprintf("cx-part%d-*", i+1)) + if err != nil { + return partNames, err + } + offset := int64(0) + for j := 0; j < i; j++ { + offset += partSizes[j] + } + if _, err := f.Seek(offset, io.SeekStart); err != nil { + err := partFile.Close() + if err != nil { + return nil, err + } + err = os.Remove(partFile.Name()) + if err != nil { + return nil, err + } + return partNames, err + } + if _, err := io.CopyN(partFile, f, size); err != nil && err != io.EOF { + err := partFile.Close() + if err != nil { + return nil, err + } + err = os.Remove(partFile.Name()) + if err != nil { + return nil, err + } + return partNames, err + } + if err := partFile.Sync(); err != nil { + return partNames, err + } + if err := partFile.Close(); err != nil { + return partNames, err + } + partNames[i] = partFile.Name() + } + return partNames, nil +} + +func closeFileVerbose(f *os.File) { + if err := f.Close(); err != nil { + logger.PrintfIfVerbose("Warning: failed to close input file - %v", err) + } +} + +func cleanUpTempParts(partList []string) { + cleanupMaxRetries := 3 + for i, partPath := range partList { + if partPath != "" { + logger.PrintIfVerbose(fmt.Sprintf("Cleaning up temporary part%d - %s", i+1, partPath)) + tries := cleanupMaxRetries + for attempt := 1; tries > 0; attempt++ { + removeErr := os.Remove(partPath) + if removeErr != nil { + if os.IsNotExist(removeErr) { + logger.PrintIfVerbose(fmt.Sprintf("Temporary part%d already removed - %s", i+1, partPath)) + break + } + logger.PrintIfVerbose(fmt.Sprintf( + "Failed to remove temporary part%d - Attempt %d/%d - %v", + i+1, + attempt, + cleanupMaxRetries, + removeErr, + )) + tries-- + Wait(attempt) + } else { + logger.PrintIfVerbose(fmt.Sprintf("Removed temporary part%d", i+1)) + break + } + } + if tries == 0 { + logger.PrintIfVerbose(fmt.Sprintf("Failed to remove temporary part%d - %s", i+1, partPath)) + } + } else { + logger.PrintIfVerbose(fmt.Sprintf("No temporary part%d to clean", i+1)) + } + } +} + +// Wait implements exponential backoff wait strategy +func Wait(attempt int) { + cleanupRetryWaitSeconds := 15 + // Calculate exponential backoff delay + waitDuration := time.Duration(cleanupRetryWaitSeconds * (1 << (attempt - 1))) // 2^(attempt-1) + logger.PrintIfVerbose(fmt.Sprintf("Waiting %d seconds before retrying...", waitDuration)) + time.Sleep(waitDuration * time.Second) +} diff --git a/internal/wrappers/uploads.go b/internal/wrappers/uploads.go index e2089c781..f778b47bd 100644 --- a/internal/wrappers/uploads.go +++ b/internal/wrappers/uploads.go @@ -2,4 +2,5 @@ package wrappers type UploadsWrapper interface { UploadFile(sourcesFile string, featureFlagsWrapper FeatureFlagsWrapper) (*string, error) + UploadFileInMultipart(path string, wrapper FeatureFlagsWrapper) (*string, error) } diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 4010effb3..6e0128832 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -2726,3 +2726,22 @@ func TestCreateScanWithNewProjectName_Assign_Groups(t *testing.T) { assert.NilError(t, err, "Groups should be assigned to newly created projects") } +func TestCreateScan_AsMultipartUpload_Success(t *testing.T) { + // Simulate a file size > 5GB by setting MaxSizeBytes to less than actual size + commands.MaxSizeBytes = 10240 // 10KB less than actual file size + defer func() { commands.MaxSizeBytes = 5 * 1024 * 1024 * 1024 }() // Reset after test + args := []string{ + "scan", "create", + flag(params.ProjectName), getProjectNameForScanTests(), + flag(params.SourcesFlag), "data/insecure.zip", + flag(params.BranchFlag), "dummy_branch", + flag(params.DebugFlag), + } + var buf bytes.Buffer + log.SetOutput(&buf) + + err, _ := executeCommand(t, args...) + assert.NilError(t, err) + log.SetOutput(os.Stderr) + assert.Assert(t, strings.Contains(buf.String(), "Uploading source code in multiple parts"), "Test for uploading file in multiple parts failed.") +}