From 187d4ed1e6f7a8aa9c83dc51f0324bd3434248d9 Mon Sep 17 00:00:00 2001 From: Bar Belity Date: Sun, 27 Jan 2019 19:00:58 +0200 Subject: [PATCH] Create RetryExecutor + add retries to client's download and upload operations --- artifactory/httpclient/httpclient.go | 8 +- artifactory/services/go/publishwithheaders.go | 2 +- .../services/go/publishwithmatrixparams.go | 2 +- artifactory/services/go/publishzipandmod.go | 4 +- artifactory/services/readfile.go | 3 +- artifactory/services/upload.go | 8 +- .../services/utils/artifactoryutils.go | 4 +- bintray/services/download.go | 9 +- bintray/services/logs/logs.go | 3 +- bintray/services/upload.go | 2 +- httpclient/client.go | 126 ++++++++++++++---- utils/retryexecutor.go | 56 ++++++++ utils/retryexecutor_test.go | 52 ++++++++ 13 files changed, 229 insertions(+), 50 deletions(-) create mode 100644 utils/retryexecutor.go create mode 100644 utils/retryexecutor_test.go diff --git a/artifactory/httpclient/httpclient.go b/artifactory/httpclient/httpclient.go index 2666efc1a..adacf13d2 100644 --- a/artifactory/httpclient/httpclient.go +++ b/artifactory/httpclient/httpclient.go @@ -141,11 +141,11 @@ func (rtc *ArtifactoryHttpClient) Send(method string, url string, content []byte return } -func (rtc *ArtifactoryHttpClient) UploadFile(localPath, url string, +func (rtc *ArtifactoryHttpClient) UploadFile(localPath, url, logMsgPrefix string, httpClientsDetails *httputils.HttpClientDetails, retries int) (resp *http.Response, body []byte, err error) { isNewToken := false for i := 0; i < 2; i++ { - resp, body, err = rtc.httpClient.UploadFile(localPath, url, *httpClientsDetails, retries) + resp, body, err = rtc.httpClient.UploadFile(localPath, url, logMsgPrefix, *httpClientsDetails, retries) if err != nil { return } @@ -160,10 +160,10 @@ func (rtc *ArtifactoryHttpClient) UploadFile(localPath, url string, return } -func (rtc *ArtifactoryHttpClient) ReadRemoteFile(downloadPath string, httpClientsDetails *httputils.HttpClientDetails, retries int) (ioReaderCloser io.ReadCloser, resp *http.Response, err error) { +func (rtc *ArtifactoryHttpClient) ReadRemoteFile(downloadPath string, httpClientsDetails *httputils.HttpClientDetails) (ioReaderCloser io.ReadCloser, resp *http.Response, err error) { isNewToken := false for i := 0; i < 2; i++ { - ioReaderCloser, resp, err = rtc.httpClient.ReadRemoteFile(downloadPath, *httpClientsDetails, retries) + ioReaderCloser, resp, err = rtc.httpClient.ReadRemoteFile(downloadPath, *httpClientsDetails) if err != nil { return } diff --git a/artifactory/services/go/publishwithheaders.go b/artifactory/services/go/publishwithheaders.go index e6972b8ec..3dba2537c 100644 --- a/artifactory/services/go/publishwithheaders.go +++ b/artifactory/services/go/publishwithheaders.go @@ -33,7 +33,7 @@ func (pwh *publishWithHeader) PublishPackage(params GoParams, client *rthttpclie if err != nil { return err } - resp, _, err := client.UploadFile(params.GetZipPath(), url, &clientDetails, 0) + resp, _, err := client.UploadFile(params.GetZipPath(), url, "", &clientDetails, 0) if err != nil { return err } diff --git a/artifactory/services/go/publishwithmatrixparams.go b/artifactory/services/go/publishwithmatrixparams.go index e8f12ede0..8f30ed799 100644 --- a/artifactory/services/go/publishwithmatrixparams.go +++ b/artifactory/services/go/publishwithmatrixparams.go @@ -40,7 +40,7 @@ func (pwmp *publishWithMatrixParams) PublishPackage(params GoParams, client *rth return err } - resp, _, err := client.UploadFile(params.GetZipPath(), url, &clientDetails, 0) + resp, _, err := client.UploadFile(params.GetZipPath(), url, "", &clientDetails, 0) if err != nil { return err } diff --git a/artifactory/services/go/publishzipandmod.go b/artifactory/services/go/publishzipandmod.go index 0d0df903b..683faa52f 100644 --- a/artifactory/services/go/publishzipandmod.go +++ b/artifactory/services/go/publishzipandmod.go @@ -41,7 +41,7 @@ func (pwa *publishZipAndModApi) PublishPackage(params GoParams, client *rthttpcl clientDetails := ArtDetails.CreateHttpClientDetails() addGoVersion(params, &zipUrl) - resp, _, err := client.UploadFile(params.GetZipPath(), zipUrl, &clientDetails, 0) + resp, _, err := client.UploadFile(params.GetZipPath(), zipUrl, "", &clientDetails, 0) if err != nil { return err } @@ -54,7 +54,7 @@ func (pwa *publishZipAndModApi) PublishPackage(params GoParams, client *rthttpcl return err } addGoVersion(params, &url) - resp, _, err = client.UploadFile(params.GetModPath(), url, &clientDetails, 0) + resp, _, err = client.UploadFile(params.GetModPath(), url, "", &clientDetails, 0) if err != nil { return err } diff --git a/artifactory/services/readfile.go b/artifactory/services/readfile.go index f8917a22b..aa8dd94d3 100644 --- a/artifactory/services/readfile.go +++ b/artifactory/services/readfile.go @@ -15,7 +15,6 @@ type ReadFileService struct { DryRun bool MinSplitSize int64 SplitCount int - Retries int } func NewReadFileService(client *rthttpclient.ArtifactoryHttpClient) *ReadFileService { @@ -56,7 +55,7 @@ func (ds *ReadFileService) ReadRemoteFile(downloadPath string) (io.ReadCloser, e return nil, err } httpClientsDetails := ds.ArtDetails.CreateHttpClientDetails() - ioReadCloser, resp, err := ds.client.ReadRemoteFile(readPath, &httpClientsDetails, ds.Retries) + ioReadCloser, resp, err := ds.client.ReadRemoteFile(readPath, &httpClientsDetails) if err != nil { return nil, err } diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 595d34642..27b76aed3 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -326,7 +326,7 @@ func (us *UploadService) uploadFile(localPath, targetPath, props string, uploadP return utils.FileInfo{}, false, err } if uploadParams.IsSymlink() && fileutils.IsFileSymlink(fileInfo) { - resp, details, body, err = us.uploadSymlink(targetPathWithProps, httpClientsDetails, uploadParams) + resp, details, body, err = us.uploadSymlink(targetPathWithProps, logMsgPrefix, httpClientsDetails, uploadParams) } else { resp, details, body, checksumDeployed, err = us.doUpload(localPath, targetPathWithProps, logMsgPrefix, httpClientsDetails, fileInfo, uploadParams) } @@ -338,12 +338,12 @@ func (us *UploadService) uploadFile(localPath, targetPath, props string, uploadP return artifact, us.DryRun || checksumDeployed || resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusOK, nil } -func (us *UploadService) uploadSymlink(targetPath string, httpClientsDetails httputils.HttpClientDetails, uploadParams UploadParams) (resp *http.Response, details *fileutils.FileDetails, body []byte, err error) { +func (us *UploadService) uploadSymlink(targetPath, logMsgPrefix string, httpClientsDetails httputils.HttpClientDetails, uploadParams UploadParams) (resp *http.Response, details *fileutils.FileDetails, body []byte, err error) { details, err = fspatterns.CreateSymlinkFileDetails() if err != nil { return } - resp, body, err = utils.UploadFile("", targetPath, &us.ArtDetails, details, httpClientsDetails, us.client, us.Retries) + resp, body, err = utils.UploadFile("", targetPath, logMsgPrefix, &us.ArtDetails, details, httpClientsDetails, us.client, us.Retries) return } @@ -363,7 +363,7 @@ func (us *UploadService) doUpload(localPath, targetPath, logMsgPrefix string, ht } if !us.DryRun && !checksumDeployed { var body []byte - resp, body, err = utils.UploadFile(localPath, targetPath, &us.ArtDetails, details, httpClientsDetails, us.client, us.Retries) + resp, body, err = utils.UploadFile(localPath, targetPath, logMsgPrefix, &us.ArtDetails, details, httpClientsDetails, us.client, us.Retries) if err != nil { return resp, details, body, checksumDeployed, err } diff --git a/artifactory/services/utils/artifactoryutils.go b/artifactory/services/utils/artifactoryutils.go index e60f9f268..0df2be733 100644 --- a/artifactory/services/utils/artifactoryutils.go +++ b/artifactory/services/utils/artifactoryutils.go @@ -20,7 +20,7 @@ import ( const ARTIFACTORY_SYMLINK = "symlink.dest" const SYMLINK_SHA1 = "symlink.destsha1" -func UploadFile(localPath, url string, artifactoryDetails *auth.ArtifactoryDetails, details *fileutils.FileDetails, +func UploadFile(localPath, url, logMsgPrefix string, artifactoryDetails *auth.ArtifactoryDetails, details *fileutils.FileDetails, httpClientsDetails httputils.HttpClientDetails, client *rthttpclient.ArtifactoryHttpClient, retries int) (*http.Response, []byte, error) { var err error if details == nil { @@ -34,7 +34,7 @@ func UploadFile(localPath, url string, artifactoryDetails *auth.ArtifactoryDetai AddChecksumHeaders(requestClientDetails.Headers, details) AddAuthHeaders(requestClientDetails.Headers, *artifactoryDetails) - return client.UploadFile(localPath, url, requestClientDetails, retries) + return client.UploadFile(localPath, url, logMsgPrefix, requestClientDetails, retries) } func AddChecksumHeaders(headers map[string]string, fileDetails *fileutils.FileDetails) { diff --git a/bintray/services/download.go b/bintray/services/download.go index 805dad135..1a19d4f5a 100644 --- a/bintray/services/download.go +++ b/bintray/services/download.go @@ -20,6 +20,8 @@ import ( "sync" ) +const BintrayDownloadRetries = 3 + func NewDownloadService(client *httpclient.HttpClient) *DownloadService { ds := &DownloadService{client: client} return ds @@ -215,7 +217,7 @@ func (ds *DownloadService) downloadBintrayFile(downloadParams *DownloadFileParam LocalPath: localPath, LocalFileName: localFileName} - resp, err := client.DownloadFile(downloadDetails, logMsgPrefix, httpClientsDetails, 0, false) + resp, err := client.DownloadFile(downloadDetails, logMsgPrefix, httpClientsDetails, BintrayDownloadRetries, false) if err != nil { return err } @@ -229,7 +231,7 @@ func (ds *DownloadService) downloadBintrayFile(downloadParams *DownloadFileParam var resp *http.Response var redirectUrl string resp, redirectUrl, err = - client.DownloadFileNoRedirect(url, localPath, localFileName, httpClientsDetails) + client.DownloadFileNoRedirect(url, localPath, localFileName, httpClientsDetails, BintrayDownloadRetries) // There are two options now. Either the file has just been downloaded as one block, or // we got a redirect to DSN download URL. In case of the later, we should download the file // concurrently from the DSN URL. @@ -242,7 +244,8 @@ func (ds *DownloadService) downloadBintrayFile(downloadParams *DownloadFileParam LocalFileName: localFileName, LocalPath: localPath, FileSize: details.Size, - SplitCount: ds.SplitCount} + SplitCount: ds.SplitCount, + Retries: BintrayDownloadRetries} resp, err = client.DownloadFileConcurrently(concurrentDownloadFlags, "", httpClientsDetails) if errorutils.CheckError(err) != nil { diff --git a/bintray/services/logs/logs.go b/bintray/services/logs/logs.go index 4b65c9c52..fbb256995 100644 --- a/bintray/services/logs/logs.go +++ b/bintray/services/logs/logs.go @@ -3,6 +3,7 @@ package logs import ( "errors" "github.com/jfrog/jfrog-client-go/bintray/auth" + "github.com/jfrog/jfrog-client-go/bintray/services" "github.com/jfrog/jfrog-client-go/bintray/services/versions" "github.com/jfrog/jfrog-client-go/httpclient" clientutils "github.com/jfrog/jfrog-client-go/utils" @@ -61,7 +62,7 @@ func (ls *LogsService) Download(versionPath *versions.Path, logName string) erro DownloadPath: downloadUrl, LocalPath: "", LocalFileName: logName} - resp, err := client.DownloadFile(details, "", httpClientsDetails, 3, false) + resp, err := client.DownloadFile(details, "", httpClientsDetails, services.BintrayDownloadRetries, false) if err != nil { return err } diff --git a/bintray/services/upload.go b/bintray/services/upload.go index 81e104489..cb3c5baaf 100644 --- a/bintray/services/upload.go +++ b/bintray/services/upload.go @@ -155,7 +155,7 @@ func uploadFile(artifact clientutils.Artifact, url, logMsgPrefix string, bintray if err != nil { return false, err } - resp, body, err := client.UploadFile(artifact.LocalPath, url, httpClientsDetails, 0) + resp, body, err := client.UploadFile(artifact.LocalPath, url, logMsgPrefix, httpClientsDetails, 0) if err != nil { return false, err } diff --git a/httpclient/client.go b/httpclient/client.go index 6d2ee12d9..86a9dff9e 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -5,6 +5,7 @@ import ( "crypto/sha1" "encoding/hex" "errors" + "fmt" "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" multifilereader "github.com/jfrog/jfrog-client-go/utils/io" @@ -30,15 +31,8 @@ type HttpClient struct { Client *http.Client } -func (jc *HttpClient) sendGetForFileDownload(url string, followRedirect bool, httpClientsDetails httputils.HttpClientDetails, currentSplit, retries int) (resp *http.Response, redirectUrl string, err error) { - for i := 0; i < retries+1; i++ { - resp, _, redirectUrl, err = jc.sendGetLeaveBodyOpen(url, followRedirect, httpClientsDetails) - if resp != nil && resp.StatusCode < 500 { - // No error and status < 500 - return - } - log.Warn("Download attempt #", i, "of part", currentSplit, "of", url, "failed -", getFailureReason(resp, err)) - } +func (jc *HttpClient) sendGetForFileDownload(url string, followRedirect bool, httpClientsDetails httputils.HttpClientDetails, currentSplit int) (resp *http.Response, redirectUrl string, err error) { + resp, _, redirectUrl, err = jc.sendGetLeaveBodyOpen(url, followRedirect, httpClientsDetails) return } @@ -152,19 +146,34 @@ func setRequestHeaders(httpClientsDetails httputils.HttpClientDetails, size int6 req.Header.Set("Content-Length", length) } -func (jc *HttpClient) UploadFile(localPath, url string, httpClientsDetails httputils.HttpClientDetails, retries int) (resp *http.Response, body []byte, err error) { - for i := 0; i < retries+1; i++ { - resp, body, err = jc.doUploadFile(localPath, url, httpClientsDetails) - if resp != nil && resp.StatusCode < 500 { - // No error and status < 500 - return - } - log.Warn("Upload attempt #", i, "to", url, "failed -", getFailureReason(resp, err)) +func (jc *HttpClient) UploadFile(localPath, url, logMsgPrefix string, httpClientsDetails httputils.HttpClientDetails, retries int) (resp *http.Response, body []byte, err error) { + retryExecutor := utils.RetryExecutor{ + MaxRetries: retries, + RetriesInterval: 0, + ErrorMessage: fmt.Sprintf("Failure occurred while uploading to %s", url), + ExecutionHandler: func() (bool, error) { + resp, body, err = jc.doUploadFile(localPath, url, logMsgPrefix, httpClientsDetails) + if err != nil { + return true, err + } + message := resp.Status + if body != nil { + message = fmt.Sprintf("%s %s", message, body) + } + // If response-code < 500, should not retry + if resp != nil && resp.StatusCode < 500 { + return false, nil + } + // Perform retry + return true, nil + }, } - return resp, body, err + + err = retryExecutor.Execute() + return } -func (jc *HttpClient) doUploadFile(localPath, url string, httpClientsDetails httputils.HttpClientDetails) (*http.Response, []byte, error) { +func (jc *HttpClient) doUploadFile(localPath, url, logMsgPrefix string, httpClientsDetails httputils.HttpClientDetails) (*http.Response, []byte, error) { var file *os.File var err error if localPath != "" { @@ -197,6 +206,7 @@ func (jc *HttpClient) doUploadFile(localPath, url string, httpClientsDetails htt return nil, nil, err } defer resp.Body.Close() + log.Info(logMsgPrefix+":", resp.Status+"...") body, err := ioutil.ReadAll(resp.Body) if errorutils.CheckError(err) != nil { return nil, nil, err @@ -206,8 +216,8 @@ func (jc *HttpClient) doUploadFile(localPath, url string, httpClientsDetails htt // Read remote file, // The caller is responsible to check if resp.StatusCode is StatusOK before reading, and to close io.ReadCloser after done reading. -func (jc *HttpClient) ReadRemoteFile(downloadPath string, httpClientsDetails httputils.HttpClientDetails, retries int) (io.ReadCloser, *http.Response, error) { - resp, _, err := jc.sendGetForFileDownload(downloadPath, true, httpClientsDetails, 0, retries) +func (jc *HttpClient) ReadRemoteFile(downloadPath string, httpClientsDetails httputils.HttpClientDetails) (io.ReadCloser, *http.Response, error) { + resp, _, err := jc.sendGetForFileDownload(downloadPath, true, httpClientsDetails, 0) if err != nil { return nil, nil, err } @@ -222,18 +232,50 @@ func (jc *HttpClient) DownloadFile(downloadFileDetails *DownloadFileDetails, log return resp, err } -func (jc *HttpClient) DownloadFileNoRedirect(downloadPath, localPath, fileName string, httpClientsDetails httputils.HttpClientDetails) (*http.Response, string, error) { +func (jc *HttpClient) DownloadFileNoRedirect(downloadPath, localPath, fileName string, httpClientsDetails httputils.HttpClientDetails, retries int) (*http.Response, string, error) { downloadFileDetails := &DownloadFileDetails{DownloadPath: downloadPath, LocalPath: localPath, FileName: fileName} - return jc.downloadFile(downloadFileDetails, "", false, httpClientsDetails, 0, false) + return jc.downloadFile(downloadFileDetails, "", false, httpClientsDetails, retries, false) } func (jc *HttpClient) downloadFile(downloadFileDetails *DownloadFileDetails, logMsgPrefix string, followRedirect bool, httpClientsDetails httputils.HttpClientDetails, retries int, isExplode bool) (resp *http.Response, redirectUrl string, err error) { - resp, redirectUrl, err = jc.sendGetForFileDownload(downloadFileDetails.DownloadPath, followRedirect, httpClientsDetails, 0, retries) + retryExecutor := utils.RetryExecutor{ + MaxRetries: retries, + RetriesInterval: 0, + ErrorMessage: fmt.Sprintf("Failure occurred while downloading %s", downloadFileDetails.DownloadPath), + LogMsgPrefix: logMsgPrefix, + ExecutionHandler: func() (bool, error) { + resp, redirectUrl, err = jc.doDownloadFile(downloadFileDetails, logMsgPrefix, followRedirect, httpClientsDetails, isExplode) + // In case followRedirect is 'false' and doDownloadFile did redirect, an error is returned and redirectUrl + // receives the redirect address. This case should not retry. + if err != nil && !followRedirect && redirectUrl != "" { + return false, err + } + // If error occurred during doDownloadFile, perform retry. + if err != nil { + return true, err + } + // If response-code < 500, should not retry + if resp != nil && resp.StatusCode < 500 { + return false, nil + } + // Perform retry + return true, nil + }, + } + + err = retryExecutor.Execute() + return +} + +func (jc *HttpClient) doDownloadFile(downloadFileDetails *DownloadFileDetails, logMsgPrefix string, followRedirect bool, + httpClientsDetails httputils.HttpClientDetails, isExplode bool) (resp *http.Response, redirectUrl string, err error) { + resp, redirectUrl, err = jc.sendGetForFileDownload(downloadFileDetails.DownloadPath, followRedirect, httpClientsDetails, 0) if err != nil { return } + log.Info(logMsgPrefix+":", resp.Status+"...") defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return resp, redirectUrl, nil @@ -528,15 +570,41 @@ func extractAndMergeChunks(chunksPaths []string, flags ConcurrentDownloadFlags, return true, nil } -func (jc *HttpClient) downloadFileRange(flags ConcurrentDownloadFlags, start, end int64, currentSplit int, logMsgPrefix string, httpClientsDetails httputils.HttpClientDetails, retries int) (string, *http.Response, error) { +func (jc *HttpClient) downloadFileRange(flags ConcurrentDownloadFlags, start, end int64, currentSplit int, logMsgPrefix string, + httpClientsDetails httputils.HttpClientDetails, retries int) (fileName string, resp *http.Response, err error) { + retryExecutor := utils.RetryExecutor{ + MaxRetries: retries, + RetriesInterval: 0, + ErrorMessage: fmt.Sprintf("Failure occurred while downloading part %d of %s", currentSplit, flags.DownloadPath), + LogMsgPrefix: fmt.Sprintf("%s[%s]: ", logMsgPrefix, strconv.Itoa(currentSplit)), + ExecutionHandler: func() (bool, error) { + fileName, resp, err = jc.doDownloadFileRange(flags, start, end, currentSplit, logMsgPrefix, httpClientsDetails) + if err != nil { + return true, err + } + // If response-code < 500, should not retry + if resp != nil && resp.StatusCode < 500 { + return false, nil + } + // Perform retry + return true, nil + }, + } + + err = retryExecutor.Execute() + return +} + +func (jc *HttpClient) doDownloadFileRange(flags ConcurrentDownloadFlags, start, end int64, currentSplit int, logMsgPrefix string, + httpClientsDetails httputils.HttpClientDetails) (fileName string, resp *http.Response, err error) { tempLocalPath, err := fileutils.GetTempDirPath() if err != nil { - return "", nil, err + return } tempFile, err := ioutil.TempFile(tempLocalPath, strconv.Itoa(currentSplit)+"_") if errorutils.CheckError(err) != nil { - return "", nil, err + return } defer tempFile.Close() @@ -544,7 +612,7 @@ func (jc *HttpClient) downloadFileRange(flags ConcurrentDownloadFlags, start, en httpClientsDetails.Headers = make(map[string]string) } httpClientsDetails.Headers["Range"] = "bytes=" + strconv.FormatInt(start, 10) + "-" + strconv.FormatInt(end-1, 10) - resp, _, err := jc.sendGetForFileDownload(flags.DownloadPath, true, httpClientsDetails, currentSplit, retries) + resp, _, err = jc.sendGetForFileDownload(flags.DownloadPath, true, httpClientsDetails, currentSplit) if err != nil { return "", nil, err } @@ -553,7 +621,7 @@ func (jc *HttpClient) downloadFileRange(flags ConcurrentDownloadFlags, start, en log.Info(logMsgPrefix+"["+strconv.Itoa(currentSplit)+"]:", resp.Status+"...") // Unexpected http response if resp.StatusCode != http.StatusPartialContent { - return "", resp, nil + return } err = os.MkdirAll(tempLocalPath, 0777) diff --git a/utils/retryexecutor.go b/utils/retryexecutor.go new file mode 100644 index 000000000..4d9336878 --- /dev/null +++ b/utils/retryexecutor.go @@ -0,0 +1,56 @@ +package utils + +import ( + "fmt" + "github.com/jfrog/jfrog-client-go/utils/log" + "time" +) + +type ExecutionHandlerFunc func() (bool, error) + +type RetryExecutor struct { + // The amount of retries to perform. + MaxRetries int + + // Number of seconds to sleep between retries. + RetriesInterval int + + // Message to display when retrying. + ErrorMessage string + + // Prefix to print at the beginning of each log. + LogMsgPrefix string + + // ExecutionHandler is the operation to run with retries. + ExecutionHandler ExecutionHandlerFunc +} + +func (runner *RetryExecutor) Execute() error { + var err error + var shouldRetry bool + for i := 0; i <= runner.MaxRetries; i++ { + // Run ExecutionHandler + shouldRetry, err = runner.ExecutionHandler() + + // If should not retry, return + if !shouldRetry { + return err + } + + log.Warn(runner.getLogRetryMessage(i, err)) + // Going to sleep for RetryInterval seconds + if runner.RetriesInterval > 0 && i < runner.MaxRetries { + time.Sleep(time.Second * time.Duration(runner.RetriesInterval)) + } + } + + return err +} + +func (runner *RetryExecutor) getLogRetryMessage(attemptNumber int, err error) (message string) { + message = fmt.Sprintf("%sAttempt %v - %s", runner.LogMsgPrefix, attemptNumber, runner.ErrorMessage) + if err != nil { + message = fmt.Sprintf("%s - %s", message, err.Error()) + } + return +} diff --git a/utils/retryexecutor_test.go b/utils/retryexecutor_test.go new file mode 100644 index 000000000..132cf5427 --- /dev/null +++ b/utils/retryexecutor_test.go @@ -0,0 +1,52 @@ +package utils + +import ( + "fmt" + "github.com/jfrog/jfrog-client-go/utils/log" + "testing" +) + +func TestRetryExecutorSuccess(t *testing.T) { + retriesToPerform := 10 + breakRetriesAt := 4 + runCount := 0 + + executor := RetryExecutor{ + MaxRetries: retriesToPerform, + RetriesInterval: 0, + ErrorMessage: "Testing RetryExecutor", + ExecutionHandler: func() (bool, error) { + runCount++ + if runCount == breakRetriesAt { + log.Warn("Breaking after", runCount-1, "retries") + return false, nil + } + return true, nil + }, + } + + executor.Execute() + if runCount != breakRetriesAt { + t.Error(fmt.Errorf("expected, %d, got: %d", breakRetriesAt, runCount)) + } +} + +func TestRetryExecutorFail(t *testing.T) { + retriesToPerform := 5 + runCount := 0 + + executor := RetryExecutor{ + MaxRetries: retriesToPerform, + RetriesInterval: 0, + ErrorMessage: "Testing RetryExecutor", + ExecutionHandler: func() (bool, error) { + runCount++ + return true, nil + }, + } + + executor.Execute() + if runCount != retriesToPerform+1 { + t.Error(fmt.Errorf("expected, %d, got: %d", retriesToPerform, runCount)) + } +}