diff --git a/clients/download.go b/clients/download.go index ebdc907e20..67615e537e 100644 --- a/clients/download.go +++ b/clients/download.go @@ -11,6 +11,8 @@ import ( "runtime" "time" + "github.com/avast/retry-go/v4" + "github.com/schollz/progressbar/v3" ) @@ -20,6 +22,8 @@ const ( PluginTypeSource PluginType = "source" PluginTypeDestination PluginType = "destination" DefaultDownloadDir = ".cq" + RetryAttempts = 5 + RetryWaitTime = 1 * time.Second ) func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, name string, version string, typ PluginType) error { @@ -88,23 +92,42 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) if err != nil { return fmt.Errorf("failed create request %s: %w", url, err) } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("failed to get url %s: %w", url, err) - } - defer resp.Body.Close() - // Check server response - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad status: %s. downloading %s", resp.Status, url) - } - fmt.Printf("Downloading %s\n", url) - bar := downloadProgressBar(resp.ContentLength, "Downloading") + err = retry.Do( + func() error { + // Do http request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to get url %s: %w", url, err) + } + + // Check server response + if resp.StatusCode != http.StatusOK { + fmt.Printf("Failed downloading %s with status code %d. Retrying\n", url, resp.StatusCode) + return fmt.Errorf("statusCode != 200") + } + defer resp.Body.Close() + + fmt.Printf("Downloading %s\n", url) + bar := downloadProgressBar(resp.ContentLength, "Downloading") + + // Writer the body to file + _, err = io.Copy(io.MultiWriter(out, bar), resp.Body) + if err != nil { + return fmt.Errorf("failed to copy body to file %s: %w", localPath, err) + } + + return nil + }, + retry.RetryIf(func(err error) bool { + return err.Error() == "statusCode != 200" + }), + retry.Attempts(RetryAttempts), + retry.Delay(RetryWaitTime), + ) - // Writer the body to file - _, err = io.Copy(io.MultiWriter(out, bar), resp.Body) if err != nil { - return fmt.Errorf("failed to copy body to file %s: %w", localPath, err) + return fmt.Errorf("failed downloading: %s", url) } return nil diff --git a/go.mod b/go.mod index d387fc51c9..1d7c147239 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cloudquery/plugin-sdk go 1.19 require ( + github.com/avast/retry-go/v4 v4.3.0 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/getsentry/sentry-go v0.14.0 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index 7077ca0645..4b2e18a2d7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/avast/retry-go/v4 v4.3.0 h1:cqI48aXx0BExKoM7XPklDpoHAg7/srPPLAfWG5z62jo= +github.com/avast/retry-go/v4 v4.3.0/go.mod h1:bqOlT4nxk4phk9buiQFaghzjpqdchOSwPgjdfdQBtdg= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=