feat: Add retry logic when downloading plugins from GitHub

clients/download.go

@@ -11,6 +11,8 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/avast/retry-go/v4"
+
 	"github.com/schollz/progressbar/v3"
 )
 
@@ -20,6 +22,8 @@ const (
 	PluginTypeSource      PluginType = "source"
 	PluginTypeDestination PluginType = "destination"
 	DefaultDownloadDir               = ".cq"
+	RetryAttempts                    = 5
+	RetryWaitTime                    = 1 * time.Second
 )
 
 func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, name string, version string, typ PluginType) error {
@@ -88,23 +92,42 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error)
 	if err != nil {
 		return fmt.Errorf("failed create request %s: %w", url, err)
 	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to get url %s: %w", url, err)
-	}
-	defer resp.Body.Close()
 
-	// Check server response
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad status: %s. downloading %s", resp.Status, url)
-	}
-	fmt.Printf("Downloading %s\n", url)
-	bar := downloadProgressBar(resp.ContentLength, "Downloading")
+	err = retry.Do(
+		func() error {
+			// Do http request
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return fmt.Errorf("failed to get url %s: %w", url, err)
+			}
+
+			// Check server response
+			if resp.StatusCode != http.StatusOK {
+				fmt.Printf("Failed downloading %s with status code %d. Retrying\n", url, resp.StatusCode)
+				return fmt.Errorf("statusCode != 200")
+			}
+			defer resp.Body.Close()
+
+			fmt.Printf("Downloading %s\n", url)
+			bar := downloadProgressBar(resp.ContentLength, "Downloading")
+
+			// Writer the body to file
+			_, err = io.Copy(io.MultiWriter(out, bar), resp.Body)
+			if err != nil {
+				return fmt.Errorf("failed to copy body to file %s: %w", localPath, err)
+			}
+
+			return nil
+		},
+		retry.RetryIf(func(err error) bool {
+			return err.Error() == "statusCode != 200"
+		}),
+		retry.Attempts(RetryAttempts),
+		retry.Delay(RetryWaitTime),
+	)
 
-	// Writer the body to file
-	_, err = io.Copy(io.MultiWriter(out, bar), resp.Body)
 	if err != nil {
-		return fmt.Errorf("failed to copy body to file %s: %w", localPath, err)
+		return fmt.Errorf("failed downloading: %s", url)
 	}
 
 	return nil

go.mod

@@ -3,6 +3,7 @@ module github.com/cloudquery/plugin-sdk
 go 1.19
 
 require (
+	github.com/avast/retry-go/v4 v4.3.0
 	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
 	github.com/getsentry/sentry-go v0.14.0
 	github.com/ghodss/yaml v1.0.0

go.sum

@@ -1,5 +1,7 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/avast/retry-go/v4 v4.3.0 h1:cqI48aXx0BExKoM7XPklDpoHAg7/srPPLAfWG5z62jo=
+github.com/avast/retry-go/v4 v4.3.0/go.mod h1:bqOlT4nxk4phk9buiQFaghzjpqdchOSwPgjdfdQBtdg=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=