From a5a0956942f4a725c29f4fe4d7399d7b5099ac0c Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Tue, 25 Oct 2022 17:58:50 +0200 Subject: [PATCH 1/5] add retry for downloading plugins from github --- clients/download.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/clients/download.go b/clients/download.go index ebdc907e20..d722953519 100644 --- a/clients/download.go +++ b/clients/download.go @@ -20,6 +20,8 @@ const ( PluginTypeSource PluginType = "source" PluginTypeDestination PluginType = "destination" DefaultDownloadDir = ".cq" + RetryAttempts = 5 + SleepTime = 1 * time.Second ) func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, name string, version string, typ PluginType) error { @@ -75,6 +77,21 @@ func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, return nil } +func retryDo(req *http.Request) (resp *http.Response, err error) { + for i := 0; i < RetryAttempts; i++ { + resp, err = http.DefaultClient.Do(req) + if resp.StatusCode != http.StatusOK { + // Close body before retry + resp.Body.Close() + time.Sleep(SleepTime) + continue + } else { + return resp, nil + } + } + return resp, fmt.Errorf("failed to download attempt %d times", RetryAttempts) +} + func downloadFile(ctx context.Context, localPath string, url string) (err error) { // Create the file out, err := os.Create(localPath) @@ -92,12 +109,18 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) if err != nil { return fmt.Errorf("failed to get url %s: %w", url, err) } - defer resp.Body.Close() // Check server response if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad status: %s. downloading %s", resp.Status, url) + // Close existed resp body before entering retry function + resp.Body.Close() + resp, err = retryDo(req) + if err != nil { + return fmt.Errorf("bad status: %s. downloading %s", resp.Status, url) + } } + defer resp.Body.Close() + fmt.Printf("Downloading %s\n", url) bar := downloadProgressBar(resp.ContentLength, "Downloading") From aae4c0fc91cf55f2398f3a4b18a2663096abadb2 Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Fri, 28 Oct 2022 19:46:48 +0200 Subject: [PATCH 2/5] implement retry with retry-go library --- clients/download.go | 63 +++++++++++++++++++-------------------------- go.mod | 1 + go.sum | 2 ++ 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/clients/download.go b/clients/download.go index d722953519..ab3f04e415 100644 --- a/clients/download.go +++ b/clients/download.go @@ -4,6 +4,7 @@ import ( "archive/zip" "context" "fmt" + "github.com/avast/retry-go/v4" "io" "net/http" "os" @@ -21,7 +22,7 @@ const ( PluginTypeDestination PluginType = "destination" DefaultDownloadDir = ".cq" RetryAttempts = 5 - SleepTime = 1 * time.Second + RetryWaitTime = 1 * time.Second ) func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, name string, version string, typ PluginType) error { @@ -77,21 +78,6 @@ func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, return nil } -func retryDo(req *http.Request) (resp *http.Response, err error) { - for i := 0; i < RetryAttempts; i++ { - resp, err = http.DefaultClient.Do(req) - if resp.StatusCode != http.StatusOK { - // Close body before retry - resp.Body.Close() - time.Sleep(SleepTime) - continue - } else { - return resp, nil - } - } - return resp, fmt.Errorf("failed to download attempt %d times", RetryAttempts) -} - func downloadFile(ctx context.Context, localPath string, url string) (err error) { // Create the file out, err := os.Create(localPath) @@ -105,29 +91,34 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) if err != nil { return fmt.Errorf("failed create request %s: %w", url, err) } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("failed to get url %s: %w", url, err) - } - - // Check server response - if resp.StatusCode != http.StatusOK { - // Close existed resp body before entering retry function - resp.Body.Close() - resp, err = retryDo(req) - if err != nil { - return fmt.Errorf("bad status: %s. downloading %s", resp.Status, url) - } - } - defer resp.Body.Close() - fmt.Printf("Downloading %s\n", url) - bar := downloadProgressBar(resp.ContentLength, "Downloading") + err = retry.Do( + func() error { + // Do http request + resp, err := http.DefaultClient.Do(req) + + // Check server response + if resp.StatusCode == http.StatusOK && err == nil { + // Request successfully + defer resp.Body.Close() + + fmt.Printf("Downloading %s\n", url) + bar := downloadProgressBar(resp.ContentLength, "Downloading") + + // Writer the body to file + _, err = io.Copy(io.MultiWriter(out, bar), resp.Body) + if err != nil { + return fmt.Errorf("failed to copy body to file %s: %w", localPath, err) + } + } + return err + }, + retry.Attempts(RetryAttempts), + retry.Delay(RetryWaitTime), + ) - // Writer the body to file - _, err = io.Copy(io.MultiWriter(out, bar), resp.Body) if err != nil { - return fmt.Errorf("failed to copy body to file %s: %w", localPath, err) + return fmt.Errorf("failed downloading: %s: %w", url, err) } return nil diff --git a/go.mod b/go.mod index 9affd99102..5b057e63da 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cloudquery/plugin-sdk go 1.19 require ( + github.com/avast/retry-go/v4 v4.3.0 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/getsentry/sentry-go v0.13.0 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index 9307e724a1..c65beaf007 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/avast/retry-go/v4 v4.3.0 h1:cqI48aXx0BExKoM7XPklDpoHAg7/srPPLAfWG5z62jo= +github.com/avast/retry-go/v4 v4.3.0/go.mod h1:bqOlT4nxk4phk9buiQFaghzjpqdchOSwPgjdfdQBtdg= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= From 468d36794aad0e92b4d74af12a62f4dfb85d1b42 Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Fri, 28 Oct 2022 20:39:08 +0200 Subject: [PATCH 3/5] add condition for retry --- clients/download.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/clients/download.go b/clients/download.go index ab3f04e415..e4757d09d1 100644 --- a/clients/download.go +++ b/clients/download.go @@ -98,21 +98,28 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) resp, err := http.DefaultClient.Do(req) // Check server response - if resp.StatusCode == http.StatusOK && err == nil { - // Request successfully - defer resp.Body.Close() - - fmt.Printf("Downloading %s\n", url) - bar := downloadProgressBar(resp.ContentLength, "Downloading") - - // Writer the body to file - _, err = io.Copy(io.MultiWriter(out, bar), resp.Body) - if err != nil { - return fmt.Errorf("failed to copy body to file %s: %w", localPath, err) - } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("statusCode != 200") } - return err + defer resp.Body.Close() + + fmt.Printf("Downloading %s\n", url) + bar := downloadProgressBar(resp.ContentLength, "Downloading") + + // Writer the body to file + _, err = io.Copy(io.MultiWriter(out, bar), resp.Body) + if err != nil { + return fmt.Errorf("failed to copy body to file %s: %w", localPath, err) + } + + return nil }, + retry.RetryIf(func(err error) bool { + if err.Error() == "statusCode != 200" { + return true + } + return false + }), retry.Attempts(RetryAttempts), retry.Delay(RetryWaitTime), ) From b8ce1c7018646391046425bb223181c625c42dc7 Mon Sep 17 00:00:00 2001 From: Chuan-Yen Chiang Date: Fri, 28 Oct 2022 20:47:07 +0200 Subject: [PATCH 4/5] add err handling for DefaultClient.Do --- clients/download.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/download.go b/clients/download.go index e4757d09d1..259e89a5f6 100644 --- a/clients/download.go +++ b/clients/download.go @@ -96,6 +96,9 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) func() error { // Do http request resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to get url %s: %w", url, err) + } // Check server response if resp.StatusCode != http.StatusOK { @@ -115,10 +118,7 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) return nil }, retry.RetryIf(func(err error) bool { - if err.Error() == "statusCode != 200" { - return true - } - return false + return err.Error() == "statusCode != 200" }), retry.Attempts(RetryAttempts), retry.Delay(RetryWaitTime), From a4a4f5f10be1204c3aaeffb5e7c113315a93df7a Mon Sep 17 00:00:00 2001 From: erezrokah Date: Thu, 3 Nov 2022 17:17:06 +0200 Subject: [PATCH 5/5] feat: Add retry progress --- clients/download.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/clients/download.go b/clients/download.go index 259e89a5f6..67615e537e 100644 --- a/clients/download.go +++ b/clients/download.go @@ -4,7 +4,6 @@ import ( "archive/zip" "context" "fmt" - "github.com/avast/retry-go/v4" "io" "net/http" "os" @@ -12,6 +11,8 @@ import ( "runtime" "time" + "github.com/avast/retry-go/v4" + "github.com/schollz/progressbar/v3" ) @@ -102,6 +103,7 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) // Check server response if resp.StatusCode != http.StatusOK { + fmt.Printf("Failed downloading %s with status code %d. Retrying\n", url, resp.StatusCode) return fmt.Errorf("statusCode != 200") } defer resp.Body.Close() @@ -125,7 +127,7 @@ func downloadFile(ctx context.Context, localPath string, url string) (err error) ) if err != nil { - return fmt.Errorf("failed downloading: %s: %w", url, err) + return fmt.Errorf("failed downloading: %s", url) } return nil