fix(retrieve): retry retrievals

pkg/retrieve/retrieve.go

@@ -2,7 +2,9 @@ package retrieve
 
 import (
 	"io"
+	"time"
 
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/vmware-tanzu/sonobuoy/pkg/client"
@@ -29,39 +31,68 @@ func NewCmdRetrieve(config *pkg.Config) *cobra.Command {
 
 			log.Info("Collecting results...")
 
-			// Get a reader that contains the tar output of the results directory.
-			reader, ec, err := config.SonobuoyClient.RetrieveResults(&client.RetrieveConfig{
-				Namespace: "sonobuoy",
-				Path:      config2.AggregatorResultsPath,
-			})
-			if err != nil {
-				log.WithError(err).Error("error retrieving results from sonobuoy")
+			if err := retrieveResultsRetry(config); err != nil {
+				log.Error(err)
 				return
 			}
 
-			// Download results into cache directory
-			err, results := retrieveResults(pkg.ResultsDirectory, reader, ec)
-			if err != nil {
-				log.WithError(err).Error("error retrieving certification results from sonobyuoy")
-				return
-			}
+			log.Info("Use the 'results' command to check the certification test summary")
+		},
+	}
+}
 
-			// Log the new files to stdout and save them to a cache for results and upload commands
-			for _, result := range results {
-				log.Infof("Results saved to %s", result)
+func retrieveResultsRetry(config *pkg.Config) error {
+	var err error
+	limit := 10 // Retry retrieve 10 times
+	pause := time.Second * 2
+	retries := 1
+	for retries <= limit {
+		err = retrieveResults(config)
+		if err != nil {
+			log.Warn(err)
+			if retries+1 < limit {
+				log.Warnf("Retrying retrieval %d more times", limit-retries)
 			}
+			time.Sleep(pause)
+			retries++
+			continue
+		}
+		return nil // Retrieved results without a problem
+	}
 
-			err = results2.SaveToResultsFile(results)
-			if err != nil {
-				log.WithError(err).Error("error saving results to cache file")
-			}
+	return errors.New("Retrieval retry limit reached")
+}
 
-			log.Info("Use the 'results' command to check the certification test summary")
-		},
+func retrieveResults(config *pkg.Config) error {
+	// Get a reader that contains the tar output of the results directory.
+	reader, ec, err := config.SonobuoyClient.RetrieveResults(&client.RetrieveConfig{
+		Namespace: "sonobuoy",
+		Path:      config2.AggregatorResultsPath,
+	})
+	if err != nil {
+		return errors.Wrap(err, "error retrieving results from sonobuoy")
+	}
+
+	// Download results into cache directory
+	err, results := writeResultsToDirectory(pkg.ResultsDirectory, reader, ec)
+	if err != nil {
+		return errors.Wrap(err, "error retrieving certification results from sonobyuoy")
+	}
+
+	// Log the new files to stdout and save them to a cache for results and upload commands
+	for _, result := range results {
+		log.Infof("Results saved to %s", result)
 	}
+
+	err = results2.SaveToResultsFile(results)
+	if err != nil {
+		return errors.Wrap(err, "error saving results to cache file")
+	}
+
+	return nil
 }
 
-func retrieveResults(outputDir string, r io.Reader, ec <-chan error) (error, []string) {
+func writeResultsToDirectory(outputDir string, r io.Reader, ec <-chan error) (error, []string) {
 	eg := &errgroup.Group{}
 	var results []string
 	eg.Go(func() error { return <-ec })