fix: retry Github 503 requests up to 3 times (#2650)

Fixes #2649

Author: parthea

internal/github/github.go

@@ -24,10 +24,36 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/google/go-github/v69/github"
 )
 
+const (
+	maxRetries = 3
+	retryDelay = 2 * time.Second
+)
+
+type retryableTransport struct {
+	transport http.RoundTripper
+}
+
+// RoundTrip implements the http.RoundTripper interface and adds retry logic
+// for transient server errors.
+func (t *retryableTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	var resp *http.Response
+	var err error
+	for i := 0; i < maxRetries; i++ {
+		resp, err = t.transport.RoundTrip(req)
+		if err == nil && resp.StatusCode != http.StatusServiceUnavailable {
+			return resp, nil
+		}
+		slog.Warn("retrying due to error", "err", err, "status_code", resp.StatusCode)
+		time.Sleep(retryDelay)
+	}
+	return resp, err
+}
+
 // PullRequest is a type alias for the go-github type.
 type PullRequest = github.PullRequest
 
@@ -57,6 +83,14 @@ func NewClient(accessToken string, repo *Repository) *Client {
 }
 
 func newClientWithHTTP(accessToken string, repo *Repository, httpClient *http.Client) *Client {
+	if httpClient == nil {
+		httpClient = &http.Client{}
+	}
+	transport := httpClient.Transport
+	if transport == nil {
+		transport = http.DefaultTransport
+	}
+	httpClient.Transport = &retryableTransport{transport: transport}
 	client := github.NewClient(httpClient)
 	if repo != nil && repo.BaseURL != "" {
 		baseURL, _ := url.Parse(repo.BaseURL)

internal/github/github_test.go

@@ -1000,6 +1000,77 @@ func TestCreateTag(t *testing.T) {
 	}
 }
 
+func TestRetryableTransport(t *testing.T) {
+	t.Parallel()
+	for _, test := range []struct {
+		name             string
+		handler          func(w http.ResponseWriter, r *http.Request, requestCount int)
+		wantStatusCode   int
+		wantErr          bool
+		wantRequestCount int
+	}{
+		{
+			name: "Success after retries",
+			handler: func(w http.ResponseWriter, r *http.Request, requestCount int) {
+				if requestCount < 3 {
+					w.WriteHeader(http.StatusServiceUnavailable)
+				} else {
+					w.WriteHeader(http.StatusOK)
+				}
+			},
+			wantStatusCode:   http.StatusOK,
+			wantErr:          false,
+			wantRequestCount: 3,
+		},
+		{
+			name: "Failure after all retries",
+			handler: func(w http.ResponseWriter, r *http.Request, _ int) {
+				w.WriteHeader(http.StatusServiceUnavailable)
+			},
+			wantStatusCode:   http.StatusServiceUnavailable,
+			wantErr:          true,
+			wantRequestCount: 3,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var requestCount int
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				requestCount++
+				test.handler(w, r, requestCount)
+			}))
+			defer server.Close()
+
+			repo := &Repository{Owner: "owner", Name: "repo"}
+			client := newClientWithHTTP("fake-token", repo, server.Client())
+			client.BaseURL, _ = url.Parse(server.URL + "/")
+
+			req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil)
+			if err != nil {
+				t.Fatalf("http.NewRequestWithContext() failed: %v", err)
+			}
+			resp, err := client.Do(t.Context(), req, nil)
+
+			if test.wantErr {
+				if err == nil {
+					t.Fatal("client.Do() err = nil, want error")
+				}
+			} else {
+				if err != nil {
+					t.Fatalf("client.Do() failed: %v", err)
+				}
+				defer resp.Body.Close()
+			}
+
+			if resp.StatusCode != test.wantStatusCode {
+				t.Errorf("client.Do() status = %d, want %d", resp.StatusCode, test.wantStatusCode)
+			}
+			if requestCount != test.wantRequestCount {
+				t.Errorf("requestCount = %d, want %d", requestCount, test.wantRequestCount)
+			}
+		})
+	}
+}
+
 func TestNewClient(t *testing.T) {
 
 	t.Parallel()