diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 46607c38d..1739edd90 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -87,11 +87,15 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { "added", len(event.RepositoriesAdded), "removed", len(event.RepositoriesRemoved), ) - if err := handleInstallationRepositoriesEvent(c.Request.Context(), gh, event, appId64); err != nil { - slog.Error("Failed to handle installation repositories event", "error", err) - c.String(http.StatusAccepted, "Failed to handle webhook event.") - return - } + + // Run in goroutine to avoid webhook timeouts for large installations + go func(ctx context.Context) { + defer logging.InheritRequestLogger(ctx)() + // Use background context so work continues after HTTP response + if err := handleInstallationRepositoriesEvent(context.Background(), gh, event, appId64); err != nil { + slog.Error("Failed to handle installation repositories event", "error", err) + } + }(c.Request.Context()) case *github.PushEvent: slog.Info("Processing PushEvent", "repo", *event.Repo.FullName, diff --git a/backend/controllers/github_callback.go b/backend/controllers/github_callback.go index 4337ce5a0..6855fedd3 100644 --- a/backend/controllers/github_callback.go +++ b/backend/controllers/github_callback.go @@ -31,6 +31,10 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) { code := "" if codeExists && len(codeParams) > 0 && len(codeParams[0]) > 0 { code = codeParams[0] + } else { + slog.Debug("No code parameter found, probably a setup update, going to return success since we are relying on webhooks now") + c.HTML(http.StatusOK, "github_success.tmpl", gin.H{}) + return } appId := c.Request.URL.Query().Get("state") diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index 280f2a2d8..6fa30b9f6 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -9,7 +9,6 @@ import ( "runtime/debug" "slices" "strconv" - "strings" "github.com/diggerhq/digger/backend/ci_backends" config2 "github.com/diggerhq/digger/backend/config" @@ -138,16 +137,27 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return nil } - // Silently skip repos without digger.yml - this is expected for org-wide installations - if strings.Contains(err.Error(), "could not find digger.yml") || - strings.Contains(err.Error(), "could not find digger.yaml") { - slog.Info("No Digger config found, skipping repo", + // Check if the error is due to missing digger config and the app is installed for all repos + if errors.Is(err, digger_config.ErrDiggerConfigNotFound) { + slog.Debug("Digger config not found, checking if app is installed for all repos", "prNumber", prNumber, "repoFullName", repoFullName, ) - return nil + isAllRepos, checkErr := utils.IsAllReposInstallation(appId, installationId) + if checkErr != nil { + slog.Warn("Failed to check if installation is for all repos", + "error", checkErr, + ) + } else if isAllRepos { + slog.Info("Digger config not found but GitHub App is installed for all repos, skipping error comment", + "prNumber", prNumber, + "repoFullName", repoFullName, + ) + return nil + } } + slog.Error("Error getting Digger config for PR", "prNumber", prNumber, "repoFullName", repoFullName, @@ -515,7 +525,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not retrieve created batch: %v", err)) return fmt.Errorf("error getting digger batch") } - + if config.CommentRenderMode == digger_config.CommentRenderModeGroupByModule { slog.Info("Using GroupByModule render mode for comments", "prNumber", prNumber) diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go index fe5bcb745..6ebd2695a 100644 --- a/backend/controllers/projects.go +++ b/backend/controllers/projects.go @@ -1026,7 +1026,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed batch"}) return } - //err = UpdateCheckStatusForBatch(d.GithubClientProvider, refreshedBatch) slog.Debug("Attempting to update GitHub Check Run for batch", "batchId", batch.ID, "checkRunId", refreshedBatch.CheckRunId, @@ -1056,7 +1055,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed job"}) return } - //err = UpdateCommitStatusForJob(d.GithubClientProvider, refreshedJob) slog.Debug("Attempting to update GitHub Check Run for job", "jobId", jobId, "checkRunId", refreshedJob.CheckRunId, diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index 2c44a7809..3870d3190 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -210,9 +210,12 @@ func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerB return fmt.Errorf("error generating realtime comment message: %v", err) } - summary, err := GenerateChecksSummaryForBatch(batch) - if err != nil { - slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + var summary = "" + if batch.Status == orchestrator_scheduler.BatchJobSucceeded || batch.Status == orchestrator_scheduler.BatchJobFailed { + summary, err = GenerateChecksSummaryForBatch(batch) + if err != nil { + slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + } } if isPlanBatch { @@ -397,11 +400,15 @@ func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob) "```\n" - summary, err := GenerateChecksSummaryForJob(job) - if err != nil { - slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + var summary = "" + if job.Status == orchestrator_scheduler.DiggerJobSucceeded || job.Status == orchestrator_scheduler.DiggerJobFailed { + summary, err = GenerateChecksSummaryForJob(job) + if err != nil { + slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + } } + slog.Debug("Updating PR status for job", "jobId", job.DiggerJobID, "status", status, "conclusion", conclusion) if isPlan { title := fmt.Sprintf("%v to create %v to update %v to delete", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted) diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 87f193dba..f92cd22f4 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -259,9 +259,9 @@ func GetCheckRunConclusionForJob(job *DiggerJob) (string, error) { return "failure", nil } slog.Error("Unknown job status in GetCheckRunConclusionForJob - this will cause GitHub API 422 error", - "jobId", job.DiggerJobID, - "jobStatus", job.Status, - "jobStatusInt", int(job.Status), - "validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"}) + "jobId", job.DiggerJobID, + "jobStatus", job.Status, + "jobStatusInt", int(job.Status), + "validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"}) return "", fmt.Errorf("unknown job status: %v", job.Status) } diff --git a/docs/ce/local-development/backend.mdx b/docs/ce/local-development/backend.mdx index 69c53a445..21895892b 100644 --- a/docs/ce/local-development/backend.mdx +++ b/docs/ce/local-development/backend.mdx @@ -1,5 +1,5 @@ --- -title: Orchestrator local setup +title: Backend (orchestrator) local setup --- The backend serves orchestration APIs, GitHub app endpoints, and internal APIs the UI relies on. @@ -51,7 +51,7 @@ The backend serves orchestration APIs, GitHub app endpoints, and internal APIs t ## GitHub app integration -- For a quick install link, set `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local` to your app's install URL (`https://github.com/apps//installations/new`). +- For a quick install link, set `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local` to your app’s install URL (`https://github.com/apps//installations/new`). - To create a new app via the backend, open `http://localhost:3000/github/setup` (requires `HOSTNAME` set to a reachable URL for callbacks). ## Troubleshooting diff --git a/docs/docs.json b/docs/docs.json index 580f98a31..9713d3488 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -223,4 +223,4 @@ "linkedin": "https://www.linkedin.com/company/diggerhq/" } } -} +} \ No newline at end of file diff --git a/go.work.sum b/go.work.sum index 9a906912f..e9d5d226e 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1454,6 +1454,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= @@ -1622,6 +1623,7 @@ github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= @@ -2031,6 +2033,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= @@ -2139,6 +2142,7 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMo github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= @@ -2179,7 +2183,6 @@ github.com/segmentio/conf v1.2.0 h1:5OT9+6OyVHLsFLsiJa/2KlqiA1m7mpdUBlkB/qYTMts= github.com/segmentio/conf v1.2.0/go.mod h1:Y3B9O/PqqWqjyxyWWseyj/quPEtMu1zDp/kVbSWWaB0= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU= github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index cfa95c4f7..26d8388ca 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -490,55 +490,7 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, options GithubCheckRu opts.Conclusion = github.String(*conclusion) } - checkRun, resp, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts) - - // Log rate limit information - if resp != nil { - limit := resp.Header.Get("X-RateLimit-Limit") - remaining := resp.Header.Get("X-RateLimit-Remaining") - reset := resp.Header.Get("X-RateLimit-Reset") - - if limit != "" && remaining != "" { - limitInt, _ := strconv.Atoi(limit) - remainingInt, _ := strconv.Atoi(remaining) - - // Calculate percentage remaining - var percentRemaining float64 - if limitInt > 0 { - percentRemaining = (float64(remainingInt) / float64(limitInt)) * 100 - } - - // Log based on severity - if remainingInt == 0 { - slog.Error("GitHub API rate limit EXHAUSTED", - "operation", "UpdateCheckRun", - "checkRunId", checkRunId, - "limit", limit, - "remaining", remaining, - "reset", reset, - "owner", owner, - "repo", repoName) - } else if percentRemaining < 20 { - slog.Warn("GitHub API rate limit getting LOW", - "operation", "UpdateCheckRun", - "checkRunId", checkRunId, - "limit", limit, - "remaining", remaining, - "percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining), - "reset", reset, - "owner", owner, - "repo", repoName) - } else { - slog.Debug("GitHub API rate limit status", - "operation", "UpdateCheckRun", - "checkRunId", checkRunId, - "limit", limit, - "remaining", remaining, - "percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining)) - } - } - } - + checkRun, _, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts) if err != nil { slog.Error("Failed to update check run", "inputCheckRunId", checkRunId,