Skip to content
Open
42 changes: 24 additions & 18 deletions backend/controllers/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,32 +66,38 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
"installationId", *event.Installation.ID,
)

if *event.Action == "deleted" {
err := handleInstallationDeletedEvent(event, appId64)
if err != nil {
slog.Error("Failed to handle installation deleted event", "error", err)
c.String(http.StatusAccepted, "Failed to handle webhook event.")
return
}
} else if *event.Action == "created" || *event.Action == "unsuspended" || *event.Action == "new_permissions_accepted" {
if err := handleInstallationUpsertEvent(c.Request.Context(), gh, event, appId64); err != nil {
slog.Error("Failed to handle installation upsert event", "error", err)
c.String(http.StatusAccepted, "Failed to handle webhook event.")
return
// Run in goroutine to avoid webhook timeouts for large installations
go func(ctx context.Context) {
defer logging.InheritRequestLogger(ctx)()
if *event.Action == "deleted" {
if err := handleInstallationDeletedEvent(event, appId64); err != nil {
slog.Error("Failed to handle installation deleted event", "error", err)
}
} else if *event.Action == "created" || *event.Action == "unsuspended" || *event.Action == "new_permissions_accepted" {
if err := handleInstallationUpsertEvent(c.Request.Context(), gh, event, appId64); err != nil {
slog.Error("Failed to handle installation upsert event", "error", err)
c.String(http.StatusAccepted, "Failed to handle webhook event.")
return
}
}
}
}(c.Request.Context())

case *github.InstallationRepositoriesEvent:
slog.Info("Processing InstallationRepositoriesEvent",
"action", event.GetAction(),
"installationId", event.Installation.GetID(),
"added", len(event.RepositoriesAdded),
"removed", len(event.RepositoriesRemoved),
)
if err := handleInstallationRepositoriesEvent(c.Request.Context(), gh, event, appId64); err != nil {
slog.Error("Failed to handle installation repositories event", "error", err)
c.String(http.StatusAccepted, "Failed to handle webhook event.")
return
}

// Run in goroutine to avoid webhook timeouts for large installations
go func(ctx context.Context) {
defer logging.InheritRequestLogger(ctx)()
// Use background context so work continues after HTTP response
if err := handleInstallationRepositoriesEvent(context.Background(), gh, event, appId64); err != nil {
slog.Error("Failed to handle installation repositories event", "error", err)
}
}(c.Request.Context())
case *github.PushEvent:
slog.Info("Processing PushEvent",
"repo", *event.Repo.FullName,
Expand Down
4 changes: 4 additions & 0 deletions backend/controllers/github_callback.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
code := ""
if codeExists && len(codeParams) > 0 && len(codeParams[0]) > 0 {
code = codeParams[0]
} else {
slog.Debug("No code parameter found, probably a setup update, going to return success since we are relying on webhooks now")
c.HTML(http.StatusOK, "github_success.tmpl", gin.H{})
return
}

appId := c.Request.URL.Query().Get("state")
Expand Down
2 changes: 1 addition & 1 deletion backend/controllers/github_comment.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
}

diggerYmlStr, ghService, config, projectsGraph, prSourceBranch, commitSha, changedFiles, err := getDiggerConfigForPR(gh, orgId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
if err != nil {
if err != nil {
slog.Error("Error getting Digger config for PR",
"issueNumber", issueNumber,
"repoFullName", repoFullName,
Expand Down
4 changes: 2 additions & 2 deletions backend/controllers/github_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -830,7 +830,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
"branch", prBranch,
"error", err,
)
return "", nil, nil, nil, nil, nil, nil, fmt.Errorf("error loading digger.yml: %v", err)
return "", nil, nil, nil, nil, nil, nil, fmt.Errorf("error loading digger.yml: %w", err)
}

return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, changedFiles, nil
Expand Down Expand Up @@ -893,7 +893,7 @@ func GetDiggerConfigForBranchOrSha(gh utils.GithubClientProvider, installationId
"branch", branch,
"error", err,
)
return "", nil, nil, nil, fmt.Errorf("error cloning and loading config %v", err)
return "", nil, nil, nil, fmt.Errorf("error cloning and loading config: %w", err)
}

projectCount := 0
Expand Down
24 changes: 17 additions & 7 deletions backend/controllers/github_pull_request.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"runtime/debug"
"slices"
"strconv"
"strings"

"github.com/diggerhq/digger/backend/ci_backends"
config2 "github.com/diggerhq/digger/backend/config"
Expand Down Expand Up @@ -138,16 +137,27 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
return nil
}

// Silently skip repos without digger.yml - this is expected for org-wide installations
if strings.Contains(err.Error(), "could not find digger.yml") ||
strings.Contains(err.Error(), "could not find digger.yaml") {
slog.Info("No Digger config found, skipping repo",
// Check if the error is due to missing digger config and the app is installed for all repos
if errors.Is(err, digger_config.ErrDiggerConfigNotFound) {
slog.Debug("Digger config not found, checking if app is installed for all repos",
"prNumber", prNumber,
"repoFullName", repoFullName,
)
return nil
isAllRepos, checkErr := utils.IsAllReposInstallation(appId, installationId)
if checkErr != nil {
slog.Warn("Failed to check if installation is for all repos",
"error", checkErr,
)
} else if isAllRepos {
slog.Info("Digger config not found but GitHub App is installed for all repos, skipping error comment",
"prNumber", prNumber,
"repoFullName", repoFullName,
)
return nil
}
}


slog.Error("Error getting Digger config for PR",
"prNumber", prNumber,
"repoFullName", repoFullName,
Expand Down Expand Up @@ -515,7 +525,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not retrieve created batch: %v", err))
return fmt.Errorf("error getting digger batch")
}

if config.CommentRenderMode == digger_config.CommentRenderModeGroupByModule {
slog.Info("Using GroupByModule render mode for comments", "prNumber", prNumber)

Expand Down
2 changes: 0 additions & 2 deletions backend/controllers/projects.go
Original file line number Diff line number Diff line change
Expand Up @@ -1026,7 +1026,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed batch"})
return
}
//err = UpdateCheckStatusForBatch(d.GithubClientProvider, refreshedBatch)
slog.Debug("Attempting to update GitHub Check Run for batch",
"batchId", batch.ID,
"checkRunId", refreshedBatch.CheckRunId,
Expand Down Expand Up @@ -1056,7 +1055,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed job"})
return
}
//err = UpdateCommitStatusForJob(d.GithubClientProvider, refreshedJob)
slog.Debug("Attempting to update GitHub Check Run for job",
"jobId", jobId,
"checkRunId", refreshedJob.CheckRunId,
Expand Down
19 changes: 13 additions & 6 deletions backend/controllers/projects_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,9 +210,12 @@ func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerB
return fmt.Errorf("error generating realtime comment message: %v", err)
}

summary, err := GenerateChecksSummaryForBatch(batch)
if err != nil {
slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
var summary = ""
if batch.Status == orchestrator_scheduler.BatchJobSucceeded || batch.Status == orchestrator_scheduler.BatchJobFailed {
summary, err = GenerateChecksSummaryForBatch(batch)
if err != nil {
slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
}
}

if isPlanBatch {
Expand Down Expand Up @@ -397,11 +400,15 @@ func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob)
"```\n"


summary, err := GenerateChecksSummaryForJob(job)
if err != nil {
slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
var summary = ""
if job.Status == orchestrator_scheduler.DiggerJobSucceeded || job.Status == orchestrator_scheduler.DiggerJobFailed {
summary, err = GenerateChecksSummaryForJob(job)
if err != nil {
slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
}
}


slog.Debug("Updating PR status for job", "jobId", job.DiggerJobID, "status", status, "conclusion", conclusion)
if isPlan {
title := fmt.Sprintf("%v to create %v to update %v to delete", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted)
Expand Down
8 changes: 4 additions & 4 deletions backend/models/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,9 @@ func GetCheckRunConclusionForJob(job *DiggerJob) (string, error) {
return "failure", nil
}
slog.Error("Unknown job status in GetCheckRunConclusionForJob - this will cause GitHub API 422 error",
"jobId", job.DiggerJobID,
"jobStatus", job.Status,
"jobStatusInt", int(job.Status),
"validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"})
"jobId", job.DiggerJobID,
"jobStatus", job.Status,
"jobStatusInt", int(job.Status),
"validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"})
return "", fmt.Errorf("unknown job status: %v", job.Status)
}
53 changes: 53 additions & 0 deletions backend/utils/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,59 @@ func GetGithubHostname() string {
return githubHostname
}

// IsAllReposInstallation checks if the GitHub App installation is configured to access all repositories
// (as opposed to a selected subset). Returns true if installation is for "all" repos.
// Note: This requires app-level JWT authentication, not installation token authentication.
func IsAllReposInstallation(appId int64, installationId int64) (bool, error) {
githubAppPrivateKey := ""
githubAppPrivateKeyB64 := os.Getenv("GITHUB_APP_PRIVATE_KEY_BASE64")
if githubAppPrivateKeyB64 != "" {
decodedBytes, err := base64.StdEncoding.DecodeString(githubAppPrivateKeyB64)
if err != nil {
slog.Error("Failed to decode GITHUB_APP_PRIVATE_KEY_BASE64", "error", err)
return false, fmt.Errorf("error decoding private key: %v", err)
}
githubAppPrivateKey = string(decodedBytes)
} else {
githubAppPrivateKey = os.Getenv("GITHUB_APP_PRIVATE_KEY")
if githubAppPrivateKey == "" {
return false, fmt.Errorf("missing GitHub app private key")
}
}

// Use app-level transport (JWT) instead of installation token
atr, err := ghinstallation.NewAppsTransport(net.DefaultTransport, appId, []byte(githubAppPrivateKey))
if err != nil {
slog.Error("Failed to create GitHub app transport",
"appId", appId,
"error", err,
)
return false, fmt.Errorf("error creating app transport: %v", err)
}

client := github.NewClient(&net.Client{Transport: atr})

installation, _, err := client.Apps.GetInstallation(context.Background(), installationId)
if err != nil {
slog.Error("Failed to get GitHub installation details",
"installationId", installationId,
"error", err,
)
return false, fmt.Errorf("error getting installation details: %v", err)
}

repositorySelection := installation.GetRepositorySelection()
isAllRepos := repositorySelection == "all"

slog.Debug("Checked installation repository selection",
"installationId", installationId,
"repositorySelection", repositorySelection,
"isAllRepos", isAllRepos,
)

return isAllRepos, nil
}

func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, repoName string, diggerJobID string) (int64, string, error) {
slog.Debug("Looking for workflow for job",
"diggerJobId", diggerJobID,
Expand Down
4 changes: 2 additions & 2 deletions docs/ce/local-development/backend.mdx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
---
title: Orchestrator local setup
title: Backend (orchestrator) local setup
---

The backend serves orchestration APIs, GitHub app endpoints, and internal APIs the UI relies on.
Expand Down Expand Up @@ -51,7 +51,7 @@ The backend serves orchestration APIs, GitHub app endpoints, and internal APIs t

## GitHub app integration

- For a quick install link, set `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local` to your app's install URL (`https://github.com/apps/<app>/installations/new`).
- For a quick install link, set `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local` to your apps install URL (`https://github.com/apps/<app>/installations/new`).
- To create a new app via the backend, open `http://localhost:3000/github/setup` (requires `HOSTNAME` set to a reachable URL for callbacks).

## Troubleshooting
Expand Down
2 changes: 1 addition & 1 deletion docs/docs.json
Original file line number Diff line number Diff line change
Expand Up @@ -223,4 +223,4 @@
"linkedin": "https://www.linkedin.com/company/diggerhq/"
}
}
}
}
5 changes: 4 additions & 1 deletion go.work.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1454,6 +1454,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
Expand Down Expand Up @@ -1622,6 +1623,7 @@ github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td
github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg=
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
Expand Down Expand Up @@ -2031,6 +2033,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
Expand Down Expand Up @@ -2139,6 +2142,7 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMo
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
Expand Down Expand Up @@ -2179,7 +2183,6 @@ github.com/segmentio/conf v1.2.0 h1:5OT9+6OyVHLsFLsiJa/2KlqiA1m7mpdUBlkB/qYTMts=
github.com/segmentio/conf v1.2.0/go.mod h1:Y3B9O/PqqWqjyxyWWseyj/quPEtMu1zDp/kVbSWWaB0=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU=
github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
Expand Down
50 changes: 1 addition & 49 deletions libs/ci/github/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -490,55 +490,7 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, options GithubCheckRu
opts.Conclusion = github.String(*conclusion)
}

checkRun, resp, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts)

// Log rate limit information
if resp != nil {
limit := resp.Header.Get("X-RateLimit-Limit")
remaining := resp.Header.Get("X-RateLimit-Remaining")
reset := resp.Header.Get("X-RateLimit-Reset")

if limit != "" && remaining != "" {
limitInt, _ := strconv.Atoi(limit)
remainingInt, _ := strconv.Atoi(remaining)

// Calculate percentage remaining
var percentRemaining float64
if limitInt > 0 {
percentRemaining = (float64(remainingInt) / float64(limitInt)) * 100
}

// Log based on severity
if remainingInt == 0 {
slog.Error("GitHub API rate limit EXHAUSTED",
"operation", "UpdateCheckRun",
"checkRunId", checkRunId,
"limit", limit,
"remaining", remaining,
"reset", reset,
"owner", owner,
"repo", repoName)
} else if percentRemaining < 20 {
slog.Warn("GitHub API rate limit getting LOW",
"operation", "UpdateCheckRun",
"checkRunId", checkRunId,
"limit", limit,
"remaining", remaining,
"percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining),
"reset", reset,
"owner", owner,
"repo", repoName)
} else {
slog.Debug("GitHub API rate limit status",
"operation", "UpdateCheckRun",
"checkRunId", checkRunId,
"limit", limit,
"remaining", remaining,
"percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining))
}
}
}

checkRun, _, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts)
if err != nil {
slog.Error("Failed to update check run",
"inputCheckRunId", checkRunId,
Expand Down
Loading
Loading