Add commit status summary table to reduce query from commit status table

models/git/commit_status.go

@@ -292,30 +292,27 @@ func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOp
 }
 
 // GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
-func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) {
+func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map[int64][]*CommitStatus, error) {
 	type result struct {
 		Index  int64
 		RepoID int64
+		SHA    string
 	}
 
-	results := make([]result, 0, len(repoIDsToLatestCommitSHAs))
+	results := make([]result, 0, len(repoSHAs))
 
 	getBase := func() *xorm.Session {
 		return db.GetEngine(ctx).Table(&CommitStatus{})
 	}
 
 	// Create a disjunction of conditions for each repoID and SHA pair
-	conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs))
-	for repoID, sha := range repoIDsToLatestCommitSHAs {
-		conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha})
+	conds := make([]builder.Cond, 0, len(repoSHAs))
+	for _, repoSHA := range repoSHAs {
+		conds = append(conds, builder.Eq{"repo_id": repoSHA.RepoID, "sha": repoSHA.SHA})
 	}
 	sess := getBase().Where(builder.Or(conds...)).
-		Select("max( `index` ) as `index`, repo_id").
-		GroupBy("context_hash, repo_id").OrderBy("max( `index` ) desc")
-
-	if !listOptions.IsListAll() {
-		sess = db.SetSessionPagination(sess, &listOptions)
-	}
+		Select("max( `index` ) as `index`, repo_id, sha").
+		GroupBy("context_hash, repo_id, sha").OrderBy("max( `index` ) desc")
 
 	err := sess.Find(&results)
 	if err != nil {
@@ -332,7 +329,7 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHA
 			cond := builder.Eq{
 				"`index`": result.Index,
 				"repo_id": result.RepoID,
-				"sha":     repoIDsToLatestCommitSHAs[result.RepoID],
+				"sha":     result.SHA,
 			}
 			conds = append(conds, cond)
 		}

models/git/commit_status_summary.go

@@ -0,0 +1,84 @@
+// Copyright 2024 Gitea. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package git
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/setting"
+	api "code.gitea.io/gitea/modules/structs"
+
+	"xorm.io/builder"
+)
+
+// CommitStatusSummary holds the latest commit Status of a single Commit
+type CommitStatusSummary struct {
+	ID     int64                 `xorm:"pk autoincr"`
+	RepoID int64                 `xorm:"INDEX UNIQUE(repo_id_sha)"`
+	SHA    string                `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
+	State  api.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
+}
+
+func init() {
+	db.RegisterModel(new(CommitStatusSummary))
+}
+
+type RepoSHA struct {
+	RepoID int64
+	SHA    string
+}
+
+func GetLatestCommitStatusForRepoAndSHAs(ctx context.Context, repoSHAs []RepoSHA) ([]*CommitStatus, error) {
+	cond := builder.NewCond()
+	for _, rs := range repoSHAs {
+		cond = cond.Or(builder.Eq{"repo_id": rs.RepoID, "sha": rs.SHA})
+	}
+
+	var summaries []CommitStatusSummary
+	if err := db.GetEngine(ctx).Where(cond).Find(&summaries); err != nil {
+		return nil, err
+	}
+
+	commitStatuses := make([]*CommitStatus, 0, len(repoSHAs))
+	for _, summary := range summaries {
+		commitStatuses = append(commitStatuses, &CommitStatus{
+			RepoID: summary.RepoID,
+			SHA:    summary.SHA,
+			State:  summary.State,
+		})
+	}
+	return commitStatuses, nil
+}
+
+func UpdateCommitStatusSummary(ctx context.Context, repoID int64, sha string) error {
+	commitStatuses, _, err := GetLatestCommitStatus(ctx, repoID, sha, db.ListOptionsAll)
+	if err != nil {
+		return err
+	}
+	state := CalcCommitStatus(commitStatuses)
+	// mysql will return 0 when update a record which state hasn't been changed which behaviour is different from other database,
+	// so we need to use insert in on duplicate
+	if setting.Database.Type.IsMySQL() {
+		_, err := db.GetEngine(ctx).Exec("INSERT INTO commit_status_summary (repo_id,sha,state) VALUES (?,?,?) ON DUPLICATE KEY UPDATE state=?",
+			repoID, sha, state.State, state.State)
+		return err
+	}
+
+	if cnt, err := db.GetEngine(ctx).Where("repo_id=? AND sha=?", repoID, sha).
+		Cols("state").
+		Update(&CommitStatusSummary{
+			State: state.State,
+		}); err != nil {
+		return err
+	} else if cnt == 0 {
+		_, err = db.GetEngine(ctx).Insert(&CommitStatusSummary{
+			RepoID: repoID,
+			SHA:    sha,
+			State:  state.State,
+		})
+		return err
+	}
+	return nil
+}

models/migrations/migrations.go

@@ -576,7 +576,10 @@ var migrations = []Migration{
 
 	// Gitea 1.22.0 ends at 294
 
+	// v294 -> v295
 	NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue),
+	// v295 -> v296
+	NewMigration("Add commit status summary table", v1_23.AddCommitStatusSummary),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_23/v295.go

@@ -0,0 +1,18 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_23 //nolint
+
+import "xorm.io/xorm"
+
+func AddCommitStatusSummary(x *xorm.Engine) error {
+	type CommitStatusSummary struct {
+		ID     int64  `xorm:"pk autoincr"`
+		RepoID int64  `xorm:"INDEX UNIQUE(repo_id_sha)"`
+		SHA    string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
+		State  string `xorm:"VARCHAR(7) NOT NULL"`
+	}
+	// there is no migrations because if there is no data on this table, it will fall back to get data
+	// from commit status
+	return x.Sync2(new(CommitStatusSummary))
+}

services/actions/commit_status.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	api "code.gitea.io/gitea/modules/structs"
 	webhook_module "code.gitea.io/gitea/modules/webhook"
+	commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
 
 	"github.com/nektos/act/pkg/jobparser"
 )
@@ -122,18 +123,13 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er
 	if err != nil {
 		return fmt.Errorf("HashTypeInterfaceFromHashString: %w", err)
 	}
-	if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
-		Repo:    repo,
-		SHA:     commitID,
-		Creator: creator,
-		CommitStatus: &git_model.CommitStatus{
-			SHA:         sha,
-			TargetURL:   fmt.Sprintf("%s/jobs/%d", run.Link(), index),
-			Description: description,
-			Context:     ctxname,
-			CreatorID:   creator.ID,
-			State:       state,
-		},
+	if err := commitstatus_service.CreateCommitStatus(ctx, repo, creator, commitID.String(), &git_model.CommitStatus{
+		SHA:         sha,
+		TargetURL:   fmt.Sprintf("%s/jobs/%d", run.Link(), index),
+		Description: description,
+		Context:     ctxname,
+		CreatorID:   creator.ID,
+		State:       state,
 	}); err != nil {
 		return fmt.Errorf("NewCommitStatus: %w", err)
 	}

services/repository/commitstatus/commitstatus.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
+	"slices"
 
 	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
@@ -59,13 +60,19 @@ func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creato
 		sha = commit.ID.String()
 	}
 
-	if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
-		Repo:         repo,
-		Creator:      creator,
-		SHA:          commit.ID,
-		CommitStatus: status,
+	if err := db.WithTx(ctx, func(ctx context.Context) error {
+		if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
+			Repo:         repo,
+			Creator:      creator,
+			SHA:          commit.ID,
+			CommitStatus: status,
+		}); err != nil {
+			return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
+		}
+
+		return git_model.UpdateCommitStatusSummary(ctx, repo.ID, commit.ID.String())
 	}); err != nil {
-		return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
+		return err
 	}
 
 	defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
@@ -114,8 +121,35 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep
 		return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
 	}
 
+	var repoSHAs []git_model.RepoSHA
+	for id, sha := range repoIDsToLatestCommitSHAs {
+		repoSHAs = append(repoSHAs, git_model.RepoSHA{RepoID: id, SHA: sha})
+	}
+
+	summaryResults, err := git_model.GetLatestCommitStatusForRepoAndSHAs(ctx, repoSHAs)
+	if err != nil {
+		return nil, fmt.Errorf("GetLatestCommitStatusForRepoAndSHAs: %v", err)
+	}
+
+	for _, summary := range summaryResults {
+		for i, repo := range repos {
+			if repo.ID == summary.RepoID {
+				results[i] = summary
+				_ = slices.DeleteFunc(repoSHAs, func(repoSHA git_model.RepoSHA) bool {
+					return repoSHA.RepoID == repo.ID
+				})
+				if results[i].State != "" {
+					if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
+						log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
+					}
+				}
+				break
+			}
+		}
+	}
+
 	// call the database O(1) times to get the commit statuses for all repos
-	repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll)
+	repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoSHAs)
 	if err != nil {
 		return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
 	}

tests/integration/pull_status_test.go

@@ -12,6 +12,9 @@ import (
 	"testing"
 
 	auth_model "code.gitea.io/gitea/models/auth"
+	git_model "code.gitea.io/gitea/models/git"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
 	api "code.gitea.io/gitea/modules/structs"
 
 	"github.com/stretchr/testify/assert"
@@ -90,6 +93,10 @@ func TestPullCreate_CommitStatus(t *testing.T) {
 			assert.True(t, ok)
 			assert.Contains(t, cls, statesIcons[status])
 		}
+
+		repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"})
+		css := unittest.AssertExistsAndLoadBean(t, &git_model.CommitStatusSummary{RepoID: repo1.ID, SHA: commitID})
+		assert.EqualValues(t, api.CommitStatusWarning, css.State)
 	})
 }