From aed6af6158d14ef188c0f534c07e8d2efbb375a2 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 5 Nov 2020 20:04:30 +0100 Subject: [PATCH 1/4] Doctor: CountOrFixUpdatableCodeCommentReplies --- cmd/doctor.go | 23 ++++++++++++++++ models/consistency.go | 61 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/cmd/doctor.go b/cmd/doctor.go index 2ca2bb5e70b6a..498cc0c07cc5c 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -155,9 +155,32 @@ var checklist = []check{ isDefault: false, f: runDoctorEnablePushOptions, }, + { + title: "Recalculate Code-comment-replies commitID", + name: "recalculate_code_comment_replies", + isDefault: true, + f: runDoctorUpdateCodeCommentReplies, + }, // more checks please append here } +func runDoctorUpdateCodeCommentReplies(ctx *cli.Context) ([]string, error) { + if ctx.Bool("fix") { + _, result, err := models.CountOrFixUpdatableCodeCommentReplies(true) + if err != nil { + return nil, err + } + return result, nil + } else { + count, _, err := models.CountOrFixUpdatableCodeCommentReplies(false) + if err != nil { + return nil, err + } + result := fmt.Sprintf("%d code comment replies without commitID exist", count) + return []string{result}, nil + } +} + func runRecreateTable(ctx *cli.Context) error { // Redirect the default golog to here golog.SetFlags(0) diff --git a/models/consistency.go b/models/consistency.go index fbb99ca80c883..2894290174d19 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -5,6 +5,7 @@ package models import ( + "fmt" "reflect" "strings" "testing" @@ -295,3 +296,63 @@ func FixNullArchivedRepository() (int64, error) { IsArchived: false, }) } + +func CountOrFixUpdatableCodeCommentReplies(fix bool) (int64, []string, error) { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return 0, nil, err + } + + var ( + start = 0 + batchSize = 100 + count int64 + result []string + ) + + for { + var comments = make([]*Comment, 0, batchSize) + if err := sess.SQL(`SELECT comment.id as id, first.commit_sha as commit_sha, first.patch as patch, first.invalidated as invalidated + FROM comment INNER JOIN ( + SELECT C.id, C.review_id, C.line, C.tree_path, C.patch, C.commit_sha, C.invalidated + FROM comment AS C + WHERE C.type = 21 + AND C.created_unix = + (SELECT MIN(comment.created_unix) + FROM comment + WHERE comment.review_id = C.review_id + AND comment.type = 21 + AND comment.line = C.line + AND comment.tree_path = C.tree_path) + ) AS first + ON comment.review_id = first.review_id + AND comment.tree_path = first.tree_path AND comment.line = first.line + WHERE comment.type = 21 + AND comment.id != first.id + AND comment.commit_sha != first.commit_sha`).Limit(batchSize, start).Find(&comments); err != nil { + return 0, nil, fmt.Errorf("failed to select: %v", err) + } + + if fix { + for _, comment := range comments { + if _, err := sess.Table("comment").Cols("commit_sha", "patch", "invalidated").Update(comment); err != nil { + return 0, nil, fmt.Errorf("failed to update comment[%d]: %v %v", comment.ID, comment, err) + } + result = append(result, fmt.Sprintf("update comment[%d]: %s\n", comment.ID, comment.CommitSHA)) + } + } + + count += int64(len(comments)) + start += len(comments) + if len(comments) < batchSize { + break + } + } + + if fix { + return count, result, sess.Commit() + } + + return count, nil, nil +} From 8456c464e39e71eef32bc88da497d2d5c2adb50a Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 5 Nov 2020 20:26:37 +0100 Subject: [PATCH 2/4] Update models/consistency.go --- models/consistency.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/consistency.go b/models/consistency.go index 2894290174d19..6e4d516b4c926 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -297,6 +297,7 @@ func FixNullArchivedRepository() (int64, error) { }) } +// CountOrFixUpdatableCodeCommentReplies count or fix CodeCommentReplies missing the CommitSHA func CountOrFixUpdatableCodeCommentReplies(fix bool) (int64, []string, error) { sess := x.NewSession() defer sess.Close() From 193430e3bd862410a808299f0cccc2de61a51c89 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 5 Nov 2020 20:32:26 +0100 Subject: [PATCH 3/4] Update cmd/doctor.go --- cmd/doctor.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/doctor.go b/cmd/doctor.go index 498cc0c07cc5c..9d1b07876b8e0 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -171,14 +171,13 @@ func runDoctorUpdateCodeCommentReplies(ctx *cli.Context) ([]string, error) { return nil, err } return result, nil - } else { - count, _, err := models.CountOrFixUpdatableCodeCommentReplies(false) - if err != nil { - return nil, err - } - result := fmt.Sprintf("%d code comment replies without commitID exist", count) - return []string{result}, nil } + count, _, err := models.CountOrFixUpdatableCodeCommentReplies(false) + if err != nil { + return nil, err + } + result := fmt.Sprintf("%d code comment replies without commitID exist", count) + return []string{result}, nil } func runRecreateTable(ctx *cli.Context) error { From 67f7bacb1ed5c781a95df714323745c88d308969 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 10 Nov 2020 00:28:38 +0100 Subject: [PATCH 4/4] apply fixes done in #13448 --- models/consistency.go | 89 +++++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/models/consistency.go b/models/consistency.go index 6e4d516b4c926..4a1d2b02a613c 100644 --- a/models/consistency.go +++ b/models/consistency.go @@ -7,9 +7,12 @@ package models import ( "fmt" "reflect" + "strconv" "strings" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" "xorm.io/builder" ) @@ -299,45 +302,70 @@ func FixNullArchivedRepository() (int64, error) { // CountOrFixUpdatableCodeCommentReplies count or fix CodeCommentReplies missing the CommitSHA func CountOrFixUpdatableCodeCommentReplies(fix bool) (int64, []string, error) { - sess := x.NewSession() - defer sess.Close() - if err := sess.Begin(); err != nil { - return 0, nil, err - } var ( start = 0 batchSize = 100 + sqlCmd string count int64 result []string ) + sqlSelect := `SELECT comment.id as id, first.commit_sha as commit_sha, first.patch as patch, first.invalidated as invalidated` + sqlTail := ` FROM comment INNER JOIN ( + SELECT C.id, C.review_id, C.line, C.tree_path, C.patch, C.commit_sha, C.invalidated + FROM comment AS C + WHERE C.type = 21 + AND C.created_unix = + (SELECT MIN(comment.created_unix) + FROM comment + WHERE comment.review_id = C.review_id + AND comment.type = 21 + AND comment.line = C.line + AND comment.tree_path = C.tree_path) + ) AS first + ON comment.review_id = first.review_id + AND comment.tree_path = first.tree_path AND comment.line = first.line + WHERE comment.type = 21 + AND comment.id != first.id + AND comment.commit_sha != first.commit_sha` + + sess := x.NewSession() + defer sess.Close() for { + if err := sess.Begin(); err != nil { + return 0, nil, err + } + + if setting.Database.UseMSSQL { + if _, err := sess.Exec(sqlSelect + " INTO #temp_comments" + sqlTail); err != nil { + return 0, nil, fmt.Errorf("unable to create temporary table") + } + } + var comments = make([]*Comment, 0, batchSize) - if err := sess.SQL(`SELECT comment.id as id, first.commit_sha as commit_sha, first.patch as patch, first.invalidated as invalidated - FROM comment INNER JOIN ( - SELECT C.id, C.review_id, C.line, C.tree_path, C.patch, C.commit_sha, C.invalidated - FROM comment AS C - WHERE C.type = 21 - AND C.created_unix = - (SELECT MIN(comment.created_unix) - FROM comment - WHERE comment.review_id = C.review_id - AND comment.type = 21 - AND comment.line = C.line - AND comment.tree_path = C.tree_path) - ) AS first - ON comment.review_id = first.review_id - AND comment.tree_path = first.tree_path AND comment.line = first.line - WHERE comment.type = 21 - AND comment.id != first.id - AND comment.commit_sha != first.commit_sha`).Limit(batchSize, start).Find(&comments); err != nil { + + switch { + case setting.Database.UseMySQL: + sqlCmd = sqlSelect + sqlTail + " LIMIT " + strconv.Itoa(batchSize) + ", " + strconv.Itoa(start) + case setting.Database.UsePostgreSQL: + fallthrough + case setting.Database.UseSQLite3: + sqlCmd = sqlSelect + sqlTail + " LIMIT " + strconv.Itoa(batchSize) + " OFFSET " + strconv.Itoa(start) + case setting.Database.UseMSSQL: + sqlCmd = "SELECT TOP " + strconv.Itoa(batchSize) + " * FROM #temp_comments WHERE " + + "(id NOT IN ( SELECT TOP " + strconv.Itoa(start) + " id FROM #temp_comments ORDER BY id )) ORDER BY id" + default: + return 0, nil, fmt.Errorf("Unsupported database type") + } + + if err := sess.SQL(sqlCmd).Find(&comments); err != nil { return 0, nil, fmt.Errorf("failed to select: %v", err) } if fix { for _, comment := range comments { - if _, err := sess.Table("comment").Cols("commit_sha", "patch", "invalidated").Update(comment); err != nil { + if _, err := sess.Table("comment").ID(comment.ID).Cols("commit_sha", "patch", "invalidated").Update(comment); err != nil { return 0, nil, fmt.Errorf("failed to update comment[%d]: %v %v", comment.ID, comment, err) } result = append(result, fmt.Sprintf("update comment[%d]: %s\n", comment.ID, comment.CommitSHA)) @@ -346,14 +374,17 @@ func CountOrFixUpdatableCodeCommentReplies(fix bool) (int64, []string, error) { count += int64(len(comments)) start += len(comments) + + if fix { + if err := sess.Commit(); err != nil { + return count, result, err + } + } + if len(comments) < batchSize { break } } - if fix { - return count, result, sess.Commit() - } - - return count, nil, nil + return count, result, nil }