From 5c01fc0b3d0ab25e6efc5a04278e9619db0b6972 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Tue, 22 Mar 2022 16:23:13 +0800 Subject: [PATCH 01/29] add safe mirror option --- models/repo/mirror.go | 11 +- options/locale/locale_en-US.ini | 2 + options/locale/locale_zh-CN.ini | 8 +- routers/web/repo/setting.go | 2 + services/forms/repo_form.go | 1 + services/mirror/mirror_pull.go | 12 ++ services/mirror/mirror_pull_safe_check.go | 202 ++++++++++++++++++++++ templates/repo/settings/options.tmpl | 7 + 8 files changed, 237 insertions(+), 8 deletions(-) create mode 100644 services/mirror/mirror_pull_safe_check.go diff --git a/models/repo/mirror.go b/models/repo/mirror.go index 8494331ff710a..c5bb0e36bf756 100644 --- a/models/repo/mirror.go +++ b/models/repo/mirror.go @@ -28,11 +28,12 @@ type RemoteMirrorer interface { // Mirror represents mirror information of a repository. type Mirror struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"INDEX"` - Repo *Repository `xorm:"-"` - Interval time.Duration - EnablePrune bool `xorm:"NOT NULL DEFAULT true"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` + Repo *Repository `xorm:"-"` + Interval time.Duration + EnablePrune bool `xorm:"NOT NULL DEFAULT true"` + EnableSafeMirror bool `xorm:"NOT NULL DEFAULT true"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0627735b64916..31deb6a92fe89 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -869,6 +869,8 @@ mirror_last_synced = Last Synchronized mirror_password_placeholder = (Unchanged) mirror_password_blank_placeholder = (Unset) mirror_password_help = Change the username to erase a stored password. +mirror_enable_safe = Safe Mirror +mirror_enable_safe_desc = Enable safe mirror to avoid delete local repository data watchers = Watchers stargazers = Stargazers forks = Forks diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index b92151447dd83..473e4044bafa0 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -690,8 +690,8 @@ last_used=上次使用在 no_activity=没有最近活动 can_read_info=读取 can_write_info=写入 -key_state_desc=7 天内使用过该密钥 -token_state_desc=7 天内使用过该密钥 +key_state_desc=7 天内使用过该密钥 +token_state_desc=7 天内使用过该密钥 principal_state_desc=7 天内使用过该规则 show_openid=在个人信息上显示 hide_openid=在个人信息上隐藏 @@ -868,11 +868,13 @@ mirror_last_synced=上次同步 mirror_password_placeholder=(未更改) mirror_password_blank_placeholder=(未设置) mirror_password_help=更改用户名以删除已储存的密码。 +mirror_enable_safe = 安全镜像 +mirror_enable_safe_desc = 开启安全镜像来避免镜像库出问题引起本地镜像仓库数据被删除 watchers=关注者 stargazers=称赞者 forks=派生仓库 pick_reaction=选择你的表情 -reactions_more=再加载 %d +reactions_more=再加载 %d unit_disabled=站点管理员已禁用此仓库单元。 language_other=其它 adopt_search=输入用户名以搜索未被收录的仓库... (留空以查找全部) diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 5f7b948b8cac5..6f0cd20d88860 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -70,6 +70,7 @@ func Settings(ctx *context.Context) { ctx.Data["SigningKeyAvailable"] = len(signing) > 0 ctx.Data["SigningSettings"] = setting.Repository.Signing ctx.Data["CodeIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled + ctx.Data["EnableSafeMirror"] = ctx.Repo.Mirror.EnableSafeMirror if ctx.User.IsAdmin { if setting.Indexer.RepoIndexerEnabled { status, err := repo_model.GetIndexerStatus(ctx.Repo.Repository, repo_model.RepoIndexerTypeCode) @@ -193,6 +194,7 @@ func SettingsPost(ctx *context.Context) { ctx.RenderWithErr(ctx.Tr("repo.mirror_interval_invalid"), tplSettingsOptions, &form) } else { ctx.Repo.Mirror.EnablePrune = form.EnablePrune + ctx.Repo.Mirror.EnableSafeMirror = form.EnableSafeMirror ctx.Repo.Mirror.Interval = interval if interval != 0 { ctx.Repo.Mirror.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(interval) diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 3760c71f2a1fd..ced27961cff9b 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -130,6 +130,7 @@ type RepoSettingForm struct { Private bool Template bool EnablePrune bool + EnableSafeMirror bool // Advanced settings EnableWiki bool diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 6c9c4a04838d2..ee3b4410e76ad 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -190,6 +190,7 @@ func pruneBrokenReferences(ctx context.Context, func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() + safeMirror := m.EnableSafeMirror timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) @@ -204,6 +205,17 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) } + // detect can safe mirror + err, canUpdate := detectCanUpdateMirror(ctx, m, gitArgs) + if err != nil { + log.Error("CreateRepositoryNotice: %v", err) + return nil, false + } + if safeMirror && !canUpdate { + log.Error("CheckSyncMirrors [repo: %-v]: can not safe mirror...", m.Repo) + return nil, false + } + stdoutBuilder := strings.Builder{} stderrBuilder := strings.Builder{} if err := git.NewCommand(ctx, gitArgs...). diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go new file mode 100644 index 0000000000000..27daa71b591fd --- /dev/null +++ b/services/mirror/mirror_pull_safe_check.go @@ -0,0 +1,202 @@ +package mirror + +import ( + admin_model "code.gitea.io/gitea/models/admin" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// get git command running stdout and stderr +func getGitCommandStdoutStderr(ctx context.Context, m *repo_model.Mirror, gitArgs []string, newRepoPath string) (string, string, error) { + stdoutBuilder := strings.Builder{} + stderrBuilder := strings.Builder{} + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + remoteAddr, remoteErr := git.GetRemoteAddress(ctx, newRepoPath, m.GetRemoteName()) + if remoteErr != nil { + log.Error("GetMirrorCanUpdate [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) + return "", "", remoteErr + } + + if err := git.NewCommand(ctx, gitArgs...). + SetDescription(fmt.Sprintf("Mirror.getMirrorCanUpdate: %s", m.Repo.FullName())). + RunWithContext(&git.RunContext{ + Timeout: timeout, + Dir: newRepoPath, + Stdout: &stdoutBuilder, + Stderr: &stderrBuilder, + }); err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() + sanitizer := util.NewURLSanitizer(remoteAddr, true) + stderrMessage := sanitizer.Replace(stderr) + stdoutMessage := sanitizer.Replace(stdout) + log.Error("CreateRepositoryNotice: %v", err) + stderrMessage = sanitizer.Replace(stderr) + stdoutMessage = sanitizer.Replace(stdout) + desc := fmt.Sprintf("Failed to get mirror repository can update '%s': %s", newRepoPath, stderrMessage) + log.Error("GetMirrorCanUpdate [repo: %-v]: failed to get mirror repository can update:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + if err = admin_model.CreateRepositoryNotice(desc); err != nil { + log.Error("GetMirrorCanUpdateNotice: %v", err) + } + } + stdoutRepoCommitCount := stdoutBuilder.String() + stderrRepoCommitCount := stdoutBuilder.String() + stderrBuilder.Reset() + stdoutBuilder.Reset() + + return stdoutRepoCommitCount, stderrRepoCommitCount, nil +} + +// detect user can update the mirror +func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string) (error, bool) { + repoPath := m.Repo.RepoPath() + newRepoPath := fmt.Sprintf("%s_update", repoPath) + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + //do copy directory recursive + err := util.CopyDir(repoPath, newRepoPath) + defer util.RemoveAll(newRepoPath) + if err != nil { + log.Error("GetMirrorCanUpdate [repo: %-v]: CopyDirectory Error %v", m.Repo, err) + return err, false + } + remoteAddr, remoteErr := git.GetRemoteAddress(ctx, newRepoPath, m.GetRemoteName()) + if remoteErr != nil { + log.Error("GetMirrorCanUpdate [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) + } + + stdoutBuilder := strings.Builder{} + stderrBuilder := strings.Builder{} + if err := git.NewCommand(ctx, gitArgs...). + SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). + RunWithContext(&git.RunContext{ + Timeout: timeout, + Dir: newRepoPath, + Stdout: &stdoutBuilder, + Stderr: &stderrBuilder, + }); err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() + + // sanitize the output, since it may contain the remote address, which may + // contain a password + sanitizer := util.NewURLSanitizer(remoteAddr, true) + stderrMessage := sanitizer.Replace(stderr) + stdoutMessage := sanitizer.Replace(stdout) + + // Now check if the error is a resolve reference due to broken reference + if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") { + log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err) + err = nil + + // Attempt prune + pruneErr := pruneBrokenReferences(ctx, m, newRepoPath, timeout, &stdoutBuilder, &stderrBuilder, sanitizer, false) + if pruneErr == nil { + // Successful prune - reattempt mirror + stderrBuilder.Reset() + stdoutBuilder.Reset() + if err = git.NewCommand(ctx, gitArgs...). + SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). + RunWithContext(&git.RunContext{ + Timeout: timeout, + Dir: newRepoPath, + Stdout: &stdoutBuilder, + Stderr: &stderrBuilder, + }); err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() + + // sanitize the output, since it may contain the remote address, which may + // contain a password + stderrMessage = sanitizer.Replace(stderr) + stdoutMessage = sanitizer.Replace(stdout) + } + } + } + + // If there is still an error (or there always was an error) + if err != nil { + log.Error("SyncMirrors [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", newRepoPath, stderrMessage) + if err = admin_model.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return nil, false + } + } + /* + gitea安全镜像仓库实现逻辑及思路 + + 思路逻辑 拉取最新的项目代码，对比当前镜像的代码count数目 + + 譬如当前镜像的count数目为1135 + 最新的代码count数目为1140 + 那么比对1135后的commit + git log -1 --skip=5 --format="%H" 那么这是未更新前的代码的commit id 对比当前镜像仓库未更新前的最后一条commit id + + + 实现思路，未更新本地镜像前记录下当前镜像项目已经提交的commit数目，另外拷贝一个文件夹进行代码更新，更新后获取提交过的commit数目， + 如果本地镜像和更新后的commit 数目一致且代码最后一条commit id一致，那么可以进行安全进更新（重复进行更新，相当于只是为了记录运行更新日志） + 如果本地镜像和更新后的commit数目不一致且后续的commit数目大于当前的项目commit数目，且skip后的项目commit id和当前的最后commit id一致，那么进行代码更新 + 如果本地镜像和更新后的commit数目小于已更新的代码数目，需进行人工干预更新，基本上已经判定作者提交了force push或者作者删库跑路或者已经商业化了 + + git rev-list HEAD --count 获取已经提交的历史记录数 + */ + gitCommitCountArgs := []string{"rev-list", "HEAD", "--count"} + stdoutNewRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, newRepoPath) + if err != nil { + return err, false + } + stdoutNewRepoCommitCount = strings.TrimSpace(stdoutNewRepoCommitCount) + stdoutRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, repoPath) + if err != nil { + return err, false + } + stdoutRepoCommitCount = strings.TrimSpace(stdoutRepoCommitCount) + var repoCommitCount, newRepoCommitCount int64 + if i, err := strconv.ParseInt(stdoutRepoCommitCount, 10, 64); err == nil { + repoCommitCount = i + } else { + return err, false + } + if i, err := strconv.ParseInt(stdoutNewRepoCommitCount, 10, 64); err == nil { + newRepoCommitCount = i + } else { + return err, false + } + if repoCommitCount > newRepoCommitCount { + // 大概率用户已经删库跑路了 + return nil, false + } else if repoCommitCount == newRepoCommitCount { + // noting to happen + return nil, true + } else { + //compare commit id + gitNewRepoLastCommitIdArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", newRepoCommitCount-newRepoCommitCount), "--format=\"%H\""} + stdoutNewRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIdArgs, newRepoPath) + if err != nil { + return err, false + } + gitRepoLastCommitIdArgs := []string{"log", "--format=\"%H\"", "-n", "1"} + stdoutRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIdArgs, repoPath) + if err != nil { + return err, false + } + if stdoutNewRepoCommitId != stdoutRepoCommitId { + return errors.New(fmt.Sprintf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitId)), false + } else { + return nil, true + } + } + return errors.New("Unknow error!"), false +} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 7695345010d42..c956e8a56fc5b 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -115,6 +115,13 @@ +
+ +
+ + +
+
From 4bdb91846dbdcbab4e0cab9c3ad04a969a8a7646 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Tue, 22 Mar 2022 19:45:58 +0800 Subject: [PATCH 02/29] remove translation and chinese comment --- options/locale/locale_zh-CN.ini | 2 -- services/mirror/mirror_pull_safe_check.go | 21 ++++----------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 473e4044bafa0..8e312ba169c91 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -868,8 +868,6 @@ mirror_last_synced=上次同步 mirror_password_placeholder=(未更改) mirror_password_blank_placeholder=(未设置) mirror_password_help=更改用户名以删除已储存的密码。 -mirror_enable_safe = 安全镜像 -mirror_enable_safe_desc = 开启安全镜像来避免镜像库出问题引起本地镜像仓库数据被删除 watchers=关注者 stargazers=称赞者 forks=派生仓库 diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 27daa71b591fd..23970517709b4 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -1,3 +1,7 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package mirror import ( @@ -134,24 +138,7 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return nil, false } } - /* - gitea安全镜像仓库实现逻辑及思路 - - 思路逻辑 拉取最新的项目代码，对比当前镜像的代码count数目 - - 譬如当前镜像的count数目为1135 - 最新的代码count数目为1140 - 那么比对1135后的commit - git log -1 --skip=5 --format="%H" 那么这是未更新前的代码的commit id 对比当前镜像仓库未更新前的最后一条commit id - - - 实现思路，未更新本地镜像前记录下当前镜像项目已经提交的commit数目，另外拷贝一个文件夹进行代码更新，更新后获取提交过的commit数目， - 如果本地镜像和更新后的commit 数目一致且代码最后一条commit id一致，那么可以进行安全进更新（重复进行更新，相当于只是为了记录运行更新日志） - 如果本地镜像和更新后的commit数目不一致且后续的commit数目大于当前的项目commit数目，且skip后的项目commit id和当前的最后commit id一致，那么进行代码更新 - 如果本地镜像和更新后的commit数目小于已更新的代码数目，需进行人工干预更新，基本上已经判定作者提交了force push或者作者删库跑路或者已经商业化了 - git rev-list HEAD --count 获取已经提交的历史记录数 - */ gitCommitCountArgs := []string{"rev-list", "HEAD", "--count"} stdoutNewRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, newRepoPath) if err != nil { From 1dc838e85b60a09f84c0b879819a42e93c7fe245 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Tue, 22 Mar 2022 20:10:14 +0800 Subject: [PATCH 03/29] add migrate for safe mirror function --- models/migrations/migrations.go | 3 +++ models/repo/mirror.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 3c8edb8eaf8ed..3c019bdef11f5 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -378,6 +378,9 @@ var migrations = []Migration{ // v211 -> v212 NewMigration("Create ForeignReference table", createForeignReferenceTable), + + // v212 -> v213 + NewMigration("Add new safe mirror option", addColorColToMirror), } // GetCurrentDBVersion returns the current db version diff --git a/models/repo/mirror.go b/models/repo/mirror.go index c5bb0e36bf756..8c2d316229a46 100644 --- a/models/repo/mirror.go +++ b/models/repo/mirror.go @@ -33,7 +33,7 @@ type Mirror struct { Repo *Repository `xorm:"-"` Interval time.Duration EnablePrune bool `xorm:"NOT NULL DEFAULT true"` - EnableSafeMirror bool `xorm:"NOT NULL DEFAULT true"` + EnableSafeMirror bool `xorm:"NOT NULL DEFAULT false"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` From a9cd1878039376bc931e8a54aa62e363d20336bf Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Tue, 22 Mar 2022 21:48:53 +0800 Subject: [PATCH 04/29] add model migration file --- models/migrations/v222.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 models/migrations/v222.go diff --git a/models/migrations/v222.go b/models/migrations/v222.go new file mode 100644 index 0000000000000..b2f406a79d3f8 --- /dev/null +++ b/models/migrations/v222.go @@ -0,0 +1,21 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "xorm.io/xorm" +) + +func addColorColToMirror(x *xorm.Engine) error{ + type Mirror struct { + EnableSafeMirror bool `xorm:"NOT NULL DEFAULT false"` + } + + if err := x.Sync2(new(Mirror)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + return nil +} From a8066a55b022adf0dfe654f0b3f737ffc7cd4bf0 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Tue, 22 Mar 2022 21:50:39 +0800 Subject: [PATCH 05/29] remove chinese comment --- services/mirror/mirror_pull_safe_check.go | 1 - 1 file changed, 1 deletion(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 23970517709b4..4fbc7e381c499 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -162,7 +162,6 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return err, false } if repoCommitCount > newRepoCommitCount { - // 大概率用户已经删库跑路了 return nil, false } else if repoCommitCount == newRepoCommitCount { // noting to happen From d46a5de7fe0c005b33f34dadbb419a23bbf9c0fb Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 10:42:08 +0800 Subject: [PATCH 06/29] replace fmt errorf --- services/mirror/mirror_pull_safe_check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 4fbc7e381c499..8ba2eaf82b84f 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -179,7 +179,7 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return err, false } if stdoutNewRepoCommitId != stdoutRepoCommitId { - return errors.New(fmt.Sprintf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitId)), false + return fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitId), false } else { return nil, true } From 93c5f50e629ae43b33d9cf5724281b8345ca1976 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 10:53:42 +0800 Subject: [PATCH 07/29] format the migration code --- models/migrations/v222.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v222.go b/models/migrations/v222.go index b2f406a79d3f8..30fd03f77be8f 100644 --- a/models/migrations/v222.go +++ b/models/migrations/v222.go @@ -9,7 +9,7 @@ import ( "xorm.io/xorm" ) -func addColorColToMirror(x *xorm.Engine) error{ +func addColorColToMirror(x *xorm.Engine) error { type Mirror struct { EnableSafeMirror bool `xorm:"NOT NULL DEFAULT false"` } From 724030a16553f84befd8c7342ac33cb2353fda06 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 10:57:45 +0800 Subject: [PATCH 08/29] remove unclear statement code --- services/mirror/mirror_pull_safe_check.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 8ba2eaf82b84f..218283a3206f0 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "context" - "errors" "fmt" "strconv" "strings" @@ -184,5 +183,4 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return nil, true } } - return errors.New("Unknow error!"), false } From ed8b20f9a93174cc8a0020b7a6e3b4f1c64d9276 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 11:02:23 +0800 Subject: [PATCH 09/29] remove some unnecessary code --- services/mirror/mirror_pull_safe_check.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 218283a3206f0..6ec08c07f766a 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -44,8 +44,6 @@ func getGitCommandStdoutStderr(ctx context.Context, m *repo_model.Mirror, gitArg stderrMessage := sanitizer.Replace(stderr) stdoutMessage := sanitizer.Replace(stdout) log.Error("CreateRepositoryNotice: %v", err) - stderrMessage = sanitizer.Replace(stderr) - stdoutMessage = sanitizer.Replace(stdout) desc := fmt.Sprintf("Failed to get mirror repository can update '%s': %s", newRepoPath, stderrMessage) log.Error("GetMirrorCanUpdate [repo: %-v]: failed to get mirror repository can update:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) if err = admin_model.CreateRepositoryNotice(desc); err != nil { From f602aab9873b2ac5d6bfd112c8e735533993071d Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 11:08:18 +0800 Subject: [PATCH 10/29] remove some unnecessary code --- services/mirror/mirror_pull_safe_check.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 6ec08c07f766a..0994153e362f9 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -177,8 +177,7 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] } if stdoutNewRepoCommitId != stdoutRepoCommitId { return fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitId), false - } else { - return nil, true } } + return nil, true } From f02d5b3ac1437fdcb0c9dd844ef4e26abbf93894 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 11:15:56 +0800 Subject: [PATCH 11/29] lint the code --- services/mirror/mirror_pull.go | 2 +- services/mirror/mirror_pull_safe_check.go | 26 +++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index ee3b4410e76ad..b7971b4d2b23c 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -206,7 +206,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo } // detect can safe mirror - err, canUpdate := detectCanUpdateMirror(ctx, m, gitArgs) + canUpdate, err := detectCanUpdateMirror(ctx, m, gitArgs) if err != nil { log.Error("CreateRepositoryNotice: %v", err) return nil, false diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 0994153e362f9..3f1425f164db9 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -59,7 +59,7 @@ func getGitCommandStdoutStderr(ctx context.Context, m *repo_model.Mirror, gitArg } // detect user can update the mirror -func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string) (error, bool) { +func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string) (bool, error) { repoPath := m.Repo.RepoPath() newRepoPath := fmt.Sprintf("%s_update", repoPath) timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second @@ -69,7 +69,7 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] defer util.RemoveAll(newRepoPath) if err != nil { log.Error("GetMirrorCanUpdate [repo: %-v]: CopyDirectory Error %v", m.Repo, err) - return err, false + return false, err } remoteAddr, remoteErr := git.GetRemoteAddress(ctx, newRepoPath, m.GetRemoteName()) if remoteErr != nil { @@ -132,52 +132,52 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] if err = admin_model.CreateRepositoryNotice(desc); err != nil { log.Error("CreateRepositoryNotice: %v", err) } - return nil, false + return false, nil } } gitCommitCountArgs := []string{"rev-list", "HEAD", "--count"} stdoutNewRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, newRepoPath) if err != nil { - return err, false + return false, err } stdoutNewRepoCommitCount = strings.TrimSpace(stdoutNewRepoCommitCount) stdoutRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, repoPath) if err != nil { - return err, false + return false, err } stdoutRepoCommitCount = strings.TrimSpace(stdoutRepoCommitCount) var repoCommitCount, newRepoCommitCount int64 if i, err := strconv.ParseInt(stdoutRepoCommitCount, 10, 64); err == nil { repoCommitCount = i } else { - return err, false + return false, err } if i, err := strconv.ParseInt(stdoutNewRepoCommitCount, 10, 64); err == nil { newRepoCommitCount = i } else { - return err, false + return false, err } if repoCommitCount > newRepoCommitCount { - return nil, false + return false, nil } else if repoCommitCount == newRepoCommitCount { // noting to happen - return nil, true + return true, nil } else { //compare commit id gitNewRepoLastCommitIdArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", newRepoCommitCount-newRepoCommitCount), "--format=\"%H\""} stdoutNewRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIdArgs, newRepoPath) if err != nil { - return err, false + return false, err } gitRepoLastCommitIdArgs := []string{"log", "--format=\"%H\"", "-n", "1"} stdoutRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIdArgs, repoPath) if err != nil { - return err, false + return false, err } if stdoutNewRepoCommitId != stdoutRepoCommitId { - return fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitId), false + return false, fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitId) } } - return nil, true + return true, nil } From 44e491261500229d22c2835e3293ecfb448228dd Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 11:24:36 +0800 Subject: [PATCH 12/29] lint the code --- services/mirror/mirror_pull_safe_check.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 3f1425f164db9..acecfb204b4d6 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -165,12 +165,13 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return true, nil } else { //compare commit id - gitNewRepoLastCommitIdArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", newRepoCommitCount-newRepoCommitCount), "--format=\"%H\""} + skipcout := newRepoCommitCount - newRepoCommitCount + gitNewRepoLastCommitIdArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", skipcout), "--format='%H'"} stdoutNewRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIdArgs, newRepoPath) if err != nil { return false, err } - gitRepoLastCommitIdArgs := []string{"log", "--format=\"%H\"", "-n", "1"} + gitRepoLastCommitIdArgs := []string{"log", "--format='%H'", "-n", "1"} stdoutRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIdArgs, repoPath) if err != nil { return false, err From 9b40deeb9d903c782b18f13e40d72667f1242e84 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 11:30:52 +0800 Subject: [PATCH 13/29] lint the code --- services/mirror/mirror_pull_safe_check.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index acecfb204b4d6..744bfb73e6790 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -165,14 +165,14 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return true, nil } else { //compare commit id - skipcout := newRepoCommitCount - newRepoCommitCount - gitNewRepoLastCommitIdArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", skipcout), "--format='%H'"} - stdoutNewRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIdArgs, newRepoPath) + skipcout := newRepoCommitCount - repoCommitCount + gitNewRepoLastCommitIDArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", skipcout), "--format='%H'"} + stdoutNewRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIDArgs, newRepoPath) if err != nil { return false, err } - gitRepoLastCommitIdArgs := []string{"log", "--format='%H'", "-n", "1"} - stdoutRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIdArgs, repoPath) + gitRepoLastCommitIDArgs := []string{"log", "--format='%H'", "-n", "1"} + stdoutRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIDArgs, repoPath) if err != nil { return false, err } From 5e16d8a9d98617c638d54f40299f62583618cc23 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 11:34:13 +0800 Subject: [PATCH 14/29] lint the code --- services/mirror/mirror_pull_safe_check.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 744bfb73e6790..f167801aba935 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -167,7 +167,7 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] //compare commit id skipcout := newRepoCommitCount - repoCommitCount gitNewRepoLastCommitIDArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", skipcout), "--format='%H'"} - stdoutNewRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIDArgs, newRepoPath) + stdoutNewRepoCommitID, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIDArgs, newRepoPath) if err != nil { return false, err } @@ -176,8 +176,8 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] if err != nil { return false, err } - if stdoutNewRepoCommitId != stdoutRepoCommitId { - return false, fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitId) + if stdoutNewRepoCommitID != stdoutRepoCommitId { + return false, fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitID) } } return true, nil From aaaac621a5b8ebc19b65c894db843207231e5161 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 11:37:10 +0800 Subject: [PATCH 15/29] lint the code --- services/mirror/mirror_pull_safe_check.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index f167801aba935..43b1abccbcf20 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -172,12 +172,12 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return false, err } gitRepoLastCommitIDArgs := []string{"log", "--format='%H'", "-n", "1"} - stdoutRepoCommitId, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIDArgs, repoPath) + stdoutRepoCommitID, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIDArgs, repoPath) if err != nil { return false, err } - if stdoutNewRepoCommitID != stdoutRepoCommitId { - return false, fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitId, stdoutNewRepoCommitID) + if stdoutNewRepoCommitID != stdoutRepoCommitID { + return false, fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitID, stdoutNewRepoCommitID) } } return true, nil From a7eb9cf8e1ec7201b3eda505ad8fa65f264a2fdb Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 17:14:30 +0800 Subject: [PATCH 16/29] abstract the code to function --- services/mirror/mirror_pull_safe_check.go | 36 +++++++++++++++-------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 43b1abccbcf20..54fb21e9f39c9 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -58,24 +58,13 @@ func getGitCommandStdoutStderr(ctx context.Context, m *repo_model.Mirror, gitArg return stdoutRepoCommitCount, stderrRepoCommitCount, nil } -// detect user can update the mirror -func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string) (bool, error) { - repoPath := m.Repo.RepoPath() - newRepoPath := fmt.Sprintf("%s_update", repoPath) +// sync new repo mirror +func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, newRepoPath string) (bool, error) { timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second - - //do copy directory recursive - err := util.CopyDir(repoPath, newRepoPath) - defer util.RemoveAll(newRepoPath) - if err != nil { - log.Error("GetMirrorCanUpdate [repo: %-v]: CopyDirectory Error %v", m.Repo, err) - return false, err - } remoteAddr, remoteErr := git.GetRemoteAddress(ctx, newRepoPath, m.GetRemoteName()) if remoteErr != nil { log.Error("GetMirrorCanUpdate [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) } - stdoutBuilder := strings.Builder{} stderrBuilder := strings.Builder{} if err := git.NewCommand(ctx, gitArgs...). @@ -135,7 +124,28 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] return false, nil } } + return false, nil +} +// detect user can update the mirror +func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string) (bool, error) { + repoPath := m.Repo.RepoPath() + newRepoPath := fmt.Sprintf("%s_update", repoPath) + + //do copy directory recursive + err := util.CopyDir(repoPath, newRepoPath) + defer util.RemoveAll(newRepoPath) + if err != nil { + log.Error("GetMirrorCanUpdate [repo: %-v]: CopyDirectory Error %v", m.Repo, err) + return false, err + } + syncStatus, err := syncRepoMirror(ctx, m, gitArgs, newRepoPath) + if err != nil { + return false, err + } + if !syncStatus { + return false, nil + } gitCommitCountArgs := []string{"rev-list", "HEAD", "--count"} stdoutNewRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, newRepoPath) if err != nil { From 6801ef158178dfd2da774f0bccbab62cc92ea978 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 17:18:05 +0800 Subject: [PATCH 17/29] abstract the code to function --- models/migrations/v222.go | 1 + services/mirror/mirror_pull_safe_check.go | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/models/migrations/v222.go b/models/migrations/v222.go index 30fd03f77be8f..60482f00435b0 100644 --- a/models/migrations/v222.go +++ b/models/migrations/v222.go @@ -6,6 +6,7 @@ package migrations import ( "fmt" + "xorm.io/xorm" ) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 54fb21e9f39c9..46f46f12a1a72 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -5,17 +5,18 @@ package mirror import ( + "context" + "fmt" + "strconv" + "strings" + "time" + admin_model "code.gitea.io/gitea/models/admin" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - "context" - "fmt" - "strconv" - "strings" - "time" ) // get git command running stdout and stderr From e3246095ba544658ffb20d1074938f28dbd7ad4a Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 17:24:33 +0800 Subject: [PATCH 18/29] format the code --- services/mirror/mirror_pull_safe_check.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 46f46f12a1a72..eec512b518dd4 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -133,7 +133,7 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] repoPath := m.Repo.RepoPath() newRepoPath := fmt.Sprintf("%s_update", repoPath) - //do copy directory recursive + // do copy directory recursive err := util.CopyDir(repoPath, newRepoPath) defer util.RemoveAll(newRepoPath) if err != nil { @@ -175,7 +175,7 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] // noting to happen return true, nil } else { - //compare commit id + // compare commit id skipcout := newRepoCommitCount - repoCommitCount gitNewRepoLastCommitIDArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", skipcout), "--format='%H'"} stdoutNewRepoCommitID, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIDArgs, newRepoPath) From fff683b0ef4771c9cf6db2d9b1aa3d6dccc78b72 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 17:33:10 +0800 Subject: [PATCH 19/29] format the code --- services/mirror/mirror_pull_safe_check.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index eec512b518dd4..d624b604a5087 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -69,7 +69,7 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, stdoutBuilder := strings.Builder{} stderrBuilder := strings.Builder{} if err := git.NewCommand(ctx, gitArgs...). - SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). + SetDescription(fmt.Sprintf("Mirror.runSyncTest: %s", m.Repo.FullName())). RunWithContext(&git.RunContext{ Timeout: timeout, Dir: newRepoPath, @@ -87,7 +87,7 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, // Now check if the error is a resolve reference due to broken reference if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") { - log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err) + log.Warn("SyncMirrorsTest [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err) err = nil // Attempt prune @@ -97,7 +97,7 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, stderrBuilder.Reset() stdoutBuilder.Reset() if err = git.NewCommand(ctx, gitArgs...). - SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). + SetDescription(fmt.Sprintf("Mirror.runSyncTest: %s", m.Repo.FullName())). RunWithContext(&git.RunContext{ Timeout: timeout, Dir: newRepoPath, @@ -117,7 +117,7 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, // If there is still an error (or there always was an error) if err != nil { - log.Error("SyncMirrors [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + log.Error("SyncMirrorsTest [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", newRepoPath, stderrMessage) if err = admin_model.CreateRepositoryNotice(desc); err != nil { log.Error("CreateRepositoryNotice: %v", err) From 061b46a64a44b2178b7f66913ad9a342007aa3eb Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 17:40:49 +0800 Subject: [PATCH 20/29] abstract the code to function --- services/mirror/mirror_pull_safe_check.go | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index d624b604a5087..3146783361171 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -66,18 +66,18 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, if remoteErr != nil { log.Error("GetMirrorCanUpdate [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) } - stdoutBuilder := strings.Builder{} - stderrBuilder := strings.Builder{} + stdoutSyncBuilder := strings.Builder{} + stderrSyncBuilder := strings.Builder{} if err := git.NewCommand(ctx, gitArgs...). - SetDescription(fmt.Sprintf("Mirror.runSyncTest: %s", m.Repo.FullName())). + SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). RunWithContext(&git.RunContext{ Timeout: timeout, Dir: newRepoPath, - Stdout: &stdoutBuilder, - Stderr: &stderrBuilder, + Stdout: &stdoutSyncBuilder, + Stderr: &stderrSyncBuilder, }); err != nil { - stdout := stdoutBuilder.String() - stderr := stderrBuilder.String() + stdout := stdoutSyncBuilder.String() + stderr := stderrSyncBuilder.String() // sanitize the output, since it may contain the remote address, which may // contain a password @@ -87,25 +87,25 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, // Now check if the error is a resolve reference due to broken reference if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") { - log.Warn("SyncMirrorsTest [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err) + log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err) err = nil // Attempt prune - pruneErr := pruneBrokenReferences(ctx, m, newRepoPath, timeout, &stdoutBuilder, &stderrBuilder, sanitizer, false) + pruneErr := pruneBrokenReferences(ctx, m, newRepoPath, timeout, &stdoutSyncBuilder, &stderrSyncBuilder, sanitizer, false) if pruneErr == nil { // Successful prune - reattempt mirror - stderrBuilder.Reset() - stdoutBuilder.Reset() + stderrSyncBuilder.Reset() + stdoutSyncBuilder.Reset() if err = git.NewCommand(ctx, gitArgs...). - SetDescription(fmt.Sprintf("Mirror.runSyncTest: %s", m.Repo.FullName())). + SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). RunWithContext(&git.RunContext{ Timeout: timeout, Dir: newRepoPath, - Stdout: &stdoutBuilder, - Stderr: &stderrBuilder, + Stdout: &stdoutSyncBuilder, + Stderr: &stderrSyncBuilder, }); err != nil { - stdout := stdoutBuilder.String() - stderr := stderrBuilder.String() + stdout := stdoutSyncBuilder.String() + stderr := stderrSyncBuilder.String() // sanitize the output, since it may contain the remote address, which may // contain a password @@ -117,7 +117,7 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, // If there is still an error (or there always was an error) if err != nil { - log.Error("SyncMirrorsTest [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + log.Error("SyncMirrors [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", newRepoPath, stderrMessage) if err = admin_model.CreateRepositoryNotice(desc); err != nil { log.Error("CreateRepositoryNotice: %v", err) From e3b10d202becdd502dd2071ee8c369dc4cb980c8 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 19:06:48 +0800 Subject: [PATCH 21/29] abstract the code to function --- services/mirror/mirror_pull_safe_check.go | 31 ++++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 3146783361171..0d83a512270c8 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -66,18 +66,19 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, if remoteErr != nil { log.Error("GetMirrorCanUpdate [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) } - stdoutSyncBuilder := strings.Builder{} - stderrSyncBuilder := strings.Builder{} - if err := git.NewCommand(ctx, gitArgs...). + stdoutBuilder := strings.Builder{} + stderrBuilder := strings.Builder{} + err := git.NewCommand(ctx, gitArgs...). SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). RunWithContext(&git.RunContext{ Timeout: timeout, Dir: newRepoPath, - Stdout: &stdoutSyncBuilder, - Stderr: &stderrSyncBuilder, - }); err != nil { - stdout := stdoutSyncBuilder.String() - stderr := stderrSyncBuilder.String() + Stdout: &stdoutBuilder, + Stderr: &stderrBuilder, + }) + if err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() // sanitize the output, since it may contain the remote address, which may // contain a password @@ -91,21 +92,21 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, err = nil // Attempt prune - pruneErr := pruneBrokenReferences(ctx, m, newRepoPath, timeout, &stdoutSyncBuilder, &stderrSyncBuilder, sanitizer, false) + pruneErr := pruneBrokenReferences(ctx, m, newRepoPath, timeout, &stdoutBuilder, &stderrBuilder, sanitizer, false) if pruneErr == nil { // Successful prune - reattempt mirror - stderrSyncBuilder.Reset() - stdoutSyncBuilder.Reset() + stderrBuilder.Reset() + stdoutBuilder.Reset() if err = git.NewCommand(ctx, gitArgs...). SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). RunWithContext(&git.RunContext{ Timeout: timeout, Dir: newRepoPath, - Stdout: &stdoutSyncBuilder, - Stderr: &stderrSyncBuilder, + Stdout: &stdoutBuilder, + Stderr: &stderrBuilder, }); err != nil { - stdout := stdoutSyncBuilder.String() - stderr := stderrSyncBuilder.String() + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() // sanitize the output, since it may contain the remote address, which may // contain a password From ad9a06262f17e53192e65dd5aa58ee396ccbf1e6 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 19:21:11 +0800 Subject: [PATCH 22/29] lint the code --- services/mirror/mirror_pull_safe_check.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 0d83a512270c8..27418b6a84c1d 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -76,13 +76,14 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, Stdout: &stdoutBuilder, Stderr: &stderrBuilder, }) + sanitizer := util.NewURLSanitizer(remoteAddr, true) + var stdout,stderr string if err != nil { - stdout := stdoutBuilder.String() - stderr := stderrBuilder.String() + stdout = stdoutBuilder.String() + stderr = stderrBuilder.String() // sanitize the output, since it may contain the remote address, which may // contain a password - sanitizer := util.NewURLSanitizer(remoteAddr, true) stderrMessage := sanitizer.Replace(stderr) stdoutMessage := sanitizer.Replace(stdout) From 8b7479ab88bc5cfec0345aa8f76af7ca27967ae5 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 19:25:12 +0800 Subject: [PATCH 23/29] lint the code --- services/mirror/mirror_pull_safe_check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 27418b6a84c1d..4547d3de1a663 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -77,7 +77,7 @@ func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, Stderr: &stderrBuilder, }) sanitizer := util.NewURLSanitizer(remoteAddr, true) - var stdout,stderr string + var stdout, stderr string if err != nil { stdout = stdoutBuilder.String() stderr = stderrBuilder.String() From b703f0acba246976d8e40c238744cf2e52ea4211 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 19:30:35 +0800 Subject: [PATCH 24/29] fix the lint error --- services/mirror/mirror_pull.go | 2 ++ services/mirror/mirror_pull_safe_check.go | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index b7971b4d2b23c..bc801cb9f6217 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -211,6 +211,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Error("CreateRepositoryNotice: %v", err) return nil, false } + newRepoPath := fmt.Sprintf("%s_update", repoPath) + util.RemoveAll(newRepoPath) if safeMirror && !canUpdate { log.Error("CheckSyncMirrors [repo: %-v]: can not safe mirror...", m.Repo) return nil, false diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 4547d3de1a663..7cf90ef2d56b3 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -137,7 +137,6 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] // do copy directory recursive err := util.CopyDir(repoPath, newRepoPath) - defer util.RemoveAll(newRepoPath) if err != nil { log.Error("GetMirrorCanUpdate [repo: %-v]: CopyDirectory Error %v", m.Repo, err) return false, err From a1cb58fa4f6c40c391890984adb3fd2b07956176 Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 19:41:30 +0800 Subject: [PATCH 25/29] reorganize the code --- services/mirror/mirror_pull.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index bc801cb9f6217..4d17ce2cad458 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -205,17 +205,23 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) } - // detect can safe mirror - canUpdate, err := detectCanUpdateMirror(ctx, m, gitArgs) - if err != nil { - log.Error("CreateRepositoryNotice: %v", err) - return nil, false - } - newRepoPath := fmt.Sprintf("%s_update", repoPath) - util.RemoveAll(newRepoPath) - if safeMirror && !canUpdate { - log.Error("CheckSyncMirrors [repo: %-v]: can not safe mirror...", m.Repo) - return nil, false + if safeMirror { + // detect can safe mirror + canUpdate, err := detectCanUpdateMirror(ctx, m, gitArgs) + newRepoPath := fmt.Sprintf("%s_update", repoPath) + // delete the temp directory + errDelete := util.RemoveAll(newRepoPath) + if errDelete != nil { + log.Error("DeleteRepositoryTempDirectoryError: %v", errDelete) + } + if err != nil { + log.Error("CheckRepositoryCanSafeMirrorError: %v", err) + } + // can not safe mirror + if !canUpdate { + log.Error("CheckSyncMirrors [repo: %-v]: can not safe mirror...", m.Repo) + return nil, false + } } stdoutBuilder := strings.Builder{} From e68e5c64edc9802fad128e23a736a0c85790689b Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 20:20:15 +0800 Subject: [PATCH 26/29] fix push mirror safe option bug --- routers/web/repo/setting.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 94a1acab1306d..f9630074ef1d0 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -70,7 +70,13 @@ func Settings(ctx *context.Context) { ctx.Data["SigningKeyAvailable"] = len(signing) > 0 ctx.Data["SigningSettings"] = setting.Repository.Signing ctx.Data["CodeIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled - ctx.Data["EnableSafeMirror"] = ctx.Repo.Mirror.EnableSafeMirror + + mirrors, err := repo_model.GetMirrorByRepoID(ctx.Repo.Repository.ID) + if err != nil { + ctx.ServerError("GetMirrorsByRepoID", err) + return + } + ctx.Data["EnableSafeMirror"] = mirrors.EnableSafeMirror if ctx.Doer.IsAdmin { if setting.Indexer.RepoIndexerEnabled { From 0ec287c3cd0aa19aea6e301be2a03354c4f7fdea Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Wed, 23 Mar 2022 20:47:59 +0800 Subject: [PATCH 27/29] fix push mirror safe option bug --- routers/web/repo/setting.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index f9630074ef1d0..5e9b7dd77f3d0 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -73,10 +73,10 @@ func Settings(ctx *context.Context) { mirrors, err := repo_model.GetMirrorByRepoID(ctx.Repo.Repository.ID) if err != nil { - ctx.ServerError("GetMirrorsByRepoID", err) - return + ctx.Data["EnableSafeMirror"] = false + } else { + ctx.Data["EnableSafeMirror"] = mirrors.EnableSafeMirror } - ctx.Data["EnableSafeMirror"] = mirrors.EnableSafeMirror if ctx.Doer.IsAdmin { if setting.Indexer.RepoIndexerEnabled { From 8f782364ffe5c9ad36dd72fc37ffee6f96b861ab Mon Sep 17 00:00:00 2001 From: zhengge2012 Date: Fri, 25 Mar 2022 20:09:35 +0800 Subject: [PATCH 28/29] lint the code and rewrite the suggestion for log --- models/migrations/migrations.go | 2 +- models/migrations/v222.go | 2 +- options/locale/locale_en-US.ini | 2 +- routers/web/repo/setting.go | 8 ++------ services/mirror/mirror_pull.go | 11 ++--------- services/mirror/mirror_pull_safe_check.go | 13 ++++++++++--- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 3c019bdef11f5..86da400d81b7d 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -380,7 +380,7 @@ var migrations = []Migration{ NewMigration("Create ForeignReference table", createForeignReferenceTable), // v212 -> v213 - NewMigration("Add new safe mirror option", addColorColToMirror), + NewMigration("Add safe mirrors", addEnableSafeMirrorColToMirror), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v222.go b/models/migrations/v222.go index 60482f00435b0..6c4896a9e3e23 100644 --- a/models/migrations/v222.go +++ b/models/migrations/v222.go @@ -10,7 +10,7 @@ import ( "xorm.io/xorm" ) -func addColorColToMirror(x *xorm.Engine) error { +func addEnableSafeMirrorColToMirror(x *xorm.Engine) error { type Mirror struct { EnableSafeMirror bool `xorm:"NOT NULL DEFAULT false"` } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 31deb6a92fe89..2c085bceecf96 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -870,7 +870,7 @@ mirror_password_placeholder = (Unchanged) mirror_password_blank_placeholder = (Unset) mirror_password_help = Change the username to erase a stored password. mirror_enable_safe = Safe Mirror -mirror_enable_safe_desc = Enable safe mirror to avoid delete local repository data +mirror_enable_safe_desc = Enable safe mirroring to avoid deleting local data such as branches or tags once they are deleted remotely watchers = Watchers stargazers = Stargazers forks = Forks diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 5e9b7dd77f3d0..80075a77b4e88 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -71,12 +71,8 @@ func Settings(ctx *context.Context) { ctx.Data["SigningSettings"] = setting.Repository.Signing ctx.Data["CodeIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled - mirrors, err := repo_model.GetMirrorByRepoID(ctx.Repo.Repository.ID) - if err != nil { - ctx.Data["EnableSafeMirror"] = false - } else { - ctx.Data["EnableSafeMirror"] = mirrors.EnableSafeMirror - } + mirror, err := repo_model.GetMirrorByRepoID(ctx.Repo.Repository.ID) + ctx.Data["EnableSafeMirror"] = err == nil && mirror.EnableSafeMirror if ctx.Doer.IsAdmin { if setting.Indexer.RepoIndexerEnabled { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 4d17ce2cad458..464e1831a48e4 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -190,7 +190,6 @@ func pruneBrokenReferences(ctx context.Context, func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() - safeMirror := m.EnableSafeMirror timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) @@ -205,21 +204,15 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) } - if safeMirror { + if m.EnableSafeMirror { // detect can safe mirror canUpdate, err := detectCanUpdateMirror(ctx, m, gitArgs) - newRepoPath := fmt.Sprintf("%s_update", repoPath) - // delete the temp directory - errDelete := util.RemoveAll(newRepoPath) - if errDelete != nil { - log.Error("DeleteRepositoryTempDirectoryError: %v", errDelete) - } if err != nil { log.Error("CheckRepositoryCanSafeMirrorError: %v", err) } // can not safe mirror if !canUpdate { - log.Error("CheckSyncMirrors [repo: %-v]: can not safe mirror...", m.Repo) + log.Error("CheckSyncMirrors [repo: %-v]: cannot sync safe mirror...", m.Repo) return nil, false } } diff --git a/services/mirror/mirror_pull_safe_check.go b/services/mirror/mirror_pull_safe_check.go index 7cf90ef2d56b3..68b090ea2c45e 100644 --- a/services/mirror/mirror_pull_safe_check.go +++ b/services/mirror/mirror_pull_safe_check.go @@ -45,10 +45,10 @@ func getGitCommandStdoutStderr(ctx context.Context, m *repo_model.Mirror, gitArg stderrMessage := sanitizer.Replace(stderr) stdoutMessage := sanitizer.Replace(stdout) log.Error("CreateRepositoryNotice: %v", err) - desc := fmt.Sprintf("Failed to get mirror repository can update '%s': %s", newRepoPath, stderrMessage) - log.Error("GetMirrorCanUpdate [repo: %-v]: failed to get mirror repository can update:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + log.Error("getGitCommandStdoutStderr [repo: %-v]: failed to check if mirror can be updated:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + desc := fmt.Sprintf("Failed to check if mirror '%s' can be updated: %s", newRepoPath, stderrMessage) if err = admin_model.CreateRepositoryNotice(desc); err != nil { - log.Error("GetMirrorCanUpdateNotice: %v", err) + log.Error("CreateRepositoryNotice: %v", err) } } stdoutRepoCommitCount := stdoutBuilder.String() @@ -137,6 +137,13 @@ func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs [] // do copy directory recursive err := util.CopyDir(repoPath, newRepoPath) + defer func() { + // delete the temp directory + errDelete := util.RemoveAll(newRepoPath) + if errDelete != nil { + log.Error("DeleteRepositoryTempDirectoryError: %v", errDelete) + } + }() if err != nil { log.Error("GetMirrorCanUpdate [repo: %-v]: CopyDirectory Error %v", m.Repo, err) return false, err From bf243eec93091f51c540fc347f130001f190a9d3 Mon Sep 17 00:00:00 2001 From: jimmy201602 Date: Tue, 7 Jun 2022 08:46:13 +0800 Subject: [PATCH 29/29] Update migrations.go --- models/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 86da400d81b7d..8da6339b1bc4e 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -379,7 +379,7 @@ var migrations = []Migration{ // v211 -> v212 NewMigration("Create ForeignReference table", createForeignReferenceTable), - // v212 -> v213 + // v216 -> v217 NewMigration("Add safe mirrors", addEnableSafeMirrorColToMirror), }