Add safe pull mirror option to avoid original repository deletion by force-push

models/migrations/migrations.go

@@ -379,6 +379,7 @@ var migrations = []Migration{
 
 	// v211 -> v212
 	NewMigration("Create ForeignReference table", createForeignReferenceTable),
+
 	// v212 -> v213
 	NewMigration("Add package tables", addPackageTables),
 	// v213 -> v214
@@ -387,6 +388,9 @@ var migrations = []Migration{
 	NewMigration("Add auto merge table", addAutoMergeTable),
 	// v215 -> v216
 	NewMigration("allow to view files in PRs", addReviewViewedFiles),
+
+	// v216 -> v217
+	NewMigration("Add safe mirrors", addEnableSafeMirrorColToMirror),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v222.go

@@ -0,0 +1,22 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"xorm.io/xorm"
+)
+
+func addEnableSafeMirrorColToMirror(x *xorm.Engine) error {
+	type Mirror struct {
+		EnableSafeMirror bool `xorm:"NOT NULL DEFAULT false"`
+	}
+
+	if err := x.Sync2(new(Mirror)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

models/repo/mirror.go

@@ -27,11 +27,12 @@ type RemoteMirrorer interface {
 
 // Mirror represents mirror information of a repository.
 type Mirror struct {
-	ID          int64       `xorm:"pk autoincr"`
-	RepoID      int64       `xorm:"INDEX"`
-	Repo        *Repository `xorm:"-"`
-	Interval    time.Duration
-	EnablePrune bool `xorm:"NOT NULL DEFAULT true"`
+	ID               int64       `xorm:"pk autoincr"`
+	RepoID           int64       `xorm:"INDEX"`
+	Repo             *Repository `xorm:"-"`
+	Interval         time.Duration
+	EnablePrune      bool `xorm:"NOT NULL DEFAULT true"`
+	EnableSafeMirror bool `xorm:"NOT NULL DEFAULT false"`
 
 	UpdatedUnix    timeutil.TimeStamp `xorm:"INDEX"`
 	NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"`

options/locale/locale_en-US.ini

@@ -874,6 +874,8 @@ mirror_last_synced = Last Synchronized
 mirror_password_placeholder = (Unchanged)
 mirror_password_blank_placeholder = (Unset)
 mirror_password_help = Change the username to erase a stored password.
+mirror_enable_safe = Safe Mirror
+mirror_enable_safe_desc = Enable safe mirroring to avoid deleting local data such as branches or tags once they are deleted remotely
 watchers = Watchers
 stargazers = Stargazers
 forks = Forks

options/locale/locale_zh-CN.ini

@@ -695,8 +695,8 @@ last_used=上次使用在
 no_activity=没有最近活动
 can_read_info=读取
 can_write_info=写入
-key_state_desc=7 天内使用过该密钥 
-token_state_desc=7 天内使用过该密钥 
+key_state_desc=7 天内使用过该密钥
+token_state_desc=7 天内使用过该密钥
 principal_state_desc=7 天内使用过该规则
 show_openid=在个人信息上显示
 hide_openid=在个人信息上隐藏
@@ -878,7 +878,7 @@ watchers=关注者
 stargazers=称赞者
 forks=派生仓库
 pick_reaction=选择你的表情
-reactions_more=再加载 %d 
+reactions_more=再加载 %d
 unit_disabled=站点管理员已禁用此仓库单元。
 language_other=其它
 adopt_search=输入用户名以搜索未被收录的仓库... (留空以查找全部)

routers/web/repo/setting.go

@@ -70,6 +70,10 @@ func Settings(ctx *context.Context) {
 	ctx.Data["SigningKeyAvailable"] = len(signing) > 0
 	ctx.Data["SigningSettings"] = setting.Repository.Signing
 	ctx.Data["CodeIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
+
+	mirror, err := repo_model.GetMirrorByRepoID(ctx.Repo.Repository.ID)
+	ctx.Data["EnableSafeMirror"] = err == nil && mirror.EnableSafeMirror
+
 	if ctx.Doer.IsAdmin {
 		if setting.Indexer.RepoIndexerEnabled {
 			status, err := repo_model.GetIndexerStatus(ctx, ctx.Repo.Repository, repo_model.RepoIndexerTypeCode)
@@ -193,6 +197,7 @@ func SettingsPost(ctx *context.Context) {
 			ctx.RenderWithErr(ctx.Tr("repo.mirror_interval_invalid"), tplSettingsOptions, &form)
 		} else {
 			ctx.Repo.Mirror.EnablePrune = form.EnablePrune
+			ctx.Repo.Mirror.EnableSafeMirror = form.EnableSafeMirror
 			ctx.Repo.Mirror.Interval = interval
 			ctx.Repo.Mirror.ScheduleNextUpdate()
 			if err := repo_model.UpdateMirror(ctx, ctx.Repo.Mirror); err != nil {

services/forms/repo_form.go

@@ -131,6 +131,7 @@ type RepoSettingForm struct {
 	Private            bool
 	Template           bool
 	EnablePrune        bool
+	EnableSafeMirror   bool
 
 	// Advanced settings
 	EnableWiki                            bool

services/mirror/mirror_pull.go

@@ -215,6 +215,19 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
 		log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr)
 	}
 
+	if m.EnableSafeMirror {
+		// detect can safe mirror
+		canUpdate, err := detectCanUpdateMirror(ctx, m, gitArgs)
+		if err != nil {
+			log.Error("CheckRepositoryCanSafeMirrorError: %v", err)
+		}
+		// can not safe mirror
+		if !canUpdate {
+			log.Error("CheckSyncMirrors [repo: %-v]: cannot sync safe mirror...", m.Repo)
+			return nil, false
+		}
+	}
+
 	stdoutBuilder := strings.Builder{}
 	stderrBuilder := strings.Builder{}
 	if err := git.NewCommand(ctx, gitArgs...).

services/mirror/mirror_pull_safe_check.go

@@ -0,0 +1,203 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mirror
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	admin_model "code.gitea.io/gitea/models/admin"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// get git command running stdout and stderr
+func getGitCommandStdoutStderr(ctx context.Context, m *repo_model.Mirror, gitArgs []string, newRepoPath string) (string, string, error) {
+	stdoutBuilder := strings.Builder{}
+	stderrBuilder := strings.Builder{}
+	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
+
+	remoteAddr, remoteErr := git.GetRemoteAddress(ctx, newRepoPath, m.GetRemoteName())
+	if remoteErr != nil {
+		log.Error("GetMirrorCanUpdate [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr)
+		return "", "", remoteErr
+	}
+
+	if err := git.NewCommand(ctx, gitArgs...).
+		SetDescription(fmt.Sprintf("Mirror.getMirrorCanUpdate: %s", m.Repo.FullName())).
+		RunWithContext(&git.RunContext{
+			Timeout: timeout,
+			Dir:     newRepoPath,
+			Stdout:  &stdoutBuilder,
+			Stderr:  &stderrBuilder,
+		}); err != nil {
+		stdout := stdoutBuilder.String()
+		stderr := stderrBuilder.String()
+		sanitizer := util.NewURLSanitizer(remoteAddr, true)
+		stderrMessage := sanitizer.Replace(stderr)
+		stdoutMessage := sanitizer.Replace(stdout)
+		log.Error("CreateRepositoryNotice: %v", err)
+		log.Error("getGitCommandStdoutStderr [repo: %-v]: failed to check if mirror can be updated:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err)
+		desc := fmt.Sprintf("Failed to check if mirror '%s' can be updated: %s", newRepoPath, stderrMessage)
+		if err = admin_model.CreateRepositoryNotice(desc); err != nil {
+			log.Error("CreateRepositoryNotice: %v", err)
+		}
+	}
+	stdoutRepoCommitCount := stdoutBuilder.String()
+	stderrRepoCommitCount := stdoutBuilder.String()
+	stderrBuilder.Reset()
+	stdoutBuilder.Reset()
+
+	return stdoutRepoCommitCount, stderrRepoCommitCount, nil
+}
+
+// sync new repo mirror
+func syncRepoMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string, newRepoPath string) (bool, error) {
+	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
+	remoteAddr, remoteErr := git.GetRemoteAddress(ctx, newRepoPath, m.GetRemoteName())
+	if remoteErr != nil {
+		log.Error("GetMirrorCanUpdate [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr)
+	}
+	stdoutBuilder := strings.Builder{}
+	stderrBuilder := strings.Builder{}
+	err := git.NewCommand(ctx, gitArgs...).
+		SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())).
+		RunWithContext(&git.RunContext{
+			Timeout: timeout,
+			Dir:     newRepoPath,
+			Stdout:  &stdoutBuilder,
+			Stderr:  &stderrBuilder,
+		})
+	sanitizer := util.NewURLSanitizer(remoteAddr, true)
+	var stdout, stderr string
+	if err != nil {
+		stdout = stdoutBuilder.String()
+		stderr = stderrBuilder.String()
+
+		// sanitize the output, since it may contain the remote address, which may
+		// contain a password
+		stderrMessage := sanitizer.Replace(stderr)
+		stdoutMessage := sanitizer.Replace(stdout)
+
+		// Now check if the error is a resolve reference due to broken reference
+		if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") {
+			log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
+			err = nil
+
+			// Attempt prune
+			pruneErr := pruneBrokenReferences(ctx, m, newRepoPath, timeout, &stdoutBuilder, &stderrBuilder, sanitizer, false)
+			if pruneErr == nil {
+				// Successful prune - reattempt mirror
+				stderrBuilder.Reset()
+				stdoutBuilder.Reset()
+				if err = git.NewCommand(ctx, gitArgs...).
+					SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())).
+					RunWithContext(&git.RunContext{
+						Timeout: timeout,
+						Dir:     newRepoPath,
+						Stdout:  &stdoutBuilder,
+						Stderr:  &stderrBuilder,
+					}); err != nil {
+					stdout := stdoutBuilder.String()
+					stderr := stderrBuilder.String()
+
+					// sanitize the output, since it may contain the remote address, which may
+					// contain a password
+					stderrMessage = sanitizer.Replace(stderr)
+					stdoutMessage = sanitizer.Replace(stdout)
+				}
+			}
+		}
+
+		// If there is still an error (or there always was an error)
+		if err != nil {
+			log.Error("SyncMirrors [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err)
+			desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", newRepoPath, stderrMessage)
+			if err = admin_model.CreateRepositoryNotice(desc); err != nil {
+				log.Error("CreateRepositoryNotice: %v", err)
+			}
+			return false, nil
+		}
+	}
+	return false, nil
+}
+
+// detect user can update the mirror
+func detectCanUpdateMirror(ctx context.Context, m *repo_model.Mirror, gitArgs []string) (bool, error) {
+	repoPath := m.Repo.RepoPath()
+	newRepoPath := fmt.Sprintf("%s_update", repoPath)
+
+	// do copy directory recursive
+	err := util.CopyDir(repoPath, newRepoPath)
+	defer func() {
+		// delete the temp directory
+		errDelete := util.RemoveAll(newRepoPath)
+		if errDelete != nil {
+			log.Error("DeleteRepositoryTempDirectoryError: %v", errDelete)
+		}
+	}()
+	if err != nil {
+		log.Error("GetMirrorCanUpdate [repo: %-v]: CopyDirectory Error %v", m.Repo, err)
+		return false, err
+	}
+	syncStatus, err := syncRepoMirror(ctx, m, gitArgs, newRepoPath)
+	if err != nil {
+		return false, err
+	}
+	if !syncStatus {
+		return false, nil
+	}
+	gitCommitCountArgs := []string{"rev-list", "HEAD", "--count"}
+	stdoutNewRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, newRepoPath)
+	if err != nil {
+		return false, err
+	}
+	stdoutNewRepoCommitCount = strings.TrimSpace(stdoutNewRepoCommitCount)
+	stdoutRepoCommitCount, _, err := getGitCommandStdoutStderr(ctx, m, gitCommitCountArgs, repoPath)
+	if err != nil {
+		return false, err
+	}
+	stdoutRepoCommitCount = strings.TrimSpace(stdoutRepoCommitCount)
+	var repoCommitCount, newRepoCommitCount int64
+	if i, err := strconv.ParseInt(stdoutRepoCommitCount, 10, 64); err == nil {
+		repoCommitCount = i
+	} else {
+		return false, err
+	}
+	if i, err := strconv.ParseInt(stdoutNewRepoCommitCount, 10, 64); err == nil {
+		newRepoCommitCount = i
+	} else {
+		return false, err
+	}
+	if repoCommitCount > newRepoCommitCount {
+		return false, nil
+	} else if repoCommitCount == newRepoCommitCount {
+		// noting to happen
+		return true, nil
+	} else {
+		// compare commit id
+		skipcout := newRepoCommitCount - repoCommitCount
+		gitNewRepoLastCommitIDArgs := []string{"log", "-1", fmt.Sprintf("--skip=%d", skipcout), "--format='%H'"}
+		stdoutNewRepoCommitID, _, err := getGitCommandStdoutStderr(ctx, m, gitNewRepoLastCommitIDArgs, newRepoPath)
+		if err != nil {
+			return false, err
+		}
+		gitRepoLastCommitIDArgs := []string{"log", "--format='%H'", "-n", "1"}
+		stdoutRepoCommitID, _, err := getGitCommandStdoutStderr(ctx, m, gitRepoLastCommitIDArgs, repoPath)
+		if err != nil {
+			return false, err
+		}
+		if stdoutNewRepoCommitID != stdoutRepoCommitID {
+			return false, fmt.Errorf("Old repo commit id: %s not match new repo id: %s", stdoutRepoCommitID, stdoutNewRepoCommitID)
+		}
+	}
+	return true, nil
+}

templates/repo/settings/options.tmpl

@@ -115,6 +115,13 @@
 									<label>{{.i18n.Tr "repo.mirror_prune_desc"}}</label>
 										</div>
 									</div>
+									<div class="inline field {{if .Err_EnableSafeMirror}}error{{end}}">
+										<label>{{.i18n.Tr "repo.mirror_enable_safe"}}</label>
+										<div class="ui checkbox">
+									<input id="enable_safe_mirror" name="enable_safe_mirror" type="checkbox" {{if .EnableSafeMirror}}checked{{end}}>
+									<label>{{.i18n.Tr "repo.mirror_enable_safe_desc"}}</label>
+										</div>
+									</div>
 									<div class="inline field {{if .Err_Interval}}error{{end}}">
 										<label for="interval">{{.i18n.Tr "repo.mirror_interval"}}</label>
 										<input id="interval" name="interval" value="{{.MirrorInterval}}">