gogs#2018 able to sync now for mirrors

- Refactor code to use sync.UniqueQueue
- Closes gogs#3509

.editorconfig

@@ -8,10 +8,14 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[*.{go,tmpl}]
+[*.go]
 indent_style = tab
 indent_size = 4
 
+[*.tmpl]
+indent_style = tab
+indent_size = 2
+
 [*.{less,yml}]
 indent_style = space
 indent_size = 2

README.md

@@ -3,7 +3,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra
 
 ![](https://github.com/gogits/gogs/blob/master/public/img/gogs-large-resize.png?raw=true)
 
-##### Current tip version: 0.9.95 (see [Releases](https://github.com/gogits/gogs/releases) for binary versions)
+##### Current tip version: 0.9.96 (see [Releases](https://github.com/gogits/gogs/releases) for binary versions)
 
 | Web | UI  | Preview  |
 |:-------------:|:-------:|:-------:|

conf/app.ini

@@ -17,8 +17,10 @@ ANSI_CHARSET =
 FORCE_PRIVATE = false
 ; Global maximum creation limit of repository per user, -1 means no limit
 MAX_CREATION_LIMIT = -1
-; Patch test queue length, make it as large as possible
-PULL_REQUEST_QUEUE_LENGTH = 10000
+; Mirror sync queue length, increase if mirror syncing starts hanging
+MIRROR_QUEUE_LENGTH = 1000
+; Patch test queue length, increase if pull request patch testing starts hanging
+PULL_REQUEST_QUEUE_LENGTH = 1000
 ; Preferred Licenses to place at the top of the List
 ; Name must match file name in conf/license or custom/conf/license
 PREFERRED_LICENSES = Apache License 2.0,MIT License
@@ -184,7 +186,7 @@ ENABLE_REVERSE_PROXY_AUTO_REGISTRATION = false
 ENABLE_CAPTCHA = true
 
 [webhook]
-; Hook task queue length
+; Hook task queue length, increase if webhook shooting starts hanging
 QUEUE_LENGTH = 1000
 ; Deliver timeout in seconds
 DELIVER_TIMEOUT = 5
@@ -389,7 +391,7 @@ GC = 60
 
 [mirror]
 ; Default interval in hours between each check
-DEFAULT_INTERVAL = 24
+DEFAULT_INTERVAL = 8
 
 [api]
 ; Max number of items will response in a page

conf/locale/locale_en-US.ini

@@ -370,6 +370,7 @@ mirror_prune_desc = Remove any remote-tracking references that no longer exist o
 mirror_interval = Mirror Interval (hour)
 mirror_address = Mirror Address
 mirror_address_desc = Please include necessary user credentials in the address.
+mirror_last_synced = Last Synced
 watchers = Watchers
 stargazers = Stargazers
 forks = Forks
@@ -631,6 +632,9 @@ settings.collaboration.undefined = Undefined
 settings.hooks = Webhooks
 settings.githooks = Git Hooks
 settings.basic_settings = Basic Settings
+settings.mirror_settings = Mirror Settings
+settings.sync_mirror = Sync Now
+settings.mirror_sync_in_progress = Mirror syncing is in progress, please refresh page in about a minute.
 settings.site = Official Site
 settings.update_settings = Update Settings
 settings.change_reponame_prompt = This change will affect how links relate to the repository.

gogs.go

@@ -17,7 +17,7 @@ import (
 	"github.com/gogits/gogs/modules/setting"
 )
 
-const APP_VER = "0.9.95.0830"
+const APP_VER = "0.9.96.0830"
 
 func init() {
 	runtime.GOMAXPROCS(runtime.NumCPU())

models/repo.go

@@ -381,7 +381,7 @@ func (repo *Repository) IssueStats(uid int64, filterMode int, isPull bool) (int6
 }
 
 func (repo *Repository) GetMirror() (err error) {
-	repo.Mirror, err = GetMirror(repo.ID)
+	repo.Mirror, err = GetMirrorByRepoID(repo.ID)
 	return err
 }
 
@@ -574,136 +574,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) {
 	return repo.cloneLink(false)
 }
 
-// Mirror represents a mirror information of repository.
-type Mirror struct {
-	ID          int64 `xorm:"pk autoincr"`
-	RepoID      int64
-	Repo        *Repository `xorm:"-"`
-	Interval    int         // Hour.
-	EnablePrune bool        `xorm:"NOT NULL DEFAULT true"`
-
-	Updated        time.Time `xorm:"-"`
-	UpdatedUnix    int64
-	NextUpdate     time.Time `xorm:"-"`
-	NextUpdateUnix int64
-
-	address string `xorm:"-"`
-}
-
-func (m *Mirror) BeforeInsert() {
-	m.NextUpdateUnix = m.NextUpdate.Unix()
-}
-
-func (m *Mirror) BeforeUpdate() {
-	m.UpdatedUnix = time.Now().Unix()
-	m.NextUpdateUnix = m.NextUpdate.Unix()
-}
-
-func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
-	var err error
-	switch colName {
-	case "repo_id":
-		m.Repo, err = GetRepositoryByID(m.RepoID)
-		if err != nil {
-			log.Error(3, "GetRepositoryByID[%d]: %v", m.ID, err)
-		}
-	case "updated_unix":
-		m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
-	case "next_updated_unix":
-		m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
-	}
-}
-
-func (m *Mirror) readAddress() {
-	if len(m.address) > 0 {
-		return
-	}
-
-	cfg, err := ini.Load(m.Repo.GitConfigPath())
-	if err != nil {
-		log.Error(4, "Load: %v", err)
-		return
-	}
-	m.address = cfg.Section("remote \"origin\"").Key("url").Value()
-}
-
-// HandleCloneUserCredentials replaces user credentials from HTTP/HTTPS URL
-// with placeholder <credentials>.
-// It will fail for any other forms of clone addresses.
-func HandleCloneUserCredentials(url string, mosaics bool) string {
-	i := strings.Index(url, "@")
-	if i == -1 {
-		return url
-	}
-	start := strings.Index(url, "://")
-	if start == -1 {
-		return url
-	}
-	if mosaics {
-		return url[:start+3] + "<credentials>" + url[i:]
-	}
-	return url[:start+3] + url[i+1:]
-}
-
-// Address returns mirror address from Git repository config without credentials.
-func (m *Mirror) Address() string {
-	m.readAddress()
-	return HandleCloneUserCredentials(m.address, false)
-}
-
-// FullAddress returns mirror address from Git repository config.
-func (m *Mirror) FullAddress() string {
-	m.readAddress()
-	return m.address
-}
-
-// SaveAddress writes new address to Git repository config.
-func (m *Mirror) SaveAddress(addr string) error {
-	configPath := m.Repo.GitConfigPath()
-	cfg, err := ini.Load(configPath)
-	if err != nil {
-		return fmt.Errorf("Load: %v", err)
-	}
-
-	cfg.Section("remote \"origin\"").Key("url").SetValue(addr)
-	return cfg.SaveToIndent(configPath, "\t")
-}
-
-func getMirror(e Engine, repoId int64) (*Mirror, error) {
-	m := &Mirror{RepoID: repoId}
-	has, err := e.Get(m)
-	if err != nil {
-		return nil, err
-	} else if !has {
-		return nil, ErrMirrorNotExist
-	}
-	return m, nil
-}
-
-// GetMirror returns mirror object by given repository ID.
-func GetMirror(repoId int64) (*Mirror, error) {
-	return getMirror(x, repoId)
-}
-
-func updateMirror(e Engine, m *Mirror) error {
-	_, err := e.Id(m.ID).AllCols().Update(m)
-	return err
-}
-
-func UpdateMirror(m *Mirror) error {
-	return updateMirror(x, m)
-}
-
-func DeleteMirrorByRepoID(repoID int64) error {
-	_, err := x.Delete(&Mirror{RepoID: repoID})
-	return err
-}
-
-func createUpdateHook(repoPath string) error {
-	return git.SetUpdateHook(repoPath,
-		fmt.Sprintf(_TPL_UPDATE_HOOK, setting.ScriptType, "\""+setting.AppPath+"\"", setting.CustomConf))
-}
-
 type MigrateRepoOptions struct {
 	Name        string
 	Description string
@@ -839,6 +709,11 @@ func cleanUpMigrateGitConfig(configPath string) error {
 	return nil
 }
 
+func createUpdateHook(repoPath string) error {
+	return git.SetUpdateHook(repoPath,
+		fmt.Sprintf(_TPL_UPDATE_HOOK, setting.ScriptType, "\""+setting.AppPath+"\"", setting.CustomConf))
+}
+
 // Finish migrating repository and/or wiki with things that don't need to be done for mirrors.
 func CleanUpMigrateInfo(repo *Repository) (*Repository, error) {
 	repoPath := repo.RepoPath()
@@ -1748,70 +1623,6 @@ const (
 	_CHECK_REPOs   = "check_repos"
 )
 
-// MirrorUpdate checks and updates mirror repositories.
-func MirrorUpdate() {
-	if taskStatusTable.IsRunning(_MIRROR_UPDATE) {
-		return
-	}
-	taskStatusTable.Start(_MIRROR_UPDATE)
-	defer taskStatusTable.Stop(_MIRROR_UPDATE)
-
-	log.Trace("Doing: MirrorUpdate")
-
-	mirrors := make([]*Mirror, 0, 10)
-	if err := x.Where("next_update_unix<=?", time.Now().Unix()).Iterate(new(Mirror), func(idx int, bean interface{}) error {
-		m := bean.(*Mirror)
-		if m.Repo == nil {
-			log.Error(4, "Disconnected mirror repository found: %d", m.ID)
-			return nil
-		}
-
-		repoPath := m.Repo.RepoPath()
-		wikiPath := m.Repo.WikiPath()
-		timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
-
-		gitArgs := []string{"remote", "update"}
-		if m.EnablePrune {
-			gitArgs = append(gitArgs, "--prune")
-		}
-
-		if _, stderr, err := process.ExecDir(
-			timeout, repoPath, fmt.Sprintf("MirrorUpdate: %s", repoPath),
-			"git", gitArgs...); err != nil {
-			desc := fmt.Sprintf("Fail to update mirror repository(%s): %s", repoPath, stderr)
-			log.Error(4, desc)
-			if err = CreateRepositoryNotice(desc); err != nil {
-				log.Error(4, "CreateRepositoryNotice: %v", err)
-			}
-			return nil
-		}
-		if m.Repo.HasWiki() {
-			if _, stderr, err := process.ExecDir(
-				timeout, wikiPath, fmt.Sprintf("MirrorUpdate: %s", wikiPath),
-				"git", "remote", "update", "--prune"); err != nil {
-				desc := fmt.Sprintf("Fail to update mirror wiki repository(%s): %s", wikiPath, stderr)
-				log.Error(4, desc)
-				if err = CreateRepositoryNotice(desc); err != nil {
-					log.Error(4, "CreateRepositoryNotice: %v", err)
-				}
-				return nil
-			}
-		}
-
-		m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
-		mirrors = append(mirrors, m)
-		return nil
-	}); err != nil {
-		log.Error(4, "MirrorUpdate: %v", err)
-	}
-
-	for i := range mirrors {
-		if err := UpdateMirror(mirrors[i]); err != nil {
-			log.Error(4, "UpdateMirror[%d]: %v", mirrors[i].ID, err)
-		}
-	}
-}
-
 // GitFsck calls 'git fsck' to check repository health.
 func GitFsck() {
 	if taskStatusTable.IsRunning(_GIT_FSCK) {