Interval between repository scans in now configurable

The default is 5 minutes, no restart is needed to change the value
(ref #18)

README.md

@@ -76,6 +76,7 @@ GeoipDatabasePath | Path to the geoip databases
 ConcurrentSync | Maximum number of server sync (rsync/ftp) do to simultaneously
 ScanInterval | Interval between rsync/ftp synchronizations (in minutes)
 CheckInterval | Interval between mirrors health's checks (in minutes)
+RepositoryScanInterval | Interval between scans of the local repository (in minutes, 0 to disable)
 Hashes | List of file hashes to computes (SHA1, SHA256, MD5)
 DisallowRedirects | Disable any mirror trying to do an HTTP redirect
 WeightDistributionRange | Multiplier of the distance to the first mirror to find other possible mirrors in order to distribute the load

config.go

@@ -14,18 +14,19 @@ import (
 
 var (
 	defaultConfig = configuration{
-		Repository:        "",
-		Templates:         "",
-		OutputMode:        "auto",
-		ListenAddress:     ":8080",
-		Gzip:              false,
-		RedisAddress:      "127.0.0.1:6379",
-		RedisPassword:     "",
-		LogDir:            "",
-		GeoipDatabasePath: "/usr/share/GeoIP/",
-		ConcurrentSync:    2,
-		ScanInterval:      30,
-		CheckInterval:     1,
+		Repository:             "",
+		Templates:              "",
+		OutputMode:             "auto",
+		ListenAddress:          ":8080",
+		Gzip:                   false,
+		RedisAddress:           "127.0.0.1:6379",
+		RedisPassword:          "",
+		LogDir:                 "",
+		GeoipDatabasePath:      "/usr/share/GeoIP/",
+		ConcurrentSync:         2,
+		ScanInterval:           30,
+		CheckInterval:          1,
+		RepositoryScanInterval: 5,
 		Hashes: hashing{
 			SHA1:   true,
 			SHA256: false,
@@ -52,6 +53,7 @@ type configuration struct {
 	ConcurrentSync          int        `yaml:"ConcurrentSync"`
 	ScanInterval            int        `yaml:"ScanInterval"`
 	CheckInterval           int        `yaml:"CheckInterval"`
+	RepositoryScanInterval  int        `yaml:"RepositoryScanInterval"`
 	Hashes                  hashing    `yaml:"Hashes"`
 	DisallowRedirects       bool       `yaml:"DisallowRedirects"`
 	WeightDistributionRange float32    `yaml:"WeightDistributionRange"`
@@ -125,6 +127,9 @@ func ReloadConfig() error {
 		return fmt.Errorf("Config: outputMode can only be set to 'auto', 'json' or 'redirect'")
 	}
 	c.Repository = strings.TrimRight(c.Repository, "/")
+	if c.RepositoryScanInterval < 0 {
+		c.RepositoryScanInterval = 0
+	}
 
 	if config != nil &&
 		(c.RedisAddress != config.RedisAddress ||

mirrorbits.conf

@@ -17,6 +17,7 @@ GeoipDatabasePath: /usr/share/GeoIP/
 ConcurrentSync: 5
 ScanInterval: 30
 CheckInterval: 1
+RepositoryScanInterval: 5
 Hashes:
     SHA1: On
     SHA256: Off

mirrors.go

@@ -35,6 +35,7 @@ type Mirror struct {
 	StateSince         int64    `redis:"stateSince" json:",omitempty" yaml:"-"`
 	Distance           float32  `redis:"-" yaml:"-"`
 	CountryFields      []string `redis:"-" json:"-" yaml:"-"`
+	Filepath           string   `redis:"-" json:"-" yaml:"-"`
 	Weight             int      `redis:"-" json:"-" yaml:"-"`
 	ComputedScore      int      `redis:"-" yaml:"-"`
 	LastSync           int64    `redis:"lastSync" yaml:"-"`

monitor.go

@@ -114,7 +114,7 @@ func checkRedirect(req *http.Request, via []*http.Request) error {
 // Main monitor loop
 func (m *Monitor) monitorLoop() {
 	m.wg.Add(1)
-	m.syncSource()
+	m.scanRepository()
 
 	mirrorUpdateEvent := make(chan string, 10)
 	m.redis.pubsub.SubscribeEvent(MIRROR_UPDATE, mirrorUpdateEvent)
@@ -144,8 +144,10 @@ func (m *Monitor) monitorLoop() {
 	}
 
 	// Setup recurrent tasks
-	sourceSyncTicker := time.NewTicker(5 * time.Minute)
+	repositoryScanInterval := -1
+	repositoryScanTicker := time.NewTicker(1 * time.Minute)
 	mirrorCheckTicker := time.NewTicker(1 * time.Second)
+	configUpdateTicker := time.NewTicker(1 * time.Second)
 
 	for {
 		select {
@@ -154,8 +156,18 @@ func (m *Monitor) monitorLoop() {
 			return
 		case id := <-mirrorUpdateEvent:
 			m.syncMirrorList(id)
-		case <-sourceSyncTicker.C:
-			m.syncSource()
+		case <-configUpdateTicker.C:
+			if repositoryScanInterval != GetConfig().RepositoryScanInterval {
+				repositoryScanInterval = GetConfig().RepositoryScanInterval
+
+				if repositoryScanInterval == 0 {
+					repositoryScanTicker.Stop()
+				} else {
+					repositoryScanTicker = time.NewTicker(time.Duration(repositoryScanInterval) * time.Minute)
+				}
+			}
+		case <-repositoryScanTicker.C:
+			m.scanRepository()
 		case <-mirrorCheckTicker.C:
 			m.mapLock.Lock()
 			for k, v := range m.mirrors {
@@ -434,7 +446,7 @@ func (m *Monitor) getRandomFile(identifier string) (file string, size int64, err
 }
 
 // Trigger a sync of the local repository
-func (m *Monitor) syncSource() {
+func (m *Monitor) scanRepository() {
 	err := ScanSource(m.redis, m.stop)
 	if err != nil {
 		log.Error("Scanning source failed: %s", err.Error())