Allow usage check to be configurable (minio#6006)

cmd/common-main.go

@@ -153,6 +153,15 @@ func handleCommonEnvVars() {
 		globalCacheExpiry = expiry
 	}
 
+	if intervalStr := os.Getenv("MINIO_USAGE_CHECK_INTERVAL"); intervalStr != "" {
+		interval, err := parseDuration(intervalStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidUsageCheckIntervalValue(err), "Invalid MINIO_USAGE_CHECK_INTERVAL value (`%s`)", intervalStr)
+		}
+		globalUsageCheckInterval = interval
+		globalIsEnvUsageCheck = true
+	}
+
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.

cmd/config-current.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"reflect"
 	"sync"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
 
@@ -39,9 +40,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "23"
+const serverConfigVersion = "24"
 
-type serverConfig = serverConfigV23
+type serverConfig = serverConfigV24
 
 var (
 	// globalServerConfig server config.
@@ -104,6 +105,17 @@ func (s *serverConfig) GetBrowser() bool {
 	return bool(s.Browser)
 }
 
+// Set new usage configuration, currently only supports configuring
+// usage check interval.
+func (s *serverConfig) SetUsageConfig(checkUsageInterval time.Duration) {
+	s.Usage = usageConfig{checkUsageInterval}
+}
+
+// Get current usage configuration.
+func (s *serverConfig) GetUsageConfig() usageConfig {
+	return s.Usage
+}
+
 // SetCacheConfig sets the current cache config
 func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
 	s.Cache.Drives = drives
@@ -141,6 +153,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "StorageClass configuration differs"
 	case !reflect.DeepEqual(s.Cache, t.Cache):
 		return "Cache configuration differs"
+	case !reflect.DeepEqual(s.Usage, t.Usage):
+		return "Usage configuration differs"
 	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
@@ -186,6 +200,7 @@ func newServerConfig() *serverConfig {
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
 		},
+		Usage:  usageConfig{globalDefaultUsageCheckInterval},
 		Notify: notifier{},
 	}
 
@@ -246,6 +261,10 @@ func newConfig() error {
 		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
 	}
 
+	if globalIsEnvUsageCheck {
+		srvCfg.SetUsageConfig(globalUsageCheckInterval)
+	}
+
 	// hold the mutex lock before a new config is assigned.
 	// Save the new config globally.
 	// unlock the mutex.
@@ -339,6 +358,9 @@ func loadConfig() error {
 		globalCacheExcludes = cacheConf.Exclude
 		globalCacheExpiry = cacheConf.Expiry
 	}
+	if !globalIsEnvUsageCheck {
+		globalUsageCheckInterval = globalServerConfig.GetUsageConfig().UsageCheckInterval
+	}
 	globalServerConfigMu.Unlock()
 
 	return nil

cmd/config-migrate.go

@@ -172,6 +172,11 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "23":
+		if err = migrateV23ToV24(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -1951,3 +1956,120 @@ func migrateV22ToV23() error {
 	logger.Info(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV23ToV24() error {
+	configFile := getConfigFile()
+
+	cv23 := &serverConfigV23{}
+	_, err := quick.Load(configFile, cv23)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘23’. %v", err)
+	}
+	if cv23.Version != "23" {
+		return nil
+	}
+
+	// Copy over fields from V23 into V24 config struct
+	srvConfig := &serverConfigV24{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "24"
+	srvConfig.Credential = cv23.Credential
+	srvConfig.Region = cv23.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv23.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv23.Notify.AMQP
+	}
+	if len(cv23.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv23.Notify.Elasticsearch
+	}
+	if len(cv23.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv23.Notify.Redis
+	}
+	if len(cv23.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv23.Notify.PostgreSQL
+	}
+	if len(cv23.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv23.Notify.Kafka
+	}
+	if len(cv23.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv23.Notify.NATS
+	}
+	if len(cv23.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv23.Notify.Webhook
+	}
+	if len(cv23.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv23.Notify.MySQL
+	}
+
+	if len(cv23.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv23.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv23.Browser
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv23.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv23.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv23.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv23.Cache.Drives
+	srvConfig.Cache.Exclude = cv23.Cache.Exclude
+	srvConfig.Cache.Expiry = cv23.Cache.Expiry
+
+	// Init usage config. For future migration, usage config needs
+	// to be copied over from previous version.
+	srvConfig.Usage = usageConfig{globalDefaultUsageCheckInterval}
+
+	if err = quick.Save(configFile, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv23.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv23.Version, srvConfig.Version)
+	return nil
+}

cmd/config-versions.go

@@ -602,3 +602,30 @@ type serverConfigV23 struct {
 	// Notification queue configuration.
 	Notify notifier `json:"notify"`
 }
+
+// serverConfigV24 is just like version '23' with addition of usage interval
+// field.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV24 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Usage configuration
+	Usage usageConfig `json:"usage"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}

cmd/disk-usage.go

@@ -18,12 +18,55 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"time"
 )
 
-const (
-	usageCheckInterval = 12 * time.Hour // 12 hours
-)
+// Captures configurable parameters of usage check.
+type usageConfig struct {
+	UsageCheckInterval time.Duration
+}
+
+// MarshalJSON - encodes to JSON data.
+func (u usageConfig) MarshalJSON() ([]byte, error) {
+	type _usageConfig struct {
+		UsageCheckInterval string `json:"interval"`
+	}
+	return json.Marshal(_usageConfig{u.UsageCheckInterval.String()})
+}
+
+// parseDuration - parse duration string
+func parseDuration(dStr string) (time.Duration, error) {
+	d, err := time.ParseDuration(dStr)
+	if err != nil {
+		return d, err
+	}
+	if d < globalMinimumUsageCheckInterval {
+		return d, fmt.Errorf("interval %s is not allowed, minimum required value is %s",
+			d, globalMinimumUsageCheckInterval)
+	}
+	return d, nil
+}
+
+// UnmarshalJSON - decodes JSON data.
+func (u *usageConfig) UnmarshalJSON(data []byte) error {
+	type _usageConfig struct {
+		UsageCheckInterval string `json:"interval"`
+	}
+	var u1 = _usageConfig{}
+	if err := json.Unmarshal(data, &u1); err != nil {
+		return err
+	}
+	if !globalIsEnvUsageCheck {
+		d, err := parseDuration(u1.UsageCheckInterval)
+		if err != nil {
+			return err
+		}
+		u.UsageCheckInterval = d
+	}
+	return nil
+}
 
 // getDiskUsage walks the file tree rooted at root, calling usageFn
 // for each file or directory in the tree, including root.