Allow configuration of retention policy in toml

retention-policy = "168h"
shard-duration = "24h"

Fixes influxdata#2676

etc/config.sample.toml

@@ -18,6 +18,12 @@ port = 8086
 # Change this option to true to disable reporting.
 reporting-disabled = false
 
+# the period of time that data will be kept around
+retention-policy = "168h"
+
+# the amount of time date is kept on the shard after the retention policy expires
+shard-duration = "24h"
+
 # Controls settings for initial start-up. Once a node is successfully started,
 # these settings are ignored.  If a node is started with the -join flag,
 # these settings are ignored.

meta/config.go

@@ -24,29 +24,39 @@ const (
 
 	// DefaultCommitTimeout is the default commit timeout for the store.
 	DefaultCommitTimeout = 50 * time.Millisecond
+
+	// DefaultRetentionPolicyPeriod is the period of time that data will be kept around
+	DefaultRetentionPolicyDuration = 24 * 7 * time.Hour
+
+	// DefaultShardDuration is the amount of time it date is kept after the retention policy
+	DefaultShardDuration = 24 * time.Hour
 )
 
 // Config represents the meta configuration.
 type Config struct {
-	Dir                 string        `toml:"dir"`
-	Hostname            string        `toml:"hostname"`
-	BindAddress         string        `toml:"bind-address"`
-	Peers               []string      `toml:"peers"`
-	RetentionAutoCreate bool          `toml:"retention-autocreate"`
-	ElectionTimeout     toml.Duration `toml:"election-timeout"`
-	HeartbeatTimeout    toml.Duration `toml:"heartbeat-timeout"`
-	LeaderLeaseTimeout  toml.Duration `toml:"leader-lease-timeout"`
-	CommitTimeout       toml.Duration `toml:"commit-timeout"`
+	Dir                     string        `toml:"dir"`
+	Hostname                string        `toml:"hostname"`
+	BindAddress             string        `toml:"bind-address"`
+	Peers                   []string      `toml:"peers"`
+	RetentionAutoCreate     bool          `toml:"retention-autocreate"`
+	ElectionTimeout         toml.Duration `toml:"election-timeout"`
+	HeartbeatTimeout        toml.Duration `toml:"heartbeat-timeout"`
+	LeaderLeaseTimeout      toml.Duration `toml:"leader-lease-timeout"`
+	CommitTimeout           toml.Duration `toml:"commit-timeout"`
+	RetentionPolicyDuration toml.Duration `toml:"retention-policy"`
+	ShardDuration           toml.Duration `toml:"shard-duration"`
 }
 
 func NewConfig() Config {
 	return Config{
-		Hostname:            DefaultHostname,
-		BindAddress:         DefaultBindAddress,
-		RetentionAutoCreate: true,
-		ElectionTimeout:     toml.Duration(DefaultElectionTimeout),
-		HeartbeatTimeout:    toml.Duration(DefaultHeartbeatTimeout),
-		LeaderLeaseTimeout:  toml.Duration(DefaultLeaderLeaseTimeout),
-		CommitTimeout:       toml.Duration(DefaultCommitTimeout),
+		Hostname:                DefaultHostname,
+		BindAddress:             DefaultBindAddress,
+		RetentionAutoCreate:     true,
+		ElectionTimeout:         toml.Duration(DefaultElectionTimeout),
+		HeartbeatTimeout:        toml.Duration(DefaultHeartbeatTimeout),
+		LeaderLeaseTimeout:      toml.Duration(DefaultLeaderLeaseTimeout),
+		CommitTimeout:           toml.Duration(DefaultCommitTimeout),
+		RetentionPolicyDuration: toml.Duration(DefaultRetentionPolicyDuration),
+		ShardDuration:           toml.Duration(DefaultShardDuration),
 	}
 }

meta/config_test.go

@@ -17,6 +17,8 @@ election-timeout = "10s"
 heartbeat-timeout = "20s"
 leader-lease-timeout = "30h"
 commit-timeout = "40m"
+retention-policy = "168h"
+shard-duration = "24h"
 `, &c); err != nil {
 		t.Fatal(err)
 	}
@@ -32,5 +34,9 @@ commit-timeout = "40m"
 		t.Fatalf("unexpected leader lease timeout: %v", c.LeaderLeaseTimeout)
 	} else if time.Duration(c.CommitTimeout) != 40*time.Minute {
 		t.Fatalf("unexpected commit timeout: %v", c.CommitTimeout)
+	} else if time.Duration(c.RetentionPolicy) != 24*7*time.Hour {
+		t.Fatalf("unexpected retention policy duration: %v", c.RetentionPolicy)
+	} else if time.Duration(c.ShardDuration) != 24*time.Hour {
+		t.Fatalf("unexpected shard duration: %v", c.ShardDuration)
 	}
 }

meta/data.go

@@ -14,9 +14,6 @@ const (
 	// DefaultRetentionPolicyReplicaN is the default value of RetentionPolicyInfo.ReplicaN.
 	DefaultRetentionPolicyReplicaN = 1
 
-	// DefaultRetentionPolicyDuration is the default value of RetentionPolicyInfo.Duration.
-	DefaultRetentionPolicyDuration = 7 * (24 * time.Hour)
-
 	// MinRetentionPolicyDuration represents the minimum duration for a policy.
 	MinRetentionPolicyDuration = time.Hour
 )
@@ -566,9 +563,10 @@ type RetentionPolicyInfo struct {
 // NewRetentionPolicyInfo returns a new instance of RetentionPolicyInfo with defaults set.
 func NewRetentionPolicyInfo(name string) *RetentionPolicyInfo {
 	return &RetentionPolicyInfo{
-		Name:     name,
-		ReplicaN: DefaultRetentionPolicyReplicaN,
-		Duration: DefaultRetentionPolicyDuration,
+		Name:               name,
+		ReplicaN:           DefaultRetentionPolicyReplicaN,
+		Duration:           DefaultRetentionPolicyDuration,
+		ShardGroupDuration: DefaultShardDuration,
 	}
 }
 

meta/store.go

@@ -94,6 +94,12 @@ type Store struct {
 	// The amount of time without an apply before sending a heartbeat.
 	CommitTimeout time.Duration
 
+	// The amount of time that data will be kept around.
+	RetentionPolicyDuration time.Duration
+
+	// The amount of time that date is kept after retention policy expires.
+	ShardDuration time.Duration
+
 	Logger *log.Logger
 }
 
@@ -110,11 +116,13 @@ func NewStore(c Config) *Store {
 
 		retentionAutoCreate: c.RetentionAutoCreate,
 
-		HeartbeatTimeout:   time.Duration(c.HeartbeatTimeout),
-		ElectionTimeout:    time.Duration(c.ElectionTimeout),
-		LeaderLeaseTimeout: time.Duration(c.LeaderLeaseTimeout),
-		CommitTimeout:      time.Duration(c.CommitTimeout),
-		Logger:             log.New(os.Stderr, "", log.LstdFlags),
+		HeartbeatTimeout:        time.Duration(c.HeartbeatTimeout),
+		ElectionTimeout:         time.Duration(c.ElectionTimeout),
+		LeaderLeaseTimeout:      time.Duration(c.LeaderLeaseTimeout),
+		CommitTimeout:           time.Duration(c.CommitTimeout),
+		RetentionPolicyDuration: time.Duration(c.RetentionPolicyDuration),
+		ShardDuration:           time.Duration(c.ShardDuration),
+		Logger:                  log.New(os.Stderr, "", log.LstdFlags),
 	}
 }
 

tsdb/config_test.go

@@ -0,0 +1,33 @@
+package tsdb
+
+import (
+	"testing"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+func TestConfig_Parse(t *testing.T) {
+	// Parse configuration.
+	var c Config
+	if _, err := toml.Decode(`
+dir = "/tmp/foo"
+retention-auto-create = true
+retention-check-enabled = true
+retention-check-period = "45m"
+retention-create-period = "10m"
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate configuration.
+	if c.Dir != "/tmp/foo" {
+		t.Fatalf("unexpected dir: %s", c.Dir)
+	} else if c.RetentionAutoCreate != true {
+		t.Fatalf("unexpected retention auto create: %v", c.RetentionAutoCreate)
+	} else if c.RetentionCheckEnabled != true {
+		t.Fatalf("unexpected retention check enabled: %v", c.RetentionCheckEnabled)
+	} else if time.Duration(c.RetentionCheckPeriod) != 45*time.Minute {
+		t.Fatalf("unexpected retention check period: %v", c.RetentionCheckPeriod)
+	}
+}