Merge pull request #4329 from influxdb/md-stress-timestamps

Add support for evenly spaced timestamps

cmd/influx_stress/examples/100B_FLAT.toml

@@ -0,0 +1,30 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "n"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "10s"
+  jitter = false
+  point_count = 1000000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 100000
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64-flat"

cmd/influx_stress/examples/100B_STD.toml

@@ -0,0 +1,30 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "n"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "1ns"
+  jitter = true
+  point_count = 100000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 100000
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"

cmd/influx_stress/examples/100M_FLAT.toml

@@ -0,0 +1,47 @@
+channel_buffer_size = 100000
+
+[write]
+  concurrency = 10
+  batch_size = 10000
+  batch_interval = "0s"
+  database = "stress"
+  precision = "s"
+  address = "localhost:8086"
+  reset_database = true
+  start_date = "2006-Jan-02"
+
+[[series]]
+  tick = "5s"
+  jitter = false
+  point_count = 10000 # number of points that will be written for each of the series
+  measurement = "cpu"
+  series_count = 10000
+
+ # tag_count = 20 # number of "generic" tags on a series (e.g. tag-key-1=tag-value, ... ,tag-key-20=tag-value)
+
+  [[series.tag]]
+    key = "host"
+    value = "server"
+
+  [[series.tag]]
+    key = "location"
+    value = "loc"
+
+  [[series.field]]
+    key = "value"
+    type = "float64"
+
+  # Doesnt work as expected
+##  [[series.field]]
+##    key = "value-2"
+##    type = "float64-inc"
+##
+##  [[series.field]]
+##    key = "value-3"
+##    type = "float64-inc+"
+
+ # Has 80% probability of being a constant value
+  [[series.field]]
+    key = "flat_value"
+    type = "float64-flat"
+

cmd/influx_stress/examples/example_3.toml → cmd/influx_stress/examples/100M_STD.toml

@@ -5,12 +5,14 @@ channel_buffer_size = 100000
   batch_size = 10000
   batch_interval = "0s"
   database = "stress"
-  precision = "n"
+  precision = "s"
   address = "localhost:8086"
   reset_database = true
-  starting_point = 1 # how far back in time to go in weeks
+  start_date = "2006-Jan-02"
 
 [[series]]
+  tick = "5s"
+  jitter = false
   point_count = 10000 # number of points that will be written for each of the series
   measurement = "cpu"
   series_count = 10000
@@ -19,13 +21,12 @@ channel_buffer_size = 100000
 
   [[series.tag]]
     key = "host"
-    value = "idk"
+    value = "server"
 
   [[series.tag]]
     key = "location"
-    value = "lame"
+    value = "loc"
 
   [[series.field]]
     key = "value"
     type = "float64"
-

cmd/influx_stress/examples/example_2.toml → cmd/influx_stress/examples/5M_STD.toml

@@ -5,13 +5,14 @@ channel_buffer_size = 100000
   batch_size = 5000
   batch_interval = "0s"
   database = "stress"
-  precision = "n"
+  precision = "s"
   address = "localhost:8086"
   reset_database = true
-  # doesnt do anything
-  starting_point = 4 # how far back in time to go in weeks
+  start_date = "2006-Jan-02"
 
 [[series]]
+  tick = "5s"
+  jitter = false
   point_count = 50 # number of points that will be written for each of the series
   measurement = "cpu"
   series_count = 100000

cmd/influx_stress/examples/example_4.toml → cmd/influx_stress/examples/long_form_date.toml

@@ -8,10 +8,11 @@ channel_buffer_size = 100000
   precision = "s"
   address = "localhost:8086"
   reset_database = true
-  # doesnt do anything
-  starting_point = 1 # how far back in time to go in weeks
+  start_date = "Jan 2, 2006 at 3:04pm (MST)"
 
 [[series]]
+  tick = "5s"
+  jitter = true
   point_count = 10000 # number of points that will be written for each of the series
   measurement = "cpu"
   series_count = 10000

cmd/influx_stress/examples/moderate_burn.toml

@@ -2,15 +2,16 @@ channel_buffer_size = 100
 
 [write]
   concurrency = 10
-  batch_size = 5000
-  batch_interval = "4s"
+  batch_size = 10000
+  batch_interval = "1s"
   database = "stress"
   precision = "n"
   address = "localhost:8086"
   reset_database = true
-  starting_point = 4 # how far back in time to go in weeks
 
 [[series]]
+  tick = "5s"
+  jitter = true
   point_count = 10000 # number of points that will be written for each of the series
   measurement = "cpu"
   series_count = 100000

cmd/influx_stress/examples/example_1.toml → cmd/influx_stress/examples/template.toml

@@ -20,14 +20,16 @@ channel_buffer_size = 100000
   address = "localhost:8086"
   # Drop and Create new DB
   reset_database = true
-  # How far back in time to start writting points in weeks
-  starting_point = 3
-  # Randomize timestamp a bit
-  jitter = 100000
+  # The date for the first point that is written into influx
+  start_date = "2006-Jan-02"
 
 # Describes the schema for series that will be
 # written
 [[series]]
+  # How much time between each timestamp
+  tick = "5s"
+  # Randomize timestamp a bit
+  jitter = false
   # number of points that will be written for each of the series
   point_count = 1000
   # name of the measurement that will be written
@@ -63,6 +65,7 @@ channel_buffer_size = 100000
     key = "default"
 
 [[series]]
+  tick = "1ns"
   point_count = 100 # number of points that will be written for each of the series
   measurement = "mem"
   series_count = 100000

stress/config.go

@@ -28,6 +28,8 @@ type field struct {
 // during a stress test
 type series struct {
 	PointCount  int     `toml:"point_count"`
+	Tick        string  `toml:"tick"`
+	Jitter      bool    `toml:"jitter"`
 	Measurement string  `toml:"measurement"`
 	SeriesCount int     `toml:"series_count"`
 	TagCount    int     `toml:"tag_count"`
@@ -45,10 +47,9 @@ type write struct {
 	BatchInterval string `toml:"batch_interval"`
 	Database      string `toml:"database"`
 	ResetDatabase bool   `toml:"reset_database"`
-	StartingPoint int    `toml:"starting_point"`
 	Address       string `toml:"address"`
 	Precision     string `toml:"precision"`
-	Jitter        int    `toml:"jitter"`
+	StartDate     string `toml:"start_date"`
 }
 
 // query is a struct that contains the logic for
@@ -85,6 +86,7 @@ type Config struct {
 	MeasurementQuery  measurementQuery `toml:"measurement_query"`
 	SeriesQuery       seriesQuery      `toml:"series_query"`
 	ChannelBufferSize int              `toml:"channel_buffer_size"`
+	SSL               bool             `toml:"ssl"`
 }
 
 // NewSeries, takes a measurement, and point count,
@@ -93,6 +95,7 @@ func NewSeries(m string, p int, sc int) series {
 	s := series{
 		PointCount:  p,
 		SeriesCount: sc,
+		Tick:        "1s",
 		Measurement: m,
 		Tags: []tag{
 			tag{
@@ -164,7 +167,7 @@ func DecodeFile(s string) (*Config, error) {
 
 // seriesIter is a struct that contains a
 // series and a count, where count is the
-// number of points that have been written
+//number of points that have been written
 // for the series `s`
 type seriesIter struct {
 	s         *series
@@ -175,16 +178,29 @@ type seriesIter struct {
 
 // writeInterval returns a timestamp for the current time
 // interval
-func (s *series) writeInterval(weeks int, i int) time.Time {
-	st := time.Duration(weeks) * 7 * 24 * time.Hour
-	w := st - (st/time.Duration(s.PointCount))*time.Duration(i)
-	return time.Now().Add(-1 * w)
+func (s *series) writeInterval(i int, start time.Time) time.Time {
+	var tick time.Duration
+	var j int
+	var err error
+
+	tick, err = time.ParseDuration(s.Tick)
+	if err != nil {
+		panic(err)
+	}
+
+	if s.Jitter {
+		j = rand.Intn(int(tick))
+	}
+
+	tick = tick*time.Duration(i) + time.Duration(j)
+
+	return start.Add(tick)
 }
 
 // Iter returns a pointer to a seriesIter
-func (s *series) Iter(weeks int, i int, p string) *seriesIter {
+func (s *series) Iter(i int, start time.Time, p string) *seriesIter {
 
-	return &seriesIter{s: s, count: -1, timestamp: s.writeInterval(weeks, i), precision: p}
+	return &seriesIter{s: s, count: -1, timestamp: s.writeInterval(i, start), precision: p}
 }
 
 // Next returns a new point for a series.
@@ -196,7 +212,7 @@ func (iter *seriesIter) Next() ([]byte, bool) {
 	buf.Write([]byte(fmt.Sprintf("%v,", iter.s.Measurement)))
 	buf.Write(iter.s.newTagSet(iter.count))
 	buf.Write([]byte(" "))
-	buf.Write(iter.s.newFieldSet())
+	buf.Write(iter.s.newFieldSet(iter.count))
 	buf.Write([]byte(" "))
 
 	switch iter.precision {
@@ -228,11 +244,21 @@ func (s *series) newTagSet(c int) []byte {
 
 // newFieldSet returns a byte array representation
 // of the field-set for a series
-func (s *series) newFieldSet() []byte {
+func (s *series) newFieldSet(c int) []byte {
 	var buf bytes.Buffer
 
 	for _, field := range s.Fields {
 		switch field.Type {
+		case "float64-flat":
+			if rand.Intn(10) > 2 {
+				buf.Write([]byte(fmt.Sprintf("%v=%v,", field.Key, 100)))
+			} else {
+				buf.Write([]byte(fmt.Sprintf("%v=%v,", field.Key, 100+rand.Intn(100))))
+			}
+		case "float64-inc+":
+			buf.Write([]byte(fmt.Sprintf("%v=%v,", field.Key, c+rand.Intn(2))))
+		case "float64-inc":
+			buf.Write([]byte(fmt.Sprintf("%v=%v,", field.Key, c)))
 		case "float64":
 			buf.Write([]byte(fmt.Sprintf("%v=%v,", field.Key, rand.Intn(1000))))
 		case "int":