Merge pull request #9801 from influxdata/bj-validate-write

Add option for unicode validation.

etc/config.sample.toml

@@ -67,6 +67,10 @@
   # log any sensitive data contained within a query.
   # query-log-enabled = true
 
+  # Validates incoming writes to ensure keys only have valid unicode characters.
+  # This setting will incur a small overhead because every key must be checked.
+  # validate-keys = false
+
   # Settings for the TSM engine
 
   # CacheMaxMemorySize is the maximum size a shard's cache can

tsdb/config.go

@@ -82,6 +82,9 @@ type Config struct {
 	// disks or when WAL write contention is seen.  A value of 0 fsyncs every write to the WAL.
 	WALFsyncDelay toml.Duration `toml:"wal-fsync-delay"`
 
+	// Enables unicode validation on series keys on write.
+	ValidateKeys bool `toml:"validate-keys"`
+
 	// Query logging
 	QueryLogEnabled bool `toml:"query-log-enabled"`
 

tsdb/shard.go

@@ -538,6 +538,9 @@ func (s *Shard) validateSeriesAndFields(points []models.Point) ([]models.Point,
 	names := make([][]byte, len(points))
 	tagsSlice := make([]models.Tags, len(points))
 
+	// Check if keys should be unicode validated.
+	validateKeys := s.options.Config.ValidateKeys
+
 	var j int
 	for i, p := range points {
 		tags := p.Tags()
@@ -553,6 +556,15 @@ func (s *Shard) validateSeriesAndFields(points []models.Point) ([]models.Point,
 			continue
 		}
 
+		// Drop any series with invalid unicode characters in the key.
+		if validateKeys && !models.ValidKeyTokens(string(p.Name()), tags) {
+			dropped++
+			if reason == "" {
+				reason = fmt.Sprintf("key contains invalid unicode: \"%s\"", string(p.Key()))
+			}
+			continue
+		}
+
 		keys[j] = p.Key()
 		names[j] = p.Name()
 		tagsSlice[j] = tags