Fix continous compaction edge case

The level planner would keep including the same TSM files to be
recompacted even if they were already quite compacted and split
across several TSM files.

Fixes #6683

CHANGELOG.md

@@ -44,6 +44,7 @@
 - [#2048](https://github.com/influxdata/influxdb/issues/2048): Check that retention policies exist before creating CQ
 - [#6702](https://github.com/influxdata/influxdb/issues/6702): Fix SELECT statement required privileges.
 - [#6701](https://github.com/influxdata/influxdb/issues/6701): Filter out sources that do not match the shard database/retention policy.
+- [#6683](https://github.com/influxdata/influxdb/issues/6683): Fix compaction planning re-compacting large TSM files
 
 ## v0.13.0 [2016-05-12]
 

tsdb/engine/tsm1/compact.go

@@ -191,7 +191,7 @@ func (c *DefaultPlanner) PlanLevel(level int) []CompactionGroup {
 	if len(cGroup) < limit {
 		return nil
 	}
-	return []CompactionGroup{cGroup[:limit]}
+	return []CompactionGroup{cGroup}
 
 }
 
@@ -276,8 +276,10 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup {
 			continue
 		}
 
-		// Skip the file if it's over the max size and contains a full block
-		if g.size() > uint64(maxTSMFileSize) && c.FileStore.BlockCount(g.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock {
+		// Skip the file if it's over the max size and contains a full block or the generation is split
+		// over multiple files.  In the latter case, that would mean the data in the file spilled over
+		// the 2GB limit.
+		if g.size() > uint64(maxTSMFileSize) && c.FileStore.BlockCount(g.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock || g.count() > 1 {
 			start = i + 1
 		}
 

tsdb/engine/tsm1/compact_test.go

@@ -1392,6 +1392,78 @@ func TestDefaultPlanner_Plan_CompactsMiddleSteps(t *testing.T) {
 	}
 }
 
+func TestDefaultPlanner_Plan_LargeSets(t *testing.T) {
+	cp := &tsm1.DefaultPlanner{
+		FileStore: &fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return []tsm1.FileStat{
+					tsm1.FileStat{
+						Path: "000000278-000000006.tsm",
+						Size: 2148340232,
+					},
+					tsm1.FileStat{
+						Path: "000000278-000000007.tsm",
+						Size: 2148356556,
+					},
+					tsm1.FileStat{
+						Path: "000000278-000000008.tsm",
+						Size: 167780181,
+					},
+					tsm1.FileStat{
+						Path: "000000446-000047040.tsm",
+						Size: 2148728539,
+					},
+					tsm1.FileStat{
+						Path: "000000446-000047041.tsm",
+						Size: 701863692,
+					},
+				}
+			},
+		},
+	}
+
+	tsm := cp.Plan(time.Now())
+	if exp, got := 0, len(tsm); got != exp {
+		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
+	}
+}
+
+func TestDefaultPlanner_Plan_LargeGeneration(t *testing.T) {
+	cp := &tsm1.DefaultPlanner{
+		FileStore: &fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return []tsm1.FileStat{
+					tsm1.FileStat{
+						Path: "000000278-000000006.tsm",
+						Size: 2148340232,
+					},
+					tsm1.FileStat{
+						Path: "000000278-000000007.tsm",
+						Size: 2148356556,
+					},
+					tsm1.FileStat{
+						Path: "000000278-000000008.tsm",
+						Size: 167780181,
+					},
+					tsm1.FileStat{
+						Path: "000000278-000047040.tsm",
+						Size: 2148728539,
+					},
+					tsm1.FileStat{
+						Path: "000000278-000047041.tsm",
+						Size: 701863692,
+					},
+				}
+			},
+		},
+	}
+
+	tsm := cp.Plan(time.Now())
+	if exp, got := 0, len(tsm); got != exp {
+		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
+	}
+}
+
 func assertValueEqual(t *testing.T, a, b tsm1.Value) {
 	if got, exp := a.UnixNano(), b.UnixNano(); got != exp {
 		t.Fatalf("time mismatch: got %v, exp %v", got, exp)