From b75d56fd387a3af2706f75c0536cf112fdf6b788 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 23 Jun 2022 13:18:33 -0500 Subject: [PATCH] Add new compactionstats output behind --vtable Patch by brandonwilliams; reviewed by edimitrov and dcapwell for CASSANDRA-17683 --- CHANGES.txt | 1 + NEWS.txt | 1 + .../tools/nodetool/CompactionStats.java | 24 +++- .../tools/nodetool/CompactionStatsTest.java | 107 ++++++++++++++++-- 4 files changed, 121 insertions(+), 12 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a8def999c53f..6ad96664b3df 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1-alpha2 + * Add new vtable output option to compactionstats (CASSANDRA-17683) * Fix commitLogUpperBound initialization in AbstractMemtableWithCommitlog (CASSANDRA-17587) * Fix widening to long in getBatchSizeFailThreshold (CASSANDRA-17650) * Fix widening from mebibytes to bytes in IntMebibytesBound (CASSANDRA-17716) diff --git a/NEWS.txt b/NEWS.txt index 9f0332c9e1af..75ea4986d584 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -95,6 +95,7 @@ New features native_transport_max_requests_per_second in cassandra.yaml. - Support for pre hashing passwords on CQL DCL commands - Expose all client options via system_views.clients and nodetool clientstats --client-options. + - Add new nodetool compactionstats --vtable option to match the sstable_tasks vtable. - Support for String concatenation has been added through the + operator. - New configuration max_hints_size_per_host to limit the size of local hints files per host in mebibytes. Setting to non-positive value disables the limit, which is the default behavior. Setting to a positive value to ensure diff --git a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java index 87262801c3a2..76bbbbf90b2e 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java +++ b/src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java @@ -44,6 +44,11 @@ public class CompactionStats extends NodeToolCmd description = "Display bytes in human readable form, i.e. KiB, MiB, GiB, TiB") private boolean humanReadable = false; + @Option(title = "vtable_output", + name = {"-V", "--vtable"}, + description = "Display fields matching vtable output") + private boolean vtableOutput = false; + @Override public void execute(NodeProbe probe) { @@ -70,17 +75,26 @@ public void execute(NodeProbe probe) } } out.println(); - reportCompactionTable(cm.getCompactions(), probe.getCompactionThroughput(), humanReadable, out); + reportCompactionTable(cm.getCompactions(), probe.getCompactionThroughput(), humanReadable, vtableOutput, out); } public static void reportCompactionTable(List> compactions, int compactionThroughput, boolean humanReadable, PrintStream out) + { + reportCompactionTable(compactions, compactionThroughput, humanReadable, false, out); + } + + public static void reportCompactionTable(List> compactions, int compactionThroughput, boolean humanReadable, boolean vtableOutput, PrintStream out) { if (!compactions.isEmpty()) { long remainingBytes = 0; TableBuilder table = new TableBuilder(); - table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "unit"); + if (vtableOutput) + table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "unit"); + else + table.add("id", "compaction type", "keyspace", "table", "completed", "total", "unit", "progress"); + for (Map c : compactions) { long total = Long.parseLong(c.get(CompactionInfo.TOTAL)); @@ -95,7 +109,11 @@ public static void reportCompactionTable(List> compactions, i String totalStr = toFileSize ? FileUtils.stringifyFileSize(total) : Long.toString(total); String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + "%"; String id = c.get(CompactionInfo.COMPACTION_ID); - table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit); + if (vtableOutput) + table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit); + else + table.add(id, taskType, keyspace, columnFamily, progressStr, totalStr, unit, percentComplete); + remainingBytes += total - completed; } table.printTo(out); diff --git a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java index 71b4c1726019..4ed1e000c5a5 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java @@ -65,7 +65,7 @@ public void testMaybeChangeDocs() " [(-pp | --print-port)] [(-pw | --password )]\n" + " [(-pwf | --password-file )]\n" + " [(-u | --username )] compactionstats\n" + - " [(-H | --human-readable)]\n" + + " [(-H | --human-readable)] [(-V | --vtable)]\n" + "\n" + "OPTIONS\n" + " -h , --host \n" + @@ -89,6 +89,9 @@ public void testMaybeChangeDocs() " -u , --username \n" + " Remote jmx agent username\n" + "\n" + + " -V, --vtable\n" + + " Display fields matching vtable output\n" + + "\n" + "\n"; assertThat(tool.getStdout()).isEqualTo(help); } @@ -123,14 +126,57 @@ public boolean isGlobal() tool.assertOnCleanExit(); String stdout = tool.getStdout(); assertThat(stdout).contains("pending tasks: 1"); + Assertions.assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress"); + String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%", + compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), bytesCompacted, bytesTotal, + CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100); + Assertions.assertThat(stdout).containsPattern(expectedStatsPattern); + + CompactionManager.instance.active.finishCompaction(compactionHolder); + tool = ToolRunner.invokeNodetool("compactionstats"); + tool.assertOnCleanExit(); + stdout = tool.getStdout(); + assertThat(stdout).contains("pending tasks: 0"); + } + + @Test + public void testCompactionStatsVtable() + { + createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))"); + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + + long bytesCompacted = 123; + long bytesTotal = 123456; + TimeUUID compactionId = nextTimeUUID(); + List sstables = IntStream.range(0, 10) + .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) + .collect(Collectors.toList()); + CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder() + { + public CompactionInfo getCompactionInfo() + { + return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables); + } + + public boolean isGlobal() + { + return false; + } + }; + + CompactionManager.instance.active.beginCompaction(compactionHolder); + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("compactionstats", "-V"); + tool.assertOnCleanExit(); + String stdout = tool.getStdout(); + assertThat(stdout).contains("pending tasks: 1"); Assertions.assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit"); String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", - CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, - OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES); + CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, + OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES); Assertions.assertThat(stdout).containsPattern(expectedStatsPattern); CompactionManager.instance.active.finishCompaction(compactionHolder); - tool = ToolRunner.invokeNodetool("compactionstats"); + tool = ToolRunner.invokeNodetool("compactionstats", "-V"); tool.assertOnCleanExit(); stdout = tool.getStdout(); assertThat(stdout).contains("pending tasks: 0"); @@ -146,8 +192,8 @@ public void testCompactionStatsHumanReadable() long bytesTotal = 123456; TimeUUID compactionId = nextTimeUUID(); List sstables = IntStream.range(0, 10) - .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) - .collect(Collectors.toList()); + .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) + .collect(Collectors.toList()); CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder() { public CompactionInfo getCompactionInfo() @@ -166,14 +212,57 @@ public boolean isGlobal() tool.assertOnCleanExit(); String stdout = tool.getStdout(); assertThat(stdout).contains("pending tasks: 1"); + Assertions.assertThat(stdout).containsPattern("id\\s+compaction type\\s+keyspace\\s+table\\s+completed\\s+total\\s+unit\\s+progress"); + String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%.2f%%", + compactionId, OperationType.COMPACTION, CQLTester.KEYSPACE, currentTable(), "123 bytes", "120.56 KiB", + CompactionInfo.Unit.BYTES, (double) bytesCompacted / bytesTotal * 100); + Assertions.assertThat(stdout).containsPattern(expectedStatsPattern); + + CompactionManager.instance.active.finishCompaction(compactionHolder); + tool = ToolRunner.invokeNodetool("compactionstats", "--human-readable"); + tool.assertOnCleanExit(); + stdout = tool.getStdout(); + assertThat(stdout).contains("pending tasks: 0"); + } + + @Test + public void testCompactionStatsVtableHumanReadable() + { + createTable("CREATE TABLE %s (pk int, ck int, PRIMARY KEY (pk, ck))"); + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + + long bytesCompacted = 123; + long bytesTotal = 123456; + TimeUUID compactionId = nextTimeUUID(); + List sstables = IntStream.range(0, 10) + .mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs)) + .collect(Collectors.toList()); + CompactionInfo.Holder compactionHolder = new CompactionInfo.Holder() + { + public CompactionInfo getCompactionInfo() + { + return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables); + } + + public boolean isGlobal() + { + return false; + } + }; + + CompactionManager.instance.active.beginCompaction(compactionHolder); + ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("compactionstats", "--vtable", "--human-readable"); + tool.assertOnCleanExit(); + String stdout = tool.getStdout(); + assertThat(stdout).contains("pending tasks: 1"); Assertions.assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit"); String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s", - CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, - OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES); + CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100, + OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES); Assertions.assertThat(stdout).containsPattern(expectedStatsPattern); CompactionManager.instance.active.finishCompaction(compactionHolder); - tool = ToolRunner.invokeNodetool("compactionstats", "--human-readable"); + tool = ToolRunner.invokeNodetool("compactionstats", "--vtable", "--human-readable"); tool.assertOnCleanExit(); stdout = tool.getStdout(); assertThat(stdout).contains("pending tasks: 0");