Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CASSANDRA-14572 Expose all table metrics in virtual tables #2958

Open
wants to merge 1 commit into
base: trunk
Choose a base branch
from
Open

CASSANDRA-14572 Expose all table metrics in virtual tables #2958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
54 changes: 54 additions & 0 deletions .build/build-compile-annotations.xml
View file
Open in desktop
@@ -0,0 +1,54 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project basedir="." name="apache-cassandra-compile-annotations"
xmlns:if="ant:if">

<!-- Include annotation processor for the VirtualTable models. -->
<property name="processor.build.classes" value="${build.classes}/annotation-processor" />
<property name="processor.manifest" value="${processor.build.classes}/MANIFEST.MF" />

<target name="annotation-processor-build" depends="init" description="Build system view annotation processor">
<antcall target="_annotation_processor_build"/>
</target>

<target name="_annotation_processor_build">
<mkdir dir="${processor.build.classes}" />
<javac includes="**/virtual/proc/*.java"
srcdir="test/annotation-processor/src/main/java"
destdir="${processor.build.classes}"
includeantruntime="true"
source="${java.default}" target="${java.default}">
<src path="${build.src}"/>
</javac>
</target>

<target name="annotation-processor-jar"
depends="annotation-processor-build">
<mkdir dir="${processor.build.classes}/META-INF" />
<manifest file="${processor.manifest}">
<attribute name="Implementation-Vendor" value="Apache"/>
<attribute name="Implementation-Title" value="Cassandra"/>
<attribute name="Implementation-Version" value="${version}"/>
</manifest>
<jar destfile="${build.dir.lib}/jars/cassandra-annotation-processor-${version}.jar"
manifest="${processor.manifest}">
<fileset dir="${processor.build.classes}"/>
</jar>
</target>
</project>
1 change: 1 addition & 0 deletions .build/checkstyle_suppressions.xml
View file
Open in desktop
Expand Up @@ -22,4 +22,5 @@

<suppressions>
<suppress checks="RegexpSinglelineJava" files="Semaphore\.java"/>
<suppress files="[\\/]gen-java[\\/]" checks="[a-zA-Z0-9]*"/>
</suppressions>
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Expand Up @@ -4,6 +4,7 @@
build
build/
src/gen-java/
test/gen-java/
src/resources/org/apache/cassandra/config/
logs/
data/
Expand Down
18 changes: 17 additions & 1 deletion build.xml
View file
Open in desktop
Expand Up @@ -77,6 +77,7 @@
<property name="test.methods" value=""/>
<property name="test.anttasks.src" value="${test.dir}/anttasks"/>
<property name="test.unit.src" value="${test.dir}/unit"/>
<property name="test.gen-java" value="${test.dir}/gen-java"/>
<property name="test.long.src" value="${test.dir}/long"/>
<property name="test.burn.src" value="${test.dir}/burn"/>
<property name="test.memory.src" value="${test.dir}/memory"/>
Expand Down Expand Up @@ -391,6 +392,7 @@
<mkdir dir="${stress.test.classes}"/>
<mkdir dir="${fqltool.test.classes}"/>
<mkdir dir="${build.src.gen-java}"/>
<mkdir dir="${test.gen-java}"/>
<mkdir dir="${build.dir.lib}"/>
<mkdir dir="${jacoco.export.dir}"/>
<mkdir dir="${jacoco.partials.dir}"/>
Expand All @@ -415,6 +417,7 @@
<delete dir="${build.dir}/tmp" />
<delete dir="${build.classes}" />
<delete dir="${build.src.gen-java}" />
<delete dir="${test.gen-java}" />
<delete dir="${version.properties.dir}" />
<delete dir="${jacoco.export.dir}" />
<delete dir="${jacoco.partials.dir}"/>
Expand Down Expand Up @@ -529,13 +532,18 @@
<src path="${build.src.gen-java}"/>
<compilerarg value="-XDignore.symbol.file"/>
<compilerarg line="${jdk11plus-javac-exports}"/>
<compilerarg line="-processor"/>
<compilerarg line="org.apache.cassandra.db.virtual.proc.SystemViewAnnotationProcessor,net.nicoulaj.compilecommand.CompileCommandProcessor"/>
<compilerarg line="-s"/>
<compilerarg line="${build.src.gen-java}"/>
<compilerarg value="-XprintProcessorInfo"/>
<classpath>
<path refid="cassandra.classpath"/>
</classpath>
</javac>
</target>

<target depends="init,gen-cql3-grammar,generate-cql-html,generate-jflex-java"
<target depends="init,gen-cql3-grammar,generate-cql-html,generate-jflex-java,annotation-processor-jar"
name="build-project">
<echo message="${ant.project.name}: ${ant.file}"/>
<!-- Order matters! -->
Expand Down Expand Up @@ -1031,6 +1039,11 @@
<pathelement location="${fqltool.build.classes}"/>
</classpath>
<compilerarg value="-XDignore.symbol.file"/>
<compilerarg line="-processor"/>
<compilerarg line="org.apache.cassandra.db.virtual.proc.SystemViewAnnotationProcessor"/>
<compilerarg line="-s"/>
<compilerarg line="${test.gen-java}"/>
<compilerarg value="-XprintProcessorInfo"/>
<!-- needed to compile org.apache.cassandra.utils.memory.BufferPoolTest -->
<compilerarg line="${jdk11plus-javac-exports}"/>
<src path="${test.anttasks.src}"/>
Expand All @@ -1040,6 +1053,7 @@
<src path="${test.memory.src}"/>
<src path="${test.microbench.src}"/>
<src path="${test.distributed.src}"/>
<src path="${test.gen-java}"/>
<src path="${test.simulator.src}"/>
<src path="${test.simulator-asm.src}"/>
<src path="${test.simulator-bootstrap.src}"/>
Expand Down Expand Up @@ -1979,6 +1993,7 @@
<classpathentry kind="src" path="conf" including="hotspot_compiler"/>
<classpathentry kind="src" output="${test.classes}" path="test/unit"/>
<classpathentry kind="src" output="${test.classes}" path="test/long"/>
<classpathentry kind="src" output="${test.classes}" path="test/gen-java"/>
<classpathentry kind="src" output="${test.classes}" path="test/microbench"/>
<classpathentry kind="src" output="${test.classes}" path="test/memory"/>
<classpathentry kind="src" output="${test.classes}" path="test/burn"/>
Expand Down Expand Up @@ -2081,4 +2096,5 @@
<import file="${build.helpers.dir}/build-cqlsh.xml"/>
<import file="${build.helpers.dir}/build-bench.xml"/>
<import file="${build.helpers.dir}/build-sonar.xml"/>
<import file="${build.helpers.dir}/build-compile-annotations.xml"/>
</project>
60 changes: 58 additions & 2 deletions doc/modules/cassandra/pages/managing/operating/virtualtables.adoc
View file
Open in desktop
Expand Up @@ -24,10 +24,11 @@ Since a virtual table is not replicated the partitioner sorts in order of partit

== Virtual Keyspaces

Apache Cassandra 4.0 has added two new keyspaces for virtual tables:
Apache Cassandra has the following virtual keyspaces:

* `system_virtual_schema`
* `system_views`.
* `system_views`
* `system_metrics`

The `system_virtual_schema` keyspace has three tables: `keyspaces`,
`columns` and `tables` for the virtual keyspace, table, and column definitions, respectively.
Expand All @@ -36,6 +37,10 @@ It is used by Cassandra internally and a user should not access it directly.

The `system_views` keyspace contains the actual virtual tables.

The `system_metrics` keyspace contains the virtual tables of all the metrics available in Apache Cassandra
which can be used to monitor the health of the cluster. These metrics are split into several tables each
containing a different set of metrics corresponding to the type of metric or the component of the system.

== Virtual Table Limitations

Before discussing virtual keyspaces and tables, note that virtual keyspaces and tables have some limitations.
Expand All @@ -55,6 +60,57 @@ In fact, mutations for virtual and regular tables cannot occur in the same batch

== Virtual Tables

=== Virtual Tables _system_metrics_ Keyspace

The `system_metrics` virtual keyspace contains the virtual tables of all the metrics available in Apache Cassandra.
Currently, the `system_metrics` keyspace contains the following virtual tables (subject to change), which can be found
in the `system_metrics.all_groups` virtual table:

[source, console]
----
cqlsh> select * from system_metrics.all_groups ;

group_name | virtual_table
-------------------+---------------------------
Batch | batch_group
BufferPool | buffer_pool_group
CIDRAuthorizer | cidr_authorizer_group
CQL | cql_group
Cache | cache_group
ChunkCache | chunk_cache_group
Client | client_group
ClientMessageSize | client_message_size_group
ClientRequest | client_request_group
ClientRequestSize | client_request_size_group
ColumnFamily | column_family_group
CommitLog | commit_log_group
Compaction | compaction_group
Connection | connection_group
DroppedMessage | dropped_message_group
HintsService | hints_service_group
Index | index_group
IndexColumnFamily | index_column_family_group
IndexTable | index_table_group
InternodeInbound | internode_inbound_group
Keyspace | keyspace_group
MemtablePool | memtable_pool_group
Messaging | messaging_group
Paxos | paxos_group
ReadRepair | read_repair_group
Repair | repair_group
Storage | storage_group
StorageProxy | storage_proxy_group
Streaming | streaming_group
TCM | tcm_group
Table | table_group
ThreadPools | thread_pools_group
TrieMemtable | trie_memtable_group

(33 rows)
----

=== Virtual Tables _system_views_ Keyspace

Each of the virtual tables in the `system_views` virtual keyspace contain different information.

The following table describes the virtual tables:
Expand Down
3 changes: 3 additions & 0 deletions ide/idea-iml-file.xml
View file
Open in desktop
Expand Up @@ -25,6 +25,9 @@
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/gen-java" isTestSource="false" generated="true" />
<sourceFolder url="file://$MODULE_DIR$/test/gen-java" isTestSource="false" generated="true" />
<sourceFolder url="file://$MODULE_DIR$/test/annotation-processor/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/test/annotation-processor/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/tools/stress/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tools/stress/test/unit" isTestSource="true" />
Expand Down
6 changes: 6 additions & 0 deletions ide/nbproject/project.xml
View file
Open in desktop
Expand Up @@ -47,6 +47,12 @@
<location>${project.dir}/test/unit</location>
<encoding>UTF-8</encoding>
</source-folder>
<source-folder>
<label>test/gen-java</label>
<type>java</type>
<location>${project.dir}/test/gen-java</location>
<encoding>UTF-8</encoding>
</source-folder>
<source-folder>
<label>test/fqltool/</label>
<type>java</type>
Expand Down
5 changes: 3 additions & 2 deletions pylib/cqlshlib/cql3handling.py
View file
Open in desktop
Expand Up @@ -34,8 +34,9 @@ def __str__(self):


SYSTEM_KEYSPACES = ('system', 'system_schema', 'system_traces', 'system_auth', 'system_distributed', 'system_views',
'system_virtual_schema', 'system_cluster_metadata')
NONALTERBALE_KEYSPACES = ('system', 'system_schema', 'system_views', 'system_virtual_schema', 'system_cluster_metadata')
'system_metrics', 'system_virtual_schema', 'system_cluster_metadata')
NONALTERBALE_KEYSPACES = ('system', 'system_schema', 'system_views', 'system_metrics', 'system_virtual_schema',
'system_cluster_metadata')


class Cql3ParsingRuleSet(CqlParsingRuleSet):
Expand Down
8 changes: 4 additions & 4 deletions pylib/cqlshlib/test/test_cqlsh_completion.py
View file
Open in desktop
Expand Up @@ -588,7 +588,7 @@ def test_complete_in_create_type(self):
self.trycompletions('CREATE TYPE foo ', choices=['(', '.'])

def test_complete_in_drop_type(self):
self.trycompletions('DROP TYPE ', choices=['IF', 'system_views.',
self.trycompletions('DROP TYPE ', choices=['IF', 'system_views.', 'system_metrics.',
'tags', 'system_traces.', 'system_distributed.', 'system_cluster_metadata.',
'phone_number', 'quote_udt', 'band_info_type', 'address', 'system.', 'system_schema.',
'system_auth.', 'system_virtual_schema.', self.cqlsh.keyspace + '.'
Expand Down Expand Up @@ -895,7 +895,7 @@ def test_complete_in_truncate(self):

def test_complete_in_use(self):
self.trycompletions('US', immediate='E ')
self.trycompletions('USE ', choices=[self.cqlsh.keyspace, 'system', 'system_auth',
self.trycompletions('USE ', choices=[self.cqlsh.keyspace, 'system', 'system_auth', 'system_metrics',
'system_distributed', 'system_schema', 'system_traces', 'system_views',
'system_virtual_schema', 'system_cluster_metadata' ])

Expand Down Expand Up @@ -991,7 +991,7 @@ def test_complete_in_alter_table(self):
'dynamic_columns',
'twenty_rows_composite_table',
'utf8_with_special_chars',
'system_traces.', 'songs', 'system_views.',
'system_traces.', 'songs', 'system_views.', 'system_metrics.',
'system_virtual_schema.',
'system_schema.', 'system_distributed.',
'system_cluster_metadata.',
Expand Down Expand Up @@ -1020,7 +1020,7 @@ def test_complete_in_alter_table(self):
def test_complete_in_alter_type(self):
self.trycompletions('ALTER TYPE I', immediate='F EXISTS ')
self.trycompletions('ALTER TYPE ', choices=['IF', 'system_views.',
'tags', 'system_traces.', 'system_distributed.',
'tags', 'system_traces.', 'system_distributed.', 'system_metrics.',
'phone_number', 'quote_udt', 'band_info_type', 'address', 'system.', 'system_schema.',
'system_auth.', 'system_virtual_schema.', 'system_cluster_metadata.', self.cqlsh.keyspace + '.'
])
Expand Down
13 changes: 11 additions & 2 deletions src/java/org/apache/cassandra/db/virtual/BatchMetricsTable.java
View file
Open in desktop
Expand Up @@ -26,10 +26,16 @@
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.metrics.BatchMetrics;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ClientWarn;

/**
* @deprecated The "batch_metrics" virtual table is no longer used, and will be removed in a future release. Please,
* use either "system_metrics.batch_group", or "system_metrics.type_histogram" virtual tables instead.
*/
@Deprecated(since = "5.0")
public class BatchMetricsTable extends AbstractVirtualTable
{

protected static final String TABLE_NAME = "batch_metrics";
private static final String PARTITIONS_PER_LOGGED_BATCH = "partitions_per_logged_batch";
private static final String PARTITIONS_PER_UNLOGGED_BATCH = "partitions_per_unlogged_batch";
private static final String PARTITIONS_PER_COUNTER_BATCH = "partitions_per_counter_batch";
Expand All @@ -40,7 +46,7 @@ public class BatchMetricsTable extends AbstractVirtualTable

BatchMetricsTable(String keyspace)
{
super(TableMetadata.builder(keyspace, "batch_metrics")
super(TableMetadata.builder(keyspace, TABLE_NAME)
.comment("Metrics specific to batch statements")
.kind(TableMetadata.Kind.VIRTUAL)
.partitioner(new LocalPartitioner(UTF8Type.instance))
Expand All @@ -61,6 +67,9 @@ public DataSet data()
addRow(result, PARTITIONS_PER_UNLOGGED_BATCH, metrics.partitionsPerUnloggedBatch.getSnapshot());
addRow(result, PARTITIONS_PER_COUNTER_BATCH, metrics.partitionsPerCounterBatch.getSnapshot());

ClientWarn.instance.warn(String.format("The \"%s\" virtual table is deprecated. " +
"Please, use either \"%s.batch_group\", or \"%s.type_histogram\" virtual tables instead.",
TABLE_NAME, metadata.keyspace, metadata.keyspace));
return result;
}

Expand Down
9 changes: 8 additions & 1 deletion src/java/org/apache/cassandra/db/virtual/CQLMetricsTable.java
View file
Open in desktop
Expand Up @@ -26,8 +26,13 @@
import org.apache.cassandra.metrics.CQLMetrics;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.service.ClientWarn;


/**
* @deprecated The "cql_metrics" virtual table is no longer used, and will be removed in a future release. Please,
* use "system_metrics.clq_group" virtual table instead.
*/
@Deprecated(since = "5.0")
final class CQLMetricsTable extends AbstractVirtualTable
{
public static final String TABLE_NAME = "cql_metrics";
Expand Down Expand Up @@ -70,6 +75,8 @@ public DataSet data()
addRow(result, PREPARED_STATEMENTS_RATIO, cqlMetrics.preparedStatementsRatio.getValue());
addRow(result, REGULAR_STATEMENTS_EXECUTED, cqlMetrics.regularStatementsExecuted.getCount());

ClientWarn.instance.warn("The \"" + TABLE_NAME + "\" virtual table is deprecated. " +
"Please, use \"system_metrics.clq_group\" virtual table instead.");
return result;
}

Expand Down