CASSANDRASC-111 Improve observability in Sidecar with Dropwizard metrics

CHANGES.txt

@@ -1,5 +1,6 @@
 1.0.0
 -----
+ * Improve observability in Sidecar (CASSANDRASC-111)
  * Improve logging for slice restore task (CASSANDRASC-107)
  * Add restore task watcher to report long running tasks (CASSANDRASC-106)
  * RestoreSliceTask could be stuck due to missing exception handling (CASSANDRASC-105)

src/main/dist/conf/sidecar.yaml

@@ -142,6 +142,15 @@ healthcheck:
   initial_delay_millis: 0
   poll_freq_millis: 30000
 
+metrics:
+  registry_name: cassandra_sidecar
+  vertx:
+    enabled: true
+    expose_via_jmx: false
+    jmx_domain_name: sidecar.vertx.jmx_domain
+    monitored_server_route_regexes:                # regex list to match server routes
+     - /api/v1/.*
+
 cassandra_input_validation:
   forbidden_keyspaces:
     - system_schema

src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java

@@ -21,9 +21,10 @@
 import java.util.List;
 
 import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-
 /**
  * Metadata of an instance
  */
@@ -58,4 +59,9 @@ public interface InstanceMetadata
      * @return a {@link CassandraAdapterDelegate} specific for the instance
      */
     @Nullable CassandraAdapterDelegate delegate();
+
+    /**
+     * @return {@link InstanceMetrics} metrics specific for the Cassandra instance
+     */
+    @NotNull InstanceMetrics metrics();
 }

src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadataImpl.java

@@ -21,9 +21,14 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
 import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate;
 import org.apache.cassandra.sidecar.common.DataObjectBuilder;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetricsImpl;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -38,6 +43,7 @@ public class InstanceMetadataImpl implements InstanceMetadata
     private final String stagingDir;
     @Nullable
     private final CassandraAdapterDelegate delegate;
+    private final InstanceMetrics metrics;
 
     protected InstanceMetadataImpl(Builder builder)
     {
@@ -47,6 +53,7 @@ protected InstanceMetadataImpl(Builder builder)
         dataDirs = Collections.unmodifiableList(builder.dataDirs);
         stagingDir = builder.stagingDir;
         delegate = builder.delegate;
+        metrics = builder.metrics;
     }
 
     @Override
@@ -85,6 +92,12 @@ public String stagingDir()
         return delegate;
     }
 
+    @Override
+    public InstanceMetrics metrics()
+    {
+        return metrics;
+    }
+
     public static Builder builder()
     {
         return new Builder();
@@ -95,12 +108,14 @@ public static Builder builder()
      */
     public static class Builder implements DataObjectBuilder<Builder, InstanceMetadataImpl>
     {
-        protected int id;
+        protected Integer id;
         protected String host;
         protected int port;
         protected List<String> dataDirs;
         protected String stagingDir;
         protected CassandraAdapterDelegate delegate;
+        protected String globalRegistryName;
+        protected InstanceMetrics metrics;
 
         protected Builder()
         {
@@ -114,6 +129,7 @@ protected Builder(InstanceMetadataImpl instanceMetadata)
             dataDirs = new ArrayList<>(instanceMetadata.dataDirs);
             stagingDir = instanceMetadata.stagingDir;
             delegate = instanceMetadata.delegate;
+            metrics = instanceMetadata.metrics;
         }
 
         @Override
@@ -188,6 +204,18 @@ public Builder delegate(CassandraAdapterDelegate delegate)
             return update(b -> b.delegate = delegate);
         }
 
+        /**
+         * Sets the {@code globalRegistryName} and returns a reference to this Builder enabling method chaining.
+         *
+         * @param registryName global {@link com.codahale.metrics.MetricRegistry} name
+         * @return a reference to this Builder
+         */
+        public Builder globalMetricRegistryName(String registryName)
+
+        {
+            return update(b -> b.globalRegistryName = registryName);
+        }
+
         /**
          * Returns a {@code InstanceMetadataImpl} built from the parameters previously set.
          *
@@ -196,7 +224,19 @@ public Builder delegate(CassandraAdapterDelegate delegate)
         @Override
         public InstanceMetadataImpl build()
         {
+            Objects.requireNonNull(id);
+            Objects.requireNonNull(globalRegistryName);
+
+            String instanceRegistryName = instanceRegistryName(globalRegistryName);
+            MetricRegistry instanceMetricRegistry = SharedMetricRegistries.getOrCreate(instanceRegistryName);
+            metrics = new InstanceMetricsImpl(instanceMetricRegistry);
+
             return new InstanceMetadataImpl(this);
         }
+
+        private String instanceRegistryName(String globalRegistryName)
+        {
+            return globalRegistryName + "_" + id;
+        }
     }
 }

src/main/java/org/apache/cassandra/sidecar/config/MetricsConfiguration.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.config;
+
+/**
+ * Configuration needed for capturing metrics.
+ */
+public interface MetricsConfiguration
+{
+    /**
+     * @return global registry name to be used for registering sidecar metrics
+     */
+    String registryName();
+
+    /**
+     * @return configuration needed for capturing metrics released by Vert.x framework.
+     */
+    VertxMetricsConfiguration vertxConfiguration();
+}

src/main/java/org/apache/cassandra/sidecar/config/SidecarConfiguration.java

@@ -51,6 +51,11 @@ public interface SidecarConfiguration
      */
     HealthCheckConfiguration healthCheckConfiguration();
 
+    /**
+     * @return configuration needed for metrics capture
+     */
+    MetricsConfiguration metricsConfiguration();
+
     /**
      * @return the configuration for Cassandra input validation
      */

src/main/java/org/apache/cassandra/sidecar/config/VertxMetricsConfiguration.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.config;
+
+import java.util.List;
+
+/**
+ * Holds configuration needed for enabling metrics capture in Vert.x framework, for analyzing from Sidecar.
+ */
+public interface VertxMetricsConfiguration
+{
+    /**
+     * @return boolean indicating whether metrics captured is enabled for Vert.x
+     */
+    boolean enabled();
+
+    /**
+     * @return boolean indicating whether Vert.x metrics will be exposed via JMX
+     */
+    boolean exposeViaJMX();
+
+    /**
+     * @return JMX domain to be used when metrics are exposed via JMX
+     */
+    String jmxDomainName();
+
+    /**
+     * @return List of regexes used for identifying what server routes will be monitored
+     */
+    List<String> monitoredServerRouteRegexes();
+}

src/main/java/org/apache/cassandra/sidecar/config/yaml/MetricsConfigurationImpl.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.config.yaml;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.cassandra.sidecar.config.MetricsConfiguration;
+import org.apache.cassandra.sidecar.config.VertxMetricsConfiguration;
+
+/**
+ * Configuration needed for capturing metrics.
+ */
+public class MetricsConfigurationImpl implements MetricsConfiguration
+{
+    public static final String DEFAULT_DROPWIZARD_REGISTRY_NAME = "cassandra_sidecar";
+    public static final VertxMetricsConfiguration DEFAULT_VERTX_METRICS_CONFIGURATION
+    = new VertxMetricsConfigurationImpl();
+
+    @JsonProperty(value = "registry_name")
+    protected final String registryName;
+    @JsonProperty(value = "vertx")
+    protected final VertxMetricsConfiguration vertxConfiguration;
+
+    public MetricsConfigurationImpl()
+    {
+        this(DEFAULT_DROPWIZARD_REGISTRY_NAME, DEFAULT_VERTX_METRICS_CONFIGURATION);
+    }
+
+    public MetricsConfigurationImpl(String registryName, VertxMetricsConfiguration vertxConfiguration)
+    {
+        this.registryName = registryName;
+        this.vertxConfiguration = vertxConfiguration;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public String registryName()
+    {
+        return registryName;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public VertxMetricsConfiguration vertxConfiguration()
+    {
+        return vertxConfiguration;
+    }
+}

src/main/java/org/apache/cassandra/sidecar/config/yaml/SidecarConfigurationImpl.java

@@ -42,6 +42,7 @@
 import org.apache.cassandra.sidecar.config.DriverConfiguration;
 import org.apache.cassandra.sidecar.config.HealthCheckConfiguration;
 import org.apache.cassandra.sidecar.config.InstanceConfiguration;
+import org.apache.cassandra.sidecar.config.MetricsConfiguration;
 import org.apache.cassandra.sidecar.config.RestoreJobConfiguration;
 import org.apache.cassandra.sidecar.config.S3ClientConfiguration;
 import org.apache.cassandra.sidecar.config.ServiceConfiguration;
@@ -72,6 +73,9 @@ public class SidecarConfigurationImpl implements SidecarConfiguration
     @JsonProperty("healthcheck")
     protected final HealthCheckConfiguration healthCheckConfiguration;
 
+    @JsonProperty("metrics")
+    protected final MetricsConfiguration metricsConfiguration;
+
     @JsonProperty("cassandra_input_validation")
     protected final CassandraInputValidationConfiguration cassandraInputValidationConfiguration;
 
@@ -93,6 +97,7 @@ protected SidecarConfigurationImpl(Builder builder)
         serviceConfiguration = builder.serviceConfiguration;
         sslConfiguration = builder.sslConfiguration;
         healthCheckConfiguration = builder.healthCheckConfiguration;
+        metricsConfiguration = builder.metricsConfiguration;
         cassandraInputValidationConfiguration = builder.cassandraInputValidationConfiguration;
         driverConfiguration = builder.driverConfiguration;
         restoreJobConfiguration = builder.restoreJobConfiguration;
@@ -159,6 +164,16 @@ public HealthCheckConfiguration healthCheckConfiguration()
         return healthCheckConfiguration;
     }
 
+    /**
+     * @return the configuration needed for metrics capture
+     */
+    @Override
+    @JsonProperty("metrics")
+    public MetricsConfiguration metricsConfiguration()
+    {
+        return metricsConfiguration;
+    }
+
     @Override
     @JsonProperty("driver_parameters")
     public DriverConfiguration driverConfiguration()
@@ -290,6 +305,7 @@ public static class Builder implements DataObjectBuilder<Builder, SidecarConfigu
         private ServiceConfiguration serviceConfiguration = new ServiceConfigurationImpl();
         private SslConfiguration sslConfiguration = null;
         private HealthCheckConfiguration healthCheckConfiguration = new HealthCheckConfigurationImpl();
+        private MetricsConfiguration metricsConfiguration = new MetricsConfigurationImpl();
         private CassandraInputValidationConfiguration cassandraInputValidationConfiguration
         = new CassandraInputValidationConfigurationImpl();
         private DriverConfiguration driverConfiguration = new DriverConfigurationImpl();
@@ -361,6 +377,17 @@ public Builder healthCheckConfiguration(HealthCheckConfiguration healthCheckConf
             return update(b -> b.healthCheckConfiguration = healthCheckConfiguration);
         }
 
+        /**
+         * Sets the {@code metricsConfiguration} and returns a reference to this Builder enabling method chaining.
+         *
+         * @param metricsConfiguration the {@code metricsConfiguration} to set
+         * @return a reference to this Builder
+         */
+        public Builder metricsConfiguration(MetricsConfiguration metricsConfiguration)
+        {
+            return update(b -> b.metricsConfiguration = metricsConfiguration);
+        }
+
         /**
          * Sets the {@code driverConfiguration} and returns a reference to this Builder enabling
          * method chaining.