Remove graphite and dropwizard support (#1847)

* remove graphite max rate metrics

* remove graphite offset commit metrics

* remove workload and consumer sender graphite metrics

* remove deserialization graphite metrics

* remove consumer graphite metrics

* remove undelivered messages graphite metrics

* remove schema client graphite metrics

* remove executor graphite metrics

* remove producer graphite metrics

* remove persistent buffer graphite metrics

* remove tracker elasticsearch graphite metrics

* remove subscription graphite metrics

* remove topic graphite metrics

* remove unused graphite metric

* fix dropwizard based tests

* remove consumers and frontend graphite config

* remove graphite client

* remove graphite dependencies

* remove graphite tests

* cleanup paths compiler

* remove metric registry

* remove dropwizard from hermes-client

* lint

* lint

* use shared throughput meters

* fix executor metrics test

* lint

* remove graphite from docs and configs

---------

Co-authored-by: Mateusz <76775507+szczygiel-m@users.noreply.github.com>

docker/docker-compose.yml

@@ -30,19 +30,6 @@ services:
       - kafka_data:/var/lib/kafka/data
       - kafka_secrets:/etc/kafka/secrets
 
-  graphite:
-    image: graphiteapp/graphite-statsd:1.1.3
-    ports:
-      - '2003-2004:2003-2004'
-      - '2023-2024:2023-2024'
-      - '8125:8125/udp'
-      - '8126:8126'
-      - '8082:80'
-    volumes:
-      - graphite_conf:/opt/graphite/conf
-      - graphite_data:/opt/graphite/storage
-      - statsd_data:/opt/statsd
-
   frontend:
     build:
       context: ../
@@ -52,7 +39,6 @@ services:
     depends_on:
       - zk
       - kafka
-      - graphite
 
   consumers:
     build:
@@ -61,7 +47,6 @@ services:
     depends_on:
       - zk
       - kafka
-      - graphite
 
   management:
     build:
@@ -72,7 +57,6 @@ services:
     depends_on:
       - zk
       - kafka
-      - graphite
 
   schema-registry:
     image: "confluentinc/cp-schema-registry:${CONFLUENT_IMAGES_TAG}"
@@ -87,9 +71,6 @@ services:
       - "8081:8081"
 
 volumes:
-  graphite_conf:
-  graphite_data:
-  statsd_data:
   zk_secrets:
   zk_data:
   zk_log:

docker/latest/consumers/consumers.yaml

@@ -8,11 +8,6 @@ consumer:
     clusters:
       - datacenter: "dc"
         brokerList: "kafka:29092"
-  graphite:
-    host: "graphite"
-  metrics:
-    metric-registry:
-      graphiteReporterEnabled: true
   workload:
     consumerPerSubscription: 1
   schema:

docker/latest/frontend/frontend.yaml

@@ -8,11 +8,6 @@ frontend:
     clusters:
       - datacenter: "dc"
         brokerList: "kafka:29092"
-  graphite:
-    host: "graphite"
-  metrics:
-    metric-registry:
-      graphiteReporterEnabled: true
   schema:
     cache:
       refreshAfterWrite: 1m

docker/latest/management/management.yaml

@@ -17,11 +17,6 @@ kafka:
       connectionTimeout: 3000
       bootstrapKafkaServer: kafka:29092
 
-graphite:
-  client:
-    enabled: true
-    externalMonitoringUrl: graphite:8082
-
 server:
   port: 8090
 

docs/docs/overview/architecture.md

@@ -16,7 +16,7 @@ Hermes integrates with multiple systems, each having different role.
 
 * **Message Store** - stores and routes messages, current implementation: Kafka
 * **Metadata Store** - shared metadata storage for all Hermes modules, current implementation: Zookeeper
-* **Metrics Store** *[optional]* - stores metrics gathered by Hermes, current implementation: Graphite
+* **Metrics Store** *[optional]* - stores metrics gathered by Hermes, currently Hermes exposes metrics in Prometheus format
 * **Tracking Store** *[optional]* - stores tracking (message trace) information, current implementation: ElasticSearch
 
 ## Message flow

docs/docs/quickstart.md

@@ -56,7 +56,7 @@ image: allegro/hermes-management:hermes-[specific version tag]
 ## Development
 
 The default `docker-compose` setup will start all hermes modules (consumers, frontend, management), together
-with its dependencies (Kafka, ZooKeeper, Graphite, Schema Registry). To run a specific module with gradle/IntelliJ,
+with its dependencies (Kafka, ZooKeeper, Schema Registry). To run a specific module with gradle/IntelliJ,
 just comment out the module in `services` section of the `docker-compose.yml` file, and start the java process locally:
 
 `./gradlew -p hermes-frontend run`
@@ -175,7 +175,6 @@ management:
     depends_on:
       - zk
       - kafka
-      - graphite
 [...]
 ```
 

hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesPublisher.java

@@ -1,6 +1,5 @@
 package pl.allegro.tech.hermes.benchmark.environment;
 
-import com.codahale.metrics.MetricRegistry;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.config.CookieSpecs;
@@ -28,14 +27,12 @@ public class HermesPublisher {
     private static final Logger logger = LoggerFactory.getLogger(HermesPublisher.class);
 
     private final CloseableHttpAsyncClient httpClient;
-    private final MetricRegistry metricRegistry;
     private final URI targetUrl;
     private final HttpEntity body;
 
-    public HermesPublisher(int maxConnectionsPerRoute, String targetUrl, String body, MetricRegistry metricRegistry)
+    public HermesPublisher(int maxConnectionsPerRoute, String targetUrl, String body)
             throws IOReactorException, UnsupportedEncodingException {
         this.targetUrl = URI.create(targetUrl);
-        this.metricRegistry = metricRegistry;
 
         RequestConfig requestConfig = RequestConfig.custom()
                 .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
@@ -71,9 +68,7 @@ public int publish() {
         try {
             Future<HttpResponse> future = httpClient.execute(httpPost, null);
             response = future.get().getStatusLine().getStatusCode();
-            metricRegistry.counter("response." + response).inc();
         } catch (RuntimeException | InterruptedException | ExecutionException exception) {
-            metricRegistry.counter("client.exceptions").inc();
             logger.error("Client exception", exception);
         }
         return response;

hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerEnvironment.java

@@ -1,6 +1,5 @@
 package pl.allegro.tech.hermes.benchmark.environment;
 
-import com.codahale.metrics.MetricRegistry;
 import org.apache.commons.io.IOUtils;
 import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Scope;
@@ -23,7 +22,6 @@ public class HermesServerEnvironment {
     public static final String BENCHMARK_TOPIC = "bench.topic";
 
     private HermesPublisher publisher;
-    private MetricRegistry metricRegistry;
 
     private HermesServer hermesServer;
 
@@ -39,10 +37,9 @@ public void setupEnvironment() throws Exception {
 
     @Setup(Level.Trial)
     public void setupPublisher() throws Exception {
-        metricRegistry = new MetricRegistry();
 
         String messageBody = loadMessageResource("completeMessage");
-        publisher = new HermesPublisher(MAX_CONNECTIONS_PER_ROUTE, "http://localhost:8080/topics/" + BENCHMARK_TOPIC, messageBody, metricRegistry);
+        publisher = new HermesPublisher(MAX_CONNECTIONS_PER_ROUTE, "http://localhost:8080/topics/" + BENCHMARK_TOPIC, messageBody);
     }
 
     @TearDown(Level.Trial)
@@ -52,7 +49,6 @@ public void shutdownServers() throws Exception {
 
     @TearDown(Level.Trial)
     public void shutdownPublisherAndReportMetrics() throws Exception {
-        reportMetrics();
         publisher.stop();
     }
 
@@ -65,7 +61,4 @@ public static String loadMessageResource(String name) throws IOException {
             .getResourceAsStream(String.format("/message/%s.json", name))));
     }
 
-    private void reportMetrics() {
-        metricRegistry.getCounters().forEach((key, value) -> logger.info(key + ": " + value.getCount()));
-    }
 }

hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerFactory.java

@@ -1,12 +1,10 @@
 package pl.allegro.tech.hermes.benchmark.environment;
 
-import com.codahale.metrics.MetricRegistry;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import io.undertow.server.HttpHandler;
 import pl.allegro.tech.hermes.api.Topic;
 import pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentWrapper;
-import pl.allegro.tech.hermes.common.metric.HermesMetrics;
 import pl.allegro.tech.hermes.common.metric.MetricsFacade;
 import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache;
 import pl.allegro.tech.hermes.frontend.config.HTTPHeadersProperties;
@@ -28,7 +26,6 @@
 import pl.allegro.tech.hermes.frontend.readiness.HealthCheckService;
 import pl.allegro.tech.hermes.frontend.server.HermesServer;
 import pl.allegro.tech.hermes.frontend.validator.MessageValidators;
-import pl.allegro.tech.hermes.metrics.PathsCompiler;
 import pl.allegro.tech.hermes.schema.DirectCompiledSchemaRepository;
 import pl.allegro.tech.hermes.schema.DirectSchemaVersionsRepository;
 import pl.allegro.tech.hermes.schema.RawSchemaClient;
@@ -52,8 +49,7 @@ class HermesServerFactory {
 
     static HermesServer provideHermesServer() throws IOException {
         ThroughputLimiter throughputLimiter = (exampleTopic, throughput) -> quotaConfirmed();
-        HermesMetrics hermesMetrics = new HermesMetrics(new MetricRegistry(), new PathsCompiler(""));
-        MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry(), hermesMetrics);
+        MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry());
         TopicsCache topicsCache = new InMemoryTopicsCache(metricsFacade, topic);
         BrokerMessageProducer brokerMessageProducer = new InMemoryBrokerMessageProducer();
         RawSchemaClient rawSchemaClient = new InMemorySchemaClient(topic.getName(), loadMessageResource("schema"), 1, 1);

hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryTopicsCache.java

@@ -1,13 +1,14 @@
 package pl.allegro.tech.hermes.benchmark.environment;
 
+import com.codahale.metrics.MetricRegistry;
 import pl.allegro.tech.hermes.api.Topic;
 import pl.allegro.tech.hermes.common.kafka.KafkaTopic;
 import pl.allegro.tech.hermes.common.kafka.KafkaTopicName;
 import pl.allegro.tech.hermes.common.kafka.KafkaTopics;
-import pl.allegro.tech.hermes.common.metric.HermesMetrics;
 import pl.allegro.tech.hermes.common.metric.MetricsFacade;
 import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache;
 import pl.allegro.tech.hermes.frontend.metric.CachedTopic;
+import pl.allegro.tech.hermes.frontend.metric.ThroughputRegistry;
 
 import java.util.List;
 import java.util.Optional;
@@ -17,12 +18,14 @@ class InMemoryTopicsCache implements TopicsCache {
     private final MetricsFacade metricsFacade;
     private final KafkaTopics kafkaTopics;
     private final Topic topic;
+    private final ThroughputRegistry throughputRegistry;
 
 
     InMemoryTopicsCache(MetricsFacade metricsFacade, Topic topic) {
         this.metricsFacade = metricsFacade;
         this.topic = topic;
         this.kafkaTopics = new KafkaTopics(new KafkaTopic(KafkaTopicName.valueOf(topic.getQualifiedName()), topic.getContentType()));
+        this.throughputRegistry = new ThroughputRegistry(metricsFacade, new MetricRegistry());
     }
 
     @Override
@@ -32,6 +35,7 @@ public Optional<CachedTopic> getTopic(String qualifiedTopicName) {
                     new CachedTopic(
                             topic,
                             metricsFacade,
+                            throughputRegistry,
                             kafkaTopics
                     )
             );

hermes-client/build.gradle

@@ -3,7 +3,6 @@ plugins {
 }
 
 dependencies {
-    compileOnly group: 'io.dropwizard.metrics', name: 'metrics-core', version: versions.dropwizard_metrics
     compileOnly group: 'io.micrometer', name: 'micrometer-core', version: versions.micrometer_metrics
     compileOnly group: 'org.glassfish.jersey.core', name: 'jersey-client', version: versions.jersey
     compileOnly group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: versions.jersey
@@ -20,7 +19,6 @@ dependencies {
     testImplementation group: 'javax.servlet', name: 'javax.servlet-api', version: '3.1.0'
     testImplementation group: 'com.jayway.jsonpath', name: 'json-path', version: '2.5.0'
 
-    testImplementation group: 'io.dropwizard.metrics', name: 'metrics-core', version: versions.dropwizard_metrics
     testImplementation group: 'io.micrometer', name: 'micrometer-core', version: versions.micrometer_metrics
     testImplementation group: 'org.glassfish.jersey.core', name: 'jersey-client', version: versions.jersey
     testImplementation group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: versions.jersey