From d46a39e573bba012e2b4e1b0b4a81f828d72e8a0 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 16 Jun 2020 13:19:16 +0300 Subject: [PATCH 01/23] Start doing endpoint for jmx metrics --- .../ui/cluster/service/ClusterService.java | 4 ++ .../kafka/ui/rest/MetricsRestController.java | 7 +++- .../main/resources/swagger/kafka-ui-api.yaml | 37 ++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index bad9625f12f..8dbbb95f3f6 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -156,4 +156,8 @@ public Flux getMessages(String clusterName, String topicName, Cons .orElse(Flux.empty()); } + + public Flux getJmxMetricsNames() { + + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java index 0b8d9396c69..f4b4b58f509 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java @@ -99,7 +99,12 @@ public Mono> getConsumerGroup(String cluste public Mono> updateTopic(String clusterId, String topicName, @Valid Mono topicFormData, ServerWebExchange exchange) { return clusterService.updateTopic(clusterId, topicName, topicFormData).map(ResponseEntity::ok); } - + + @Override + public Mono>> getJmxMetricsNames(String clusterName, String brokerId, ServerWebExchange exchange){ + return clusterService; + } + private Mono parseConsumerPosition(SeekType seekType, List seekTo) { return Mono.justOrEmpty(seekTo) .defaultIfEmpty(Collections.emptyList()) diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index e21372e1183..98db8a48516 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -264,6 +264,33 @@ paths: schema: $ref: '#/components/schemas/ConsumerGroupDetails' + /api/clusters/{clusterName}/brokers/{brokerId}: + get: + tags: + - /api/clusters + summary: get all JmxMetricsNames + operationId: getJmxMetricsNames + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + - name: brokerId + in: path + required: true + schema: + type: string + responses: + 200: + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/JmxMetrics' + /api/clusters/{clusterName}/consumerGroups: get: tags: @@ -506,4 +533,12 @@ components: consumers: type: array items: - $ref: '#/components/schemas/ConsumerTopicPartitionDetail' \ No newline at end of file + $ref: '#/components/schemas/ConsumerTopicPartitionDetail' + + JmxMetrics: + type: object + properties: + metricName: + type: array + items: + type: string \ No newline at end of file From f427d01ea3de7c47153bd4ca86b1412a7b03172f Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 23 Jun 2020 10:24:10 +0300 Subject: [PATCH 02/23] Added endpoint for getting jmx metric per broker --- .../ui/cluster/mapper/ClusterMapper.java | 6 +-- .../cluster/model/InternalClusterMetrics.java | 2 + .../ui/cluster/model/InternalJmxMetric.java | 14 +++++ .../ui/cluster/service/ClusterService.java | 9 +++- .../kafka/ui/cluster/util/JmxClusterUtil.java | 52 +++++++++++++++---- .../kafka/ui/kafka/KafkaService.java | 29 +++++++---- .../kafka/ui/rest/MetricsRestController.java | 10 ++-- .../main/resources/swagger/kafka-ui-api.yaml | 37 ++++++++----- 8 files changed, 117 insertions(+), 42 deletions(-) create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java index e3725a42fe9..4b53862ac28 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java @@ -1,10 +1,7 @@ package com.provectus.kafka.ui.cluster.mapper; import com.provectus.kafka.ui.cluster.config.ClustersProperties; -import com.provectus.kafka.ui.cluster.model.InternalClusterMetrics; -import com.provectus.kafka.ui.cluster.model.InternalTopic; -import com.provectus.kafka.ui.cluster.model.InternalTopicConfig; -import com.provectus.kafka.ui.cluster.model.KafkaCluster; +import com.provectus.kafka.ui.cluster.model.*; import com.provectus.kafka.ui.model.*; import org.mapstruct.Mapper; import org.mapstruct.Mapping; @@ -26,4 +23,5 @@ public interface ClusterMapper { Topic toTopic(InternalTopic topic); TopicDetails toTopicDetails(InternalTopic topic); TopicConfig toTopicConfig(InternalTopicConfig topic); + JmxMetric toJmxMetric(InternalJmxMetric metric); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java index 35671ea7b19..fd63e7aae6e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java @@ -4,6 +4,7 @@ import lombok.Data; import java.math.BigDecimal; +import java.util.List; import java.util.Map; @@ -24,5 +25,6 @@ public class InternalClusterMetrics { private final int segmentCount; private final long segmentSize; private final Map internalBrokerMetrics; + private final List jmxMetricsNames; private final int zooKeeperStatus; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java new file mode 100644 index 00000000000..6d5b613633a --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java @@ -0,0 +1,14 @@ +package com.provectus.kafka.ui.cluster.model; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder(toBuilder = true) +public class InternalJmxMetric { + + private String name; + private String type; + private String topic; + private String canonicalName; +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index 43c823305fd..75e199aff2d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -124,6 +124,7 @@ public Flux getBrokers (String clusterName) { .map(n -> n.stream().map(node -> { Broker broker = new Broker(); broker.setId(node.idString()); + broker.setHost(node.host()); return broker; }).collect(Collectors.toList()))) .flatMapMany(Flux::fromIterable); @@ -154,7 +155,11 @@ public Flux getMessages(String clusterName, String topicName, Cons } - public Flux getJmxMetricsNames() { - + public Mono getJmxMetric(String clusterName, Integer nodeId, JmxMetric metric) { + return clustersStorage.getClusterByName(clusterName) + .map(c -> kafkaService.getOrCreateAdminClient(c) + .flatMap(a -> ClusterUtil.toMono(a.getAdminClient().describeCluster().nodes()) + .map(n -> n.stream().filter(s -> s.id() == nodeId).findFirst().orElseThrow().host())) + .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, metric))).orElseThrow(); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 02e3c28e995..9ef089f189f 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -1,5 +1,7 @@ package com.provectus.kafka.ui.cluster.util; +import com.provectus.kafka.ui.cluster.model.InternalJmxMetric; +import com.provectus.kafka.ui.model.JmxMetric; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.pool2.KeyedObjectPool; @@ -8,12 +10,12 @@ import javax.management.*; import javax.management.remote.JMXConnector; import java.io.IOException; -import java.math.BigDecimal; import java.net.MalformedURLException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Component @Slf4j @@ -30,20 +32,46 @@ public class JmxClusterUtil { private static final String BYTES_IN_PER_SEC_MBEAN_OBJECT_NAME = "kafka.server:type=BrokerTopicMetrics,name=" + BYTES_IN_PER_SEC; private static final String BYTES_OUT_PER_SEC_MBEAN_OBJECT_NAME = "kafka.server:type=BrokerTopicMetrics,name=" + BYTES_OUT_PER_SEC; - private static final List attrNames = Arrays.asList("OneMinuteRate", "FiveMinuteRate", "FifteenMinuteRate"); + public List getJmxMetricsNames(int jmxPort, String jmxHost) { + String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; + List result = new ArrayList<>(); + JMXConnector srv = null; + try { + srv = pool.borrowObject(jmxUrl); + MBeanServerConnection msc = srv.getMBeanServerConnection(); + var jmxMetrics = msc.queryNames(null, null).stream().filter(q -> q.getCanonicalName().startsWith("kafka.server")).collect(Collectors.toList()); + jmxMetrics.forEach(j -> { + InternalJmxMetric.InternalJmxMetricBuilder internalMetric = InternalJmxMetric.builder(); + internalMetric.name(j.getKeyPropertyList().computeIfAbsent("name", s -> null)); + internalMetric.topic(j.getKeyPropertyList().computeIfAbsent("topic", s -> null)); + internalMetric.type(j.getKeyPropertyList().computeIfAbsent("type", s -> null)); + internalMetric.canonicalName(j.getCanonicalName()); + result.add(internalMetric.build()); + }); + } catch (IOException ioe) { + log.error("Cannot get jmxMetricsNames, {}", jmxUrl, ioe); + } catch (Exception e) { + log.error("Cannot get JmxConnection from pool, {}", jmxUrl, e); + } + return result; + } - public Map getJmxTrafficMetrics(int jmxPort, String jmxHost, String metricName) { + public JmxMetric getJmxMetric(int jmxPort, String jmxHost, String canonicalName) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; - Map result = new HashMap<>(); + + var result = new JmxMetric(); JMXConnector srv = null; try { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); - ObjectName name = metricName.equals(BYTES_IN_PER_SEC) ? new ObjectName(BYTES_IN_PER_SEC_MBEAN_OBJECT_NAME) : - new ObjectName(BYTES_OUT_PER_SEC_MBEAN_OBJECT_NAME); - for (String attrName : attrNames) { - result.put(attrName, BigDecimal.valueOf((Double) msc.getAttribute(name, attrName))); + Map resultAttr = new HashMap<>(); + ObjectName name = new ObjectName(canonicalName); + var attrNames = msc.getMBeanInfo(name).getAttributes(); + for (MBeanAttributeInfo attrName : attrNames) { + resultAttr.put(attrName.getName(), msc.getAttribute(name, attrName.getName())); } + result.setCanonicalName(canonicalName); + result.setValue(resultAttr); pool.returnObject(jmxUrl, srv); } catch (MalformedURLException url) { log.error("Cannot create JmxServiceUrl from {}", jmxUrl); @@ -71,4 +99,10 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { log.error("Cannot invalidate object in pool, {}", url); } } + + public String getParamFromName(String param, String name) { + int paramValueBeginIndex = name.indexOf(param) + param.length() + 1; + int paramValueEndIndex = name.indexOf(',', paramValueBeginIndex); + return paramValueEndIndex != -1 ? name.substring(paramValueBeginIndex, paramValueEndIndex) : name.substring(paramValueBeginIndex); + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 0f6eb011a60..2dfadc645ee 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -3,10 +3,7 @@ import com.provectus.kafka.ui.cluster.model.*; import com.provectus.kafka.ui.cluster.util.ClusterUtil; import com.provectus.kafka.ui.cluster.util.JmxClusterUtil; -import com.provectus.kafka.ui.model.ConsumerGroup; -import com.provectus.kafka.ui.model.ServerStatus; -import com.provectus.kafka.ui.model.Topic; -import com.provectus.kafka.ui.model.TopicFormData; +import com.provectus.kafka.ui.model.*; import com.provectus.kafka.ui.zookeeper.ZookeeperService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -26,7 +23,6 @@ import reactor.util.function.Tuple2; import reactor.util.function.Tuples; -import java.math.BigDecimal; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -166,12 +162,14 @@ private Mono getClusterMetrics(KafkaCluster cluster, Adm c -> { InternalClusterMetrics.InternalClusterMetricsBuilder metricsBuilder = InternalClusterMetrics.builder(); metricsBuilder.brokerCount(brokers.size()).activeControllers(c != null ? 1 : 0); - Map bytesInPerSec = jmxClusterUtil.getJmxTrafficMetrics(cluster.getJmxPort(), c.host(), JmxClusterUtil.BYTES_IN_PER_SEC); - Map bytesOutPerSec = jmxClusterUtil.getJmxTrafficMetrics(cluster.getJmxPort(), c.host(), JmxClusterUtil.BYTES_OUT_PER_SEC); + List metrics = jmxClusterUtil.getJmxMetricsNames(cluster.getJmxPort(), c.host()); + +// Map bytesOutPerSec = jmxClusterUtil.getJmxTrafficMetrics(cluster.getJmxPort(), c.host(), JmxClusterUtil.BYTES_OUT_PER_SEC); metricsBuilder .internalBrokerMetrics((brokers.stream().map(Node::id).collect(Collectors.toMap(k -> k, v -> InternalBrokerMetrics.builder().build())))) - .bytesOutPerSec(bytesOutPerSec) - .bytesInPerSec(bytesInPerSec); +// .bytesOutPerSec(bytesOutPerSec) +// .bytesInPerSec(bytesInPerSec) + .jmxMetricsNames(metrics); return metricsBuilder.build(); } ) @@ -351,4 +349,17 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern }) ); } + + public JmxMetric getJmxMetric (KafkaCluster cluster, int jmxPort, String host, JmxMetric metric) { + var jmxMetric = cluster.getMetrics().getJmxMetricsNames().stream().filter(c -> { + var foundTopic = false; + var found = jmxClusterUtil.getParamFromName("name", metric.getCanonicalName()).equals(c.getName()) + && jmxClusterUtil.getParamFromName("type", metric.getCanonicalName()).equals(c.getType()); + if (found && c.getTopic() != null) { + foundTopic = c.getTopic().equals(jmxClusterUtil.getParamFromName("topic", metric.getCanonicalName())); + } + return found && foundTopic; + }).findFirst().orElseThrow(); + return jmxClusterUtil.getJmxMetric(jmxPort, host, jmxMetric.getCanonicalName()); + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java index f4b4b58f509..498a310bb88 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java @@ -5,7 +5,6 @@ import com.provectus.kafka.ui.cluster.service.ClusterService; import com.provectus.kafka.ui.model.*; import lombok.RequiredArgsConstructor; - import org.apache.commons.lang3.tuple.Pair; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -14,12 +13,11 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import javax.validation.Valid; import java.util.Collections; import java.util.List; import java.util.function.Function; -import javax.validation.Valid; - @RestController @RequiredArgsConstructor public class MetricsRestController implements ApiClustersApi { @@ -101,8 +99,10 @@ public Mono> updateTopic(String clusterId, String topicNam } @Override - public Mono>> getJmxMetricsNames(String clusterName, String brokerId, ServerWebExchange exchange){ - return clusterService; + public Mono> getJmxMetric(String clusterName, Integer host, Mono metric, ServerWebExchange exchange){ + return metric + .flatMap(m -> clusterService.getJmxMetric(clusterName, host, m) + .map(ResponseEntity::ok)); } private Mono parseConsumerPosition(SeekType seekType, List seekTo) { diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 0e54b54af8b..a1d0f1e33ad 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -264,12 +264,12 @@ paths: schema: $ref: '#/components/schemas/ConsumerGroupDetails' - /api/clusters/{clusterName}/brokers/{brokerId}: - get: + /api/clusters/{clusterName}/metrics/brokers/{brokerId}: + post: tags: - /api/clusters - summary: get all JmxMetricsNames - operationId: getJmxMetricsNames + summary: get specific JmxMetric + operationId: getJmxMetric parameters: - name: clusterName in: path @@ -280,16 +280,19 @@ paths: in: path required: true schema: - type: string + type: integer + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/JmxMetric' responses: 200: description: OK content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/JmxMetrics' + $ref: '#/components/schemas/JmxMetric' /api/clusters/{clusterName}/consumerGroups: get: @@ -338,6 +341,10 @@ components: type: object additionalProperties: type: number + jmxMetricsNames: + type: array + items: + $ref: '#/components/schemas/JmxMetric' required: - id - name @@ -452,6 +459,8 @@ components: properties: id: type: string + host: + type: string ConsumerGroup: type: object @@ -539,10 +548,12 @@ components: items: $ref: '#/components/schemas/ConsumerTopicPartitionDetail' - JmxMetrics: + JmxMetric: type: object properties: - metricName: - type: array - items: - type: string \ No newline at end of file + canonicalName: + type: string + value: + type: object + additionalProperties: + type: object \ No newline at end of file From a24b21830c0ac9d827214dc2ab18167c0dd8d254 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 23 Jun 2020 14:03:16 +0300 Subject: [PATCH 03/23] Cluster jmx metrics sum endpoit added --- .../ui/cluster/service/ClusterService.java | 22 ++++++++++++++ .../kafka/ui/cluster/util/JmxClusterUtil.java | 9 ++---- .../kafka/ui/kafka/KafkaService.java | 10 ++----- .../kafka/ui/rest/MetricsRestController.java | 9 +++++- .../main/resources/swagger/kafka-ui-api.yaml | 29 +++++++++++++++++-- kafka-ui-react-app/package-lock.json | 9 ++++-- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index 75e199aff2d..06fda32c7b9 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -12,12 +12,14 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.StringDeserializer; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.math.BigDecimal; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -162,4 +164,24 @@ public Mono getJmxMetric(String clusterName, Integer nodeId, JmxMetri .map(n -> n.stream().filter(s -> s.id() == nodeId).findFirst().orElseThrow().host())) .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, metric))).orElseThrow(); } + + public Mono getClusterJmxMetric(String clusterName, JmxMetric metric) { + return clustersStorage.getClusterByName(clusterName) + .map(c -> kafkaService.getOrCreateAdminClient(c) + .flatMap(eac -> ClusterUtil.toMono(eac.getAdminClient().describeCluster().nodes())) + .flatMapIterable(n -> n.stream().flatMap(node -> Stream.of(node.host())).collect(Collectors.toList())) + .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, metric)) + .collectList() + .map(s -> s.stream().filter(metric1 -> metric1.getCanonicalName().equals(metric.getCanonicalName())) + .collect(Collectors.toList()) + .stream().reduce((jmx, jmx1) -> { + if (jmx.getCanonicalName().equals(jmx1.getCanonicalName())) { + jmx.getValue().keySet().forEach(k -> jmx1.getValue().compute(k, (k1, v1) -> + ((BigDecimal) v1).add(((BigDecimal) jmx1.getValue().get(k))))); + } + return jmx; + }) + .orElseThrow()) + ).orElseThrow(); + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 9ef089f189f..883963071b1 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -27,15 +27,10 @@ public class JmxClusterUtil { private static final String JMX_URL = "service:jmx:rmi:///jndi/rmi://"; private static final String JMX_SERVICE_TYPE = "jmxrmi"; - public static final String BYTES_IN_PER_SEC = "BytesInPerSec"; - public static final String BYTES_OUT_PER_SEC = "BytesOutPerSec"; - private static final String BYTES_IN_PER_SEC_MBEAN_OBJECT_NAME = "kafka.server:type=BrokerTopicMetrics,name=" + BYTES_IN_PER_SEC; - private static final String BYTES_OUT_PER_SEC_MBEAN_OBJECT_NAME = "kafka.server:type=BrokerTopicMetrics,name=" + BYTES_OUT_PER_SEC; - public List getJmxMetricsNames(int jmxPort, String jmxHost) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; List result = new ArrayList<>(); - JMXConnector srv = null; + JMXConnector srv; try { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); @@ -100,7 +95,7 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - public String getParamFromName(String param, String name) { + public static String getParamFromName(String param, String name) { int paramValueBeginIndex = name.indexOf(param) + param.length() + 1; int paramValueEndIndex = name.indexOf(',', paramValueBeginIndex); return paramValueEndIndex != -1 ? name.substring(paramValueBeginIndex, paramValueEndIndex) : name.substring(paramValueBeginIndex); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 2dfadc645ee..fdd19a27a0a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -163,12 +163,8 @@ private Mono getClusterMetrics(KafkaCluster cluster, Adm InternalClusterMetrics.InternalClusterMetricsBuilder metricsBuilder = InternalClusterMetrics.builder(); metricsBuilder.brokerCount(brokers.size()).activeControllers(c != null ? 1 : 0); List metrics = jmxClusterUtil.getJmxMetricsNames(cluster.getJmxPort(), c.host()); - -// Map bytesOutPerSec = jmxClusterUtil.getJmxTrafficMetrics(cluster.getJmxPort(), c.host(), JmxClusterUtil.BYTES_OUT_PER_SEC); metricsBuilder .internalBrokerMetrics((brokers.stream().map(Node::id).collect(Collectors.toMap(k -> k, v -> InternalBrokerMetrics.builder().build())))) -// .bytesOutPerSec(bytesOutPerSec) -// .bytesInPerSec(bytesInPerSec) .jmxMetricsNames(metrics); return metricsBuilder.build(); } @@ -353,10 +349,10 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern public JmxMetric getJmxMetric (KafkaCluster cluster, int jmxPort, String host, JmxMetric metric) { var jmxMetric = cluster.getMetrics().getJmxMetricsNames().stream().filter(c -> { var foundTopic = false; - var found = jmxClusterUtil.getParamFromName("name", metric.getCanonicalName()).equals(c.getName()) - && jmxClusterUtil.getParamFromName("type", metric.getCanonicalName()).equals(c.getType()); + var found = JmxClusterUtil.getParamFromName("name", metric.getCanonicalName()).equals(c.getName()) + && JmxClusterUtil.getParamFromName("type", metric.getCanonicalName()).equals(c.getType()); if (found && c.getTopic() != null) { - foundTopic = c.getTopic().equals(jmxClusterUtil.getParamFromName("topic", metric.getCanonicalName())); + foundTopic = c.getTopic().equals(JmxClusterUtil.getParamFromName("topic", metric.getCanonicalName())); } return found && foundTopic; }).findFirst().orElseThrow(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java index 498a310bb88..ee5dc01f90b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java @@ -99,12 +99,19 @@ public Mono> updateTopic(String clusterId, String topicNam } @Override - public Mono> getJmxMetric(String clusterName, Integer host, Mono metric, ServerWebExchange exchange){ + public Mono> getBrokerJmxMetric(String clusterName, Integer host, Mono metric, ServerWebExchange exchange){ return metric .flatMap(m -> clusterService.getJmxMetric(clusterName, host, m) .map(ResponseEntity::ok)); } + @Override + public Mono> getClusterJmxMetric(String clusterName, Mono metric, ServerWebExchange exchange){ + return metric + .flatMap(m -> clusterService.getClusterJmxMetric(clusterName, m) + .map(ResponseEntity::ok)); + } + private Mono parseConsumerPosition(SeekType seekType, List seekTo) { return Mono.justOrEmpty(seekTo) .defaultIfEmpty(Collections.emptyList()) diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index a1d0f1e33ad..48dd6811416 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -268,8 +268,8 @@ paths: post: tags: - /api/clusters - summary: get specific JmxMetric - operationId: getJmxMetric + summary: get specific JmxMetric for broker + operationId: getBrokerJmxMetric parameters: - name: clusterName in: path @@ -294,6 +294,31 @@ paths: schema: $ref: '#/components/schemas/JmxMetric' + /api/clusters/{clusterName}/metrics: + post: + tags: + - /api/clusters + summary: get specific JmxMetric for cluster + operationId: getClusterJmxMetric + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/JmxMetric' + responses: + 200: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/JmxMetric' + /api/clusters/{clusterName}/consumerGroups: get: tags: diff --git a/kafka-ui-react-app/package-lock.json b/kafka-ui-react-app/package-lock.json index c4db16d76c1..d51927152cd 100644 --- a/kafka-ui-react-app/package-lock.json +++ b/kafka-ui-react-app/package-lock.json @@ -3369,7 +3369,8 @@ }, "kind-of": { "version": "6.0.2", - "resolved": "", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", + "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==", "dev": true } } @@ -6917,7 +6918,8 @@ }, "kind-of": { "version": "6.0.2", - "resolved": "", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", + "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==", "dev": true } } @@ -16156,7 +16158,8 @@ }, "kind-of": { "version": "6.0.2", - "resolved": "", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", + "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==", "dev": true } } From 90df65bce43da082907b8cf9c7423080ea29d818 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Mon, 29 Jun 2020 09:28:38 +0300 Subject: [PATCH 04/23] Added endpoints for cluster metrics and broker metrics --- .../ui/cluster/service/ClusterService.java | 15 ++++---- .../kafka/ui/cluster/util/JmxClusterUtil.java | 37 ++++++++++++++++--- .../kafka/ui/kafka/KafkaService.java | 8 ++-- .../kafka/ui/rest/MetricsRestController.java | 15 ++++---- .../main/resources/swagger/kafka-ui-api.yaml | 26 ++++++------- 5 files changed, 63 insertions(+), 38 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index 06fda32c7b9..11deea03fbb 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -5,6 +5,7 @@ import com.provectus.kafka.ui.cluster.model.ConsumerPosition; import com.provectus.kafka.ui.cluster.model.KafkaCluster; import com.provectus.kafka.ui.cluster.util.ClusterUtil; +import com.provectus.kafka.ui.cluster.util.JmxClusterUtil; import com.provectus.kafka.ui.kafka.KafkaService; import com.provectus.kafka.ui.model.*; import lombok.RequiredArgsConstructor; @@ -12,14 +13,12 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.StringDeserializer; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.math.BigDecimal; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -157,27 +156,27 @@ public Flux getMessages(String clusterName, String topicName, Cons } - public Mono getJmxMetric(String clusterName, Integer nodeId, JmxMetric metric) { + public Mono getJmxMetric(String clusterName, Integer nodeId, String canonicalName) { return clustersStorage.getClusterByName(clusterName) .map(c -> kafkaService.getOrCreateAdminClient(c) .flatMap(a -> ClusterUtil.toMono(a.getAdminClient().describeCluster().nodes()) .map(n -> n.stream().filter(s -> s.id() == nodeId).findFirst().orElseThrow().host())) - .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, metric))).orElseThrow(); + .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, canonicalName))).orElseThrow(); } - public Mono getClusterJmxMetric(String clusterName, JmxMetric metric) { + public Mono getClusterJmxMetric(String clusterName, String canonicalName) { return clustersStorage.getClusterByName(clusterName) .map(c -> kafkaService.getOrCreateAdminClient(c) .flatMap(eac -> ClusterUtil.toMono(eac.getAdminClient().describeCluster().nodes())) .flatMapIterable(n -> n.stream().flatMap(node -> Stream.of(node.host())).collect(Collectors.toList())) - .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, metric)) + .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, canonicalName)) .collectList() - .map(s -> s.stream().filter(metric1 -> metric1.getCanonicalName().equals(metric.getCanonicalName())) + .map(s -> s.stream().filter(metric1 -> JmxClusterUtil.metricNamesEquals(metric1.getCanonicalName(), canonicalName)) .collect(Collectors.toList()) .stream().reduce((jmx, jmx1) -> { if (jmx.getCanonicalName().equals(jmx1.getCanonicalName())) { jmx.getValue().keySet().forEach(k -> jmx1.getValue().compute(k, (k1, v1) -> - ((BigDecimal) v1).add(((BigDecimal) jmx1.getValue().get(k))))); + JmxClusterUtil.metricValueReduce(v1, jmx1.getValue().get(k)))); } return jmx; }) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 883963071b1..d0a1df4d3d0 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -10,11 +10,9 @@ import javax.management.*; import javax.management.remote.JMXConnector; import java.io.IOException; +import java.math.BigDecimal; import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Component @@ -30,7 +28,7 @@ public class JmxClusterUtil { public List getJmxMetricsNames(int jmxPort, String jmxHost) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; List result = new ArrayList<>(); - JMXConnector srv; + JMXConnector srv = null; try { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); @@ -43,10 +41,13 @@ public List getJmxMetricsNames(int jmxPort, String jmxHost) { internalMetric.canonicalName(j.getCanonicalName()); result.add(internalMetric.build()); }); + pool.returnObject(jmxUrl, srv); } catch (IOException ioe) { log.error("Cannot get jmxMetricsNames, {}", jmxUrl, ioe); + closeConnectionExceptionally(jmxUrl, srv); } catch (Exception e) { log.error("Cannot get JmxConnection from pool, {}", jmxUrl, e); + closeConnectionExceptionally(jmxUrl, srv); } return result; } @@ -96,8 +97,34 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } public static String getParamFromName(String param, String name) { + if (!name.contains(param)) { + return null; + } int paramValueBeginIndex = name.indexOf(param) + param.length() + 1; int paramValueEndIndex = name.indexOf(',', paramValueBeginIndex); return paramValueEndIndex != -1 ? name.substring(paramValueBeginIndex, paramValueEndIndex) : name.substring(paramValueBeginIndex); } + + public static boolean metricNamesEquals (String metric1Name, String metric2Name) { + boolean result = false; + try { + result = Objects.equals(getParamFromName("name", metric1Name), getParamFromName("name", metric2Name)) + && + Objects.equals(getParamFromName("type", metric1Name), getParamFromName("type", metric2Name)) + && + getParamFromName("topic", metric1Name) == null || + Objects.equals(getParamFromName("topic", metric1Name), getParamFromName("topic", metric2Name)); + } catch (NullPointerException npe) { + log.error("Cannot compare {} with {}, npe caught", metric1Name, metric2Name); + } + return result; + } + + public static Object metricValueReduce(Object value1, Object value2) { + if (value1 instanceof Number) { + return new BigDecimal(value1.toString()).add(new BigDecimal(value2.toString())); + } else { + return value1; + } + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index fdd19a27a0a..7041da36fcc 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -346,13 +346,13 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern ); } - public JmxMetric getJmxMetric (KafkaCluster cluster, int jmxPort, String host, JmxMetric metric) { + public JmxMetric getJmxMetric (KafkaCluster cluster, int jmxPort, String host, String canonicalName) { var jmxMetric = cluster.getMetrics().getJmxMetricsNames().stream().filter(c -> { var foundTopic = false; - var found = JmxClusterUtil.getParamFromName("name", metric.getCanonicalName()).equals(c.getName()) - && JmxClusterUtil.getParamFromName("type", metric.getCanonicalName()).equals(c.getType()); + var found = JmxClusterUtil.getParamFromName("name", canonicalName).equals(c.getName()) + && JmxClusterUtil.getParamFromName("type", canonicalName).equals(c.getType()); if (found && c.getTopic() != null) { - foundTopic = c.getTopic().equals(JmxClusterUtil.getParamFromName("topic", metric.getCanonicalName())); + foundTopic = c.getTopic().equals(JmxClusterUtil.getParamFromName("topic", canonicalName)); } return found && foundTopic; }).findFirst().orElseThrow(); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java index ee5dc01f90b..e348cb96f67 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java @@ -99,17 +99,16 @@ public Mono> updateTopic(String clusterId, String topicNam } @Override - public Mono> getBrokerJmxMetric(String clusterName, Integer host, Mono metric, ServerWebExchange exchange){ - return metric - .flatMap(m -> clusterService.getJmxMetric(clusterName, host, m) - .map(ResponseEntity::ok)); + public Mono> getBrokerJmxMetric(String clusterName, Integer host, String canonicalName, ServerWebExchange exchange){ + return + clusterService.getJmxMetric(clusterName, host, canonicalName) + .map(ResponseEntity::ok); } @Override - public Mono> getClusterJmxMetric(String clusterName, Mono metric, ServerWebExchange exchange){ - return metric - .flatMap(m -> clusterService.getClusterJmxMetric(clusterName, m) - .map(ResponseEntity::ok)); + public Mono> getClusterJmxMetric(String clusterName, String canonicalName, ServerWebExchange exchange){ + return clusterService.getClusterJmxMetric(clusterName, canonicalName) + .map(ResponseEntity::ok); } private Mono parseConsumerPosition(SeekType seekType, List seekTo) { diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 48dd6811416..2a12bdde131 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -264,8 +264,8 @@ paths: schema: $ref: '#/components/schemas/ConsumerGroupDetails' - /api/clusters/{clusterName}/metrics/brokers/{brokerId}: - post: + /api/clusters/{clusterName}/brokers/{brokerId}/metrics/{canonicalName}: + get: tags: - /api/clusters summary: get specific JmxMetric for broker @@ -281,11 +281,11 @@ paths: required: true schema: type: integer - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/JmxMetric' + - name: canonicalName + in: path + required: true + schema: + type: string responses: 200: description: OK @@ -295,7 +295,7 @@ paths: $ref: '#/components/schemas/JmxMetric' /api/clusters/{clusterName}/metrics: - post: + get: tags: - /api/clusters summary: get specific JmxMetric for cluster @@ -306,11 +306,11 @@ paths: required: true schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/JmxMetric' + - name: canonicalName + in: path + required: true + schema: + type: string responses: 200: description: OK From b320aa66647f64fcdca27b3d259e130d62342050 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Mon, 29 Jun 2020 09:29:16 +0300 Subject: [PATCH 05/23] Cleared some code --- .../com/provectus/kafka/ui/cluster/util/ClusterUtil.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java index 1b014544bfa..ef4b1a65d89 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java @@ -1,7 +1,7 @@ package com.provectus.kafka.ui.cluster.util; -import com.provectus.kafka.ui.cluster.model.*; import com.provectus.kafka.ui.cluster.deserialization.RecordDeserializer; +import com.provectus.kafka.ui.cluster.model.*; import com.provectus.kafka.ui.model.*; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.*; @@ -28,10 +28,6 @@ @Slf4j public class ClusterUtil { - - - - private static final String CLUSTER_VERSION_PARAM_KEY = "inter.broker.protocol.version"; private static final ZoneId UTC_ZONE_ID = ZoneId.of("UTC"); From 319d027c9c031b38afc80268da326c5e37d14836 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Mon, 29 Jun 2020 11:14:43 +0300 Subject: [PATCH 06/23] Fixed jmxmetrics names --- .../ui/cluster/mapper/ClusterMapper.java | 7 ++--- .../ui/cluster/service/ConsumingService.java | 31 +++++++++---------- .../main/resources/swagger/kafka-ui-api.yaml | 18 +++++------ 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java index 4b53862ac28..c7dfa9c0b86 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java @@ -5,20 +5,17 @@ import com.provectus.kafka.ui.model.*; import org.mapstruct.Mapper; import org.mapstruct.Mapping; -import org.mapstruct.ValueMapping; @Mapper(componentModel = "spring") public interface ClusterMapper { - KafkaCluster toKafkaCluster(ClustersProperties.Cluster clusterProperties); - @Mapping(target = "brokerCount", source = "metrics.brokerCount") @Mapping(target = "onlinePartitionCount", source = "metrics.onlinePartitionCount") @Mapping(target = "topicCount", source = "metrics.topicCount") - @Mapping(target = "bytesInPerSec", source = "metrics.bytesInPerSec") - @Mapping(target = "bytesOutPerSec", source = "metrics.bytesOutPerSec") + @Mapping(target = "jmxMetricsNames", source = "metrics.jmxMetricsNames") Cluster toCluster(KafkaCluster cluster); + KafkaCluster toKafkaCluster(ClustersProperties.Cluster clusterProperties); BrokersMetrics toBrokerMetrics(InternalClusterMetrics metrics); Topic toTopic(InternalTopic topic); TopicDetails toTopicDetails(InternalTopic topic); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ConsumingService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ConsumingService.java index 90b8166e68e..35a09e4b94b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ConsumingService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ConsumingService.java @@ -1,21 +1,5 @@ package com.provectus.kafka.ui.cluster.service; -import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; - -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.utils.Bytes; -import org.springframework.stereotype.Service; - import com.provectus.kafka.ui.cluster.deserialization.DeserializationService; import com.provectus.kafka.ui.cluster.deserialization.RecordDeserializer; import com.provectus.kafka.ui.cluster.model.ConsumerPosition; @@ -24,11 +8,24 @@ import com.provectus.kafka.ui.kafka.KafkaService; import com.provectus.kafka.ui.model.SeekType; import com.provectus.kafka.ui.model.TopicMessage; - +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.Bytes; +import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; import reactor.core.scheduler.Schedulers; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + @Service @Log4j2 @RequiredArgsConstructor diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 2a12bdde131..bcd0f5ff4ad 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -358,18 +358,10 @@ components: type: integer topicCount: type: integer - bytesInPerSec: - type: object - additionalProperties: - type: number - bytesOutPerSec: - type: object - additionalProperties: - type: number jmxMetricsNames: type: array items: - $ref: '#/components/schemas/JmxMetric' + $ref: '#/components/schemas/JmxMetricName' required: - id - name @@ -581,4 +573,10 @@ components: value: type: object additionalProperties: - type: object \ No newline at end of file + type: object + + JmxMetricName: + type: object + properties: + canonicalName: + type: string \ No newline at end of file From 0c72e079aaf99be9277ee0a35d0734cb0790df22 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 30 Jun 2020 22:29:42 +0300 Subject: [PATCH 07/23] Changed to all values in metrics --- .../ui/cluster/mapper/ClusterMapper.java | 3 +- .../cluster/model/InternalClusterMetrics.java | 3 +- .../ui/cluster/model/InternalJmxMetric.java | 14 -- .../ui/cluster/service/ClusterService.java | 35 +---- .../kafka/ui/cluster/util/JmxClusterUtil.java | 49 ++----- .../kafka/ui/kafka/KafkaService.java | 48 ++++--- .../kafka/ui/rest/MetricsRestController.java | 20 +-- .../main/resources/swagger/kafka-ui-api.yaml | 131 +++++++++--------- 8 files changed, 120 insertions(+), 183 deletions(-) delete mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java index c7dfa9c0b86..7416b3e751d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java @@ -12,7 +12,7 @@ public interface ClusterMapper { @Mapping(target = "brokerCount", source = "metrics.brokerCount") @Mapping(target = "onlinePartitionCount", source = "metrics.onlinePartitionCount") @Mapping(target = "topicCount", source = "metrics.topicCount") - @Mapping(target = "jmxMetricsNames", source = "metrics.jmxMetricsNames") + @Mapping(target = "jmxMetrics", source = "metrics.jmxMetrics") Cluster toCluster(KafkaCluster cluster); KafkaCluster toKafkaCluster(ClustersProperties.Cluster clusterProperties); @@ -20,5 +20,4 @@ public interface ClusterMapper { Topic toTopic(InternalTopic topic); TopicDetails toTopicDetails(InternalTopic topic); TopicConfig toTopicConfig(InternalTopicConfig topic); - JmxMetric toJmxMetric(InternalJmxMetric metric); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java index fd63e7aae6e..3f95212a6fe 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java @@ -1,5 +1,6 @@ package com.provectus.kafka.ui.cluster.model; +import com.provectus.kafka.ui.model.JmxMetric; import lombok.Builder; import lombok.Data; @@ -25,6 +26,6 @@ public class InternalClusterMetrics { private final int segmentCount; private final long segmentSize; private final Map internalBrokerMetrics; - private final List jmxMetricsNames; + private final List jmxMetrics; private final int zooKeeperStatus; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java deleted file mode 100644 index 6d5b613633a..00000000000 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalJmxMetric.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.provectus.kafka.ui.cluster.model; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder(toBuilder = true) -public class InternalJmxMetric { - - private String name; - private String type; - private String topic; - private String canonicalName; -} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index 11deea03fbb..eedcf36909e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -31,6 +31,7 @@ public class ClusterService { private final ClusterMapper clusterMapper; private final KafkaService kafkaService; private final ConsumingService consumingService; + private final JmxClusterUtil jmxClusterUtil; public List getClusters() { return clustersStorage.getKafkaClusters() @@ -39,10 +40,12 @@ public List getClusters() { .collect(Collectors.toList()); } - public Optional getBrokersMetrics(String name) { + public Mono getBrokersMetrics(String name, Integer id) { return clustersStorage.getClusterByName(name) .map(KafkaCluster::getMetrics) - .map(clusterMapper::toBrokerMetrics); + .map(s -> kafkaService.getJmxMetric(name, id) + .map(j -> s.toBuilder().jmxMetrics(j).build())) + .map(s -> s.map(clusterMapper::toBrokerMetrics)).orElseThrow(); } public List getTopics(String name) { @@ -155,32 +158,4 @@ public Flux getMessages(String clusterName, String topicName, Cons .orElse(Flux.empty()); } - - public Mono getJmxMetric(String clusterName, Integer nodeId, String canonicalName) { - return clustersStorage.getClusterByName(clusterName) - .map(c -> kafkaService.getOrCreateAdminClient(c) - .flatMap(a -> ClusterUtil.toMono(a.getAdminClient().describeCluster().nodes()) - .map(n -> n.stream().filter(s -> s.id() == nodeId).findFirst().orElseThrow().host())) - .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, canonicalName))).orElseThrow(); - } - - public Mono getClusterJmxMetric(String clusterName, String canonicalName) { - return clustersStorage.getClusterByName(clusterName) - .map(c -> kafkaService.getOrCreateAdminClient(c) - .flatMap(eac -> ClusterUtil.toMono(eac.getAdminClient().describeCluster().nodes())) - .flatMapIterable(n -> n.stream().flatMap(node -> Stream.of(node.host())).collect(Collectors.toList())) - .map(host -> kafkaService.getJmxMetric(c, c.getJmxPort(), host, canonicalName)) - .collectList() - .map(s -> s.stream().filter(metric1 -> JmxClusterUtil.metricNamesEquals(metric1.getCanonicalName(), canonicalName)) - .collect(Collectors.toList()) - .stream().reduce((jmx, jmx1) -> { - if (jmx.getCanonicalName().equals(jmx1.getCanonicalName())) { - jmx.getValue().keySet().forEach(k -> jmx1.getValue().compute(k, (k1, v1) -> - JmxClusterUtil.metricValueReduce(v1, jmx1.getValue().get(k)))); - } - return jmx; - }) - .orElseThrow()) - ).orElseThrow(); - } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index d0a1df4d3d0..7a3166d166a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -1,6 +1,5 @@ package com.provectus.kafka.ui.cluster.util; -import com.provectus.kafka.ui.cluster.model.InternalJmxMetric; import com.provectus.kafka.ui.model.JmxMetric; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -25,21 +24,19 @@ public class JmxClusterUtil { private static final String JMX_URL = "service:jmx:rmi:///jndi/rmi://"; private static final String JMX_SERVICE_TYPE = "jmxrmi"; - public List getJmxMetricsNames(int jmxPort, String jmxHost) { + public List getJmxMetrics(int jmxPort, String jmxHost) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; - List result = new ArrayList<>(); + List result = new ArrayList<>(); JMXConnector srv = null; try { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); var jmxMetrics = msc.queryNames(null, null).stream().filter(q -> q.getCanonicalName().startsWith("kafka.server")).collect(Collectors.toList()); jmxMetrics.forEach(j -> { - InternalJmxMetric.InternalJmxMetricBuilder internalMetric = InternalJmxMetric.builder(); - internalMetric.name(j.getKeyPropertyList().computeIfAbsent("name", s -> null)); - internalMetric.topic(j.getKeyPropertyList().computeIfAbsent("topic", s -> null)); - internalMetric.type(j.getKeyPropertyList().computeIfAbsent("type", s -> null)); - internalMetric.canonicalName(j.getCanonicalName()); - result.add(internalMetric.build()); + JmxMetric metric = new JmxMetric(); + metric.setCanonicalName(j.getCanonicalName()); + metric.setValue(getJmxMetric(jmxPort, jmxHost, j.getCanonicalName())); + result.add(metric); }); pool.returnObject(jmxUrl, srv); } catch (IOException ioe) { @@ -52,22 +49,20 @@ public List getJmxMetricsNames(int jmxPort, String jmxHost) { return result; } - public JmxMetric getJmxMetric(int jmxPort, String jmxHost, String canonicalName) { + private Map getJmxMetric(int jmxPort, String jmxHost, String canonicalName) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; - var result = new JmxMetric(); + Map resultAttr = new HashMap<>(); JMXConnector srv = null; try { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); - Map resultAttr = new HashMap<>(); + ObjectName name = new ObjectName(canonicalName); var attrNames = msc.getMBeanInfo(name).getAttributes(); for (MBeanAttributeInfo attrName : attrNames) { resultAttr.put(attrName.getName(), msc.getAttribute(name, attrName.getName())); } - result.setCanonicalName(canonicalName); - result.setValue(resultAttr); pool.returnObject(jmxUrl, srv); } catch (MalformedURLException url) { log.error("Cannot create JmxServiceUrl from {}", jmxUrl); @@ -85,7 +80,7 @@ public JmxMetric getJmxMetric(int jmxPort, String jmxHost, String canonicalName) log.error("Error while retrieving connection {} from pool", jmxUrl); closeConnectionExceptionally(jmxUrl, srv); } - return result; + return resultAttr; } private void closeConnectionExceptionally(String url, JMXConnector srv) { @@ -96,30 +91,6 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - public static String getParamFromName(String param, String name) { - if (!name.contains(param)) { - return null; - } - int paramValueBeginIndex = name.indexOf(param) + param.length() + 1; - int paramValueEndIndex = name.indexOf(',', paramValueBeginIndex); - return paramValueEndIndex != -1 ? name.substring(paramValueBeginIndex, paramValueEndIndex) : name.substring(paramValueBeginIndex); - } - - public static boolean metricNamesEquals (String metric1Name, String metric2Name) { - boolean result = false; - try { - result = Objects.equals(getParamFromName("name", metric1Name), getParamFromName("name", metric2Name)) - && - Objects.equals(getParamFromName("type", metric1Name), getParamFromName("type", metric2Name)) - && - getParamFromName("topic", metric1Name) == null || - Objects.equals(getParamFromName("topic", metric1Name), getParamFromName("topic", metric2Name)); - } catch (NullPointerException npe) { - log.error("Cannot compare {} with {}, npe caught", metric1Name, metric2Name); - } - return result; - } - public static Object metricValueReduce(Object value1, Object value2) { if (value1 instanceof Number) { return new BigDecimal(value1.toString()).add(new BigDecimal(value2.toString())); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 7041da36fcc..cbdef338e10 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -42,6 +42,7 @@ public class KafkaService { private final Map adminClientCache = new ConcurrentHashMap<>(); private final Map> leadersCache = new ConcurrentHashMap<>(); private final JmxClusterUtil jmxClusterUtil; + private final ClustersStorage clustersStorage; @SneakyThrows public Mono getUpdatedCluster(KafkaCluster cluster) { @@ -158,18 +159,18 @@ private Mono> getTopicsData(AdminClient adminClient) { private Mono getClusterMetrics(KafkaCluster cluster, AdminClient client) { return ClusterUtil.toMono(client.describeCluster().nodes()) .flatMap(brokers -> - ClusterUtil.toMono(client.describeCluster().controller()).map( - c -> { + ClusterUtil.toMono(client.describeCluster().controller()).flatMap( + c -> + getClusterJmxMetric(cluster.getName()).map(jmxMetric -> { InternalClusterMetrics.InternalClusterMetricsBuilder metricsBuilder = InternalClusterMetrics.builder(); metricsBuilder.brokerCount(brokers.size()).activeControllers(c != null ? 1 : 0); - List metrics = jmxClusterUtil.getJmxMetricsNames(cluster.getJmxPort(), c.host()); metricsBuilder .internalBrokerMetrics((brokers.stream().map(Node::id).collect(Collectors.toMap(k -> k, v -> InternalBrokerMetrics.builder().build())))) - .jmxMetricsNames(metrics); + .jmxMetrics(jmxMetric); return metricsBuilder.build(); } ) - ); + )); } @@ -346,16 +347,31 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern ); } - public JmxMetric getJmxMetric (KafkaCluster cluster, int jmxPort, String host, String canonicalName) { - var jmxMetric = cluster.getMetrics().getJmxMetricsNames().stream().filter(c -> { - var foundTopic = false; - var found = JmxClusterUtil.getParamFromName("name", canonicalName).equals(c.getName()) - && JmxClusterUtil.getParamFromName("type", canonicalName).equals(c.getType()); - if (found && c.getTopic() != null) { - foundTopic = c.getTopic().equals(JmxClusterUtil.getParamFromName("topic", canonicalName)); - } - return found && foundTopic; - }).findFirst().orElseThrow(); - return jmxClusterUtil.getJmxMetric(jmxPort, host, jmxMetric.getCanonicalName()); + public Mono> getClusterJmxMetric(String clusterName) { + return clustersStorage.getClusterByName(clusterName) + .map(c -> getOrCreateAdminClient(c) + .flatMap(eac -> ClusterUtil.toMono(eac.getAdminClient().describeCluster().nodes())) + .flatMapIterable(n -> n.stream().flatMap(node -> Stream.of(node.host())).collect(Collectors.toList())) + .map(host -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), host)) + .collectList() + .map(s -> s.stream().reduce((s1, s2) -> { + s1.forEach(j1 -> { + s2.forEach(j2 -> { + if (j1.getCanonicalName().equals(j2.getCanonicalName())) { + j1.getValue().keySet().forEach(k -> j2.getValue().compute(k, (k1, v1) -> + JmxClusterUtil.metricValueReduce(j1, j2.getValue().get(k1)))); + } + }); + }); + return s1; + }).orElseThrow())).orElseThrow(); + } + + public Mono> getJmxMetric(String clusterName, Integer nodeId) { + return clustersStorage.getClusterByName(clusterName) + .map(c -> getOrCreateAdminClient(c) + .flatMap(a -> ClusterUtil.toMono(a.getAdminClient().describeCluster().nodes()) + .map(n -> n.stream().filter(s -> s.id() == nodeId).findFirst().orElseThrow().host())) + .map(host -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), host))).orElseThrow(); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java index e348cb96f67..9af52b3a29a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/rest/MetricsRestController.java @@ -30,12 +30,10 @@ public Mono>> getClusters(ServerWebExchange exchang } @Override - public Mono> getBrokersMetrics(String clusterName, ServerWebExchange exchange) { - return Mono.just( - clusterService.getBrokersMetrics(clusterName) + public Mono> getBrokersMetrics(String clusterName, Integer id, ServerWebExchange exchange) { + return clusterService.getBrokersMetrics(clusterName, id) .map(ResponseEntity::ok) - .orElse(ResponseEntity.notFound().build()) - ); + .onErrorReturn(ResponseEntity.notFound().build()); } @Override @@ -98,18 +96,6 @@ public Mono> updateTopic(String clusterId, String topicNam return clusterService.updateTopic(clusterId, topicName, topicFormData).map(ResponseEntity::ok); } - @Override - public Mono> getBrokerJmxMetric(String clusterName, Integer host, String canonicalName, ServerWebExchange exchange){ - return - clusterService.getJmxMetric(clusterName, host, canonicalName) - .map(ResponseEntity::ok); - } - - @Override - public Mono> getClusterJmxMetric(String clusterName, String canonicalName, ServerWebExchange exchange){ - return clusterService.getClusterJmxMetric(clusterName, canonicalName) - .map(ResponseEntity::ok); - } private Mono parseConsumerPosition(SeekType seekType, List seekTo) { return Mono.justOrEmpty(seekTo) diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index bcd0f5ff4ad..044d8354ace 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -52,7 +52,7 @@ paths: items: $ref: '#/components/schemas/Broker' - /api/clusters/{clusterName}/metrics/broker: + /api/clusters/{clusterName}/metrics/broker/{id}: get: tags: - /api/clusters @@ -64,6 +64,11 @@ paths: required: true schema: type: string + - name: id + in: path + required: true + schema: + type: integer responses: 200: description: OK @@ -264,60 +269,60 @@ paths: schema: $ref: '#/components/schemas/ConsumerGroupDetails' - /api/clusters/{clusterName}/brokers/{brokerId}/metrics/{canonicalName}: - get: - tags: - - /api/clusters - summary: get specific JmxMetric for broker - operationId: getBrokerJmxMetric - parameters: - - name: clusterName - in: path - required: true - schema: - type: string - - name: brokerId - in: path - required: true - schema: - type: integer - - name: canonicalName - in: path - required: true - schema: - type: string - responses: - 200: - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/JmxMetric' +# /api/clusters/{clusterName}/brokers/{brokerId}/metrics/{canonicalName}: +# get: +# tags: +# - /api/clusters +# summary: get specific JmxMetric for broker +# operationId: getBrokerJmxMetric +# parameters: +# - name: clusterName +# in: path +# required: true +# schema: +# type: string +# - name: brokerId +# in: path +# required: true +# schema: +# type: integer +# - name: canonicalName +# in: path +# required: true +# schema: +# type: string +# responses: +# 200: +# description: OK +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/JmxMetric' - /api/clusters/{clusterName}/metrics: - get: - tags: - - /api/clusters - summary: get specific JmxMetric for cluster - operationId: getClusterJmxMetric - parameters: - - name: clusterName - in: path - required: true - schema: - type: string - - name: canonicalName - in: path - required: true - schema: - type: string - responses: - 200: - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/JmxMetric' +# /api/clusters/{clusterName}/metrics: +# get: +# tags: +# - /api/clusters +# summary: get specific JmxMetric for cluster +# operationId: getClusterJmxMetric +# parameters: +# - name: clusterName +# in: path +# required: true +# schema: +# type: string +# - name: canonicalName +# in: path +# required: true +# schema: +# type: string +# responses: +# 200: +# description: OK +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/JmxMetric' /api/clusters/{clusterName}/consumerGroups: get: @@ -358,10 +363,10 @@ components: type: integer topicCount: type: integer - jmxMetricsNames: + jmxMetrics: type: array items: - $ref: '#/components/schemas/JmxMetricName' + $ref: '#/components/schemas/JmxMetric' required: - id - name @@ -392,6 +397,10 @@ components: type: integer segmentZise: type: integer + jmxMetrics: + type: array + items: + $ref: '#/components/schemas/JmxMetric' Topic: type: object @@ -571,12 +580,6 @@ components: canonicalName: type: string value: - type: object + type: string additionalProperties: - type: object - - JmxMetricName: - type: object - properties: - canonicalName: - type: string \ No newline at end of file + type: object \ No newline at end of file From e5c1bb81c507334786f652b01d16dafa134e3184 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 30 Jun 2020 22:36:02 +0300 Subject: [PATCH 08/23] Removed redundant imports --- .../com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java | 5 ++++- .../com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java | 5 ++++- .../com/provectus/kafka/ui/zookeeper/ZookeeperService.java | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java index 7416b3e751d..d5971d2f33b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java @@ -1,7 +1,10 @@ package com.provectus.kafka.ui.cluster.mapper; import com.provectus.kafka.ui.cluster.config.ClustersProperties; -import com.provectus.kafka.ui.cluster.model.*; +import com.provectus.kafka.ui.cluster.model.InternalClusterMetrics; +import com.provectus.kafka.ui.cluster.model.InternalTopic; +import com.provectus.kafka.ui.cluster.model.InternalTopicConfig; +import com.provectus.kafka.ui.cluster.model.KafkaCluster; import com.provectus.kafka.ui.model.*; import org.mapstruct.Mapper; import org.mapstruct.Mapping; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 7a3166d166a..6fd95ec0914 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -11,7 +11,10 @@ import java.io.IOException; import java.math.BigDecimal; import java.net.MalformedURLException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Component diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/zookeeper/ZookeeperService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/zookeeper/ZookeeperService.java index c116dc5ad85..26b6f300b2a 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/zookeeper/ZookeeperService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/zookeeper/ZookeeperService.java @@ -1,6 +1,5 @@ package com.provectus.kafka.ui.zookeeper; -import com.provectus.kafka.ui.cluster.model.ClustersStorage; import com.provectus.kafka.ui.cluster.model.KafkaCluster; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; From 4e624c1939051556099cd11d90931cee59b10ee1 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 30 Jun 2020 22:39:02 +0300 Subject: [PATCH 09/23] Renamed param constant --- .../com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 6fd95ec0914..fd9d4251c4f 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -26,6 +26,7 @@ public class JmxClusterUtil { private static final String JMX_URL = "service:jmx:rmi:///jndi/rmi://"; private static final String JMX_SERVICE_TYPE = "jmxrmi"; + private static final String KAFKA_SERVER_PARAM = "kafka.server"; public List getJmxMetrics(int jmxPort, String jmxHost) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; @@ -34,7 +35,7 @@ public List getJmxMetrics(int jmxPort, String jmxHost) { try { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); - var jmxMetrics = msc.queryNames(null, null).stream().filter(q -> q.getCanonicalName().startsWith("kafka.server")).collect(Collectors.toList()); + var jmxMetrics = msc.queryNames(null, null).stream().filter(q -> q.getCanonicalName().startsWith(KAFKA_SERVER_PARAM)).collect(Collectors.toList()); jmxMetrics.forEach(j -> { JmxMetric metric = new JmxMetric(); metric.setCanonicalName(j.getCanonicalName()); From 40e21c4575f90b9a1e48aad819e7eb4184edac2e Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Sat, 4 Jul 2020 16:18:59 +0300 Subject: [PATCH 10/23] Changed to calculate brokers and clusters metrics in one place --- docker/kafka-clusters-only.yaml | 36 ++++++------ .../cluster/model/InternalBrokerMetrics.java | 4 ++ .../ui/cluster/service/ClusterService.java | 12 ++-- .../kafka/ui/cluster/util/JmxClusterUtil.java | 5 +- .../kafka/ui/kafka/KafkaService.java | 48 +++++++++++++--- .../main/resources/swagger/kafka-ui-api.yaml | 55 ------------------- 6 files changed, 72 insertions(+), 88 deletions(-) diff --git a/docker/kafka-clusters-only.yaml b/docker/kafka-clusters-only.yaml index f37569d4d7b..b3d0900f06d 100644 --- a/docker/kafka-clusters-only.yaml +++ b/docker/kafka-clusters-only.yaml @@ -23,26 +23,26 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka0:9092,PLAINTEXT_HOST://localhost:29091 #,PLAIN://kafka0:29090 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT #,PLAIN:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 JMX_PORT: 9997 KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9997 - kafka01: - image: confluentinc/cp-kafka:5.1.0 - depends_on: - - zookeeper0 - ports: - - 29093:29093 - - 9999:9999 - environment: - KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: zookeeper0:2183 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka01:9092,PLAINTEXT_HOST://localhost:29093,PLAIN://kafka0:29090 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,PLAIN:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 - JMX_PORT: 9997 - KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9997 + # kafka01: + # image: confluentinc/cp-kafka:5.1.0 + # depends_on: + # - zookeeper0 + # ports: + # - 29093:29093 + # - 9999:9999 + # environment: + # KAFKA_BROKER_ID: 2 + # KAFKA_ZOOKEEPER_CONNECT: zookeeper0:2183 + # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka01:9092,PLAINTEXT_HOST://localhost:29093,PLAIN://kafka0:29090 + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,PLAIN:PLAINTEXT + # KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 + # JMX_PORT: 9997 + # KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9997 kafka-init-topics0: image: confluentinc/cp-kafka:5.1.0 @@ -102,7 +102,7 @@ services: depends_on: - zookeeper0 - kafka0 - - kafka01 + # - kafka01 ports: - 8085:8085 environment: diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java index 734ef31b07c..33de7471684 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java @@ -1,10 +1,14 @@ package com.provectus.kafka.ui.cluster.model; +import com.provectus.kafka.ui.model.JmxMetric; import lombok.Builder; import lombok.Data; +import java.util.List; + @Data @Builder(toBuilder = true) public class InternalBrokerMetrics { private final Long segmentSize; + private final List jmxMetrics; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index 8db4e123b25..d0c0cefe3bb 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -31,7 +31,6 @@ public class ClusterService { private final ClusterMapper clusterMapper; private final KafkaService kafkaService; private final ConsumingService consumingService; - private final JmxClusterUtil jmxClusterUtil; public List getClusters() { return clustersStorage.getKafkaClusters() @@ -41,11 +40,14 @@ public List getClusters() { } public Mono getBrokersMetrics(String name, Integer id) { - return clustersStorage.getClusterByName(name) + return Mono.just(clustersStorage.getClusterByName(name) .map(KafkaCluster::getMetrics) - .map(s -> kafkaService.getJmxMetric(name, id) - .map(j -> s.toBuilder().jmxMetrics(j).build())) - .map(s -> s.map(clusterMapper::toBrokerMetrics)).orElseThrow(); + .map(s -> { + var brokerMetrics = clusterMapper.toBrokerMetrics(s); + brokerMetrics.setJmxMetrics(s.getInternalBrokerMetrics().get(id).getJmxMetrics()); + brokerMetrics.setSegmentZise(Long.valueOf(s.getSegmentSize()).intValue()); + return brokerMetrics; + }).orElseThrow()); } public List getTopics(String name) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index fd9d4251c4f..4cf71de40ea 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -65,7 +65,10 @@ private Map getJmxMetric(int jmxPort, String jmxHost, String can ObjectName name = new ObjectName(canonicalName); var attrNames = msc.getMBeanInfo(name).getAttributes(); for (MBeanAttributeInfo attrName : attrNames) { - resultAttr.put(attrName.getName(), msc.getAttribute(name, attrName.getName())); + var value = msc.getAttribute(name, attrName.getName()); + if (value instanceof Number) { + resultAttr.put(attrName.getName(), value); + } } pool.returnObject(jmxUrl, srv); } catch (MalformedURLException url) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index cbdef338e10..97e2c0d5fba 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.utils.Bytes; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @@ -46,9 +47,10 @@ public class KafkaService { @SneakyThrows public Mono getUpdatedCluster(KafkaCluster cluster) { - return getOrCreateAdminClient(cluster).flatMap( + return getOrCreateAdminClient(cluster) + .flatMap( ac -> getClusterMetrics(cluster, ac.getAdminClient()) - + .flatMap(i -> fillJmxMetrics(i, cluster.getName())) .flatMap( clusterMetrics -> getTopicsData(ac.getAdminClient()).flatMap( topics -> loadTopicsConfig(ac.getAdminClient(), topics.stream().map(InternalTopic::getName).collect(Collectors.toList())) @@ -159,18 +161,16 @@ private Mono> getTopicsData(AdminClient adminClient) { private Mono getClusterMetrics(KafkaCluster cluster, AdminClient client) { return ClusterUtil.toMono(client.describeCluster().nodes()) .flatMap(brokers -> - ClusterUtil.toMono(client.describeCluster().controller()).flatMap( - c -> - getClusterJmxMetric(cluster.getName()).map(jmxMetric -> { + ClusterUtil.toMono(client.describeCluster().controller()).map( + c -> { InternalClusterMetrics.InternalClusterMetricsBuilder metricsBuilder = InternalClusterMetrics.builder(); metricsBuilder.brokerCount(brokers.size()).activeControllers(c != null ? 1 : 0); metricsBuilder - .internalBrokerMetrics((brokers.stream().map(Node::id).collect(Collectors.toMap(k -> k, v -> InternalBrokerMetrics.builder().build())))) - .jmxMetrics(jmxMetric); + .internalBrokerMetrics((brokers.stream().map(Node::id).collect(Collectors.toMap(k -> k, v -> InternalBrokerMetrics.builder().build())))); return metricsBuilder.build(); } ) - )); + ); } @@ -331,7 +331,7 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern var brokerSegmentSize = log.get(e.getKey()).values().stream() .mapToLong(v -> v.replicaInfos.values().stream() .mapToLong(r -> r.size).sum()).sum(); - InternalBrokerMetrics tempBrokerMetrics = InternalBrokerMetrics.builder().segmentSize(brokerSegmentSize).build(); + InternalBrokerMetrics tempBrokerMetrics = e.getValue().toBuilder().segmentSize(brokerSegmentSize).build(); return Collections.singletonMap(e.getKey(), tempBrokerMetrics); }); @@ -374,4 +374,34 @@ public Mono> getJmxMetric(String clusterName, Integer nodeId) { .map(n -> n.stream().filter(s -> s.id() == nodeId).findFirst().orElseThrow().host())) .map(host -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), host))).orElseThrow(); } + + private Mono fillJmxMetrics (InternalClusterMetrics internalClusterMetrics, String clusterName) { + return Flux.fromIterable(internalClusterMetrics.getInternalBrokerMetrics().keySet()) + .flatMap(id -> getJmxMetric(clusterName, id) + .map(jmx -> { + var jmxMetric = internalClusterMetrics.getInternalBrokerMetrics().get(id).toBuilder().jmxMetrics(jmx).build(); + var tempBrokerMetrics = internalClusterMetrics.getInternalBrokerMetrics(); + tempBrokerMetrics.put(id, jmxMetric); + return internalClusterMetrics.toBuilder().internalBrokerMetrics(tempBrokerMetrics).build(); + })).collectList() + .map(s -> s.stream().reduce((s1, s2) -> { + s1.getInternalBrokerMetrics().putAll(s2.getInternalBrokerMetrics().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + return s1; + }).orElseThrow()) + .map(i -> { + var tempMetrics = new HashMap<>(i.getInternalBrokerMetrics()); + tempMetrics.values().stream().flatMap(s -> Stream.of(s.getJmxMetrics())).reduce((s1, s2) -> { + s1.forEach(j1 -> { + s2.forEach(j2 -> { + if (j1.getCanonicalName().equals(j2.getCanonicalName())) { + j1.getValue().keySet().forEach(k -> j2.getValue().compute(k, (k1, v1) -> + JmxClusterUtil.metricValueReduce(j1, j2.getValue().get(k1)))); + } + }); + }); + return s1; + }); + return i.toBuilder().jmxMetrics(tempMetrics.values().stream().findFirst().orElseThrow().getJmxMetrics()).build(); + }); + } } diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index eaf88ca001a..9500d5ed159 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -273,61 +273,6 @@ paths: schema: $ref: '#/components/schemas/ConsumerGroupDetails' -# /api/clusters/{clusterName}/brokers/{brokerId}/metrics/{canonicalName}: -# get: -# tags: -# - /api/clusters -# summary: get specific JmxMetric for broker -# operationId: getBrokerJmxMetric -# parameters: -# - name: clusterName -# in: path -# required: true -# schema: -# type: string -# - name: brokerId -# in: path -# required: true -# schema: -# type: integer -# - name: canonicalName -# in: path -# required: true -# schema: -# type: string -# responses: -# 200: -# description: OK -# content: -# application/json: -# schema: -# $ref: '#/components/schemas/JmxMetric' - -# /api/clusters/{clusterName}/metrics: -# get: -# tags: -# - /api/clusters -# summary: get specific JmxMetric for cluster -# operationId: getClusterJmxMetric -# parameters: -# - name: clusterName -# in: path -# required: true -# schema: -# type: string -# - name: canonicalName -# in: path -# required: true -# schema: -# type: string -# responses: -# 200: -# description: OK -# content: -# application/json: -# schema: -# $ref: '#/components/schemas/JmxMetric' - /api/clusters/{clusterName}/consumerGroups: get: tags: From 6035003ea071b707efb174ba88d3d8a311061bc4 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Sat, 4 Jul 2020 16:19:51 +0300 Subject: [PATCH 11/23] Removed redundant imports --- .../com/provectus/kafka/ui/cluster/service/ClusterService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index d0c0cefe3bb..a260713aff5 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -5,7 +5,6 @@ import com.provectus.kafka.ui.cluster.model.ConsumerPosition; import com.provectus.kafka.ui.cluster.model.KafkaCluster; import com.provectus.kafka.ui.cluster.util.ClusterUtil; -import com.provectus.kafka.ui.cluster.util.JmxClusterUtil; import com.provectus.kafka.ui.kafka.KafkaService; import com.provectus.kafka.ui.model.*; import lombok.RequiredArgsConstructor; From bceb161ebaecc9f23ca6a34cc4fa689f4daa9a0c Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Sat, 4 Jul 2020 16:24:27 +0300 Subject: [PATCH 12/23] Fixed some mistakes --- .../ui/cluster/service/ClusterService.java | 3 +- .../kafka/ui/cluster/util/ClusterUtil.java | 2 +- .../kafka/ui/kafka/KafkaService.java | 28 +++---------------- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index a260713aff5..9eab77188f2 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -51,7 +51,7 @@ public Mono getBrokersMetrics(String name, Integer id) { public List getTopics(String name) { return clustersStorage.getClusterByName(name) - .map( c -> + .map(c -> c.getTopics().values().stream() .map(clusterMapper::toTopic) .collect(Collectors.toList()) @@ -157,6 +157,5 @@ public Flux getMessages(String clusterName, String topicName, Cons return clustersStorage.getClusterByName(clusterName) .map(c -> consumingService.loadMessages(c, topicName, consumerPosition, query, limit)) .orElse(Flux.empty()); - } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java index ef4b1a65d89..fc593330f10 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/ClusterUtil.java @@ -52,7 +52,7 @@ public static Mono toMono(KafkaFuture future, String topicName) { })); } - public static ConsumerGroup convertToConsumerGroup(ConsumerGroupDescription c, KafkaCluster cluster) { + public static ConsumerGroup convertToConsumerGroup(ConsumerGroupDescription c) { ConsumerGroup consumerGroup = new ConsumerGroup(); consumerGroup.setConsumerGroupId(c.groupId()); consumerGroup.setNumConsumers(c.members().size()); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 97e2c0d5fba..652c398e8e7 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -248,7 +248,7 @@ public Mono> getConsumerGroups(KafkaCluster cluster) { .flatMap(s -> ClusterUtil.toMono(ac.getAdminClient() .describeConsumerGroups(s.stream().map(ConsumerGroupListing::groupId).collect(Collectors.toList())).all())) .map(s -> s.values().stream() - .map(c -> ClusterUtil.convertToConsumerGroup(c, cluster)).collect(Collectors.toList()))); + .map(ClusterUtil::convertToConsumerGroup).collect(Collectors.toList()))); } public KafkaConsumer createConsumer(KafkaCluster cluster) { @@ -347,26 +347,6 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern ); } - public Mono> getClusterJmxMetric(String clusterName) { - return clustersStorage.getClusterByName(clusterName) - .map(c -> getOrCreateAdminClient(c) - .flatMap(eac -> ClusterUtil.toMono(eac.getAdminClient().describeCluster().nodes())) - .flatMapIterable(n -> n.stream().flatMap(node -> Stream.of(node.host())).collect(Collectors.toList())) - .map(host -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), host)) - .collectList() - .map(s -> s.stream().reduce((s1, s2) -> { - s1.forEach(j1 -> { - s2.forEach(j2 -> { - if (j1.getCanonicalName().equals(j2.getCanonicalName())) { - j1.getValue().keySet().forEach(k -> j2.getValue().compute(k, (k1, v1) -> - JmxClusterUtil.metricValueReduce(j1, j2.getValue().get(k1)))); - } - }); - }); - return s1; - }).orElseThrow())).orElseThrow(); - } - public Mono> getJmxMetric(String clusterName, Integer nodeId) { return clustersStorage.getClusterByName(clusterName) .map(c -> getOrCreateAdminClient(c) @@ -390,7 +370,7 @@ private Mono fillJmxMetrics (InternalClusterMetrics inte }).orElseThrow()) .map(i -> { var tempMetrics = new HashMap<>(i.getInternalBrokerMetrics()); - tempMetrics.values().stream().flatMap(s -> Stream.of(s.getJmxMetrics())).reduce((s1, s2) -> { + var resultMetrics = tempMetrics.values().stream().flatMap(s -> Stream.of(s.getJmxMetrics())).reduce((s1, s2) -> { s1.forEach(j1 -> { s2.forEach(j2 -> { if (j1.getCanonicalName().equals(j2.getCanonicalName())) { @@ -400,8 +380,8 @@ private Mono fillJmxMetrics (InternalClusterMetrics inte }); }); return s1; - }); - return i.toBuilder().jmxMetrics(tempMetrics.values().stream().findFirst().orElseThrow().getJmxMetrics()).build(); + }).orElseThrow(); + return i.toBuilder().jmxMetrics(resultMetrics).build(); }); } } From c64579c23a27cb9b7dfebd4d0472b9052ad70c9c Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 7 Jul 2020 12:54:57 +0300 Subject: [PATCH 13/23] Replaced multiple method usage into single --- .../provectus/kafka/ui/kafka/KafkaConstants.java | 5 ----- .../provectus/kafka/ui/kafka/KafkaService.java | 16 ++++++++-------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaConstants.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaConstants.java index 9d13a6c95bc..0454ed57dcf 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaConstants.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaConstants.java @@ -10,11 +10,6 @@ public final class KafkaConstants { private KafkaConstants() { } - public static String IN_BYTE_PER_SEC_METRIC = "incoming-byte-rate"; - public static String IN_BYTE_PER_SEC_METRIC_DESCRIPTION = "The number of bytes read off all sockets per second"; - public static String OUT_BYTE_PER_SEC_METRIC = "outgoing-byte-rate"; - public static String OUT_BYTE_PER_SEC_METRIC_DESCRIPTION = "The number of outgoing bytes sent to all servers per second"; - public static Map TOPIC_DEFAULT_CONFIGS = Map.ofEntries( new AbstractMap.SimpleEntry<>(CLEANUP_POLICY_CONFIG, CLEANUP_POLICY_DELETE), new AbstractMap.SimpleEntry<>(COMPRESSION_TYPE_CONFIG, "producer"), diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 652c398e8e7..d4cd4ca3e1c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -158,7 +158,7 @@ private Mono> getTopicsData(AdminClient adminClient) { .map( m -> m.values().stream().map(ClusterUtil::mapToInternalTopic).collect(Collectors.toList())); } - private Mono getClusterMetrics(KafkaCluster cluster, AdminClient client) { + private Mono getClusterMetrics(AdminClient client) { return ClusterUtil.toMono(client.describeCluster().nodes()) .flatMap(brokers -> ClusterUtil.toMono(client.describeCluster().controller()).map( @@ -347,23 +347,23 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern ); } - public Mono> getJmxMetric(String clusterName, Integer nodeId) { + public List getJmxMetric(String clusterName, Node node) { return clustersStorage.getClusterByName(clusterName) - .map(c -> getOrCreateAdminClient(c) - .flatMap(a -> ClusterUtil.toMono(a.getAdminClient().describeCluster().nodes()) - .map(n -> n.stream().filter(s -> s.id() == nodeId).findFirst().orElseThrow().host())) - .map(host -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), host))).orElseThrow(); + .map(c -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), node.host())).orElseThrow(); } private Mono fillJmxMetrics (InternalClusterMetrics internalClusterMetrics, String clusterName) { return Flux.fromIterable(internalClusterMetrics.getInternalBrokerMetrics().keySet()) - .flatMap(id -> getJmxMetric(clusterName, id) + .flatMap(id -> getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) + .flatMap(ac -> ClusterUtil.toMono(ac.getAdminClient().describeCluster().nodes())) + .map(node -> getJmxMetric(clusterName, node.stream().filter(n -> n.id() == id).findFirst().orElseThrow())) .map(jmx -> { var jmxMetric = internalClusterMetrics.getInternalBrokerMetrics().get(id).toBuilder().jmxMetrics(jmx).build(); var tempBrokerMetrics = internalClusterMetrics.getInternalBrokerMetrics(); tempBrokerMetrics.put(id, jmxMetric); return internalClusterMetrics.toBuilder().internalBrokerMetrics(tempBrokerMetrics).build(); - })).collectList() + }) + ).collectList() .map(s -> s.stream().reduce((s1, s2) -> { s1.getInternalBrokerMetrics().putAll(s2.getInternalBrokerMetrics().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); return s1; From 51a6a74d2cae62908d834621b876f806b2d72ff3 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Tue, 7 Jul 2020 14:57:59 +0300 Subject: [PATCH 14/23] Fixed mulptiple call --- .../provectus/kafka/ui/kafka/KafkaService.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index d4cd4ca3e1c..f854f88007f 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -19,7 +19,6 @@ import org.apache.kafka.common.utils.Bytes; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @@ -49,7 +48,7 @@ public class KafkaService { public Mono getUpdatedCluster(KafkaCluster cluster) { return getOrCreateAdminClient(cluster) .flatMap( - ac -> getClusterMetrics(cluster, ac.getAdminClient()) + ac -> getClusterMetrics(ac.getAdminClient()) .flatMap(i -> fillJmxMetrics(i, cluster.getName())) .flatMap( clusterMetrics -> getTopicsData(ac.getAdminClient()).flatMap( topics -> @@ -353,17 +352,18 @@ public List getJmxMetric(String clusterName, Node node) { } private Mono fillJmxMetrics (InternalClusterMetrics internalClusterMetrics, String clusterName) { - return Flux.fromIterable(internalClusterMetrics.getInternalBrokerMetrics().keySet()) - .flatMap(id -> getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) + return getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) + .flatMap(ac -> ClusterUtil.toMono(ac.getAdminClient().describeCluster().nodes())) + .flatMapIterable(nodes -> nodes) + .flatMap(broker -> getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) .flatMap(ac -> ClusterUtil.toMono(ac.getAdminClient().describeCluster().nodes())) - .map(node -> getJmxMetric(clusterName, node.stream().filter(n -> n.id() == id).findFirst().orElseThrow())) + .map(node -> getJmxMetric(clusterName, node.stream().filter(n -> n.id() == broker.id()).findFirst().orElseThrow())) .map(jmx -> { - var jmxMetric = internalClusterMetrics.getInternalBrokerMetrics().get(id).toBuilder().jmxMetrics(jmx).build(); + var jmxMetric = internalClusterMetrics.getInternalBrokerMetrics().get(broker.id()).toBuilder().jmxMetrics(jmx).build(); var tempBrokerMetrics = internalClusterMetrics.getInternalBrokerMetrics(); - tempBrokerMetrics.put(id, jmxMetric); + tempBrokerMetrics.put(broker.id(), jmxMetric); return internalClusterMetrics.toBuilder().internalBrokerMetrics(tempBrokerMetrics).build(); - }) - ).collectList() + })).collectList() .map(s -> s.stream().reduce((s1, s2) -> { s1.getInternalBrokerMetrics().putAll(s2.getInternalBrokerMetrics().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); return s1; From b218600eb2d3659f233dfa77dafa112ce8638c91 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Thu, 9 Jul 2020 12:01:31 +0300 Subject: [PATCH 15/23] Removed cluster level metrics, now only broker-level metrics in cluster --- .../kafka/ui/cluster/util/JmxClusterUtil.java | 16 ++++---- .../kafka/ui/kafka/KafkaService.java | 40 ++++++------------- .../main/resources/swagger/kafka-ui-api.yaml | 2 +- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 4cf71de40ea..c1c883a84df 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -53,10 +53,10 @@ public List getJmxMetrics(int jmxPort, String jmxHost) { return result; } - private Map getJmxMetric(int jmxPort, String jmxHost, String canonicalName) { + private Map getJmxMetric(int jmxPort, String jmxHost, String canonicalName) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; - Map resultAttr = new HashMap<>(); + Map resultAttr = new HashMap<>(); JMXConnector srv = null; try { srv = pool.borrowObject(jmxUrl); @@ -66,8 +66,10 @@ private Map getJmxMetric(int jmxPort, String jmxHost, String can var attrNames = msc.getMBeanInfo(name).getAttributes(); for (MBeanAttributeInfo attrName : attrNames) { var value = msc.getAttribute(name, attrName.getName()); - if (value instanceof Number) { - resultAttr.put(attrName.getName(), value); + if (value instanceof BigDecimal) { + resultAttr.put(attrName.getName(), (BigDecimal) value); + } else if (value instanceof Integer) { + resultAttr.put(attrName.getName(), new BigDecimal((Integer) value)); } } pool.returnObject(jmxUrl, srv); @@ -98,11 +100,11 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - public static Object metricValueReduce(Object value1, Object value2) { - if (value1 instanceof Number) { + public static BigDecimal metricValueReduce(Number value1, Number value2) { + if (value1 instanceof Integer) { return new BigDecimal(value1.toString()).add(new BigDecimal(value2.toString())); } else { - return value1; + return new BigDecimal(value1.longValue()).add(new BigDecimal(value2.longValue())); } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index f854f88007f..ff7607cefad 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -352,36 +352,22 @@ public List getJmxMetric(String clusterName, Node node) { } private Mono fillJmxMetrics (InternalClusterMetrics internalClusterMetrics, String clusterName) { + return fillBrokerMetrics(internalClusterMetrics, clusterName); + } + + private Mono fillBrokerMetrics(InternalClusterMetrics internalClusterMetrics, String clusterName) { return getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) .flatMap(ac -> ClusterUtil.toMono(ac.getAdminClient().describeCluster().nodes())) .flatMapIterable(nodes -> nodes) - .flatMap(broker -> getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) - .flatMap(ac -> ClusterUtil.toMono(ac.getAdminClient().describeCluster().nodes())) - .map(node -> getJmxMetric(clusterName, node.stream().filter(n -> n.id() == broker.id()).findFirst().orElseThrow())) - .map(jmx -> { - var jmxMetric = internalClusterMetrics.getInternalBrokerMetrics().get(broker.id()).toBuilder().jmxMetrics(jmx).build(); - var tempBrokerMetrics = internalClusterMetrics.getInternalBrokerMetrics(); - tempBrokerMetrics.put(broker.id(), jmxMetric); - return internalClusterMetrics.toBuilder().internalBrokerMetrics(tempBrokerMetrics).build(); - })).collectList() - .map(s -> s.stream().reduce((s1, s2) -> { - s1.getInternalBrokerMetrics().putAll(s2.getInternalBrokerMetrics().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - return s1; - }).orElseThrow()) - .map(i -> { - var tempMetrics = new HashMap<>(i.getInternalBrokerMetrics()); - var resultMetrics = tempMetrics.values().stream().flatMap(s -> Stream.of(s.getJmxMetrics())).reduce((s1, s2) -> { - s1.forEach(j1 -> { - s2.forEach(j2 -> { - if (j1.getCanonicalName().equals(j2.getCanonicalName())) { - j1.getValue().keySet().forEach(k -> j2.getValue().compute(k, (k1, v1) -> - JmxClusterUtil.metricValueReduce(j1, j2.getValue().get(k1)))); - } - }); - }); - return s1; - }).orElseThrow(); - return i.toBuilder().jmxMetrics(resultMetrics).build(); + .map(broker -> { + var jmx = getJmxMetric(clusterName, broker); + Map result = new HashMap<>(); + result.put(broker.id(), internalClusterMetrics.getInternalBrokerMetrics().get(broker.id()).toBuilder().jmxMetrics(jmx).build()); + return result; + }) + .collectList() + .map(s -> { var brokerMetrics = ClusterUtil.toSingleMap(s.stream()); + return internalClusterMetrics.toBuilder().internalBrokerMetrics(brokerMetrics).build(); }); } } diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 9500d5ed159..88ffbd24492 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -531,4 +531,4 @@ components: value: type: string additionalProperties: - type: object \ No newline at end of file + type: number \ No newline at end of file From 21ca9a077684c42a1b890d6b8f6f0d54b3fd81ae Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Thu, 9 Jul 2020 14:54:43 +0300 Subject: [PATCH 16/23] Just small fixes --- .../com/provectus/kafka/ui/kafka/KafkaService.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index ff7607cefad..70316461934 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -164,8 +164,6 @@ private Mono getClusterMetrics(AdminClient client) { c -> { InternalClusterMetrics.InternalClusterMetricsBuilder metricsBuilder = InternalClusterMetrics.builder(); metricsBuilder.brokerCount(brokers.size()).activeControllers(c != null ? 1 : 0); - metricsBuilder - .internalBrokerMetrics((brokers.stream().map(Node::id).collect(Collectors.toMap(k -> k, v -> InternalBrokerMetrics.builder().build())))); return metricsBuilder.build(); } ) @@ -361,13 +359,10 @@ private Mono fillBrokerMetrics(InternalClusterMetrics in .flatMapIterable(nodes -> nodes) .map(broker -> { var jmx = getJmxMetric(clusterName, broker); - Map result = new HashMap<>(); - result.put(broker.id(), internalClusterMetrics.getInternalBrokerMetrics().get(broker.id()).toBuilder().jmxMetrics(jmx).build()); - return result; + return Map.of(broker.id(), InternalBrokerMetrics.builder(). + jmxMetrics(jmx).build()); }) .collectList() - .map(s -> { var brokerMetrics = ClusterUtil.toSingleMap(s.stream()); - return internalClusterMetrics.toBuilder().internalBrokerMetrics(brokerMetrics).build(); - }); + .map(s -> internalClusterMetrics.toBuilder().internalBrokerMetrics(ClusterUtil.toSingleMap(s.stream())).build()); } } From 04ca14d98a6bf76275a0b56e592f2530ed8bea21 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Thu, 9 Jul 2020 14:58:32 +0300 Subject: [PATCH 17/23] removed redundant variable --- .../java/com/provectus/kafka/ui/kafka/KafkaService.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 70316461934..517d846e1b6 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -357,11 +357,8 @@ private Mono fillBrokerMetrics(InternalClusterMetrics in return getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) .flatMap(ac -> ClusterUtil.toMono(ac.getAdminClient().describeCluster().nodes())) .flatMapIterable(nodes -> nodes) - .map(broker -> { - var jmx = getJmxMetric(clusterName, broker); - return Map.of(broker.id(), InternalBrokerMetrics.builder(). - jmxMetrics(jmx).build()); - }) + .map(broker -> Map.of(broker.id(), InternalBrokerMetrics.builder(). + jmxMetrics(getJmxMetric(clusterName, broker)).build())) .collectList() .map(s -> internalClusterMetrics.toBuilder().internalBrokerMetrics(ClusterUtil.toSingleMap(s.stream())).build()); } From 105ef892780b82c193849e03321d6f1b1fd440a6 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Thu, 9 Jul 2020 15:04:13 +0300 Subject: [PATCH 18/23] Renamed method for cluster level metrics --- .../com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index c1c883a84df..011af2f60fe 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -100,7 +100,7 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - public static BigDecimal metricValueReduce(Number value1, Number value2) { + public static BigDecimal metricValueSum(Number value1, Number value2) { if (value1 instanceof Integer) { return new BigDecimal(value1.toString()).add(new BigDecimal(value2.toString())); } else { From 4edc5f8428776a2a151ab104cefc48454e6e405a Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Wed, 15 Jul 2020 11:00:54 +0300 Subject: [PATCH 19/23] Fixed after PR and added sum for number cluster metrics by num and persec keywords in canonicalname --- docker/kafka-clusters-only.yaml | 34 +++++----- .../ui/cluster/service/ClusterService.java | 4 +- .../kafka/ui/cluster/util/JmxClusterUtil.java | 66 ++++++++++++------- .../kafka/ui/kafka/KafkaService.java | 27 ++++++-- 4 files changed, 83 insertions(+), 48 deletions(-) diff --git a/docker/kafka-clusters-only.yaml b/docker/kafka-clusters-only.yaml index b3d0900f06d..b2ddd507ae1 100644 --- a/docker/kafka-clusters-only.yaml +++ b/docker/kafka-clusters-only.yaml @@ -27,22 +27,22 @@ services: JMX_PORT: 9997 KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9997 - # kafka01: - # image: confluentinc/cp-kafka:5.1.0 - # depends_on: - # - zookeeper0 - # ports: - # - 29093:29093 - # - 9999:9999 - # environment: - # KAFKA_BROKER_ID: 2 - # KAFKA_ZOOKEEPER_CONNECT: zookeeper0:2183 - # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka01:9092,PLAINTEXT_HOST://localhost:29093,PLAIN://kafka0:29090 - # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,PLAIN:PLAINTEXT - # KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 - # JMX_PORT: 9997 - # KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9997 + kafka01: + image: confluentinc/cp-kafka:5.1.0 + depends_on: + - zookeeper0 + ports: + - 29093:29093 + - 9999:9999 + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper0:2183 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka01:9092,PLAINTEXT_HOST://localhost:29093,PLAIN://kafka0:29090 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,PLAIN:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 + JMX_PORT: 9997 + KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9997 kafka-init-topics0: image: confluentinc/cp-kafka:5.1.0 @@ -102,7 +102,7 @@ services: depends_on: - zookeeper0 - kafka0 - # - kafka01 + - kafka01 ports: - 8085:8085 environment: diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index 9eab77188f2..686b89cacbb 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -39,14 +39,14 @@ public List getClusters() { } public Mono getBrokersMetrics(String name, Integer id) { - return Mono.just(clustersStorage.getClusterByName(name) + return Mono.justOrEmpty(clustersStorage.getClusterByName(name) .map(KafkaCluster::getMetrics) .map(s -> { var brokerMetrics = clusterMapper.toBrokerMetrics(s); brokerMetrics.setJmxMetrics(s.getInternalBrokerMetrics().get(id).getJmxMetrics()); brokerMetrics.setSegmentZise(Long.valueOf(s.getSegmentSize()).intValue()); return brokerMetrics; - }).orElseThrow()); + })); } public List getTopics(String name) { diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 011af2f60fe..b139fb41c40 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -1,8 +1,11 @@ package com.provectus.kafka.ui.cluster.util; +import com.provectus.kafka.ui.cluster.model.InternalClusterMetrics; import com.provectus.kafka.ui.model.JmxMetric; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.pool2.KeyedObjectPool; import org.springframework.stereotype.Component; @@ -36,12 +39,12 @@ public List getJmxMetrics(int jmxPort, String jmxHost) { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); var jmxMetrics = msc.queryNames(null, null).stream().filter(q -> q.getCanonicalName().startsWith(KAFKA_SERVER_PARAM)).collect(Collectors.toList()); - jmxMetrics.forEach(j -> { + for (ObjectName jmxMetric : jmxMetrics) { JmxMetric metric = new JmxMetric(); - metric.setCanonicalName(j.getCanonicalName()); - metric.setValue(getJmxMetric(jmxPort, jmxHost, j.getCanonicalName())); + metric.setCanonicalName(jmxMetric.getCanonicalName()); + metric.setValue(getJmxMetric(jmxMetric.getCanonicalName(), msc, srv, jmxUrl)); result.add(metric); - }); + }; pool.returnObject(jmxUrl, srv); } catch (IOException ioe) { log.error("Cannot get jmxMetricsNames, {}", jmxUrl, ioe); @@ -53,26 +56,17 @@ public List getJmxMetrics(int jmxPort, String jmxHost) { return result; } - private Map getJmxMetric(int jmxPort, String jmxHost, String canonicalName) { - String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; - + private Map getJmxMetric(String canonicalName, MBeanServerConnection msc, JMXConnector srv, String jmxUrl) { Map resultAttr = new HashMap<>(); - JMXConnector srv = null; try { - srv = pool.borrowObject(jmxUrl); - MBeanServerConnection msc = srv.getMBeanServerConnection(); - ObjectName name = new ObjectName(canonicalName); var attrNames = msc.getMBeanInfo(name).getAttributes(); for (MBeanAttributeInfo attrName : attrNames) { var value = msc.getAttribute(name, attrName.getName()); - if (value instanceof BigDecimal) { - resultAttr.put(attrName.getName(), (BigDecimal) value); - } else if (value instanceof Integer) { - resultAttr.put(attrName.getName(), new BigDecimal((Integer) value)); + if (value instanceof Number) { + resultAttr.put(attrName.getName(), new BigDecimal(value.toString())); } } - pool.returnObject(jmxUrl, srv); } catch (MalformedURLException url) { log.error("Cannot create JmxServiceUrl from {}", jmxUrl); closeConnectionExceptionally(jmxUrl, srv); @@ -100,11 +94,39 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - public static BigDecimal metricValueSum(Number value1, Number value2) { - if (value1 instanceof Integer) { - return new BigDecimal(value1.toString()).add(new BigDecimal(value2.toString())); - } else { - return new BigDecimal(value1.longValue()).add(new BigDecimal(value2.longValue())); - } + private static Pair getCanonicalAndMetricName(String value) { + return Pair.of(value.substring(0, value.indexOf("+")), value.substring(value.indexOf("+") + 1)); + } + + public static List>> squashIntoNameMetricPair(InternalClusterMetrics internalClusterMetrics) { + return internalClusterMetrics.getInternalBrokerMetrics().values().stream() + .map(c -> + c.getJmxMetrics().stream() + .filter(j -> StringUtils.containsIgnoreCase(j.getCanonicalName(), "num") || StringUtils.containsIgnoreCase(j.getCanonicalName(), "persec")) + .map(j -> j.getValue().entrySet().stream() + .map(e -> Pair.of(j.getCanonicalName() + "+" + e.getKey(), e.getValue())) + .collect(Collectors.toList())) + .collect(Collectors.toList()) + ) + .collect(Collectors.toList()) + .stream().flatMap(List::stream).collect(Collectors.toList()) + .stream().flatMap(List::stream).collect(Collectors.toList()) + .stream().map(c -> { + var pairNames = JmxClusterUtil.getCanonicalAndMetricName(c.getKey()); + return Pair.of(pairNames.getKey(), Pair.of(pairNames.getValue(), c.getValue())); + }).collect(Collectors.toList()); + } + + public static JmxMetric reduceJmxMetrics (List metrics) { + var result = List.copyOf(metrics); + return result.stream().reduce((j1, j2) -> { + var temp1 = new HashMap<>(j1.getValue()); + var temp2 = new HashMap<>(j2.getValue()); + temp2.forEach((k, v) -> temp1.merge(k, v, BigDecimal::add)); + var mergedMetric = new JmxMetric(); + mergedMetric.setCanonicalName(j1.getCanonicalName()); + mergedMetric.setValue(temp1); + return mergedMetric; + }).orElse(null); } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 517d846e1b6..2b37080ec3c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -49,7 +49,7 @@ public Mono getUpdatedCluster(KafkaCluster cluster) { return getOrCreateAdminClient(cluster) .flatMap( ac -> getClusterMetrics(ac.getAdminClient()) - .flatMap(i -> fillJmxMetrics(i, cluster.getName())) + .flatMap(i -> fillJmxMetrics(i, cluster.getName(), ac.getAdminClient())) .flatMap( clusterMetrics -> getTopicsData(ac.getAdminClient()).flatMap( topics -> loadTopicsConfig(ac.getAdminClient(), topics.stream().map(InternalTopic::getName).collect(Collectors.toList())) @@ -346,20 +346,33 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern public List getJmxMetric(String clusterName, Node node) { return clustersStorage.getClusterByName(clusterName) - .map(c -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), node.host())).orElseThrow(); + .map(c -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), node.host())).orElse(Collections.emptyList()); } - private Mono fillJmxMetrics (InternalClusterMetrics internalClusterMetrics, String clusterName) { - return fillBrokerMetrics(internalClusterMetrics, clusterName); + private Mono fillJmxMetrics (InternalClusterMetrics internalClusterMetrics, String clusterName, AdminClient ac) { + return fillBrokerMetrics(internalClusterMetrics, clusterName, ac).map(this::calculateClusterMetrics); } - private Mono fillBrokerMetrics(InternalClusterMetrics internalClusterMetrics, String clusterName) { - return getOrCreateAdminClient(clustersStorage.getClusterByName(clusterName).orElseThrow()) - .flatMap(ac -> ClusterUtil.toMono(ac.getAdminClient().describeCluster().nodes())) + private Mono fillBrokerMetrics(InternalClusterMetrics internalClusterMetrics, String clusterName, AdminClient ac) { + return ClusterUtil.toMono(ac.describeCluster().nodes()) .flatMapIterable(nodes -> nodes) .map(broker -> Map.of(broker.id(), InternalBrokerMetrics.builder(). jmxMetrics(getJmxMetric(clusterName, broker)).build())) .collectList() .map(s -> internalClusterMetrics.toBuilder().internalBrokerMetrics(ClusterUtil.toSingleMap(s.stream())).build()); } + + private InternalClusterMetrics calculateClusterMetrics(InternalClusterMetrics internalClusterMetrics) { + return internalClusterMetrics.toBuilder().jmxMetrics( + JmxClusterUtil.squashIntoNameMetricPair(internalClusterMetrics) + .stream().map(c -> { + JmxMetric jmx = new JmxMetric(); + jmx.setCanonicalName(c.getKey()); + jmx.setValue(Map.of(c.getValue().getKey(), c.getValue().getValue())); + return jmx; + }).collect(Collectors.groupingBy(JmxMetric::getCanonicalName, Collectors.toList())) + .values().stream().map(JmxClusterUtil::reduceJmxMetrics) + .filter(Objects::nonNull) + .collect(Collectors.toList())).build(); + } } From 5beb6ae937ecb1f4bd7775b85bc49bf3d25bfb6f Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Thu, 23 Jul 2020 10:12:30 +0300 Subject: [PATCH 20/23] Added metricdto object --- docker/kafka-clusters-only.yaml | 1 - .../kafka/ui/cluster/model/MetricDto.java | 14 ++++ .../kafka/ui/cluster/util/JmxClusterUtil.java | 67 ++++++++++--------- .../kafka/ui/kafka/KafkaService.java | 11 +-- 4 files changed, 57 insertions(+), 36 deletions(-) create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/MetricDto.java diff --git a/docker/kafka-clusters-only.yaml b/docker/kafka-clusters-only.yaml index b2ddd507ae1..6d8f1fdf5c0 100644 --- a/docker/kafka-clusters-only.yaml +++ b/docker/kafka-clusters-only.yaml @@ -79,7 +79,6 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092,PLAINTEXT_HOST://localhost:29092,PLAIN://localhost:29090 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,PLAIN:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2 JMX_PORT: 9998 KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9998 diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/MetricDto.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/MetricDto.java new file mode 100644 index 00000000000..44b5b896e8f --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/MetricDto.java @@ -0,0 +1,14 @@ +package com.provectus.kafka.ui.cluster.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.math.BigDecimal; + +@Getter +@AllArgsConstructor +public class MetricDto { + private String canonicalName; + private String metricName; + private BigDecimal value; +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index b139fb41c40..c532898f8ea 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -1,11 +1,11 @@ package com.provectus.kafka.ui.cluster.util; +import com.google.common.collect.ImmutableList; import com.provectus.kafka.ui.cluster.model.InternalClusterMetrics; +import com.provectus.kafka.ui.cluster.model.MetricDto; import com.provectus.kafka.ui.model.JmxMetric; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.pool2.KeyedObjectPool; import org.springframework.stereotype.Component; @@ -18,6 +18,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; @Component @@ -31,6 +32,16 @@ public class JmxClusterUtil { private static final String JMX_SERVICE_TYPE = "jmxrmi"; private static final String KAFKA_SERVER_PARAM = "kafka.server"; + + private static final ImmutableList metricsNames = ImmutableList.of( + "MessagesInPerSec", "BytesInPerSec", "ReplicationBytesInPerSec", "RequestsPerSec", "ErrorsPerSec", "MessageConversionsPerSec", "BytesOutPerSec", "ReplicationBytesOutPerSec", "NoKeyCompactedTopicRecordsPerSec", + "InvalidMagicNumberRecordsPerSec", "InvalidMessageCrcRecordsPerSec", "InvalidOffsetOrSequenceRecordsPerSec", + "UncleanLeaderElectionsPerSec", "IsrShrinksPerSec", "IsrExpandsPerSec", "ReassignmentBytesOutPerSec", "ReassignmentBytesInPerSec", + "ProduceMessageConversionsPerSec", "FailedFetchRequestsPerSec", "ZooKeeperSyncConnectsPerSec", "BytesRejectedPerSec", + "IsrShrinksPerSec", "ReplicationBytesOutPerSec", "ZooKeeperAuthFailuresPerSec", "TotalFetchRequestsPerSec", "FailedIsrUpdatesPerSec", + "IncrementalFetchSessionEvictionsPerSec", "FetchMessageConversionsPerSec", "TotalFetchRequestsPerSec", "FailedProduceRequestsPerSec" + ); + public List getJmxMetrics(int jmxPort, String jmxHost) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; List result = new ArrayList<>(); @@ -64,6 +75,7 @@ private Map getJmxMetric(String canonicalName, MBeanServerCo for (MBeanAttributeInfo attrName : attrNames) { var value = msc.getAttribute(name, attrName.getName()); if (value instanceof Number) { + if (!(value instanceof Double) || !((Double) value).isInfinite()) resultAttr.put(attrName.getName(), new BigDecimal(value.toString())); } } @@ -94,39 +106,34 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - private static Pair getCanonicalAndMetricName(String value) { - return Pair.of(value.substring(0, value.indexOf("+")), value.substring(value.indexOf("+") + 1)); - } - - public static List>> squashIntoNameMetricPair(InternalClusterMetrics internalClusterMetrics) { + public static List squashIntoNameMetricPair(InternalClusterMetrics internalClusterMetrics) { return internalClusterMetrics.getInternalBrokerMetrics().values().stream() .map(c -> c.getJmxMetrics().stream() - .filter(j -> StringUtils.containsIgnoreCase(j.getCanonicalName(), "num") || StringUtils.containsIgnoreCase(j.getCanonicalName(), "persec")) + .filter(j -> isSameMetric(j.getCanonicalName())) .map(j -> j.getValue().entrySet().stream() - .map(e -> Pair.of(j.getCanonicalName() + "+" + e.getKey(), e.getValue())) - .collect(Collectors.toList())) - .collect(Collectors.toList()) - ) - .collect(Collectors.toList()) - .stream().flatMap(List::stream).collect(Collectors.toList()) - .stream().flatMap(List::stream).collect(Collectors.toList()) - .stream().map(c -> { - var pairNames = JmxClusterUtil.getCanonicalAndMetricName(c.getKey()); - return Pair.of(pairNames.getKey(), Pair.of(pairNames.getValue(), c.getValue())); - }).collect(Collectors.toList()); + .map(e -> new MetricDto(j.getCanonicalName(), e.getKey(), e.getValue())))) + .flatMap(Function.identity()).flatMap(Function.identity()).collect(Collectors.toList()); } - public static JmxMetric reduceJmxMetrics (List metrics) { - var result = List.copyOf(metrics); - return result.stream().reduce((j1, j2) -> { - var temp1 = new HashMap<>(j1.getValue()); - var temp2 = new HashMap<>(j2.getValue()); - temp2.forEach((k, v) -> temp1.merge(k, v, BigDecimal::add)); - var mergedMetric = new JmxMetric(); - mergedMetric.setCanonicalName(j1.getCanonicalName()); - mergedMetric.setValue(temp1); - return mergedMetric; - }).orElse(null); + public static JmxMetric reduceJmxMetrics (JmxMetric metric1, JmxMetric metric2) { + var result = new JmxMetric(); + Map jmx1 = new HashMap<>(metric1.getValue()); + Map jmx2 = new HashMap<>(metric2.getValue()); + jmx1.forEach((k, v) -> jmx2.merge(k, v, BigDecimal::add)); + result.setCanonicalName(metric1.getCanonicalName()); + result.setValue(jmx2); + return result; + } + + private static boolean isSameMetric (String metric) { + if (metric.contains("name=")) { + int beginIndex = metric.indexOf("name="); + int endIndex = metric.indexOf(',', beginIndex); + endIndex = endIndex < 0 ? metric.length() - 1 : endIndex; + return metricsNames.contains(metric.substring(beginIndex + 5, endIndex)); + } else { + return false; + } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 2b37080ec3c..9569fd5ea4c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -367,12 +367,13 @@ private InternalClusterMetrics calculateClusterMetrics(InternalClusterMetrics in JmxClusterUtil.squashIntoNameMetricPair(internalClusterMetrics) .stream().map(c -> { JmxMetric jmx = new JmxMetric(); - jmx.setCanonicalName(c.getKey()); - jmx.setValue(Map.of(c.getValue().getKey(), c.getValue().getValue())); + jmx.setCanonicalName(c.getCanonicalName()); + jmx.setValue(Map.of(c.getMetricName(), c.getValue())); return jmx; - }).collect(Collectors.groupingBy(JmxMetric::getCanonicalName, Collectors.toList())) - .values().stream().map(JmxClusterUtil::reduceJmxMetrics) - .filter(Objects::nonNull) + }).collect(Collectors.groupingBy(JmxMetric::getCanonicalName, Collectors.reducing(JmxClusterUtil::reduceJmxMetrics))) + .values().stream() + .filter(Optional::isPresent) + .map(Optional::get) .collect(Collectors.toList())).build(); } } From 5ef0269ec0136187ff7e17f1bb116f52fb6516aa Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Thu, 23 Jul 2020 10:26:39 +0300 Subject: [PATCH 21/23] Added list of metrics to enum --- .../kafka/ui/cluster/config/Config.java | 9 +++++ .../kafka/ui/cluster/util/JmxClusterUtil.java | 25 +++++--------- .../ui/cluster/util/JmxMetricsNames.java | 33 +++++++++++++++++++ .../kafka/ui/kafka/KafkaService.java | 4 +-- 4 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/config/Config.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/config/Config.java index 83c86553c96..5b6a623b006 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/config/Config.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/config/Config.java @@ -1,5 +1,6 @@ package com.provectus.kafka.ui.cluster.config; +import com.provectus.kafka.ui.cluster.util.JmxMetricsNames; import com.provectus.kafka.ui.cluster.util.JmxPoolFactory; import org.apache.commons.pool2.KeyedObjectPool; import org.apache.commons.pool2.impl.GenericKeyedObjectPool; @@ -9,6 +10,9 @@ import org.springframework.jmx.export.MBeanExporter; import javax.management.remote.JMXConnector; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Configuration public class Config { @@ -35,4 +39,9 @@ public MBeanExporter exporter() exporter.setExcludedBeans("pool"); return exporter; } + + @Bean + public List jmxMetricsNames() { + return Stream.of(JmxMetricsNames.values()).map(Enum::name).collect(Collectors.toList()); + } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index c532898f8ea..0c2b09b7e8b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -1,6 +1,5 @@ package com.provectus.kafka.ui.cluster.util; -import com.google.common.collect.ImmutableList; import com.provectus.kafka.ui.cluster.model.InternalClusterMetrics; import com.provectus.kafka.ui.cluster.model.MetricDto; import com.provectus.kafka.ui.model.JmxMetric; @@ -27,20 +26,12 @@ public class JmxClusterUtil { private final KeyedObjectPool pool; + private final List jmxMetricsNames; private static final String JMX_URL = "service:jmx:rmi:///jndi/rmi://"; private static final String JMX_SERVICE_TYPE = "jmxrmi"; private static final String KAFKA_SERVER_PARAM = "kafka.server"; - - - private static final ImmutableList metricsNames = ImmutableList.of( - "MessagesInPerSec", "BytesInPerSec", "ReplicationBytesInPerSec", "RequestsPerSec", "ErrorsPerSec", "MessageConversionsPerSec", "BytesOutPerSec", "ReplicationBytesOutPerSec", "NoKeyCompactedTopicRecordsPerSec", - "InvalidMagicNumberRecordsPerSec", "InvalidMessageCrcRecordsPerSec", "InvalidOffsetOrSequenceRecordsPerSec", - "UncleanLeaderElectionsPerSec", "IsrShrinksPerSec", "IsrExpandsPerSec", "ReassignmentBytesOutPerSec", "ReassignmentBytesInPerSec", - "ProduceMessageConversionsPerSec", "FailedFetchRequestsPerSec", "ZooKeeperSyncConnectsPerSec", "BytesRejectedPerSec", - "IsrShrinksPerSec", "ReplicationBytesOutPerSec", "ZooKeeperAuthFailuresPerSec", "TotalFetchRequestsPerSec", "FailedIsrUpdatesPerSec", - "IncrementalFetchSessionEvictionsPerSec", "FetchMessageConversionsPerSec", "TotalFetchRequestsPerSec", "FailedProduceRequestsPerSec" - ); + private static final String NAME_METRIC_FIELD = "name="; public List getJmxMetrics(int jmxPort, String jmxHost) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; @@ -106,7 +97,7 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - public static List squashIntoNameMetricPair(InternalClusterMetrics internalClusterMetrics) { + public List squashIntoNameMetricPair(InternalClusterMetrics internalClusterMetrics) { return internalClusterMetrics.getInternalBrokerMetrics().values().stream() .map(c -> c.getJmxMetrics().stream() @@ -116,7 +107,7 @@ public static List squashIntoNameMetricPair(InternalClusterMetrics in .flatMap(Function.identity()).flatMap(Function.identity()).collect(Collectors.toList()); } - public static JmxMetric reduceJmxMetrics (JmxMetric metric1, JmxMetric metric2) { + public JmxMetric reduceJmxMetrics (JmxMetric metric1, JmxMetric metric2) { var result = new JmxMetric(); Map jmx1 = new HashMap<>(metric1.getValue()); Map jmx2 = new HashMap<>(metric2.getValue()); @@ -126,12 +117,12 @@ public static JmxMetric reduceJmxMetrics (JmxMetric metric1, JmxMetric metric2) return result; } - private static boolean isSameMetric (String metric) { - if (metric.contains("name=")) { - int beginIndex = metric.indexOf("name="); + private boolean isSameMetric (String metric) { + if (metric.contains(NAME_METRIC_FIELD)) { + int beginIndex = metric.indexOf(NAME_METRIC_FIELD); int endIndex = metric.indexOf(',', beginIndex); endIndex = endIndex < 0 ? metric.length() - 1 : endIndex; - return metricsNames.contains(metric.substring(beginIndex + 5, endIndex)); + return jmxMetricsNames.contains(metric.substring(beginIndex + 5, endIndex)); } else { return false; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java new file mode 100644 index 00000000000..23e353adc87 --- /dev/null +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java @@ -0,0 +1,33 @@ +package com.provectus.kafka.ui.cluster.util; + +import com.google.common.collect.ImmutableList; + +public enum JmxMetricsNames { + MessagesInPerSec, + BytesInPerSec, + ReplicationBytesInPerSec, + RequestsPerSec, + ErrorsPerSec, + MessageConversionsPerSec, + BytesOutPerSec, + ReplicationBytesOutPerSec, + NoKeyCompactedTopicRecordsPerSec, + InvalidMagicNumberRecordsPerSec, + InvalidMessageCrcRecordsPerSec, + InvalidOffsetOrSequenceRecordsPerSec, + UncleanLeaderElectionsPerSec, + IsrShrinksPerSec, + IsrExpandsPerSec, + ReassignmentBytesOutPerSec, + ReassignmentBytesInPerSec, + ProduceMessageConversionsPerSec, + FailedFetchRequestsPerSec, + ZooKeeperSyncConnectsPerSec, + BytesRejectedPerSec, + ZooKeeperAuthFailuresPerSec, + TotalFetchRequestsPerSec, + FailedIsrUpdatesPerSec, + IncrementalFetchSessionEvictionsPerSec, + FetchMessageConversionsPerSec, + FailedProduceRequestsPerSec +} diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 9569fd5ea4c..03cd299a94e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -364,13 +364,13 @@ private Mono fillBrokerMetrics(InternalClusterMetrics in private InternalClusterMetrics calculateClusterMetrics(InternalClusterMetrics internalClusterMetrics) { return internalClusterMetrics.toBuilder().jmxMetrics( - JmxClusterUtil.squashIntoNameMetricPair(internalClusterMetrics) + jmxClusterUtil.squashIntoNameMetricPair(internalClusterMetrics) .stream().map(c -> { JmxMetric jmx = new JmxMetric(); jmx.setCanonicalName(c.getCanonicalName()); jmx.setValue(Map.of(c.getMetricName(), c.getValue())); return jmx; - }).collect(Collectors.groupingBy(JmxMetric::getCanonicalName, Collectors.reducing(JmxClusterUtil::reduceJmxMetrics))) + }).collect(Collectors.groupingBy(JmxMetric::getCanonicalName, Collectors.reducing(jmxClusterUtil::reduceJmxMetrics))) .values().stream() .filter(Optional::isPresent) .map(Optional::get) From 9b04938774b514a653277ca1001a92d0273fb97e Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Thu, 23 Jul 2020 10:35:20 +0300 Subject: [PATCH 22/23] Renames and optimizings --- .../com/provectus/kafka/ui/cluster/model/InternalTopic.java | 1 - .../com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java | 2 +- .../com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java | 2 -- .../main/java/com/provectus/kafka/ui/kafka/KafkaService.java | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalTopic.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalTopic.java index 95783e5b3f4..164c1073250 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalTopic.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalTopic.java @@ -21,7 +21,6 @@ public class InternalTopic { private final int inSyncReplicas; private final int replicationFactor; private final int underReplicatedPartitions; - //TODO: find way to fill private final long segmentSize; private final int segmentCount; private final Map partitionSegmentSize; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 0c2b09b7e8b..1dc783b7d9c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -97,7 +97,7 @@ private void closeConnectionExceptionally(String url, JMXConnector srv) { } } - public List squashIntoNameMetricPair(InternalClusterMetrics internalClusterMetrics) { + public List convertToMetricDto(InternalClusterMetrics internalClusterMetrics) { return internalClusterMetrics.getInternalBrokerMetrics().values().stream() .map(c -> c.getJmxMetrics().stream() diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java index 23e353adc87..62197dc60c3 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxMetricsNames.java @@ -1,7 +1,5 @@ package com.provectus.kafka.ui.cluster.util; -import com.google.common.collect.ImmutableList; - public enum JmxMetricsNames { MessagesInPerSec, BytesInPerSec, diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 03cd299a94e..7faa50df7fd 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -364,7 +364,7 @@ private Mono fillBrokerMetrics(InternalClusterMetrics in private InternalClusterMetrics calculateClusterMetrics(InternalClusterMetrics internalClusterMetrics) { return internalClusterMetrics.toBuilder().jmxMetrics( - jmxClusterUtil.squashIntoNameMetricPair(internalClusterMetrics) + jmxClusterUtil.convertToMetricDto(internalClusterMetrics) .stream().map(c -> { JmxMetric jmx = new JmxMetric(); jmx.setCanonicalName(c.getCanonicalName()); From c81c51dfb051e9e0774af62c6c12af9b088a1442 Mon Sep 17 00:00:00 2001 From: Roman Nedzvetskiy Date: Mon, 27 Jul 2020 10:28:05 +0300 Subject: [PATCH 23/23] Renamed jmxmetrics objects param to metrics --- .../kafka/ui/cluster/mapper/ClusterMapper.java | 2 +- .../ui/cluster/model/InternalBrokerMetrics.java | 4 ++-- .../ui/cluster/model/InternalClusterMetrics.java | 5 ++--- .../kafka/ui/cluster/service/ClusterService.java | 2 +- .../kafka/ui/cluster/util/JmxClusterUtil.java | 12 ++++++------ .../com/provectus/kafka/ui/kafka/KafkaService.java | 8 ++++---- .../src/main/resources/swagger/kafka-ui-api.yaml | 10 +++++----- 7 files changed, 21 insertions(+), 22 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java index d5971d2f33b..8414fe42f7d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/mapper/ClusterMapper.java @@ -15,7 +15,7 @@ public interface ClusterMapper { @Mapping(target = "brokerCount", source = "metrics.brokerCount") @Mapping(target = "onlinePartitionCount", source = "metrics.onlinePartitionCount") @Mapping(target = "topicCount", source = "metrics.topicCount") - @Mapping(target = "jmxMetrics", source = "metrics.jmxMetrics") + @Mapping(target = "metrics", source = "metrics.metrics") Cluster toCluster(KafkaCluster cluster); KafkaCluster toKafkaCluster(ClustersProperties.Cluster clusterProperties); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java index 33de7471684..5350aa3c24d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalBrokerMetrics.java @@ -1,6 +1,6 @@ package com.provectus.kafka.ui.cluster.model; -import com.provectus.kafka.ui.model.JmxMetric; +import com.provectus.kafka.ui.model.Metric; import lombok.Builder; import lombok.Data; @@ -10,5 +10,5 @@ @Builder(toBuilder = true) public class InternalBrokerMetrics { private final Long segmentSize; - private final List jmxMetrics; + private final List jmxMetrics; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java index 74039d98d83..6603816d441 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/model/InternalClusterMetrics.java @@ -1,10 +1,9 @@ package com.provectus.kafka.ui.cluster.model; -import com.provectus.kafka.ui.model.JmxMetric; +import com.provectus.kafka.ui.model.Metric; import lombok.Builder; import lombok.Data; -import java.math.BigDecimal; import java.util.List; import java.util.Map; @@ -26,6 +25,6 @@ public class InternalClusterMetrics { private final int segmentCount; private final long segmentSize; private final Map internalBrokerMetrics; - private final List jmxMetrics; + private final List metrics; private final int zooKeeperStatus; } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java index 686b89cacbb..63daa181fdc 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/service/ClusterService.java @@ -43,7 +43,7 @@ public Mono getBrokersMetrics(String name, Integer id) { .map(KafkaCluster::getMetrics) .map(s -> { var brokerMetrics = clusterMapper.toBrokerMetrics(s); - brokerMetrics.setJmxMetrics(s.getInternalBrokerMetrics().get(id).getJmxMetrics()); + brokerMetrics.setMetrics(s.getInternalBrokerMetrics().get(id).getJmxMetrics()); brokerMetrics.setSegmentZise(Long.valueOf(s.getSegmentSize()).intValue()); return brokerMetrics; })); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java index 1dc783b7d9c..8554a712c61 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/cluster/util/JmxClusterUtil.java @@ -2,7 +2,7 @@ import com.provectus.kafka.ui.cluster.model.InternalClusterMetrics; import com.provectus.kafka.ui.cluster.model.MetricDto; -import com.provectus.kafka.ui.model.JmxMetric; +import com.provectus.kafka.ui.model.Metric; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.pool2.KeyedObjectPool; @@ -33,16 +33,16 @@ public class JmxClusterUtil { private static final String KAFKA_SERVER_PARAM = "kafka.server"; private static final String NAME_METRIC_FIELD = "name="; - public List getJmxMetrics(int jmxPort, String jmxHost) { + public List getJmxMetrics(int jmxPort, String jmxHost) { String jmxUrl = JMX_URL + jmxHost + ":" + jmxPort + "/" + JMX_SERVICE_TYPE; - List result = new ArrayList<>(); + List result = new ArrayList<>(); JMXConnector srv = null; try { srv = pool.borrowObject(jmxUrl); MBeanServerConnection msc = srv.getMBeanServerConnection(); var jmxMetrics = msc.queryNames(null, null).stream().filter(q -> q.getCanonicalName().startsWith(KAFKA_SERVER_PARAM)).collect(Collectors.toList()); for (ObjectName jmxMetric : jmxMetrics) { - JmxMetric metric = new JmxMetric(); + Metric metric = new Metric(); metric.setCanonicalName(jmxMetric.getCanonicalName()); metric.setValue(getJmxMetric(jmxMetric.getCanonicalName(), msc, srv, jmxUrl)); result.add(metric); @@ -107,8 +107,8 @@ public List convertToMetricDto(InternalClusterMetrics internalCluster .flatMap(Function.identity()).flatMap(Function.identity()).collect(Collectors.toList()); } - public JmxMetric reduceJmxMetrics (JmxMetric metric1, JmxMetric metric2) { - var result = new JmxMetric(); + public Metric reduceJmxMetrics (Metric metric1, Metric metric2) { + var result = new Metric(); Map jmx1 = new HashMap<>(metric1.getValue()); Map jmx2 = new HashMap<>(metric2.getValue()); jmx1.forEach((k, v) -> jmx2.merge(k, v, BigDecimal::add)); diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java index 7faa50df7fd..a4f5b1bd0a0 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/kafka/KafkaService.java @@ -344,7 +344,7 @@ private Mono updateSegmentMetrics(AdminClient ac, Intern ); } - public List getJmxMetric(String clusterName, Node node) { + public List getJmxMetric(String clusterName, Node node) { return clustersStorage.getClusterByName(clusterName) .map(c -> jmxClusterUtil.getJmxMetrics(c.getJmxPort(), node.host())).orElse(Collections.emptyList()); } @@ -363,14 +363,14 @@ private Mono fillBrokerMetrics(InternalClusterMetrics in } private InternalClusterMetrics calculateClusterMetrics(InternalClusterMetrics internalClusterMetrics) { - return internalClusterMetrics.toBuilder().jmxMetrics( + return internalClusterMetrics.toBuilder().metrics( jmxClusterUtil.convertToMetricDto(internalClusterMetrics) .stream().map(c -> { - JmxMetric jmx = new JmxMetric(); + Metric jmx = new Metric(); jmx.setCanonicalName(c.getCanonicalName()); jmx.setValue(Map.of(c.getMetricName(), c.getValue())); return jmx; - }).collect(Collectors.groupingBy(JmxMetric::getCanonicalName, Collectors.reducing(jmxClusterUtil::reduceJmxMetrics))) + }).collect(Collectors.groupingBy(Metric::getCanonicalName, Collectors.reducing(jmxClusterUtil::reduceJmxMetrics))) .values().stream() .filter(Optional::isPresent) .map(Optional::get) diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index 88ffbd24492..3787240f44d 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -312,10 +312,10 @@ components: type: integer topicCount: type: integer - jmxMetrics: + metrics: type: array items: - $ref: '#/components/schemas/JmxMetric' + $ref: '#/components/schemas/Metric' required: - id - name @@ -346,10 +346,10 @@ components: type: integer segmentZise: type: integer - jmxMetrics: + metrics: type: array items: - $ref: '#/components/schemas/JmxMetric' + $ref: '#/components/schemas/Metric' Topic: type: object @@ -523,7 +523,7 @@ components: items: $ref: '#/components/schemas/ConsumerTopicPartitionDetail' - JmxMetric: + Metric: type: object properties: canonicalName: