diff --git a/config/kraft/broker.properties b/config/kraft/broker.properties index 2c5818a193..026dd3e1eb 100644 --- a/config/kraft/broker.properties +++ b/config/kraft/broker.properties @@ -189,6 +189,33 @@ s3.stream.set.object.compaction.max.num=500 # and catch up read s3.network.baseline.bandwidth=104857600 +# Set to true to enable metrics collection +s3.telemetry.metrics.enable=false + +# The metrics level to record, supported values are INFO, DEBUG +s3.telemetry.metrics.level=INFO + +# The metrics exporter type, supported values are otlp, prometheus, log. Use comma to separate multiple exporters. +s3.telemetry.metrics.exporter.type=otlp + +# The Prometheus HTTP server host and port, if exporter type is set to prometheus +# s3.metrics.exporter.prom.host=127.0.0.1 +# s3.metrics.exporter.prom.port=9090 + +# Set to true to enable exporting tracing data to OTel Collector +s3.telemetry.tracer.enable=false + +# The OTel Collector endpoint, if exporter type is set to otlp or tracing is enabled +# s3.telemetry.exporter.otlp.endpoint=http://${your_host_name}:4317 + +# Set following configurations for batching +s3.telemetry.tracer.span.scheduled.delay.ms=1000 +s3.telemetry.tracer.span.max.queue.size=5120 +s3.telemetry.tracer.span.max.batch.size=1024 + +# Metrics report interval +s3.telemetry.exporter.report.interval.ms=5000 + ############################# Settings for Auto Balancer ############################# # The metric reporter to collect and report metrics for Auto Balancer metric.reporters=kafka.autobalancer.metricsreporter.AutoBalancerMetricsReporter diff --git a/config/kraft/server.properties b/config/kraft/server.properties index e417e80385..3d390a6794 100644 --- a/config/kraft/server.properties +++ b/config/kraft/server.properties @@ -195,6 +195,33 @@ s3.stream.set.object.compaction.max.num=500 # and catch up read s3.network.baseline.bandwidth=104857600 +# Set to true to enable metrics collection +s3.telemetry.metrics.enable=false + +# The metrics level to record, supported values are INFO, DEBUG +s3.telemetry.metrics.level=INFO + +# The metrics exporter type, supported values are otlp, prometheus, log. Use comma to separate multiple exporters. +s3.telemetry.metrics.exporter.type=otlp + +# The Prometheus HTTP server host and port, if exporter type is set to prometheus +# s3.metrics.exporter.prom.host=127.0.0.1 +# s3.metrics.exporter.prom.port=9090 + +# Set to true to enable exporting tracing data to OTel Collector +s3.telemetry.tracer.enable=false + +# The OTel Collector endpoint, if exporter type is set to otlp or tracing is enabled +# s3.telemetry.exporter.otlp.endpoint=http://${your_host_name}:4317 + +# Set following configurations for batching +s3.telemetry.tracer.span.scheduled.delay.ms=1000 +s3.telemetry.tracer.span.max.queue.size=5120 +s3.telemetry.tracer.span.max.batch.size=1024 + +# Metrics report interval +s3.telemetry.exporter.report.interval.ms=5000 + ############################# Settings for Auto Balancer ############################# # The metric reporter to collect and report metrics for Auto Balancer metric.reporters=kafka.autobalancer.metricsreporter.AutoBalancerMetricsReporter diff --git a/core/src/main/scala/kafka/log/stream/s3/telemetry/TelemetryManager.java b/core/src/main/scala/kafka/log/stream/s3/telemetry/TelemetryManager.java index f9c6e8247e..04b0b8089f 100644 --- a/core/src/main/scala/kafka/log/stream/s3/telemetry/TelemetryManager.java +++ b/core/src/main/scala/kafka/log/stream/s3/telemetry/TelemetryManager.java @@ -17,6 +17,7 @@ package kafka.log.stream.s3.telemetry; +import com.automq.stream.s3.metrics.MetricsLevel; import com.automq.stream.s3.metrics.S3StreamMetricsManager; import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; import io.opentelemetry.api.common.AttributeKey; @@ -59,6 +60,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -139,24 +141,37 @@ private void init() { autoCloseables.addAll(GarbageCollector.registerObservers(openTelemetrySdk)); Meter meter = openTelemetrySdk.getMeter(TelemetryConstants.TELEMETRY_SCOPE_NAME); + S3StreamMetricsManager.setMetricsLevel(metricsLevel()); S3StreamMetricsManager.initMetrics(meter, TelemetryConstants.KAFKA_METRICS_PREFIX); S3StreamMetricsManager.initAttributesBuilder(() -> { - //TODO: cache and reuse attributes - //TODO: support metrics level AttributesBuilder builder = attributesBuilderSupplier.get(); labelMap.forEach(builder::put); return builder; }); } - LOGGER.info("Instrument manager initialized with metrics: {}, trace: {} report interval: {}", - kafkaConfig.s3MetricsEnable(), kafkaConfig.s3TracerEnable(), kafkaConfig.s3ExporterReportIntervalMs()); + LOGGER.info("Instrument manager initialized with metrics: {} (level: {}), trace: {} report interval: {}", + kafkaConfig.s3MetricsEnable(), kafkaConfig.s3MetricsLevel(), kafkaConfig.s3TracerEnable(), kafkaConfig.s3ExporterReportIntervalMs()); } public static OpenTelemetrySdk getOpenTelemetrySdk() { return openTelemetrySdk; } + private MetricsLevel metricsLevel() { + String levelStr = kafkaConfig.s3MetricsLevel(); + if (StringUtils.isBlank(levelStr)) { + return MetricsLevel.INFO; + } + try { + String up = levelStr.toUpperCase(Locale.ENGLISH); + return MetricsLevel.valueOf(up); + } catch (Exception e) { + LOGGER.error("illegal metrics level: {}", levelStr); + return MetricsLevel.INFO; + } + } + private SdkTracerProvider getTraceProvider(Resource resource) { OtlpGrpcSpanExporter spanExporter = OtlpGrpcSpanExporter.builder() .setEndpoint(kafkaConfig.s3ExporterOTLPEndpoint()) @@ -185,6 +200,7 @@ private SdkMeterProvider getMetricsProvider(Resource resource) { } String[] exporterTypeArray = exporterTypes.split(","); for (String exporterType : exporterTypeArray) { + exporterType = exporterType.trim(); switch (exporterType) { case "otlp": initOTLPExporter(sdkMeterProviderBuilder, kafkaConfig); diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index e5885ddbf6..4b1e1915a5 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -729,6 +729,7 @@ object KafkaConfig { val S3TracerEnableProp = "s3.telemetry.tracer.enable" val S3ExporterOTLPEndpointProp = "s3.telemetry.exporter.otlp.endpoint" val S3ExporterReportIntervalMsProp = "s3.telemetry.exporter.report.interval.ms" + val S3MetricsLevelProp = "s3.telemetry.metrics.level" val S3MetricsExporterTypeProp = "s3.telemetry.metrics.exporter.type" val S3MetricsExporterPromHostProp = "s3.metrics.exporter.prom.host" val S3MetricsExporterPromPortProp = "s3.metrics.exporter.prom.port" @@ -775,6 +776,7 @@ object KafkaConfig { val S3FailoverEnableDoc = "Failover mode: if enable, the controller will scan failed node and failover the failed node" val S3MetricsEnableDoc = "Whether to enable metrics exporter for s3stream." val S3TracerEnableDoc = "Whether to enable tracer exporter for s3stream." + val S3MetricsLevelDoc = "The metrics level, supported value: INFO, DEBUG" val S3MetricsExporterTypeDoc = "The enabled S3 metrics exporters type, seperated by comma. Supported type: otlp, prometheus, log" val S3ExporterOTLPEndpointDoc = "The endpoint of OTLP collector" val S3ExporterReportIntervalMsDoc = "The interval in milliseconds to report telemetry" @@ -1620,6 +1622,7 @@ object KafkaConfig { .define(S3FailoverEnableProp, BOOLEAN, false, MEDIUM, S3FailoverEnableDoc) .define(S3MetricsEnableProp, BOOLEAN, true, MEDIUM, S3MetricsEnableDoc) .define(S3TracerEnableProp, BOOLEAN, false, MEDIUM, S3TracerEnableDoc) + .define(S3MetricsLevelProp, STRING, "INFO", MEDIUM, S3MetricsLevelDoc) .define(S3MetricsExporterTypeProp, STRING, null, MEDIUM, S3MetricsExporterTypeDoc) .define(S3ExporterOTLPEndpointProp, STRING, null, MEDIUM, S3ExporterOTLPEndpointDoc) .define(S3MetricsExporterPromHostProp, STRING, null, MEDIUM, S3MetricsExporterPromHostDoc) @@ -2202,6 +2205,7 @@ class KafkaConfig private(doLog: Boolean, val props: java.util.Map[_, _], dynami val s3FailoverEnable = getBoolean(KafkaConfig.S3FailoverEnableProp) val s3MetricsEnable = getBoolean(KafkaConfig.S3MetricsEnableProp) val s3TracerEnable = getBoolean(KafkaConfig.S3TracerEnableProp) + val s3MetricsLevel = getString(KafkaConfig.S3MetricsLevelProp) val s3MetricsExporterType = getString(KafkaConfig.S3MetricsExporterTypeProp) val s3ExporterOTLPEndpoint = getString(KafkaConfig.S3ExporterOTLPEndpointProp) val s3ExporterReportIntervalMs = getInt(KafkaConfig.S3ExporterReportIntervalMsProp) diff --git a/docker/telemetry/grafana/provisioning/dashboards/automq-for-kafka-dashboard.json b/docker/telemetry/grafana/provisioning/dashboards/automq-for-kafka-dashboard.json index e3a696b5dd..6f78ee0794 100644 --- a/docker/telemetry/grafana/provisioning/dashboards/automq-for-kafka-dashboard.json +++ b/docker/telemetry/grafana/provisioning/dashboards/automq-for-kafka-dashboard.json @@ -30,7 +30,7 @@ "y": 0 }, "id": 44, - "title": "System Overview", + "title": "JVM Overview", "type": "row" }, { @@ -120,7 +120,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "${namespace}_jvm_cpu_recent_utilization_ratio{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"}", + "expr": "${namespace}jvm_cpu_recent_utilization_ratio{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -190,7 +190,7 @@ } ] }, - "unit": "bytes" + "unit": "percentunit" }, "overrides": [ { @@ -200,12 +200,8 @@ }, "properties": [ { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "percentunit" + "id": "displayName", + "value": "${__field.labels.service_name}_${__field.labels.service_instance_id}" } ] } @@ -238,9 +234,9 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(service_namespace, service_name, service_instance_id) (${namespace}_jvm_memory_used_bytes{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"})", + "expr": "sum by(service_namespace, service_name, service_instance_id) (${namespace}jvm_memory_used_bytes{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"})", "fullMetaSearch": false, - "hide": false, + "hide": true, "includeNullMetadata": true, "instant": false, "legendFormat": "{{service_name}}_{{service_instance_id}}", @@ -255,7 +251,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(service_name, service_instance_id, service_namespace) (${namespace}_jvm_memory_limit_bytes{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"})", + "expr": "sum by(service_name, service_instance_id, service_namespace) (${namespace}jvm_memory_limit_bytes{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"})", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, @@ -277,7 +273,7 @@ "type": "math" } ], - "title": "Memory Usage", + "title": "Heap Memory Utilization", "type": "timeseries" }, { @@ -389,7 +385,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, service_name, service_instance_id, jvm_gc_action) (rate(${namespace}_jvm_gc_duration_seconds_bucket{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, service_name, service_instance_id, jvm_gc_action) (rate(${namespace}jvm_gc_duration_seconds_bucket{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"}[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -406,7 +402,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(jvm_gc_action, service_name, service_instance_id) (rate(${namespace}_jvm_gc_duration_seconds_count{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"}[$__rate_interval]))", + "expr": "sum by(jvm_gc_action, service_name, service_instance_id) (rate(${namespace}jvm_gc_duration_seconds_count{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\"}[$__rate_interval]))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -420,6 +416,141 @@ "title": "JVM GC Monitor", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "utilization" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(service_namespace, service_name, service_instance_id) (${namespace}jvm_memory_limit_bytes{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\", jvm_memory_type=\"heap\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service_name}}_{{service_instance_id}}_limit", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(service_namespace, service_name, service_instance_id) (${namespace}jvm_memory_used_bytes{service_namespace=\"$cluster_id\", service_name=~\"$node_type\", service_instance_id=~\"$node_id\", jvm_memory_type=\"heap\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service_name}}_{{service_instance_id}}_usage", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Heap Memory Usage", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { @@ -519,7 +650,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(operation_name, operation_type) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=~\"$OperationType\", operation_name!~\"append_wal|read_block_cache|read_log_cache\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval]))", + "expr": "sum by(operation_name, operation_type) (rate(${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=~\"$OperationType\", operation_name!~\"append_wal|read_block_cache|read_log_cache\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -592,7 +723,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(status) (${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=\"S3Request\", operation_name=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\", status=~\".*\"})", + "expr": "sum by(status) (${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=\"S3Request\", operation_name=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\", status=~\".*\"})", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -692,7 +823,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=~\"$OperationType\", operation_name!~\"append_wal|read_block_cache|read_log_cache\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=~\"$OperationType\", operation_name!~\"append_wal|read_block_cache|read_log_cache\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -804,7 +935,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=~\"append_log_cache|append_callback|append_wal_full\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=~\"append_log_cache|append_callback|append_wal_full\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -820,7 +951,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=~\"append_wal\", stage=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=~\"append_wal\", stage=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -837,7 +968,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_operation_count_total{operation_type=\"S3Storage\", operation_name=\"append_log_cache_full\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}operation_count_total{operation_type=\"S3Storage\", operation_name=\"append_log_cache_full\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -938,7 +1069,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, stage) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=\"append_wal\", stage=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, stage) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=\"append_wal\", stage=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -954,7 +1085,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=~\"append_callback|append_log_cache\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=~\"append_callback|append_log_cache\"}[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1015,7 +1146,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1053,7 +1185,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=~\"read_log_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=~\"read_log_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1113,7 +1245,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1152,7 +1285,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, status) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=~\"read_log_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, status) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=~\"read_log_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1212,7 +1345,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1251,7 +1385,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "${namespace}_${prefix}_wal_start_offset{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "expr": "${namespace}${prefix}wal_start_offset{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, @@ -1268,7 +1402,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "${namespace}_${prefix}_wal_trimmed_offset{cluster_id=\"$cluster_id\"}", + "expr": "${namespace}${prefix}wal_trimmed_offset{cluster_id=\"$cluster_id\"}", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, @@ -1303,7 +1437,7 @@ "refId": "C" } ], - "title": "Delta WAL Size", + "title": "Delta WAL Size(DEBUG only)", "type": "timeseries" }, { @@ -1366,7 +1500,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1404,7 +1539,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=\"read_block_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=\"S3Storage\", operation_name=\"read_block_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1464,7 +1599,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1503,7 +1639,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, status) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=\"read_block_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, status) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=\"read_block_cache\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1563,7 +1699,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1571,7 +1708,7 @@ } ] }, - "unit": "bytes" + "unit": "ns" }, "overrides": [] }, @@ -1581,6 +1718,206 @@ "x": 0, "y": 83 }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, status) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=\"read_ahead\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{operation_name}}_{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Read Ahead Latency P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(default_kafka_stream_read_ahead_size_bytes_sum{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{node_type}}_{{node_id}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Read Ahead Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 95 + }, "id": 36, "options": { "legend": { @@ -1602,7 +1939,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "${namespace}_${prefix}_block_cache_size_bytes{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "expr": "${namespace}${prefix}block_cache_size_bytes{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1613,7 +1950,7 @@ "useBackend": false } ], - "title": "Block Cache Size", + "title": "Block Cache Size (DEBUG only)", "type": "timeseries" }, { @@ -1663,7 +2000,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1671,7 +2009,7 @@ } ] }, - "unit": "ns" + "unit": "bytes" }, "overrides": [] }, @@ -1679,9 +2017,9 @@ "h": 12, "w": 12, "x": 12, - "y": 83 + "y": 95 }, - "id": 40, + "id": 50, "options": { "legend": { "calcs": [], @@ -1701,18 +2039,18 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, status) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=\"S3Storage\", operation_name=\"read_ahead\", status=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "editorMode": "code", + "expr": "${namespace}${prefix}available_inflight_read_ahead_size_bytes{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{operation_name}}_{{status}}", + "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], - "title": "Read Ahead Latency P99", + "title": "Available Inflight Read Ahead Size (DEBUG only)", "type": "timeseries" }, { @@ -1721,7 +2059,7 @@ "h": 1, "w": 24, "x": 0, - "y": 95 + "y": 107 }, "id": 7, "panels": [], @@ -1775,7 +2113,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1791,7 +2130,7 @@ "h": 12, "w": 12, "x": 0, - "y": 96 + "y": 108 }, "id": 4, "options": { @@ -1814,7 +2153,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_network_inbound_usage_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}network_inbound_usage_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1830,7 +2169,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_network_outbound_usage_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}network_outbound_usage_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1891,7 +2230,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1907,7 +2247,7 @@ "h": 12, "w": 12, "x": 12, - "y": 96 + "y": 108 }, "id": 25, "options": { @@ -1930,7 +2270,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_download_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}download_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1946,7 +2286,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_upload_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}upload_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -2007,7 +2347,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2023,7 +2364,7 @@ "h": 12, "w": 12, "x": 0, - "y": 108 + "y": 120 }, "id": 5, "options": { @@ -2046,7 +2387,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "${namespace}_${prefix}_network_inbound_available_bandwidth_bytes{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "expr": "${namespace}${prefix}network_inbound_available_bandwidth_bytes{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2062,7 +2403,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "${namespace}_${prefix}_network_outbound_available_bandwidth_bytes{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "expr": "${namespace}${prefix}network_outbound_available_bandwidth_bytes{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -2123,7 +2464,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2131,7 +2473,7 @@ } ] }, - "unit": "none" + "unit": "ns" }, "overrides": [] }, @@ -2139,9 +2481,9 @@ "h": 12, "w": 12, "x": 12, - "y": 108 + "y": 120 }, - "id": 11, + "id": 41, "options": { "legend": { "calcs": [], @@ -2161,10 +2503,10 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "${namespace}_${prefix}_network_inbound_limiter_queue_size{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "editorMode": "builder", + "expr": "histogram_quantile(0.99, sum by(le, cluster_id, node_id, node_type) (rate(${namespace}${prefix}network_inbound_limiter_queue_time_nanoseconds_bucket{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, - "includeNullMetadata": true, + "includeNullMetadata": false, "instant": false, "legendFormat": "inbound", "range": true, @@ -2177,11 +2519,11 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "${namespace}_${prefix}_network_outbound_limiter_queue_size{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "editorMode": "builder", + "expr": "histogram_quantile(0.99, sum by(le, cluster_id, node_id, node_type) (rate(${namespace}${prefix}network_outbound_limiter_queue_time_nanoseconds_bucket{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "hide": false, - "includeNullMetadata": true, + "includeNullMetadata": false, "instant": false, "legendFormat": "outbound", "range": true, @@ -2189,7 +2531,7 @@ "useBackend": false } ], - "title": "S3 Network Limiter", + "title": "S3Stream Network Queue Time", "type": "timeseries" }, { @@ -2239,7 +2581,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2255,7 +2598,7 @@ "h": 12, "w": 12, "x": 0, - "y": 120 + "y": 132 }, "id": 29, "options": { @@ -2278,7 +2621,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "${namespace}_${prefix}_available_s3_inflight_read_quota{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "expr": "${namespace}${prefix}available_s3_inflight_read_quota{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -2295,7 +2638,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "${namespace}_${prefix}_available_s3_inflight_write_quota{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", + "expr": "${namespace}${prefix}available_s3_inflight_write_quota{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -2306,7 +2649,7 @@ "useBackend": false } ], - "title": "Available S3 Inflight Quota", + "title": "Available S3 Inflight Quota (DEBUG only)", "type": "timeseries" }, { @@ -2356,7 +2699,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2364,7 +2708,7 @@ } ] }, - "unit": "ns" + "unit": "none" }, "overrides": [] }, @@ -2372,9 +2716,9 @@ "h": 12, "w": 12, "x": 12, - "y": 120 + "y": 132 }, - "id": 41, + "id": 11, "options": { "legend": { "calcs": [], @@ -2394,10 +2738,10 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le, cluster_id, node_id, node_type) (rate(${namespace}_${prefix}_network_inbound_limiter_queue_time_nanoseconds_bucket{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "editorMode": "code", + "expr": "${namespace}${prefix}network_inbound_limiter_queue_size{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, - "includeNullMetadata": false, + "includeNullMetadata": true, "instant": false, "legendFormat": "inbound", "range": true, @@ -2410,11 +2754,11 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le, cluster_id, node_id, node_type) (rate(${namespace}_${prefix}_network_outbound_limiter_queue_time_nanoseconds_bucket{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "editorMode": "code", + "expr": "${namespace}${prefix}network_outbound_limiter_queue_size{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}", "fullMetaSearch": false, "hide": false, - "includeNullMetadata": false, + "includeNullMetadata": true, "instant": false, "legendFormat": "outbound", "range": true, @@ -2422,7 +2766,7 @@ "useBackend": false } ], - "title": "S3Stream Network Queue Time", + "title": "S3 Network Limiter Queue Size (DEBUG only)", "type": "timeseries" }, { @@ -2431,7 +2775,7 @@ "h": 1, "w": 24, "x": 0, - "y": 132 + "y": 144 }, "id": 8, "panels": [], @@ -2485,7 +2829,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2500,7 +2845,7 @@ "h": 14, "w": 12, "x": 0, - "y": 133 + "y": 145 }, "id": 12, "options": { @@ -2523,7 +2868,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_object_count_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}object_count_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2583,7 +2928,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2599,7 +2945,7 @@ "h": 14, "w": 12, "x": 12, - "y": 133 + "y": 145 }, "id": 14, "options": { @@ -2622,7 +2968,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(stage, le) (rate(${namespace}_${prefix}_object_stage_cost_nanoseconds_bucket{stage=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(stage, le) (rate(${namespace}${prefix}object_stage_cost_nanoseconds_bucket{stage=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2660,7 +3006,7 @@ "h": 14, "w": 3, "x": 0, - "y": 147 + "y": 159 }, "id": 38, "options": { @@ -2695,7 +3041,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(size) (${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=\"S3Request\", operation_name=\"get_object\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"})", + "expr": "sum by(size) (${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=\"S3Request\", operation_name=\"get_object\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"})", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2755,7 +3101,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2771,7 +3118,7 @@ "h": 14, "w": 9, "x": 3, - "y": 147 + "y": 159 }, "id": 26, "options": { @@ -2794,7 +3141,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, size) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=\"S3Request\", operation_name=\"get_object\", size=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, operation_type, operation_name, size) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=\"S3Request\", operation_name=\"get_object\", size=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2832,7 +3179,7 @@ "h": 14, "w": 3, "x": 12, - "y": 147 + "y": 159 }, "id": 39, "options": { @@ -2867,7 +3214,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(size) (${namespace}_${prefix}_operation_latency_nanoseconds_count{operation_type=\"S3Request\", operation_name=~\"put_object|upload_part\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"})", + "expr": "sum by(size) (${namespace}${prefix}operation_latency_nanoseconds_count{operation_type=\"S3Request\", operation_name=~\"put_object|upload_part\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"})", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2927,7 +3274,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2943,7 +3291,7 @@ "h": 14, "w": 9, "x": 15, - "y": 147 + "y": 159 }, "id": 37, "options": { @@ -2966,7 +3314,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, size) (rate(${namespace}_${prefix}_operation_latency_nanoseconds_bucket{operation_type=\"S3Request\", operation_name=~\"put_object|upload_part\", size=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le, size) (rate(${namespace}${prefix}operation_latency_nanoseconds_bucket{operation_type=\"S3Request\", operation_name=~\"put_object|upload_part\", size=~\".*\", cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2985,7 +3333,7 @@ "h": 1, "w": 24, "x": 0, - "y": 161 + "y": 173 }, "id": 10, "panels": [], @@ -3039,7 +3387,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3055,7 +3404,7 @@ "h": 14, "w": 12, "x": 0, - "y": 162 + "y": 174 }, "id": 15, "options": { @@ -3078,7 +3427,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_compaction_read_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}compaction_read_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -3094,7 +3443,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(${namespace}_${prefix}_compaction_write_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", + "expr": "rate(${namespace}${prefix}compaction_write_size_bytes_total{cluster_id=\"$cluster_id\", node_id=\"$node_id\", node_type=\"$node_type\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -3110,16 +3459,15 @@ } ], "refresh": "5s", - "schemaVersion": 39, + "schemaVersion": 38, "tags": [], "templating": { "list": [ { "current": { - "isNone": true, "selected": false, - "text": "None", - "value": "" + "text": "xJGnLhprSXareHmTpNkpHg", + "value": "xJGnLhprSXareHmTpNkpHg" }, "datasource": { "type": "prometheus", @@ -3137,7 +3485,7 @@ "query": "label_values(default_jvm_cpu_recent_utilization_ratio,service_namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, @@ -3145,39 +3493,39 @@ }, { "current": { - "selected": true, - "text": "default", - "value": "default" + "selected": false, + "text": "default_", + "value": "default_" }, "hide": 0, "name": "namespace", "options": [ { "selected": true, - "text": "default", - "value": "default" + "text": "default_", + "value": "default_" } ], - "query": "default", + "query": "default_", "skipUrlSync": false, "type": "textbox" }, { "current": { - "selected": true, - "text": "kafka_stream", - "value": "kafka_stream" + "selected": false, + "text": "kafka_stream_", + "value": "kafka_stream_" }, "hide": 0, "name": "prefix", "options": [ { "selected": true, - "text": "kafka_stream", - "value": "kafka_stream" + "text": "kafka_stream_", + "value": "kafka_stream_" } ], - "query": "kafka_stream", + "query": "kafka_stream_", "skipUrlSync": false, "type": "textbox" }, @@ -3248,7 +3596,7 @@ "query": "label_values(default_jvm_cpu_recent_utilization_ratio,service_instance_id)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, @@ -3279,7 +3627,7 @@ "query": "label_values(default_jvm_cpu_recent_utilization_ratio,service_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, @@ -3306,13 +3654,13 @@ ] }, "time": { - "from": "now-15m", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "automq-for-kafka", "uid": "c25319fc-1031-4843-aaca-813f9d2b8438", - "version": 11, + "version": 21, "weekStart": "" } \ No newline at end of file diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 3f948885f4..389ae3c852 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -128,7 +128,7 @@ versions += [ zookeeper: "3.6.3", zstd: "1.5.2-1", commonLang: "3.12.0", - s3stream: "0.11.0-SNAPSHOT", + s3stream: "0.12.0-SNAPSHOT", opentelemetry: "1.32.0", opentelemetryAlpha: "1.32.0-alpha", oshi: "6.4.7"