Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v23.2.x] metrics: fix kafka_max_offset on read replicas #16275

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/v/cluster/partition_probe.cc
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,14 @@ void replicated_partition_probe::setup_public_metrics(const model::ntp& ntp) {
sm::make_gauge(
"max_offset",
[this] {
// TODO: merge code with replicated_partition.h?
if (_partition.is_read_replica_mode_enabled()) {
if (_partition.cloud_data_available()) {
return _partition.next_cloud_offset();
}
// This is a read replica and there's no data in the cloud.
return model::offset(0);
}
auto log_offset = _partition.high_watermark();
auto translator = _partition.get_offset_translator_state();

Expand Down
23 changes: 19 additions & 4 deletions tests/rptest/services/redpanda.py
Original file line number Diff line number Diff line change
Expand Up @@ -954,10 +954,25 @@ def metric_sum(self,
for sample in family.samples:
if sample.name != metric_name:
continue
if ns and sample.labels["namespace"] != ns:
continue
if topic and sample.labels["topic"] != topic:
continue
labels = sample.labels
if ns:
if "redpanda_namespace" in labels:
if labels["redpanda_namespace"] != ns:
continue
elif "namespace" in labels:
if labels["namespace"] != ns:
continue
else:
assert False, f"Missing namespace label: {sample}"
if topic:
if "redpanda_topic" in labels:
if labels["redpanda_topic"] != topic:
continue
elif "topic" in labels:
if labels["topic"] != topic:
continue
else:
assert False, f"Missing topic label: {sample}"
count += int(sample.value)
return count

Expand Down
9 changes: 8 additions & 1 deletion tests/rptest/tests/read_replica_e2e_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from ducktape.mark import matrix
from ducktape.tests.test import TestContext

from rptest.services.redpanda import CloudStorageType, RedpandaService, get_cloud_storage_type, make_redpanda_service
from rptest.services.redpanda import CloudStorageType, MetricsEndpoint, RedpandaService, get_cloud_storage_type, make_redpanda_service
from rptest.services.redpanda_installer import InstallOptions, RedpandaInstaller
from rptest.tests.end_to_end import EndToEndTest
from rptest.utils.expect_rate import ExpectRate, RateTarget
Expand Down Expand Up @@ -238,6 +238,13 @@ def _setup_read_replica(self,
backoff_sec=5,
err_msg="Could not create read replica topic. Most likely " +
"because topic manifest is not in S3.")
if num_messages > 0:
wait_until(lambda: self.second_cluster.metric_sum(
"redpanda_kafka_max_offset",
topic=self.topic_name,
metrics_endpoint=MetricsEndpoint.PUBLIC_METRICS) > 0,
timeout_sec=10,
backoff_sec=1)

def _bucket_usage(self) -> BucketUsage:
assert self.redpanda and self.redpanda.cloud_storage_client
Expand Down
Loading