From 1d6b9469e4ff51f170c9454af6a504fcbec2ebe6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 9 Jun 2021 16:35:44 -0500 Subject: [PATCH 1/5] Add metrics for how often events are soft_failed Spawned from missing messages we were seeing on `matrix.org` from a federated Gtiter bridged room, https://gitlab.com/gitterHQ/webapp/-/issues/2770. The underlying issue in Synapse is tracked by https://github.com/matrix-org/synapse/issues/10066 where the message and join event race and the message is `soft_failed` before the `join` event reaches the remote federated server. Less soft_failed events = better and usually this should only trigger for events where people are doing bad things and trying to fuzz and fake everything. --- synapse/handlers/federation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6e40e2c2165e..c36620257fa4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -33,6 +33,7 @@ ) import attr +from prometheus_client import Counter from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json from unpaddedbase64 import decode_base64 @@ -101,6 +102,8 @@ logger = logging.getLogger(__name__) +soft_failed_event_counter = Counter("synapse_handlers_federation_soft_failed_event", "") + @attr.s(slots=True) class _NewEventInfo: @@ -2498,6 +2501,7 @@ async def _check_for_soft_fail( event_auth.check(room_version_obj, event, auth_events=current_auth_events) except AuthError as e: logger.warning("Soft-failing %r because %s", event, e) + soft_failed_event_counter.inc() event.internal_metadata.soft_failed = True async def on_get_missing_events( From c71f2c11ce2594516147ce2bf998aac5bc85314c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 9 Jun 2021 16:49:10 -0500 Subject: [PATCH 2/5] Add changelog --- changelog.d/10156.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/10156.misc diff --git a/changelog.d/10156.misc b/changelog.d/10156.misc new file mode 100644 index 000000000000..bbe266abb08f --- /dev/null +++ b/changelog.d/10156.misc @@ -0,0 +1 @@ +Add `synapse_handlers_federation_soft_failed_event` metric to track how often events are `soft_failed`. From 7d97ee8681e2d3c8e89e9153b07b0f1b57fc9911 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 10 Jun 2021 20:49:38 -0500 Subject: [PATCH 3/5] Update metric name to follow prometheus standards See https://prometheus.io/docs/practices/naming/ Background for existing event names: https://github.com/matrix-org/synapse/pull/10156#discussion_r649359350 --- changelog.d/10156.misc | 2 +- synapse/handlers/federation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/10156.misc b/changelog.d/10156.misc index bbe266abb08f..40fd82a3463f 100644 --- a/changelog.d/10156.misc +++ b/changelog.d/10156.misc @@ -1 +1 @@ -Add `synapse_handlers_federation_soft_failed_event` metric to track how often events are `soft_failed`. +Add `synapse_federation_soft_failed_events_total` metric to track how often events are `soft_failed`. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c36620257fa4..e17f4d6fb60e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -102,7 +102,7 @@ logger = logging.getLogger(__name__) -soft_failed_event_counter = Counter("synapse_handlers_federation_soft_failed_event", "") +soft_failed_event_counter = Counter("synapse_federation_soft_failed_events_total", "") @attr.s(slots=True) From fa4cf59ab7f7451a1860cef1abb2e1a3560d59da Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 10 Jun 2021 20:50:42 -0500 Subject: [PATCH 4/5] Update soft failed to be human readable in changelog --- changelog.d/10156.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/10156.misc b/changelog.d/10156.misc index 40fd82a3463f..92a188b87bb7 100644 --- a/changelog.d/10156.misc +++ b/changelog.d/10156.misc @@ -1 +1 @@ -Add `synapse_federation_soft_failed_events_total` metric to track how often events are `soft_failed`. +Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. From 2970bacd70f3d1c5dfa62dedc57c8d6469eb93fe Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 10 Jun 2021 20:58:57 -0500 Subject: [PATCH 5/5] Fill in metric description --- synapse/handlers/federation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e17f4d6fb60e..66470634859f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -102,7 +102,10 @@ logger = logging.getLogger(__name__) -soft_failed_event_counter = Counter("synapse_federation_soft_failed_events_total", "") +soft_failed_event_counter = Counter( + "synapse_federation_soft_failed_events_total", + "Events received over federation that we marked as soft_failed", +) @attr.s(slots=True)