Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add an awful secondary timeout to fix wedged requests #3875

Merged
merged 2 commits into from Sep 14, 2018
Merged
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -0,0 +1 @@
Mitigate outbound federation randomly becoming wedged
@@ -42,6 +42,7 @@
)
from synapse.http.endpoint import matrix_federation_endpoint
from synapse.util import logcontext
from synapse.util.async_helpers import timeout_no_seriously
from synapse.util.logcontext import make_deferred_yieldable
from synapse.util.metrics import Measure

@@ -228,6 +229,16 @@ def _request(self, destination, method, path,
)
request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())

# Sometimes the timeout above doesn't work, so lets hack yet
# another layer of timeouts in in the vain hope that at some
# point the world made sense and this really really really
# should work.
request_deferred = timeout_no_seriously(
request_deferred,
timeout=_sec_timeout * 2,
reactor=self.hs.get_reactor(),
)

with Measure(self.clock, "outbound_request"):
response = yield make_deferred_yieldable(
request_deferred,
@@ -438,3 +438,54 @@ def _cancelled_to_timed_out_error(value, timeout):
value.trap(CancelledError)
raise DeferredTimeoutError(timeout, "Deferred")
return value


def timeout_no_seriously(deferred, timeout, reactor):
"""The in build twisted deferred addTimeout (and the method above)
completely fail to time things out under some unknown circumstances.
Lets try a different way of timing things out and maybe that will make
things work?!
TODO: Kill this with fire.
"""

new_d = defer.Deferred()

timed_out = [False]

def time_it_out():
timed_out[0] = True
deferred.cancel()

if not new_d.called:
new_d.errback(DeferredTimeoutError(timeout, "Deferred"))

delayed_call = reactor.callLater(timeout, time_it_out)

def convert_cancelled(value):
if timed_out[0]:
return _cancelled_to_timed_out_error(value, timeout)
return value

deferred.addBoth(convert_cancelled)

def cancel_timeout(result):
# stop the pending call to cancel the deferred if it's been fired
if delayed_call.active():
delayed_call.cancel()
return result

deferred.addBoth(cancel_timeout)

def success_cb(val):
if not new_d.called:
new_d.callback(val)

def failure_cb(val):
if not new_d.called:
new_d.errback(val)

deferred.addCallbacks(success_cb, failure_cb)

return new_d
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.