/
retry.ex
139 lines (113 loc) · 4.57 KB
/
retry.ex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
defmodule Tesla.Middleware.Retry do
@moduledoc """
Retry using exponential backoff and full jitter. This middleware only retries in the
case of connection errors (`nxdomain`, `connrefused` etc). Application error
checking for retry can be customized through `:should_retry` option by
providing a function in returning a boolean.
## Backoff algorithm
The backoff algorithm optimizes for tight bounds on completing a request successfully.
It does this by first calculating an exponential backoff factor based on the
number of retries that have been performed. It then multiplies this factor against the
base delay. The total maximum delay is found by taking the minimum of either the calculated delay
or the maximum delay specified. This creates an upper bound on the maximum delay
we can see.
In order to find the actual delay value we apply additive noise which is proportional to the
current desired delay. This ensures that the actual delay is kept within the expected order
of magnitude, while still having some randomness, which ensures that our retried requests
don't "harmonize" making it harder for the downstream service to heal.
## Example
```
defmodule MyClient do
use Tesla
plug Tesla.Middleware.Retry,
delay: 500,
max_retries: 10,
max_delay: 4_000,
should_retry: fn
{:ok, %{status: status}} when status in [400, 500] -> true
{:ok, _} -> false
{:error, _} -> true
end
end
```
## Options
- `:delay` - The base delay in milliseconds (positive integer, defaults to 50)
- `:max_retries` - maximum number of retries (non-negative integer, defaults to 5)
- `:max_delay` - maximum delay in milliseconds (positive integer, defaults to 5000)
- `:should_retry` - function to determine if request should be retried
- `:jitter_factor` - additive noise proportionality constant (float between 0 and 1, defaults to 0.2)
"""
# Not necessary in Elixir 1.10+
use Bitwise, skip_operators: true
@behaviour Tesla.Middleware
@defaults [
delay: 50,
max_retries: 5,
max_delay: 5_000,
jitter_factor: 0.2
]
@impl Tesla.Middleware
def call(env, next, opts) do
opts = opts || []
context = %{
retries: 0,
delay: integer_opt!(opts, :delay, 1),
max_retries: integer_opt!(opts, :max_retries, 0),
max_delay: integer_opt!(opts, :max_delay, 1),
should_retry: Keyword.get(opts, :should_retry, &match?({:error, _}, &1)),
jitter_factor: float_opt!(opts, :jitter_factor, 0, 1)
}
retry(env, next, context)
end
# If we have max retries set to 0 don't retry
defp retry(env, next, %{max_retries: 0}), do: Tesla.run(env, next)
# If we're on our last retry then just run and don't handle the error
defp retry(env, next, %{max_retries: max, retries: max}) do
Tesla.run(env, next)
end
# Otherwise we retry if we get a retriable error
defp retry(env, next, context) do
res = Tesla.run(env, next)
if context.should_retry.(res) do
backoff(context.max_delay, context.delay, context.retries, context.jitter_factor)
context = update_in(context, [:retries], &(&1 + 1))
retry(env, next, context)
else
res
end
end
# Exponential backoff with jitter
defp backoff(cap, base, attempt, jitter_factor) do
factor = Bitwise.bsl(1, attempt)
max_sleep = min(cap, base * factor)
# This ensures that the delay's order of magnitude is kept intact,
# while still having some jitter. Generates a value x where 1-jitter_factor <= x <= 1 + jitter_factor
jitter = 1 + 2 * jitter_factor * :rand.uniform() - jitter_factor
# The actual delay is in the range max_sleep * (1 - jitter_factor), max_sleep * (1 + jitter_factor)
delay = trunc(max_sleep + jitter)
:timer.sleep(delay)
end
defp integer_opt!(opts, key, min) do
case Keyword.fetch(opts, key) do
{:ok, value} when is_integer(value) and value >= min -> value
{:ok, invalid} -> invalid_integer(key, invalid, min)
:error -> @defaults[key]
end
end
defp float_opt!(opts, key, min, max) do
case Keyword.fetch(opts, key) do
{:ok, value} when is_float(value) and value >= min and value <= max -> value
{:ok, invalid} -> invalid_float(key, invalid, min, max)
:error -> @defaults[key]
end
end
defp invalid_integer(key, value, min) do
raise(ArgumentError, "expected :#{key} to be an integer >= #{min}, got #{inspect(value)}")
end
defp invalid_float(key, value, min, max) do
raise(
ArgumentError,
"expected :#{key} to be a float >= #{min} and <= #{max}, got #{inspect(value)}"
)
end
end