Skip to content
This repository was archived by the owner on Dec 24, 2022. It is now read-only.

Commit 8e4c2f5

Browse files
mythzmythz
committed
Implement RetryReconnectOnFailedMasters & enable by default
1 parent 65646a3 commit 8e4c2f5

File tree

3 files changed

+74
-40
lines changed

3 files changed

+74
-40
lines changed

‎src/ServiceStack.Redis/RedisConfig.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,12 @@ public class RedisConfig
7272
/// </summary>
7373
public static bool VerifyMasterConnections = true;
7474

75+
/// <summary>
76+
/// Whether to retry re-connecting on same connection if not a master instance (default true)
77+
/// For Managed Services (e.g. AWS ElastiCache) which eventually restores master instances on same host
78+
/// </summary>
79+
public static bool RetryReconnectOnFailedMasters = true;
80+
7581
/// <summary>
7682
/// The ConnectTimeout on clients used to find the next available host (default 200ms)
7783
/// </summary>
@@ -126,6 +132,7 @@ public static void Reset()
126132
BackOffMultiplier = 10;
127133
BufferPoolMaxSize = 500000;
128134
VerifyMasterConnections = true;
135+
RetryReconnectOnFailedMasters = true;
129136
HostLookupTimeoutMs = 200;
130137
AssumeServerVersion = null;
131138
DeactivatedClientsExpiry = TimeSpan.Zero;

‎src/ServiceStack.Redis/RedisManagerPool.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public RedisPoolConfig()
3434
}
3535

3636
/// <summary>
37-
/// Provides thread-safe pooling of redis client connections. All connections are treaded as read and write hosts.
37+
/// Provides thread-safe pooling of redis client connections. All connections are treated as read and write hosts.
3838
/// </summary>
3939
public partial class RedisManagerPool
4040
: IRedisClientsManager, IRedisFailover, IHandleClientDispose, IHasRedisResolver, IRedisClientCacheManager
@@ -178,8 +178,8 @@ private RedisClient GetClient(bool forAsync)
178178
lock (clients)
179179
{
180180
//Create new client outside of pool when max pool size exceeded
181-
//Reverting free-slot not needed when -1 since slwo wasn't reserved or
182-
//when existingClient changed (failover) since no longer reserver
181+
//Reverting free-slot not needed when -1 since slow wasn't reserved or
182+
//when existingClient changed (failover) since no longer reserved
183183
var stillReserved = inactivePoolIndex >= 0 && inactivePoolIndex < clients.Length &&
184184
clients[inactivePoolIndex] == existingClient;
185185
if (inactivePoolIndex == -1 || !stillReserved)
@@ -246,7 +246,7 @@ public override void Dispose() { }
246246
private int GetInActiveClient(out RedisClient inactiveClient)
247247
{
248248
//this will loop through all hosts in readClients once even though there are 2 for loops
249-
//both loops are used to try to get the prefered host according to the round robin algorithm
249+
//both loops are used to try to get the preferred host according to the round robin algorithm
250250
var readWriteTotal = RedisResolver.ReadWriteHostsCount;
251251
var desiredIndex = poolIndex % clients.Length;
252252
for (int x = 0; x < readWriteTotal; x++)

‎src/ServiceStack.Redis/RedisResolver.cs‎

Lines changed: 63 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.IO;
34
using System.Linq;
45
using System.Threading;
56
using ServiceStack.Logging;
@@ -77,51 +78,77 @@ public virtual RedisClient CreateRedisClient(RedisEndpoint config, bool master)
7778

7879
if (master && RedisConfig.VerifyMasterConnections)
7980
{
80-
var role = client.GetServerRole();
81-
if (role != RedisServerRole.Master)
81+
var firstAttempt = DateTime.UtcNow;
82+
Exception firstEx = null;
83+
var retryTimeSpan = TimeSpan.FromMilliseconds(config.RetryTimeout);
84+
var i = 0;
85+
while (DateTime.UtcNow - firstAttempt < retryTimeSpan)
8286
{
83-
Interlocked.Increment(ref RedisState.TotalInvalidMasters);
84-
log.Error("Redis Master Host '{0}' is {1}. Resetting allHosts...".Fmt(config.GetHostString(), role));
85-
var newMasters = new List<RedisEndpoint>();
86-
var newReplicas = new List<RedisEndpoint>();
87-
RedisClient masterClient = null;
88-
foreach (var hostConfig in allHosts)
87+
try
8988
{
90-
try
91-
{
92-
var testClient = ClientFactory(hostConfig);
93-
testClient.ConnectTimeout = RedisConfig.HostLookupTimeoutMs;
94-
var testRole = testClient.GetServerRole();
95-
switch (testRole)
96-
{
97-
case RedisServerRole.Master:
98-
newMasters.Add(hostConfig);
99-
if (masterClient == null)
100-
masterClient = testClient;
101-
break;
102-
case RedisServerRole.Slave:
103-
newReplicas.Add(hostConfig);
104-
break;
105-
}
106-
107-
}
108-
catch { /* skip */ }
89+
client = GetValidMaster(client, config);
90+
return client;
91+
}
92+
catch (Exception ex)
93+
{
94+
if (!RedisConfig.RetryReconnectOnFailedMasters)
95+
throw;
96+
97+
firstEx ??= ex;
98+
ExecUtils.SleepBackOffMultiplier(++i);
10999
}
100+
}
101+
throw new TimeoutException($"Could not resolve master instance within {config.RetryTimeout}ms RetryTimeout", firstEx);
102+
}
103+
104+
return client;
105+
}
110106

111-
if (masterClient == null)
107+
protected RedisClient GetValidMaster(RedisClient client, RedisEndpoint config)
108+
{
109+
var role = client.GetServerRole();
110+
if (role != RedisServerRole.Master)
111+
{
112+
Interlocked.Increment(ref RedisState.TotalInvalidMasters);
113+
log.Error("Redis Master Host '{0}' is {1}. Resetting allHosts...".Fmt(config.GetHostString(), role));
114+
var newMasters = new List<RedisEndpoint>();
115+
var newReplicas = new List<RedisEndpoint>();
116+
RedisClient masterClient = null;
117+
foreach (var hostConfig in allHosts)
118+
{
119+
try
112120
{
113-
Interlocked.Increment(ref RedisState.TotalNoMastersFound);
114-
var errorMsg = "No master found in: " + string.Join(", ", allHosts.Map(x => x.GetHostString()));
115-
log.Error(errorMsg);
116-
throw new Exception(errorMsg);
121+
var testClient = ClientFactory(hostConfig);
122+
testClient.ConnectTimeout = RedisConfig.HostLookupTimeoutMs;
123+
var testRole = testClient.GetServerRole();
124+
switch (testRole)
125+
{
126+
case RedisServerRole.Master:
127+
newMasters.Add(hostConfig);
128+
if (masterClient == null)
129+
masterClient = testClient;
130+
break;
131+
case RedisServerRole.Slave:
132+
newReplicas.Add(hostConfig);
133+
break;
134+
}
135+
117136
}
137+
catch { /* skip */ }
138+
}
118139

119-
ResetMasters(newMasters);
120-
ResetSlaves(newReplicas);
121-
return masterClient;
140+
if (masterClient == null)
141+
{
142+
Interlocked.Increment(ref RedisState.TotalNoMastersFound);
143+
var errorMsg = "No master found in: " + string.Join(", ", allHosts.Map(x => x.GetHostString()));
144+
log.Error(errorMsg);
145+
throw new InvalidDataException(errorMsg);
122146
}
123-
}
124147

148+
ResetMasters(newMasters);
149+
ResetSlaves(newReplicas);
150+
return masterClient;
151+
}
125152
return client;
126153
}
127154

0 commit comments

Comments
 (0)