Skip to content

Commit

Permalink
Add retry support for RedisIOErrors
Browse files Browse the repository at this point in the history 
[#173252885]

Signed-off-by: Emily Berk <eberk@vmware.com>
  • Loading branch information
@margocrawf
margocrawf authored and Emily Berk committed Jul 23, 2020
1 parent e6a1271 commit 31fed1f
Show file tree
Hide file tree
Showing 3 changed files with 189 additions and 3 deletions.
57 changes: 57 additions & 0 deletions src/filters/oidc/redis_retry_wrapper.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ absl::optional<std::string> oidc::RedisRetryWrapper::hget(const absl::string_vie
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {
if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand All @@ -36,6 +43,13 @@ oidc::RedisRetryWrapper::hmget(const absl::string_view key, const std::vector<st
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {
if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand All @@ -51,6 +65,14 @@ bool RedisRetryWrapper::hset(const absl::string_view key, const absl::string_vie
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {

if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand All @@ -68,6 +90,13 @@ void RedisRetryWrapper::hmset(const absl::string_view key,
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {
if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand All @@ -85,6 +114,13 @@ bool RedisRetryWrapper::hsetnx(const absl::string_view key,
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {
if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand All @@ -100,6 +136,13 @@ long long RedisRetryWrapper::del(const absl::string_view key) {
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {
if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand All @@ -115,6 +158,13 @@ bool RedisRetryWrapper::expireat(const absl::string_view key, long long timestam
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {
if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand All @@ -130,6 +180,13 @@ long long RedisRetryWrapper::hdel(absl::string_view key, std::vector<std::string
}
spdlog::error("{}: redis connection closed error, throwing error", __func__);
throw RedisError(err.what());
} catch (const RedisIoError &err) {
if (retries < 3) {
spdlog::trace("{}: redis connection timed out, retrying", __func__);
continue;
}
spdlog::error("{}: redis timed out, throwing error", __func__);
throw RedisError(err.what());
}
}
}
Expand Down
5 changes: 5 additions & 0 deletions src/filters/oidc/redis_wrapper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ class RedisClosedError : public std::runtime_error {
using std::runtime_error::runtime_error;
};

class RedisIoError : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};

} // namespace oidc
} // namespace filters
} // namespace authservice
Expand Down
130 changes: 127 additions & 3 deletions test/filters/oidc/redis_retry_wrapper_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ ACTION(ThrowRedisClosedError) {
throw RedisClosedError("redis is closed");
}

ACTION(ThrowRedisIoError) {
throw RedisIoError("redis timed out");
}

ACTION(ThrowRedisError) {
throw RedisError("redis problem");
}
Expand All @@ -71,6 +75,21 @@ TEST_F(RedisRetryWrapperTest, hget_WhenRedisWrapperThrowsMoreThan3RedisClosedExc
ASSERT_THROW(redis_retry_wrapper->hget(session_id, key_1), RedisError);
}

TEST_F(RedisRetryWrapperTest, hget_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, hget(Eq(session_id), Eq(key_1)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return(val_1));
ASSERT_EQ(val_1, redis_retry_wrapper->hget(session_id, key_1));
}

TEST_F(RedisRetryWrapperTest, hget_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hget(Eq(session_id), Eq(key_1)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->hget(session_id, key_1), RedisError);
}

TEST_F(RedisRetryWrapperTest, hget_WhenRedisWrapperThrowsRedisError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hget(Eq(session_id), Eq(key_1))).WillOnce(ThrowRedisError());
ASSERT_THROW(redis_retry_wrapper->hget(session_id, key_1), RedisError);
Expand All @@ -97,6 +116,21 @@ TEST_F(RedisRetryWrapperTest, hmget_WhenRedisWrapperThrowsMoreThan3RedisClosedEx
ASSERT_THROW(redis_retry_wrapper->hmget(session_id, list_of_keys), RedisError);
}

TEST_F(RedisRetryWrapperTest, hmget_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, hmget(Eq(session_id), Eq(list_of_keys)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return(response_map));
ASSERT_EQ(response_map, redis_retry_wrapper->hmget(session_id, list_of_keys));
}

TEST_F(RedisRetryWrapperTest, hmget_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hmget(Eq(session_id), Eq(list_of_keys)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->hmget(session_id, list_of_keys), RedisError);
}

TEST_F(RedisRetryWrapperTest, hmget_WhenRedisWrapperThrowsRedisError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hmget(Eq(session_id), Eq(list_of_keys))).WillOnce(ThrowRedisError());
ASSERT_THROW(redis_retry_wrapper->hmget(session_id, list_of_keys), RedisError);
Expand All @@ -122,6 +156,20 @@ TEST_F(RedisRetryWrapperTest, hset_WhenRedisWrapperThrowsMoreThan3RedisClosedExc
.Times(4).WillRepeatedly(ThrowRedisClosedError());
ASSERT_THROW(redis_retry_wrapper->hset(session_id, key_1, val_1), RedisError);
}
TEST_F(RedisRetryWrapperTest, hset_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, hset(Eq(session_id), Eq(key_1), Eq(val_1)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return(true));
ASSERT_EQ(true, redis_retry_wrapper->hset(session_id, key_1, val_1));
}

TEST_F(RedisRetryWrapperTest, hset_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hset(Eq(session_id), Eq(key_1), Eq(val_1)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->hset(session_id, key_1, val_1), RedisError);
}

TEST_F(RedisRetryWrapperTest, hset_WhenRedisWrapperThrowsRedisError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hset(Eq(session_id), Eq(key_1), Eq(val_1))).WillOnce(ThrowRedisError());
Expand Down Expand Up @@ -149,6 +197,21 @@ TEST_F(RedisRetryWrapperTest, hmset_WhenRedisWrapperThrowsMoreThan3RedisClosedEx
ASSERT_THROW(redis_retry_wrapper->hmset(session_id, keys_to_vals), RedisError);
}

TEST_F(RedisRetryWrapperTest, hmset_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, hmset(Eq(session_id), Eq(keys_to_vals)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return());
redis_retry_wrapper->hmset(session_id, keys_to_vals);
}

TEST_F(RedisRetryWrapperTest, hmset_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hmset(Eq(session_id), Eq(keys_to_vals)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->hmset(session_id, keys_to_vals), RedisError);
}

TEST_F(RedisRetryWrapperTest, hmset_WhenRedisWrapperThrowsRedisError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hmset(Eq(session_id), Eq(keys_to_vals))).WillOnce(ThrowRedisError());
ASSERT_THROW(redis_retry_wrapper->hmset(session_id, keys_to_vals), RedisError);
Expand All @@ -175,6 +238,21 @@ TEST_F(RedisRetryWrapperTest, hsetnx_WhenRedisWrapperThrowsMoreThan3RedisClosedE
ASSERT_THROW(redis_retry_wrapper->hsetnx(session_id, key_1, val_1), RedisError);
}

TEST_F(RedisRetryWrapperTest, hsetnx_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, hsetnx(Eq(session_id), Eq(key_1), Eq(val_1)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return(true));
ASSERT_TRUE(redis_retry_wrapper->hsetnx(session_id, key_1, val_1));
}

TEST_F(RedisRetryWrapperTest, hsetnx_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hsetnx(Eq(session_id), Eq(key_1), Eq(val_1)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->hsetnx(session_id, key_1, val_1), RedisError);
}

TEST_F(RedisRetryWrapperTest, hsetnx_WhenRedisWrapperThrowsRedisError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hsetnx(Eq(session_id), Eq(key_1), Eq(val_1))).WillOnce(ThrowRedisError());
ASSERT_THROW(redis_retry_wrapper->hsetnx(session_id, key_1, val_1), RedisError);
Expand All @@ -200,6 +278,21 @@ TEST_F(RedisRetryWrapperTest, del_WhenRedisWrapperThrowsMoreThan3RedisClosedExce
ASSERT_THROW(redis_retry_wrapper->del(session_id), RedisError);
}

TEST_F(RedisRetryWrapperTest, del_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, del(Eq(session_id)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return(1));
ASSERT_EQ(redis_retry_wrapper->del(session_id), 1);
}

TEST_F(RedisRetryWrapperTest, del_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, del(Eq(session_id)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->del(session_id), RedisError);
}

TEST_F(RedisRetryWrapperTest, del_WhenRedisWrapperThrowsRedisError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, del(Eq(session_id))).WillOnce(ThrowRedisError());
ASSERT_THROW(redis_retry_wrapper->del(session_id), RedisError);
Expand All @@ -225,6 +318,21 @@ TEST_F(RedisRetryWrapperTest, expireat_WhenRedisWrapperThrowsMoreThan3RedisClose
ASSERT_THROW(redis_retry_wrapper->expireat(session_id, 1L), RedisError);
}

TEST_F(RedisRetryWrapperTest, expireat_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, expireat(Eq(session_id), Eq(1)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return(true));
ASSERT_TRUE(redis_retry_wrapper->expireat(session_id, 1));
}

TEST_F(RedisRetryWrapperTest, expireat_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, expireat(Eq(session_id), Eq(1)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->expireat(session_id, 1), RedisError);
}

TEST_F(RedisRetryWrapperTest, expireat_WhenRedisWrapperThrowsRedisError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, expireat(Eq(session_id), Eq(1))).WillOnce(ThrowRedisError());
ASSERT_THROW(redis_retry_wrapper->expireat(session_id, 1L), RedisError);
Expand All @@ -241,12 +349,28 @@ TEST_F(RedisRetryWrapperTest, hdel_WhenRedisWrapperThrowsAClosedException_ItRetr
.WillOnce(ThrowRedisClosedError())
.WillOnce(ThrowRedisClosedError())
.WillOnce(ThrowRedisClosedError())
.WillOnce(Return(true));
ASSERT_TRUE(redis_retry_wrapper->hdel(session_id, list_of_keys));
.WillOnce(Return(2));
ASSERT_EQ(redis_retry_wrapper->hdel(session_id, list_of_keys), 2);
}

TEST_F(RedisRetryWrapperTest, hdel_WhenRedisWrapperThrowsMoreThan3RedisClosedExceptions_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hdel(Eq(session_id), Eq(list_of_keys))).Times(4).WillRepeatedly(ThrowRedisClosedError());
EXPECT_CALL(*redis_wrapper_mock_, hdel(Eq(session_id), Eq(list_of_keys))).Times(4).WillRepeatedly(
ThrowRedisClosedError());
ASSERT_THROW(redis_retry_wrapper->hdel(session_id, list_of_keys), RedisError);
}

TEST_F(RedisRetryWrapperTest, hdel_WhenRedisWrapperThrowsAIoError_ItRetries) {
EXPECT_CALL(*redis_wrapper_mock_, hdel(Eq(session_id), Eq(list_of_keys)))
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(ThrowRedisIoError())
.WillOnce(Return(2));
ASSERT_EQ(redis_retry_wrapper->hdel(session_id, list_of_keys), 2);
}

TEST_F(RedisRetryWrapperTest, hdel_WhenRedisWrapperThrowsMoreThan3RedisIoError_ItThrowsRedisError) {
EXPECT_CALL(*redis_wrapper_mock_, hdel(Eq(session_id), Eq(list_of_keys)))
.Times(4).WillRepeatedly(ThrowRedisIoError());
ASSERT_THROW(redis_retry_wrapper->hdel(session_id, list_of_keys), RedisError);
}

Expand Down

0 comments on commit 31fed1f

Please sign in to comment.