Retry REDIS_REPLY_ERROR for RedisClient::GetNextJobID (ray-project#33733

)

Encountered check failure `redis_client.cc:73: Check failed: reply->type == REDIS_REPLY_INTEGER Expected integer, found Redis type 6 for JobCounter`. This PR retries REDIS_REPLY_ERROR which is 6 and also prints out the error message.


Signed-off-by: Jiajun Yao <jeromeyjj@gmail.com>

src/ray/gcs/redis_client.cc

@@ -66,7 +66,19 @@ static int DoGetNextJobID(redisContext *context) {
   redisReply *reply = nullptr;
   bool under_retry_limit = RunRedisCommandWithRetries(
       context, cmd.c_str(), &reply, [](const redisReply *reply) {
-        return reply != nullptr && reply->type != REDIS_REPLY_NIL;
+        if (reply == nullptr) {
+          RAY_LOG(WARNING) << "Didn't get reply for " << cmd;
+          return false;
+        }
+        if (reply->type == REDIS_REPLY_NIL) {
+          RAY_LOG(WARNING) << "Got nil reply for " << cmd;
+          return false;
+        }
+        if (reply->type == REDIS_REPLY_ERROR) {
+          RAY_LOG(WARNING) << "Got error reply for " << cmd << " Error is " << reply->str;
+          return false;
+        }
+        return true;
       });
   RAY_CHECK(reply);
   RAY_CHECK(under_retry_limit) << "No entry found for JobCounter";