clients check for the health of the cache servers

distcache/cache_client.py

@@ -1,5 +1,7 @@
 """
 Implements distcache client. It interacts with the users.
+
+Note: This package lets all the individual client discover the health of the servers themselves.
 """
 
 import socket
@@ -9,9 +11,6 @@
 from distcache import utils
 
 
-# Each client has list of all servers.
-# Do servers register with the client?
-
 class CacheClient:
     """
     Implements cache client. It responds to user requests.
@@ -31,6 +30,9 @@ def __init__(self):
         self.servers = self.config.get_server_pool()
         self.ring = ConsistentHashing(self.config.server_pool)
 
+        self.missed_response = {}  # Keeps track of consecutive missed response for each server
+        self.threshold_missed_response = 3  # If k consecutive response is negative for a server, declare it dead
+
     def _get_server_for_key(self, key):
         """
         :return: client_socket for the given key
@@ -55,6 +57,16 @@ def execute_query(self, key, message):
         self.client_socket.connect(server_address)
         response = utils.send_receive_ack(message, self.client_socket, self.HEADER_LENGTH, self.FORMAT)
         self.client_socket.close()
+
+        # Health check
+        if not response:
+            self.missed_response[server_address] = self.missed_response.get(server_address, 0)
+            if self.missed_response[server_address] == self.threshold_missed_response:
+                # This server is now pronounced dead
+                self.ring.remove_node(server_address)
+        else:
+            self.missed_response[server_address] = 0
+
         return response
 
     def set(self, key, value):

usage/reconstructing_cache.py

@@ -1,10 +1,4 @@
 from distcache.cache_server import CacheServer
 
 server = CacheServer(5)
-server.spawn()
-# server.spawn()
-# server.spawn()
-
-server.reconstruct_from_log()
-
-server.close()
+server.replay_log()