Permalink
Browse files

First implementation of Redis Sentinel.

This commit implements the first, beta quality implementation of Redis
Sentinel, a distributed monitoring system for Redis with notification
and automatic failover capabilities.

More info at http://redis.io/topics/sentinel
  • Loading branch information...
1 parent 03f412d commit 6b5daa2df2a0711a25746cb025927dc3deb7717e @antirez committed Jul 23, 2012
Showing with 2,614 additions and 44 deletions.
  1. +2 −0 .gitignore
  2. +41 −0 sentinel.conf
  3. +16 −9 src/Makefile
  4. +15 −0 src/anet.c
  5. +11 −0 src/config.c
  6. +72 −35 src/redis.c
  7. +18 −0 src/redis.h
  8. +2,439 −0 src/sentinel.c
View
@@ -4,6 +4,7 @@
*.log
redis-cli
redis-server
+redis-sentinel
redis-benchmark
redis-check-dump
redis-check-aof
@@ -24,3 +25,4 @@ deps/lua/src/luac
deps/lua/src/liblua.a
.make-*
.prerequisites
+*.dSYM
View
@@ -0,0 +1,41 @@
+# Example sentienl.conf
@anydot
anydot Jul 23, 2012 Contributor

Should be "sentinel.conf"

@antirez
antirez Jul 23, 2012 Owner

Thank you, fixed

+
+# sentinel monitor <name> <ip> <port> quorum. Tells Sentinel to monitor this
+# slave, and to consider it in O_DOWN (Objectively Down) state only if at
+# least two sentinels agree.
@anydot
anydot Jul 23, 2012 Contributor

Should be "least quorum sentinels agree."

@antirez
antirez Jul 23, 2012 Owner

Thanks, fixed.

+#
+# Note: master name should not include special characters or spaces.
+# The valid charset is A-z 0-9 and the three characters ".-_".
+sentinel monitor mymaster 127.0.0.1 6379 2
+
+# Number of milliseconds the master (or any attached slave or sentinel) should
+# be unreachable (as in, not acceptable reply to PING, continuously, for the
+# specified period) in order to consider it in S_DOWN state (Subjectively
+# Down).
+#
+# Default is 30 seconds.
+sentinel down-after-milliseconds mymaster 30000
+
+# Specify if this Sentinel can start the failover for this master.
+sentinel can-failover mymaster yes
+
+# How many slaves we can reconfigure to point to the new slave simultaneously
+# during the failover. Use a low number if you use the slaves to serve query
+# to avoid that all the slaves will be unreachable at about the same
+# time while performing the synchronization with the master.
+sentinel parallel-syncs mymaster 1
+
+# Specifies the failover timeout in milliseconds. When this time has elapsed
+# without any progress in the failover process, it is considered concluded by
+# the sentinel even if not all the attached slaves were correctly configured
+# to replicate with the new master (however a "best effort" SLAVEOF command
+# is sent to all the slaves before).
+#
+# Also when 25% of this time has elapsed without any advancement, and there
+# is a leader switch (the sentinel did not started the failover but is now
+# elected as leader), the sentinel will continue the failover doing a
+# "takeover".
+#
+# Default is 15 minutes.
+sentinel failover-timeout mymaster 900000
+
View
@@ -78,6 +78,7 @@ endif
REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)
REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS)
+REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL)
PREFIX?=/usr/local
INSTALL_BIN= $(PREFIX)/bin
@@ -93,10 +94,12 @@ ENDCOLOR="\033[0m"
ifndef V
QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR) 1>&2;
QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2;
+QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2;
endif
REDIS_SERVER_NAME= redis-server
-REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o
+REDIS_SENTINEL_NAME= redis-sentinel
+REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o
REDIS_CLI_NAME= redis-cli
REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o
REDIS_BENCHMARK_NAME= redis-benchmark
@@ -106,7 +109,7 @@ REDIS_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o crc64.o
REDIS_CHECK_AOF_NAME= redis-check-aof
REDIS_CHECK_AOF_OBJ= redis-check-aof.o
-all: $(REDIS_SERVER_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME)
+all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME)
@echo ""
@echo "Hint: To run 'make test' is a good idea ;)"
@echo ""
@@ -151,7 +154,11 @@ endif
# redis-server
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ)
- $(REDIS_LD) -o $@ $^ ../deps/lua/src/liblua.a $(FINAL_LIBS)
+ $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a $(FINAL_LIBS)
+
+# redis-sentinel
+$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
+ $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME)
# redis-cli
$(REDIS_CLI_NAME): $(REDIS_CLI_OBJ)
@@ -176,7 +183,7 @@ $(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ)
$(REDIS_CC) -c $<
clean:
- rm -rf $(REDIS_SERVER_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html
+ rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html
.PHONY: clean
@@ -217,8 +224,8 @@ src/help.h:
install: all
mkdir -p $(INSTALL_BIN)
- $(INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN)
- $(INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN)
- $(INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN)
- $(INSTALL) $(REDIS_CHECK_DUMP_NAME) $(INSTALL_BIN)
- $(INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN)
+ $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN)
+ $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN)
+ $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN)
+ $(REDIS_INSTALL) $(REDIS_CHECK_DUMP_NAME) $(INSTALL_BIN)
+ $(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN)
View
@@ -367,3 +367,18 @@ int anetPeerToString(int fd, char *ip, int *port) {
if (port) *port = ntohs(sa.sin_port);
return 0;
}
+
+int anetSockName(int fd, char *ip, int *port) {
+ struct sockaddr_in sa;
+ socklen_t salen = sizeof(sa);
+
+ if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) {
@anydot
anydot Jul 26, 2012 Contributor

Asside of "false" branch (ie success), in this branch there's no check if port and/or IP is not null -> possible crash when getsockname will fail.

+ *port = 0;
+ ip[0] = '?';
+ ip[1] = '\0';
+ return -1;
+ }
+ if (ip) strcpy(ip,inet_ntoa(sa.sin_addr));
+ if (port) *port = ntohs(sa.sin_port);
+ return 0;
+}
View
@@ -354,6 +354,17 @@ void loadServerConfigFromString(char *config) {
if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
+ } else if (!strcasecmp(argv[0],"sentinel")) {
+ /* argc == 1 is handled by main() as we need to enter the sentinel
+ * mode ASAP. */
+ if (argc != 1) {
+ if (!server.sentinel_mode) {
+ err = "sentinel directive while not in sentinel mode";
+ goto loaderr;
+ }
+ err = sentinelHandleConfiguration(argv+1,argc-1);
+ if (err) goto loaderr;
+ }
} else {
err = "Bad directive or wrong number of arguments"; goto loaderr;
}
View
@@ -821,13 +821,8 @@ void clientsCron(void) {
* a macro is used: run_with_period(milliseconds) { .... }
*/
-/* Using the following macro you can run code inside serverCron() with the
- * specified period, specified in milliseconds.
- * The actual resolution depends on REDIS_HZ. */
-#define run_with_period(_ms_) if (!(loops % ((_ms_)/(1000/REDIS_HZ))))
-
int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
- int j, loops = server.cronloops;
+ int j;
REDIS_NOTUSED(eventLoop);
REDIS_NOTUSED(id);
REDIS_NOTUSED(clientData);
@@ -896,11 +891,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
}
/* Show information about connected clients */
- run_with_period(5000) {
- redisLog(REDIS_VERBOSE,"%d clients connected (%d slaves), %zu bytes in use",
- listLength(server.clients)-listLength(server.slaves),
- listLength(server.slaves),
- zmalloc_used_memory());
+ if (!server.sentinel_mode) {
+ run_with_period(5000) {
+ redisLog(REDIS_VERBOSE,
+ "%d clients connected (%d slaves), %zu bytes in use",
+ listLength(server.clients)-listLength(server.slaves),
+ listLength(server.slaves),
+ zmalloc_used_memory());
+ }
}
/* We need to do a few operations on clients asynchronously. */
@@ -985,6 +983,11 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
if (server.cluster_enabled) clusterCron();
}
+ /* Run the sentinel timer if we are in sentinel mode. */
@anydot
anydot Jul 26, 2012 Contributor

Isn't:

if (server.sentinel_mode) {
run_with_period(100)
sentinelTimer();
}

Better and more inline with previous sentinel-related if above?

+ run_with_period(100) {
+ if (server.sentinel_mode) sentinelTimer();
+ }
+
server.cronloops++;
return 1000/REDIS_HZ;
}
@@ -2444,21 +2447,26 @@ void usage() {
fprintf(stderr," ./redis-server /etc/redis/6379.conf\n");
fprintf(stderr," ./redis-server --port 7777\n");
fprintf(stderr," ./redis-server --port 7777 --slaveof 127.0.0.1 8888\n");
- fprintf(stderr," ./redis-server /etc/myredis.conf --loglevel verbose\n");
+ fprintf(stderr," ./redis-server /etc/myredis.conf --loglevel verbose\n\n");
+ fprintf(stderr,"Sentinel mode:\n");
+ fprintf(stderr," ./redis-server /etc/sentinel.conf --sentinel\n");
exit(1);
}
void redisAsciiArt(void) {
#include "asciilogo.h"
char *buf = zmalloc(1024*16);
+ char *mode = "stand alone";
+
+ if (server.cluster_enabled) mode = "cluster";
+ else if (server.sentinel_mode) mode = "sentinel";
snprintf(buf,1024*16,ascii_logo,
REDIS_VERSION,
redisGitSHA1(),
strtol(redisGitDirty(),NULL,10) > 0,
(sizeof(long) == 8) ? "64" : "32",
- server.cluster_enabled ? "cluster" : "stand alone",
- server.port,
+ mode, server.port,
(long) getpid()
);
redisLogRaw(REDIS_NOTICE|REDIS_LOG_RAW,buf);
@@ -2496,17 +2504,53 @@ void setupSignalHandlers(void) {
void memtest(size_t megabytes, int passes);
+/* Returns 1 if there is --sentinel among the arguments or if
+ * argv[0] is exactly "redis-sentinel". */
+int checkForSentinelMode(int argc, char **argv) {
+ int j;
+
+ if (strstr(argv[0],"redis-sentinel") != NULL) return 1;
+ for (j = 1; j < argc; j++)
+ if (!strcmp(argv[j],"--sentinel")) return 1;
@anydot
anydot Jul 26, 2012 Contributor

Could result in hard-to-find configuration errors as --sentinel would match even when it's find in parameter of some cfg variable. (Surely user error but it could confuse user :-) )

+ return 0;
+}
+
+/* Function called at startup to load RDB or AOF file in memory. */
+void loadDataFromDisk(void) {
+ long long start = ustime();
+ if (server.aof_state == REDIS_AOF_ON) {
+ if (loadAppendOnlyFile(server.aof_filename) == REDIS_OK)
+ redisLog(REDIS_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000);
+ } else {
+ if (rdbLoad(server.rdb_filename) == REDIS_OK) {
+ redisLog(REDIS_NOTICE,"DB loaded from disk: %.3f seconds",
+ (float)(ustime()-start)/1000000);
+ } else if (errno != ENOENT) {
+ redisLog(REDIS_WARNING,"Fatal error loading the DB. Exiting.");
+ exit(1);
+ }
+ }
+}
+
int main(int argc, char **argv) {
- long long start;
struct timeval tv;
/* We need to initialize our libraries, and the server configuration. */
zmalloc_enable_thread_safeness();
srand(time(NULL)^getpid());
gettimeofday(&tv,NULL);
dictSetHashFunctionSeed(tv.tv_sec^tv.tv_usec^getpid());
+ server.sentinel_mode = checkForSentinelMode(argc,argv);
initServerConfig();
+ /* We need to init sentinel right now as parsing the configuration file
+ * in sentinel mode will have the effect of populating the sentinel
+ * data structures with master nodes to monitor. */
+ if (server.sentinel_mode) {
+ initSentinelConfig();
+ initSentinel();
+ }
+
if (argc >= 2) {
int j = 1; /* First option to parse in argv[] */
sds options = sdsempty();
@@ -2558,27 +2602,20 @@ int main(int argc, char **argv) {
initServer();
if (server.daemonize) createPidFile();
redisAsciiArt();
- redisLog(REDIS_WARNING,"Server started, Redis version " REDIS_VERSION);
-#ifdef __linux__
- linuxOvercommitMemoryWarning();
-#endif
- start = ustime();
- if (server.aof_state == REDIS_AOF_ON) {
- if (loadAppendOnlyFile(server.aof_filename) == REDIS_OK)
- redisLog(REDIS_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000);
- } else {
- if (rdbLoad(server.rdb_filename) == REDIS_OK) {
- redisLog(REDIS_NOTICE,"DB loaded from disk: %.3f seconds",
- (float)(ustime()-start)/1000000);
- } else if (errno != ENOENT) {
- redisLog(REDIS_WARNING,"Fatal error loading the DB. Exiting.");
- exit(1);
- }
+
+ if (!server.sentinel_mode) {
+ /* Things only needed when not runnign in Sentinel mode. */
+ redisLog(REDIS_WARNING,"Server started, Redis version " REDIS_VERSION);
+ #ifdef __linux__
+ linuxOvercommitMemoryWarning();
+ #endif
+ loadDataFromDisk();
+ if (server.ipfd > 0)
+ redisLog(REDIS_NOTICE,"The server is now ready to accept connections on port %d", server.port);
+ if (server.sofd > 0)
+ redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket);
}
- if (server.ipfd > 0)
- redisLog(REDIS_NOTICE,"The server is now ready to accept connections on port %d", server.port);
- if (server.sofd > 0)
- redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket);
+
aeSetBeforeSleepProc(server.el,beforeSleep);
aeMain(server.el);
aeDeleteEventLoop(server.el);
View
@@ -257,6 +257,11 @@
#define REDIS_PROPAGATE_AOF 1
#define REDIS_PROPAGATE_REPL 2
+/* Using the following macro you can run code inside serverCron() with the
+ * specified period, specified in milliseconds.
+ * The actual resolution depends on REDIS_HZ. */
+#define run_with_period(_ms_) if (!(server.cronloops%((_ms_)/(1000/REDIS_HZ))))
+
/* We can print the stacktrace, so our assert is defined this way: */
#define redisAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_redisAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),_exit(1)))
#define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1)))
@@ -579,6 +584,7 @@ struct redisServer {
int arch_bits; /* 32 or 64 depending on sizeof(long) */
int cronloops; /* Number of times the cron function run */
char runid[REDIS_RUN_ID_SIZE+1]; /* ID always different at every exec. */
+ int sentinel_mode; /* True if this instance is a Sentinel. */
/* Networking */
int port; /* TCP listening port */
char *bindaddr; /* Bind address or NULL */
@@ -1115,6 +1121,12 @@ void clusterCron(void);
clusterNode *getNodeByQuery(redisClient *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask);
void clusterPropagatePublish(robj *channel, robj *message);
+/* Sentinel */
+void initSentinelConfig(void);
+void initSentinel(void);
+void sentinelTimer(void);
+char *sentinelHandleConfiguration(char **argv, int argc);
+
/* Scripting */
void scriptingInit(void);
@@ -1280,4 +1292,10 @@ void enableWatchdog(int period);
void disableWatchdog(void);
void watchdogScheduleSignal(int period);
void redisLogHexDump(int level, char *descr, void *value, size_t len);
+
+#define redisDebug(fmt, ...) \
+ printf("DEBUG %s:%d > " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+#define redisDebugMark() \
+ printf("-- MARK %s:%d --\n", __FILE__, __LINE__)
+
#endif
Oops, something went wrong.

0 comments on commit 6b5daa2

Please sign in to comment.