Redis 5.0 with Stream Feture.

README.md

@@ -119,7 +119,7 @@ parameter (the path of the configuration file):
 It is possible to alter the Redis configuration by passing parameters directly
 as options using the command line. Examples:
 
-    % ./redis-server --port 9999 --slaveof 127.0.0.1 6379
+    % ./redis-server --port 9999 --replicaof 127.0.0.1 6379
     % ./redis-server /etc/redis/6379.conf --loglevel debug
 
 All the options in redis.conf are also supported as options using the command
@@ -245,7 +245,7 @@ A few important fields in this structure are:
 * `server.db` is an array of Redis databases, where data is stored.
 * `server.commands` is the command table.
 * `server.clients` is a linked list of clients connected to the server.
-* `server.master` is a special client, the master, if the instance is a slave.
+* `server.master` is a special client, the master, if the instance is a replica.
 
 There are tons of other fields. Most fields are commented directly inside
 the structure definition.
@@ -323,7 +323,7 @@ Inside server.c you can find code that handles other vital things of the Redis s
 networking.c
 ---
 
-This file defines all the I/O functions with clients, masters and slaves
+This file defines all the I/O functions with clients, masters and replicas
 (which in Redis are just special clients):
 
 * `createClient()` allocates and initializes a new client.
@@ -390,16 +390,16 @@ replication.c
 
 This is one of the most complex files inside Redis, it is recommended to
 approach it only after getting a bit familiar with the rest of the code base.
-In this file there is the implementation of both the master and slave role
+In this file there is the implementation of both the master and replica role
 of Redis.
 
-One of the most important functions inside this file is `replicationFeedSlaves()` that writes commands to the clients representing slave instances connected
-to our master, so that the slaves can get the writes performed by the clients:
+One of the most important functions inside this file is `replicationFeedSlaves()` that writes commands to the clients representing replica instances connected
+to our master, so that the replicas can get the writes performed by the clients:
 this way their data set will remain synchronized with the one in the master.
 
 This file also implements both the `SYNC` and `PSYNC` commands that are
 used in order to perform the first synchronization between masters and
-slaves, or to continue the replication after a disconnection.
+replicas, or to continue the replication after a disconnection.
 
 Other C files
 ---

runtest

@@ -11,4 +11,4 @@ then
     echo "You need tcl 8.5 or newer in order to run the Redis test"
     exit 1
 fi
-$TCLSH tests/test_helper.tcl $*
+$TCLSH tests/test_helper.tcl "${@}"

sentinel.conf

@@ -20,6 +20,21 @@
 # The port that this sentinel instance will run on
 port 26379
 
+# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis-sentinel.pid when
+# daemonized.
+daemonize no
+
+# When running daemonized, Redis Sentinel writes a pid file in
+# /var/run/redis-sentinel.pid by default. You can specify a custom pid file
+# location here.
+pidfile /var/run/redis-sentinel.pid
+
+# Specify the log file name. Also the empty string can be used to force
+# Sentinel to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile ""
+
 # sentinel announce-ip <ip>
 # sentinel announce-port <port>
 #
@@ -58,23 +73,23 @@ dir /tmp
 # be elected by the majority of the known Sentinels in order to
 # start a failover, so no failover can be performed in minority.
 #
-# Slaves are auto-discovered, so you don't need to specify slaves in
+# Replicas are auto-discovered, so you don't need to specify replicas in
 # any way. Sentinel itself will rewrite this configuration file adding
-# the slaves using additional configuration options.
+# the replicas using additional configuration options.
 # Also note that the configuration file is rewritten when a
-# slave is promoted to master.
+# replica is promoted to master.
 #
 # Note: master name should not include special characters or spaces.
 # The valid charset is A-z 0-9 and the three characters ".-_".
 sentinel monitor mymaster 127.0.0.1 6379 2
 
 # sentinel auth-pass <master-name> <password>
 #
-# Set the password to use to authenticate with the master and slaves.
+# Set the password to use to authenticate with the master and replicas.
 # Useful if there is a password set in the Redis instances to monitor.
 #
-# Note that the master password is also used for slaves, so it is not
-# possible to set a different password in masters and slaves instances
+# Note that the master password is also used for replicas, so it is not
+# possible to set a different password in masters and replicas instances
 # if you want to be able to monitor these instances with Sentinel.
 #
 # However you can have Redis instances without the authentication enabled
@@ -89,19 +104,19 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 
 # sentinel down-after-milliseconds <master-name> <milliseconds>
 #
-# Number of milliseconds the master (or any attached slave or sentinel) should
+# Number of milliseconds the master (or any attached replica or sentinel) should
 # be unreachable (as in, not acceptable reply to PING, continuously, for the
 # specified period) in order to consider it in S_DOWN state (Subjectively
 # Down).
 #
 # Default is 30 seconds.
 sentinel down-after-milliseconds mymaster 30000
 
-# sentinel parallel-syncs <master-name> <numslaves>
+# sentinel parallel-syncs <master-name> <numreplicas>
 #
-# How many slaves we can reconfigure to point to the new slave simultaneously
-# during the failover. Use a low number if you use the slaves to serve query
-# to avoid that all the slaves will be unreachable at about the same
+# How many replicas we can reconfigure to point to the new replica simultaneously
+# during the failover. Use a low number if you use the replicas to serve query
+# to avoid that all the replicas will be unreachable at about the same
 # time while performing the synchronization with the master.
 sentinel parallel-syncs mymaster 1
 
@@ -113,18 +128,18 @@ sentinel parallel-syncs mymaster 1
 #   already tried against the same master by a given Sentinel, is two
 #   times the failover timeout.
 #
-# - The time needed for a slave replicating to a wrong master according
+# - The time needed for a replica replicating to a wrong master according
 #   to a Sentinel current configuration, to be forced to replicate
 #   with the right master, is exactly the failover timeout (counting since
 #   the moment a Sentinel detected the misconfiguration).
 #
 # - The time needed to cancel a failover that is already in progress but
 #   did not produced any configuration change (SLAVEOF NO ONE yet not
-#   acknowledged by the promoted slave).
+#   acknowledged by the promoted replica).
 #
-# - The maximum time a failover in progress waits for all the slaves to be
-#   reconfigured as slaves of the new master. However even after this time
-#   the slaves will be reconfigured by the Sentinels anyway, but not with
+# - The maximum time a failover in progress waits for all the replicas to be
+#   reconfigured as replicas of the new master. However even after this time
+#   the replicas will be reconfigured by the Sentinels anyway, but not with
 #   the exact parallel-syncs progression as specified.
 #
 # Default is 3 minutes.
@@ -185,7 +200,7 @@ sentinel failover-timeout mymaster 180000
 # <role> is either "leader" or "observer"
 # 
 # The arguments from-ip, from-port, to-ip, to-port are used to communicate
-# the old address of the master and the new address of the elected slave
+# the old address of the master and the new address of the elected replica
 # (now a master).
 #
 # This script should be resistant to multiple invocations.
@@ -213,12 +228,17 @@ sentinel deny-scripts-reconfig yes
 #
 # In such case it is possible to tell Sentinel to use different command names
 # instead of the normal ones. For example if the master "mymaster", and the
-# associated slaves, have "CONFIG" all renamed to "GUESSME", I could use:
+# associated replicas, have "CONFIG" all renamed to "GUESSME", I could use:
 #
-# sentinel rename-command mymaster CONFIG GUESSME
+# SENTINEL rename-command mymaster CONFIG GUESSME
 #
 # After such configuration is set, every time Sentinel would use CONFIG it will
 # use GUESSME instead. Note that there is no actual need to respect the command
 # case, so writing "config guessme" is the same in the example above.
 #
 # SENTINEL SET can also be used in order to perform this configuration at runtime.
+#
+# In order to set a command back to its original name (undo the renaming), it
+# is possible to just rename a command to itsef:
+#
+# SENTINEL rename-command mymaster CONFIG CONFIG

src/Makefile

@@ -144,7 +144,7 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o
 REDIS_CLI_NAME=redis-cli
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark

src/aof.c

@@ -798,7 +798,9 @@ int loadAppendOnlyFile(char *filename) {
     }
 
     /* This point can only be reached when EOF is reached without errors.
-     * If the client is in the middle of a MULTI/EXEC, log error and quit. */
+     * If the client is in the middle of a MULTI/EXEC, handle it as it was
+     * a short read, even if technically the protocol is correct: we want
+     * to remove the unprocessed tail and continue. */
     if (fakeClient->flags & CLIENT_MULTI) goto uxeof;
 
 loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */

src/bio.c

@@ -204,14 +204,14 @@ void *bioProcessBackgroundJobs(void *arg) {
         }
         zfree(job);
 
-        /* Unblock threads blocked on bioWaitStepOfType() if any. */
-        pthread_cond_broadcast(&bio_step_cond[type]);
-
         /* Lock again before reiterating the loop, if there are no longer
          * jobs to process we'll block again in pthread_cond_wait(). */
         pthread_mutex_lock(&bio_mutex[type]);
         listDelNode(bio_jobs[type],ln);
         bio_pending[type]--;
+
+        /* Unblock threads blocked on bioWaitStepOfType() if any. */
+        pthread_cond_broadcast(&bio_step_cond[type]);
     }
 }
 

src/blocked.c

@@ -126,12 +126,37 @@ void processUnblockedClients(void) {
          * the code is conceptually more correct this way. */
         if (!(c->flags & CLIENT_BLOCKED)) {
             if (c->querybuf && sdslen(c->querybuf) > 0) {
-                processInputBuffer(c);
+                processInputBufferAndReplicate(c);
             }
         }
     }
 }
 
+/* This function will schedule the client for reprocessing at a safe time.
+ *
+ * This is useful when a client was blocked for some reason (blocking opeation,
+ * CLIENT PAUSE, or whatever), because it may end with some accumulated query
+ * buffer that needs to be processed ASAP:
+ *
+ * 1. When a client is blocked, its readable handler is still active.
+ * 2. However in this case it only gets data into the query buffer, but the
+ *    query is not parsed or executed once there is enough to proceed as
+ *    usually (because the client is blocked... so we can't execute commands).
+ * 3. When the client is unblocked, without this function, the client would
+ *    have to write some query in order for the readable handler to finally
+ *    call processQueryBuffer*() on it.
+ * 4. With this function instead we can put the client in a queue that will
+ *    process it for queries ready to be executed at a safe time.
+ */
+void queueClientForReprocessing(client *c) {
+    /* The client may already be into the unblocked list because of a previous
+     * blocking operation, don't add back it into the list multiple times. */
+    if (!(c->flags & CLIENT_UNBLOCKED)) {
+        c->flags |= CLIENT_UNBLOCKED;
+        listAddNodeTail(server.unblocked_clients,c);
+    }
+}
+
 /* Unblock a client calling the right function depending on the kind
  * of operation the client is blocking for. */
 void unblockClient(client *c) {
@@ -152,12 +177,7 @@ void unblockClient(client *c) {
     server.blocked_clients_by_type[c->btype]--;
     c->flags &= ~CLIENT_BLOCKED;
     c->btype = BLOCKED_NONE;
-    /* The client may already be into the unblocked list because of a previous
-     * blocking operation, don't add back it into the list multiple times. */
-    if (!(c->flags & CLIENT_UNBLOCKED)) {
-        c->flags |= CLIENT_UNBLOCKED;
-        listAddNodeTail(server.unblocked_clients,c);
-    }
+    queueClientForReprocessing(c);
 }
 
 /* This function gets called when a blocked client timed out in order to
@@ -195,7 +215,7 @@ void disconnectAllBlockedClients(void) {
         if (c->flags & CLIENT_BLOCKED) {
             addReplySds(c,sdsnew(
                 "-UNBLOCKED force unblock from blocking operation, "
-                "instance state changed (master -> slave?)\r\n"));
+                "instance state changed (master -> replica?)\r\n"));
             unblockClient(c);
             c->flags |= CLIENT_CLOSE_AFTER_REPLY;
         }
@@ -269,7 +289,7 @@ void handleClientsBlockedOnKeys(void) {
                         robj *dstkey = receiver->bpop.target;
                         int where = (receiver->lastcmd &&
                                      receiver->lastcmd->proc == blpopCommand) ?
-                                    LIST_HEAD : LIST_TAIL;
+                                     LIST_HEAD : LIST_TAIL;
                         robj *value = listTypePop(o,where);
 
                         if (value) {
@@ -285,7 +305,7 @@ void handleClientsBlockedOnKeys(void) {
                             {
                                 /* If we failed serving the client we need
                                  * to also undo the POP operation. */
-                                    listTypePush(o,value,where);
+                                listTypePush(o,value,where);
                             }
 
                             if (dstkey) decrRefCount(dstkey);