Skip to content

Commit

Permalink
Merge unstable into 6.2
Browse files Browse the repository at this point in the history
  • Loading branch information
@oranagra
oranagra committed Jan 12, 2021
2 parents b8c67ce + 4f8458d commit b34fc03
Show file tree
Hide file tree
Showing 86 changed files with 5,000 additions and 1,657 deletions.
2 changes: 1 addition & 1 deletion deps/hiredis/async_private.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
#define _EL_CLEANUP(ctx) do { \
if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \
ctx->ev.cleanup = NULL; \
} while(0);
} while(0)

static inline void refreshTimeout(redisAsyncContext *ctx) {
#define REDIS_TIMER_ISSET(tvp) \
Expand Down
18 changes: 14 additions & 4 deletions redis.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,9 +196,12 @@ tcp-keepalive 300
#
# tls-cluster yes

# Explicitly specify TLS versions to support. Allowed values are case insensitive
# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or
# any combination. To enable only TLSv1.2 and TLSv1.3, use:
# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended
# that older formally deprecated versions are kept disabled to reduce the attack surface.
# You can explicitly specify TLS versions to support.
# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2",
# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination.
# To enable only TLSv1.2 and TLSv1.3, use:
#
# tls-protocols "TLSv1.2 TLSv1.3"

Expand Down Expand Up @@ -688,7 +691,7 @@ replica-priority 100

# Redis implements server assisted support for client side caching of values.
# This is implemented using an invalidation table that remembers, using
# 16 millions of slots, what clients may have certain subsets of keys. In turn
# a radix key indexed by key name, what clients have which keys. In turn
# this is used in order to send invalidation messages to clients. Please
# check this page to understand more about the feature:
#
Expand Down Expand Up @@ -1973,3 +1976,10 @@ jemalloc-bg-thread yes
#
# Set bgsave child process to cpu affinity 1,10,11
# bgsave_cpulist 1,10-11

# In some cases redis will emit warnings and even refuse to start if it detects
# that the system is in bad state, it is possible to suppress these warnings
# by setting the following config which takes a space delimited list of warnings
# to suppress
#
# ignore-warnings ARM64-COW-BUG
22 changes: 16 additions & 6 deletions src/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,18 @@ else
ifeq ($(uname_S),Darwin)
# Darwin
FINAL_LIBS+= -ldl
# Homebrew's OpenSSL is not linked to /usr/local to avoid
# conflicts with the system's LibreSSL installation so it
# must be referenced explicitly during build.
ifeq ($(uname_M),arm64)
# Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX
OPENSSL_CFLAGS=-I/opt/homebrew/opt/openssl/include
OPENSSL_LDFLAGS=-L/opt/homebrew/opt/openssl/lib
else
# Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX
OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include
OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib
endif
else
ifeq ($(uname_S),AIX)
# AIX
Expand Down Expand Up @@ -260,11 +270,11 @@ endif

REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o
REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o
REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o
REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o
REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX)
REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX)

Expand Down Expand Up @@ -336,7 +346,7 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ)
$(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS)

dict-benchmark: dict.c zmalloc.c sds.c siphash.c
dict-benchmark: dict.c zmalloc.c sds.c siphash.c mt19937-64.c
$(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS)

DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
Expand Down Expand Up @@ -405,8 +415,8 @@ install: all
$(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN)
$(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN)
$(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN)
$(REDIS_INSTALL) $(REDIS_CHECK_RDB_NAME) $(INSTALL_BIN)
$(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN)
@ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_RDB_NAME)
@ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_AOF_NAME)
@ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME)

uninstall:
Expand Down
46 changes: 29 additions & 17 deletions src/acl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,15 +174,15 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) {
return sdsnewlen(hex,HASH_PASSWORD_LEN);
}

/* Given a hash and the hash length, returns C_OK if it is a valid password
/* Given a hash and the hash length, returns C_OK if it is a valid password
* hash, or C_ERR otherwise. */
int ACLCheckPasswordHash(unsigned char *hash, int hashlen) {
if (hashlen != HASH_PASSWORD_LEN) {
return C_ERR;
return C_ERR;
}

/* Password hashes can only be characters that represent
* hexadecimal values, which are numbers and lowercase
* hexadecimal values, which are numbers and lowercase
* characters 'a' through 'f'. */
for(int i = 0; i < HASH_PASSWORD_LEN; i++) {
char c = hash[i];
Expand Down Expand Up @@ -2184,18 +2184,30 @@ void aclCommand(client *c) {
}
} else if (c->argc == 2 && !strcasecmp(sub,"help")) {
const char *help[] = {
"LOAD -- Reload users from the ACL file.",
"SAVE -- Save the current config to the ACL file.",
"LIST -- Show user details in config file format.",
"USERS -- List all the registered usernames.",
"SETUSER <username> [attribs ...] -- Create or modify a user.",
"GETUSER <username> -- Get the user details.",
"DELUSER <username> [...] -- Delete a list of users.",
"CAT -- List available categories.",
"CAT <category> -- List commands inside category.",
"GENPASS [<bits>] -- Generate a secure user password.",
"WHOAMI -- Return the current connection username.",
"LOG [<count> | RESET] -- Show the ACL log entries.",
"CAT [<category>]",
" List all commands that belong to <category>, or all command categories",
" when no category is specified.",
"DELUSER <username> [<username> ...]",
" Delete a list of users.",
"GETUSER <username>",
" Get the user's details.",
"GENPASS [<bits>]",
" Generate a secure 256-bit user password. The optional `bits` argument can",
" be used to specify a different size.",
"LIST",
" Show users details in config file format.",
"LOAD",
" Reload users from the ACL file.",
"LOG [<count> | RESET]",
" Show the ACL log entries.",
"SAVE",
" Save the current config to the ACL file.",
"SETUSER <username> <attribute> [<attribute> ...]",
" Create or modify a user with the specified attributes.",
"USERS",
" List all the registered usernames.",
"WHOAMI",
" Return the current connection username.",
NULL
};
addReplyHelp(c,help);
Expand Down Expand Up @@ -2224,7 +2236,7 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd) {
void authCommand(client *c) {
/* Only two or three argument forms are allowed. */
if (c->argc > 3) {
addReply(c,shared.syntaxerr);
addReplyErrorObject(c,shared.syntaxerr);
return;
}

Expand Down
80 changes: 56 additions & 24 deletions src/aof.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,29 +206,27 @@ int aofFsyncInProgress(void) {
/* Starts a background task that performs fsync() against the specified
* file descriptor (the one of the AOF file) in another thread. */
void aof_background_fsync(int fd) {
bioCreateBackgroundJob(BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL);
bioCreateFsyncJob(fd);
}

/* Kills an AOFRW child process if exists */
void killAppendOnlyChild(void) {
int statloc;
/* No AOFRW child? return. */
if (server.aof_child_pid == -1) return;
if (server.child_type != CHILD_TYPE_AOF) return;
/* Kill AOFRW child, wait for child exit. */
serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld",
(long) server.aof_child_pid);
if (kill(server.aof_child_pid,SIGUSR1) != -1) {
while(wait3(&statloc,0,NULL) != server.aof_child_pid);
(long) server.child_pid);
if (kill(server.child_pid,SIGUSR1) != -1) {
while(wait3(&statloc,0,NULL) != server.child_pid);
}
/* Reset the buffer accumulating changes while the child saves. */
aofRewriteBufferReset();
aofRemoveTempFile(server.aof_child_pid);
server.aof_child_pid = -1;
aofRemoveTempFile(server.child_pid);
resetChildState();
server.aof_rewrite_time_start = -1;
/* Close pipes used for IPC between the two processes. */
aofClosePipes();
closeChildInfoPipe();
updateDictResizePolicy();
}

/* Called when the user switches from "appendonly yes" to "appendonly no"
Expand Down Expand Up @@ -265,14 +263,14 @@ int startAppendOnly(void) {
strerror(errno));
return C_ERR;
}
if (hasActiveChildProcess() && server.aof_child_pid == -1) {
if (hasActiveChildProcess() && server.child_type != CHILD_TYPE_AOF) {
server.aof_rewrite_scheduled = 1;
serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible.");
} else {
/* If there is a pending AOF rewrite, we need to switch it off and
* start a new one: the old one cannot be reused because it is not
* accumulating the AOF buffer. */
if (server.aof_child_pid != -1) {
if (server.child_type == CHILD_TYPE_AOF) {
serverLog(LL_WARNING,"AOF was enabled but there is already an AOF rewriting in background. Stopping background AOF and starting a rewrite now.");
killAppendOnlyChild();
}
Expand Down Expand Up @@ -646,7 +644,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
* accumulate the differences between the child DB and the current one
* in a buffer, so that when the child process will do its work we
* can append the differences to the new append only file. */
if (server.aof_child_pid != -1)
if (server.child_type == CHILD_TYPE_AOF)
aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf));

sdsfree(buf);
Expand Down Expand Up @@ -1427,6 +1425,8 @@ int rewriteAppendOnlyFileRio(rio *aof) {
dictEntry *de;
size_t processed = 0;
int j;
long key_count = 0;
long long cow_updated_time = 0;

for (j = 0; j < server.dbnum; j++) {
char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
Expand Down Expand Up @@ -1486,6 +1486,19 @@ int rewriteAppendOnlyFileRio(rio *aof) {
processed = aof->processed_bytes;
aofReadDiffFromParent();
}

/* Update COW info every 1 second (approximately).
* in order to avoid calling mstime() on each iteration, we will
* check the diff every 1024 keys */
if ((key_count & 1023) == 0) {
key_count = 0;
long long now = mstime();
if (now - cow_updated_time >= 1000) {
sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
cow_updated_time = now;
}
}
key_count++;
}
dictReleaseIterator(di);
di = NULL;
Expand Down Expand Up @@ -1579,8 +1592,31 @@ int rewriteAppendOnlyFile(char *filename) {
serverLog(LL_NOTICE,
"Concatenating %.2f MB of AOF diff received from parent.",
(double) sdslen(server.aof_child_diff) / (1024*1024));
if (rioWrite(&aof,server.aof_child_diff,sdslen(server.aof_child_diff)) == 0)
goto werr;

/* Now we write the entire AOF buffer we received from the parent
* via the pipe during the life of this fork child.
* once a second, we'll take a break and send updated COW info to the parent */
size_t bytes_to_write = sdslen(server.aof_child_diff);
const char *buf = server.aof_child_diff;
long long cow_updated_time = mstime();

while (bytes_to_write) {
/* We write the AOF buffer in chunk of 8MB so that we can check the time in between them */
size_t chunk_size = bytes_to_write < (8<<20) ? bytes_to_write : (8<<20);

if (rioWrite(&aof,buf,chunk_size) == 0)
goto werr;

bytes_to_write -= chunk_size;
buf += chunk_size;

/* Update COW info */
long long now = mstime();
if (now - cow_updated_time >= 1000) {
sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
cow_updated_time = now;
}
}

/* Make sure data will not remain on the OS's output buffers */
if (fflush(fp)) goto werr;
Expand Down Expand Up @@ -1703,7 +1739,6 @@ int rewriteAppendOnlyFileBackground(void) {

if (hasActiveChildProcess()) return C_ERR;
if (aofCreatePipes() != C_OK) return C_ERR;
openChildInfoPipe();
if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) {
char tmpfile[256];

Expand All @@ -1712,15 +1747,14 @@ int rewriteAppendOnlyFileBackground(void) {
redisSetCpuAffinity(server.aof_rewrite_cpulist);
snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
sendChildCOWInfo(CHILD_TYPE_AOF, "AOF rewrite");
sendChildCOWInfo(CHILD_TYPE_AOF, 1, "AOF rewrite");
exitFromChild(0);
} else {
exitFromChild(1);
}
} else {
/* Parent */
if (childpid == -1) {
closeChildInfoPipe();
serverLog(LL_WARNING,
"Can't rewrite append only file in background: fork: %s",
strerror(errno));
Expand All @@ -1731,8 +1765,7 @@ int rewriteAppendOnlyFileBackground(void) {
"Background append only file rewriting started by pid %ld",(long) childpid);
server.aof_rewrite_scheduled = 0;
server.aof_rewrite_time_start = time(NULL);
server.aof_child_pid = childpid;
updateDictResizePolicy();

/* We set appendseldb to -1 in order to force the next call to the
* feedAppendOnlyFile() to issue a SELECT command, so the differences
* accumulated by the parent into server.aof_rewrite_buf will start
Expand All @@ -1745,7 +1778,7 @@ int rewriteAppendOnlyFileBackground(void) {
}

void bgrewriteaofCommand(client *c) {
if (server.aof_child_pid != -1) {
if (server.child_type == CHILD_TYPE_AOF) {
addReplyError(c,"Background append only file rewriting already in progress");
} else if (hasActiveChildProcess()) {
server.aof_rewrite_scheduled = 1;
Expand Down Expand Up @@ -1803,7 +1836,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
* rewritten AOF. */
latencyStartMonitor(latency);
snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof",
(int)server.aof_child_pid);
(int)server.child_pid);
newfd = open(tmpfile,O_WRONLY|O_APPEND);
if (newfd == -1) {
serverLog(LL_WARNING,
Expand Down Expand Up @@ -1909,7 +1942,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
server.aof_state = AOF_ON;

/* Asynchronously close the overwritten AOF. */
if (oldfd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL);
if (oldfd != -1) bioCreateCloseJob(oldfd);

serverLog(LL_VERBOSE,
"Background AOF rewrite signal handler took %lldus", ustime()-now);
Expand All @@ -1931,8 +1964,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
cleanup:
aofClosePipes();
aofRewriteBufferReset();
aofRemoveTempFile(server.aof_child_pid);
server.aof_child_pid = -1;
aofRemoveTempFile(server.child_pid);
server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start;
server.aof_rewrite_time_start = -1;
/* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */
Expand Down
Loading

0 comments on commit b34fc03

Please sign in to comment.