Skip to content

Commit 697af43

Browse files
committed
initial changes needed to turn the current VM code into a cache system. Tons of work to do still.
1 parent 33388d4 commit 697af43

File tree

4 files changed

+43
-75
lines changed

4 files changed

+43
-75
lines changed

Diff for: src/diskstore.c

+12-3
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,17 @@
1616
* directory will contain in the average 15258 entires, that is ok with
1717
* most filesystems implementation.
1818
*
19-
* The actaul implementation of this disk store is highly related to the
20-
* filesystem implementation. This implementation may be replaced by
19+
* Note that since Redis supports multiple databases, the actual key name
20+
* is:
21+
*
22+
* /0b/ee/<dbid>_0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
23+
*
24+
* so for instance if the key is inside DB 0:
25+
*
26+
* /0b/ee/0_0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
27+
*
28+
* The actaul implementation of this disk store is highly dependant to the
29+
* filesystem implementation itself. This implementation may be replaced by
2130
* a B+TREE implementation in future implementations.
2231
*
2332
* Data ok every key is serialized using the same format used for .rdb
@@ -68,7 +77,7 @@
6877
int dsOpen(void) {
6978
struct stat sb;
7079
int retval;
71-
char *path = server.diskstore_path;
80+
char *path = server.ds_path;
7281

7382
if ((retval = stat(path,&sb) == -1) && errno != ENOENT) {
7483
redisLog(REDIS_WARNING, "Error opening disk store at %s: %s",

Diff for: src/networking.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ void _addReplyStringToList(redisClient *c, char *s, size_t len) {
167167

168168
void addReply(redisClient *c, robj *obj) {
169169
if (_installWriteEvent(c) != REDIS_OK) return;
170-
redisAssert(!server.vm_enabled || obj->storage == REDIS_VM_MEMORY);
170+
redisAssert(!server.ds_enabled || obj->storage == REDIS_VM_MEMORY);
171171

172172
/* This is an important place where we can avoid copy-on-write
173173
* when there is a saving child running, avoiding touching the
@@ -460,7 +460,7 @@ void freeClient(redisClient *c) {
460460
/* Remove from the list of clients waiting for swapped keys, or ready
461461
* to be restarted, but not yet woken up again. */
462462
if (c->flags & REDIS_IO_WAIT) {
463-
redisAssert(server.vm_enabled);
463+
redisAssert(server.ds_enabled);
464464
if (listLength(c->io_keys) == 0) {
465465
ln = listSearchKey(server.io_ready_clients,c);
466466

@@ -474,7 +474,7 @@ void freeClient(redisClient *c) {
474474
dontWaitForSwappedKey(c,ln->value);
475475
}
476476
}
477-
server.vm_blocked_clients--;
477+
server.cache_blocked_clients--;
478478
}
479479
listRelease(c->io_keys);
480480
/* Master/slave cleanup.

Diff for: src/redis.c

+24-63
Original file line numberDiff line numberDiff line change
@@ -618,27 +618,12 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
618618
* in order to guarantee a strict consistency. */
619619
if (server.masterhost == NULL) activeExpireCycle();
620620

621-
/* Swap a few keys on disk if we are over the memory limit and VM
622-
* is enbled. Try to free objects from the free list first. */
623-
if (vmCanSwapOut()) {
624-
while (server.vm_enabled && zmalloc_used_memory() >
625-
server.vm_max_memory)
626-
{
627-
int retval = (server.vm_max_threads == 0) ?
628-
vmSwapOneObjectBlocking() :
629-
vmSwapOneObjectThreaded();
630-
if (retval == REDIS_ERR && !(loops % 300) &&
631-
zmalloc_used_memory() >
632-
(server.vm_max_memory+server.vm_max_memory/10))
633-
{
634-
redisLog(REDIS_WARNING,"WARNING: vm-max-memory limit exceeded by more than 10%% but unable to swap more objects out!");
635-
}
636-
/* Note that when using threade I/O we free just one object,
637-
* because anyway when the I/O thread in charge to swap this
638-
* object out will finish, the handler of completed jobs
639-
* will try to swap more objects if we are still out of memory. */
640-
if (retval == REDIS_ERR || server.vm_max_threads > 0) break;
641-
}
621+
/* Remove a few cached objects from memory if we are over the
622+
* configured memory limit */
623+
while (server.ds_enabled && zmalloc_used_memory() >
624+
server.cache_max_memory)
625+
{
626+
cacheFreeOneEntry();
642627
}
643628

644629
/* Replication cron function -- used to reconnect to master and
@@ -656,8 +641,8 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
656641
listNode *ln;
657642
redisClient *c;
658643

659-
/* Awake clients that got all the swapped keys they requested */
660-
if (server.vm_enabled && listLength(server.io_ready_clients)) {
644+
/* Awake clients that got all the on disk keys they requested */
645+
if (server.ds_enabled && listLength(server.io_ready_clients)) {
661646
listIter li;
662647

663648
listRewind(server.io_ready_clients,&li);
@@ -668,7 +653,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
668653
/* Resume the client. */
669654
listDelNode(server.io_ready_clients,ln);
670655
c->flags &= (~REDIS_IO_WAIT);
671-
server.vm_blocked_clients--;
656+
server.cache_blocked_clients--;
672657
aeCreateFileEvent(server.el, c->fd, AE_READABLE,
673658
readQueryFromClient, c);
674659
cmd = lookupCommand(c->argv[0]->ptr);
@@ -787,13 +772,10 @@ void initServerConfig() {
787772
server.maxmemory = 0;
788773
server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU;
789774
server.maxmemory_samples = 3;
790-
server.vm_enabled = 0;
791-
server.vm_swap_file = zstrdup("/tmp/redis-%p.vm");
792-
server.vm_page_size = 256; /* 256 bytes per page */
793-
server.vm_pages = 1024*1024*100; /* 104 millions of pages */
794-
server.vm_max_memory = 1024LL*1024*1024*1; /* 1 GB of RAM */
795-
server.vm_max_threads = 4;
796-
server.vm_blocked_clients = 0;
775+
server.ds_enabled = 0;
776+
server.ds_path = zstrdup("/tmp/redis.ds");
777+
server.cache_max_memory = 64LL*1024*1024; /* 64 MB of RAM */
778+
server.cache_blocked_clients = 0;
797779
server.hash_max_zipmap_entries = REDIS_HASH_MAX_ZIPMAP_ENTRIES;
798780
server.hash_max_zipmap_value = REDIS_HASH_MAX_ZIPMAP_VALUE;
799781
server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES;
@@ -873,7 +855,7 @@ void initServer() {
873855
server.db[j].expires = dictCreate(&keyptrDictType,NULL);
874856
server.db[j].blocking_keys = dictCreate(&keylistDictType,NULL);
875857
server.db[j].watched_keys = dictCreate(&keylistDictType,NULL);
876-
if (server.vm_enabled)
858+
if (server.ds_enabled)
877859
server.db[j].io_keys = dictCreate(&keylistDictType,NULL);
878860
server.db[j].id = j;
879861
}
@@ -911,7 +893,7 @@ void initServer() {
911893
}
912894
}
913895

914-
if (server.vm_enabled) vmInit();
896+
if (server.ds_enabled) dsInit();
915897
}
916898

917899
/* Populates the Redis Command Table starting from the hard coded list
@@ -1050,8 +1032,8 @@ int processCommand(redisClient *c) {
10501032
queueMultiCommand(c,cmd);
10511033
addReply(c,shared.queued);
10521034
} else {
1053-
if (server.vm_enabled && server.vm_max_threads > 0 &&
1054-
blockClientOnSwappedKeys(c,cmd)) return REDIS_ERR;
1035+
if (server.ds_enabled && blockClientOnSwappedKeys(c,cmd))
1036+
return REDIS_ERR;
10551037
call(c,cmd);
10561038
}
10571039
return REDIS_OK;
@@ -1072,7 +1054,6 @@ int prepareForShutdown() {
10721054
if (server.appendonly) {
10731055
/* Append only file: fsync() the AOF and exit */
10741056
aof_fsync(server.appendfd);
1075-
if (server.vm_enabled) unlink(server.vm_swap_file);
10761057
} else if (server.saveparamslen > 0) {
10771058
/* Snapshotting. Perform a SYNC SAVE and exit */
10781059
if (rdbSave(server.dbfilename) != REDIS_OK) {
@@ -1185,7 +1166,7 @@ sds genRedisInfoString(void) {
11851166
"hash_max_zipmap_value:%zu\r\n"
11861167
"pubsub_channels:%ld\r\n"
11871168
"pubsub_patterns:%u\r\n"
1188-
"vm_enabled:%d\r\n"
1169+
"ds_enabled:%d\r\n"
11891170
"role:%s\r\n"
11901171
,REDIS_VERSION,
11911172
redisGitSHA1(),
@@ -1228,7 +1209,7 @@ sds genRedisInfoString(void) {
12281209
server.hash_max_zipmap_value,
12291210
dictSize(server.pubsub_channels),
12301211
listLength(server.pubsub_patterns),
1231-
server.vm_enabled != 0,
1212+
server.ds_enabled != 0,
12321213
server.masterhost == NULL ? "master" : "slave"
12331214
);
12341215
if (server.masterhost) {
@@ -1255,33 +1236,13 @@ sds genRedisInfoString(void) {
12551236
);
12561237
}
12571238
}
1258-
if (server.vm_enabled) {
1239+
if (server.ds_enabled) {
12591240
lockThreadedIO();
12601241
info = sdscatprintf(info,
1261-
"vm_conf_max_memory:%llu\r\n"
1262-
"vm_conf_page_size:%llu\r\n"
1263-
"vm_conf_pages:%llu\r\n"
1264-
"vm_stats_used_pages:%llu\r\n"
1265-
"vm_stats_swapped_objects:%llu\r\n"
1266-
"vm_stats_swappin_count:%llu\r\n"
1267-
"vm_stats_swappout_count:%llu\r\n"
1268-
"vm_stats_io_newjobs_len:%lu\r\n"
1269-
"vm_stats_io_processing_len:%lu\r\n"
1270-
"vm_stats_io_processed_len:%lu\r\n"
1271-
"vm_stats_io_active_threads:%lu\r\n"
1272-
"vm_stats_blocked_clients:%lu\r\n"
1273-
,(unsigned long long) server.vm_max_memory,
1274-
(unsigned long long) server.vm_page_size,
1275-
(unsigned long long) server.vm_pages,
1276-
(unsigned long long) server.vm_stats_used_pages,
1277-
(unsigned long long) server.vm_stats_swapped_objects,
1278-
(unsigned long long) server.vm_stats_swapins,
1279-
(unsigned long long) server.vm_stats_swapouts,
1280-
(unsigned long) listLength(server.io_newjobs),
1281-
(unsigned long) listLength(server.io_processing),
1282-
(unsigned long) listLength(server.io_processed),
1283-
(unsigned long) server.io_active_threads,
1284-
(unsigned long) server.vm_blocked_clients
1242+
"cache_max_memory:%llu\r\n"
1243+
"cache_blocked_clients:%lu\r\n"
1244+
,(unsigned long long) server.cache_max_memory,
1245+
(unsigned long) server.cache_blocked_clients
12851246
);
12861247
unlockThreadedIO();
12871248
}

Diff for: src/redis.h

+4-6
Original file line numberDiff line numberDiff line change
@@ -440,19 +440,17 @@ struct redisServer {
440440
int maxmemory_samples;
441441
/* Blocked clients */
442442
unsigned int bpop_blocked_clients;
443-
unsigned int vm_blocked_clients;
443+
unsigned int cache_blocked_clients;
444444
list *unblocked_clients;
445445
/* Sort parameters - qsort_r() is only available under BSD so we
446446
* have to take this state global, in order to pass it to sortCompare() */
447447
int sort_desc;
448448
int sort_alpha;
449449
int sort_bypattern;
450450
/* Virtual memory configuration */
451-
int vm_enabled;
452-
char *vm_swap_file;
453-
off_t vm_page_size;
454-
off_t vm_pages;
455-
unsigned long long vm_max_memory;
451+
int ds_enabled; /* backend disk in redis.conf */
452+
char *ds_path; /* location of the disk store on disk */
453+
unsigned long long cache_max_memory;
456454
/* Zip structure config */
457455
size_t hash_max_zipmap_entries;
458456
size_t hash_max_zipmap_value;

0 commit comments

Comments
 (0)