diff --git a/redis.conf b/redis.conf index 84f492b88e70..84f45fdb7187 100644 --- a/redis.conf +++ b/redis.conf @@ -1704,8 +1704,8 @@ notify-keyspace-events "" # Hashes are encoded using a memory efficient data structure when they have a # small number of entries, and the biggest entry does not exceed a given # threshold. These thresholds can be configured using the following directives. -hash-max-ziplist-entries 512 -hash-max-ziplist-value 64 +hash-max-listpack-entries 512 +hash-max-listpack-value 64 # Lists are also encoded in a special way to save a lot of space. # The number of entries allowed per internal list node can be specified diff --git a/src/aof.c b/src/aof.c index 3c02c8aaa56f..2663b4f7e321 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1105,12 +1105,12 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { * * The function returns 0 on error, non-zero on success. */ static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) { - if (hi->encoding == OBJ_ENCODING_ZIPLIST) { + if (hi->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *vstr = NULL; unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; - hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll); + hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll); if (vstr) return rioWriteBulkString(r, (char*)vstr, vlen); else diff --git a/src/config.c b/src/config.c index 87d4576c3bd0..954a96e75961 100644 --- a/src/config.c +++ b/src/config.c @@ -2636,11 +2636,11 @@ standardConfig configs[] = { createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), /* Size_t configs */ - createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("hash-max-listpack-entries", "hash-max-ziplist-entries", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ - createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("hash-max-listpack-value", "hash-max-ziplist-value", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), diff --git a/src/db.c b/src/db.c index b4ecc76771d8..635dc972f50b 100644 --- a/src/db.c +++ b/src/db.c @@ -935,7 +935,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { while(intsetGet(o->ptr,pos++,&ll)) listAddNodeTail(keys,createStringObjectFromLongLong(ll)); cursor = 0; - } else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { + } else if (o->type == OBJ_ZSET) { unsigned char *p = ziplistIndex(o->ptr,0); unsigned char *vstr; unsigned int vlen; @@ -949,6 +949,18 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { p = ziplistNext(o->ptr,p); } cursor = 0; + } else if (o->type == OBJ_HASH) { + unsigned char *p = lpFirst(o->ptr); + unsigned char *vstr; + int64_t vlen; + unsigned char intbuf[LP_INTBUF_SIZE]; + + while(p) { + vstr = lpGet(p,&vlen,intbuf); + listAddNodeTail(keys, createStringObject((char*)vstr,vlen)); + p = lpNext(o->ptr,p); + } + cursor = 0; } else { serverPanic("Not handled encoding in SCAN."); } diff --git a/src/defrag.c b/src/defrag.c index 5d18c9079a00..c804af3701f9 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -864,7 +864,7 @@ long defragKey(redisDb *db, dictEntry *de) { serverPanic("Unknown sorted set encoding"); } } else if (ob->type == OBJ_HASH) { - if (ob->encoding == OBJ_ENCODING_ZIPLIST) { + if (ob->encoding == OBJ_ENCODING_LISTPACK) { if ((newzl = activeDefragAlloc(ob->ptr))) defragged++, ob->ptr = newzl; } else if (ob->encoding == OBJ_ENCODING_HT) { diff --git a/src/listpack.c b/src/listpack.c index 8672f63c811e..9bb1c8c95d7e 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -55,6 +55,7 @@ #define LP_ENCODING_7BIT_UINT 0 #define LP_ENCODING_7BIT_UINT_MASK 0x80 #define LP_ENCODING_IS_7BIT_UINT(byte) (((byte)&LP_ENCODING_7BIT_UINT_MASK)==LP_ENCODING_7BIT_UINT) +#define LP_ENCODING_7BIT_UINT_ENTRY_SIZE 2 #define LP_ENCODING_6BIT_STR 0x80 #define LP_ENCODING_6BIT_STR_MASK 0xC0 @@ -63,6 +64,7 @@ #define LP_ENCODING_13BIT_INT 0xC0 #define LP_ENCODING_13BIT_INT_MASK 0xE0 #define LP_ENCODING_IS_13BIT_INT(byte) (((byte)&LP_ENCODING_13BIT_INT_MASK)==LP_ENCODING_13BIT_INT) +#define LP_ENCODING_13BIT_INT_ENTRY_SIZE 3 #define LP_ENCODING_12BIT_STR 0xE0 #define LP_ENCODING_12BIT_STR_MASK 0xF0 @@ -71,18 +73,22 @@ #define LP_ENCODING_16BIT_INT 0xF1 #define LP_ENCODING_16BIT_INT_MASK 0xFF #define LP_ENCODING_IS_16BIT_INT(byte) (((byte)&LP_ENCODING_16BIT_INT_MASK)==LP_ENCODING_16BIT_INT) +#define LP_ENCODING_16BIT_INT_ENTRY_SIZE 4 #define LP_ENCODING_24BIT_INT 0xF2 #define LP_ENCODING_24BIT_INT_MASK 0xFF #define LP_ENCODING_IS_24BIT_INT(byte) (((byte)&LP_ENCODING_24BIT_INT_MASK)==LP_ENCODING_24BIT_INT) +#define LP_ENCODING_24BIT_INT_ENTRY_SIZE 5 #define LP_ENCODING_32BIT_INT 0xF3 #define LP_ENCODING_32BIT_INT_MASK 0xFF #define LP_ENCODING_IS_32BIT_INT(byte) (((byte)&LP_ENCODING_32BIT_INT_MASK)==LP_ENCODING_32BIT_INT) +#define LP_ENCODING_32BIT_INT_ENTRY_SIZE 6 #define LP_ENCODING_64BIT_INT 0xF4 #define LP_ENCODING_64BIT_INT_MASK 0xFF #define LP_ENCODING_IS_64BIT_INT(byte) (((byte)&LP_ENCODING_64BIT_INT_MASK)==LP_ENCODING_64BIT_INT) +#define LP_ENCODING_64BIT_INT_ENTRY_SIZE 10 #define LP_ENCODING_32BIT_STR 0xF0 #define LP_ENCODING_32BIT_STR_MASK 0xFF @@ -246,6 +252,58 @@ unsigned char* lpShrinkToFit(unsigned char *lp) { } } +/* Stores the integer encoded representation of 'v' in the 'intenc' buffer. */ +static inline void lpEncodeIntegerGetType(int64_t v, unsigned char *intenc, uint64_t *enclen) { + if (v >= 0 && v <= 127) { + /* Single byte 0-127 integer. */ + intenc[0] = v; + *enclen = 1; + } else if (v >= -4096 && v <= 4095) { + /* 13 bit integer. */ + if (v < 0) v = ((int64_t)1<<13)+v; + intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT; + intenc[1] = v&0xff; + *enclen = 2; + } else if (v >= -32768 && v <= 32767) { + /* 16 bit integer. */ + if (v < 0) v = ((int64_t)1<<16)+v; + intenc[0] = LP_ENCODING_16BIT_INT; + intenc[1] = v&0xff; + intenc[2] = v>>8; + *enclen = 3; + } else if (v >= -8388608 && v <= 8388607) { + /* 24 bit integer. */ + if (v < 0) v = ((int64_t)1<<24)+v; + intenc[0] = LP_ENCODING_24BIT_INT; + intenc[1] = v&0xff; + intenc[2] = (v>>8)&0xff; + intenc[3] = v>>16; + *enclen = 4; + } else if (v >= -2147483648 && v <= 2147483647) { + /* 32 bit integer. */ + if (v < 0) v = ((int64_t)1<<32)+v; + intenc[0] = LP_ENCODING_32BIT_INT; + intenc[1] = v&0xff; + intenc[2] = (v>>8)&0xff; + intenc[3] = (v>>16)&0xff; + intenc[4] = v>>24; + *enclen = 5; + } else { + /* 64 bit integer. */ + uint64_t uv = v; + intenc[0] = LP_ENCODING_64BIT_INT; + intenc[1] = uv&0xff; + intenc[2] = (uv>>8)&0xff; + intenc[3] = (uv>>16)&0xff; + intenc[4] = (uv>>24)&0xff; + intenc[5] = (uv>>32)&0xff; + intenc[6] = (uv>>40)&0xff; + intenc[7] = (uv>>48)&0xff; + intenc[8] = uv>>56; + *enclen = 9; + } +} + /* Given an element 'ele' of size 'size', determine if the element can be * represented inside the listpack encoded as integer, and returns * LP_ENCODING_INT if so. Otherwise returns LP_ENCODING_STR if no integer @@ -257,57 +315,10 @@ unsigned char* lpShrinkToFit(unsigned char *lp) { * Regardless of the returned encoding, 'enclen' is populated by reference to * the number of bytes that the string or integer encoded element will require * in order to be represented. */ -int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, uint64_t *enclen) { +static inline int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, uint64_t *enclen) { int64_t v; if (lpStringToInt64((const char*)ele, size, &v)) { - if (v >= 0 && v <= 127) { - /* Single byte 0-127 integer. */ - intenc[0] = v; - *enclen = 1; - } else if (v >= -4096 && v <= 4095) { - /* 13 bit integer. */ - if (v < 0) v = ((int64_t)1<<13)+v; - intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT; - intenc[1] = v&0xff; - *enclen = 2; - } else if (v >= -32768 && v <= 32767) { - /* 16 bit integer. */ - if (v < 0) v = ((int64_t)1<<16)+v; - intenc[0] = LP_ENCODING_16BIT_INT; - intenc[1] = v&0xff; - intenc[2] = v>>8; - *enclen = 3; - } else if (v >= -8388608 && v <= 8388607) { - /* 24 bit integer. */ - if (v < 0) v = ((int64_t)1<<24)+v; - intenc[0] = LP_ENCODING_24BIT_INT; - intenc[1] = v&0xff; - intenc[2] = (v>>8)&0xff; - intenc[3] = v>>16; - *enclen = 4; - } else if (v >= -2147483648 && v <= 2147483647) { - /* 32 bit integer. */ - if (v < 0) v = ((int64_t)1<<32)+v; - intenc[0] = LP_ENCODING_32BIT_INT; - intenc[1] = v&0xff; - intenc[2] = (v>>8)&0xff; - intenc[3] = (v>>16)&0xff; - intenc[4] = v>>24; - *enclen = 5; - } else { - /* 64 bit integer. */ - uint64_t uv = v; - intenc[0] = LP_ENCODING_64BIT_INT; - intenc[1] = uv&0xff; - intenc[2] = (uv>>8)&0xff; - intenc[3] = (uv>>16)&0xff; - intenc[4] = (uv>>24)&0xff; - intenc[5] = (uv>>32)&0xff; - intenc[6] = (uv>>40)&0xff; - intenc[7] = (uv>>48)&0xff; - intenc[8] = uv>>56; - *enclen = 9; - } + lpEncodeIntegerGetType(v, intenc, enclen); return LP_ENCODING_INT; } else { if (size < 64) *enclen = 1+size; @@ -322,7 +333,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui * The function returns the number of bytes used to encode it, from * 1 to 5. If 'buf' is NULL the function just returns the number of bytes * needed in order to encode the backlen. */ -unsigned long lpEncodeBacklen(unsigned char *buf, uint64_t l) { +static inline unsigned long lpEncodeBacklen(unsigned char *buf, uint64_t l) { if (l <= 127) { if (buf) buf[0] = l; return 1; @@ -361,7 +372,7 @@ unsigned long lpEncodeBacklen(unsigned char *buf, uint64_t l) { /* Decode the backlen and returns it. If the encoding looks invalid (more than * 5 bytes are used), UINT64_MAX is returned to report the problem. */ -uint64_t lpDecodeBacklen(unsigned char *p) { +static inline uint64_t lpDecodeBacklen(unsigned char *p) { uint64_t val = 0; uint64_t shift = 0; do { @@ -378,7 +389,7 @@ uint64_t lpDecodeBacklen(unsigned char *p) { * buffer 's'. The function should be called with 'buf' having always enough * space for encoding the string. This is done by calling lpEncodeGetType() * before calling this function. */ -void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) { +static inline void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) { if (len < 64) { buf[0] = len | LP_ENCODING_6BIT_STR; memcpy(buf+1,s,len); @@ -403,7 +414,7 @@ void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) { * str), so should only be called when we know 'p' was already validated by * lpCurrentEncodedSizeBytes or ASSERT_INTEGRITY_LEN (possibly since 'p' is * a return value of another function that validated its return. */ -uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) { +static inline uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) { if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1; if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1+LP_ENCODING_6BIT_STR_LEN(p); if (LP_ENCODING_IS_13BIT_INT(p[0])) return 2; @@ -421,7 +432,7 @@ uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) { * This includes just the encoding byte, and the bytes needed to encode the length * of the element (excluding the element data itself) * If the element encoding is wrong then 0 is returned. */ -uint32_t lpCurrentEncodedSizeBytes(unsigned char *p) { +static inline uint32_t lpCurrentEncodedSizeBytes(unsigned char *p) { if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1; if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1; if (LP_ENCODING_IS_13BIT_INT(p[0])) return 1; @@ -492,7 +503,7 @@ unsigned char *lpLast(unsigned char *lp) { * needed. As a side effect of calling this function, the listpack header * could be modified, because if the count is found to be already within * the 'numele' header field range, the new value is set. */ -uint32_t lpLength(unsigned char *lp) { +unsigned long lpLength(unsigned char *lp) { uint32_t numele = lpGetNumElements(lp); if (numele != LP_HDR_NUMELE_UNKNOWN) return numele; @@ -534,6 +545,10 @@ uint32_t lpLength(unsigned char *lp) { * by the function, than to pass a buffer (and convert it back to a number) * is of course useless. * + * If 'entry_size' is not NULL, *entry_size is set to the entry length of the + * listpack element pointed by 'p'. This includes the encoding bytes, length + * bytes, the element data itself, and the backlen bytes. + * * If the function is called against a badly encoded ziplist, so that there * is no valid way to parse it, the function returns like if there was an * integer encoded with value 12345678900000000 + , this may @@ -545,7 +560,7 @@ uint32_t lpLength(unsigned char *lp) { * assumed to be valid, so that would be a very high API cost. However a function * in order to check the integrity of the listpack at load time is provided, * check lpIsValid(). */ -unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { +static inline unsigned char *lpGetWithSize(unsigned char *p, int64_t *count, unsigned char *intbuf, uint64_t *entry_size) { int64_t val; uint64_t uval, negstart, negmax; @@ -554,24 +569,29 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { negstart = UINT64_MAX; /* 7 bit ints are always positive. */ negmax = 0; uval = p[0] & 0x7f; + if (entry_size) *entry_size = LP_ENCODING_7BIT_UINT_ENTRY_SIZE; } else if (LP_ENCODING_IS_6BIT_STR(p[0])) { *count = LP_ENCODING_6BIT_STR_LEN(p); + if (entry_size) *entry_size = 1 + *count + lpEncodeBacklen(NULL, *count + 1); return p+1; } else if (LP_ENCODING_IS_13BIT_INT(p[0])) { uval = ((p[0]&0x1f)<<8) | p[1]; negstart = (uint64_t)1<<12; negmax = 8191; + if (entry_size) *entry_size = LP_ENCODING_13BIT_INT_ENTRY_SIZE; } else if (LP_ENCODING_IS_16BIT_INT(p[0])) { uval = (uint64_t)p[1] | (uint64_t)p[2]<<8; negstart = (uint64_t)1<<15; negmax = UINT16_MAX; + if (entry_size) *entry_size = LP_ENCODING_16BIT_INT_ENTRY_SIZE; } else if (LP_ENCODING_IS_24BIT_INT(p[0])) { uval = (uint64_t)p[1] | (uint64_t)p[2]<<8 | (uint64_t)p[3]<<16; negstart = (uint64_t)1<<23; negmax = UINT32_MAX>>8; + if (entry_size) *entry_size = LP_ENCODING_24BIT_INT_ENTRY_SIZE; } else if (LP_ENCODING_IS_32BIT_INT(p[0])) { uval = (uint64_t)p[1] | (uint64_t)p[2]<<8 | @@ -579,6 +599,7 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { (uint64_t)p[4]<<24; negstart = (uint64_t)1<<31; negmax = UINT32_MAX; + if (entry_size) *entry_size = LP_ENCODING_32BIT_INT_ENTRY_SIZE; } else if (LP_ENCODING_IS_64BIT_INT(p[0])) { uval = (uint64_t)p[1] | (uint64_t)p[2]<<8 | @@ -590,11 +611,14 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { (uint64_t)p[8]<<56; negstart = (uint64_t)1<<63; negmax = UINT64_MAX; + if (entry_size) *entry_size = LP_ENCODING_64BIT_INT_ENTRY_SIZE; } else if (LP_ENCODING_IS_12BIT_STR(p[0])) { *count = LP_ENCODING_12BIT_STR_LEN(p); + if (entry_size) *entry_size = 2 + *count + lpEncodeBacklen(NULL, *count + 2); return p+2; } else if (LP_ENCODING_IS_32BIT_STR(p[0])) { *count = LP_ENCODING_32BIT_STR_LEN(p); + if (entry_size) *entry_size = 5 + *count + lpEncodeBacklen(NULL, *count + 5); return p+5; } else { uval = 12345678900000000ULL + p[0]; @@ -626,17 +650,105 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { } } -/* Insert, delete or replace the specified element 'ele' of length 'len' at - * the specified position 'p', with 'p' being a listpack element pointer - * obtained with lpFirst(), lpLast(), lpNext(), lpPrev() or lpSeek(). +unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { + return lpGetWithSize(p, count, intbuf, NULL); +} + +/* This is just a wrapper to lpGet() that is able to get entry value directly. + * When the function returns NULL, it populates the integer value by reference in 'lval'. + * Otherwise if the element is encoded as a string a pointer to the string (pointing + * inside the listpack itself) is returned, and 'slen' is set to the length of the + * string. */ +unsigned char *lpGetValue(unsigned char *p, unsigned int *slen, long long *lval) { + unsigned char *vstr; + int64_t ele_len; + + vstr = lpGet(p, &ele_len, NULL); + if (vstr) { + *slen = ele_len; + } else { + *lval = ele_len; + } + return vstr; +} + +/* Find pointer to the entry equal to the specified entry. Skip 'skip' entries + * between every comparison. Returns NULL when the field could not be found. */ +unsigned char *lpFind(unsigned char *lp, unsigned char *p, unsigned char *s, + uint32_t slen, unsigned int skip) { + int skipcnt = 0; + unsigned char vencoding = 0; + unsigned char *value; + int64_t ll, vll; + uint64_t entry_size = 123456789; /* initialized to avoid warning. */ + uint32_t lp_bytes = lpBytes(lp); + + assert(p); + while (p) { + if (skipcnt == 0) { + value = lpGetWithSize(p, &ll, NULL, &entry_size); + if (value) { + if (slen == ll && memcmp(value, s, slen) == 0) { + return p; + } + } else { + /* Find out if the searched field can be encoded. Note that + * we do it only the first time, once done vencoding is set + * to non-zero and vll is set to the integer value. */ + if (vencoding == 0) { + /* If the entry can be encoded as integer we set it to + * 1, else set it to UCHAR_MAX, so that we don't retry + * again the next time. */ + if (slen >= 32 || slen == 0 || !lpStringToInt64((const char*)s, slen, &vll)) { + vencoding = UCHAR_MAX; + } else { + vencoding = 1; + } + } + + /* Compare current entry with specified entry, do it only + * if vencoding != UCHAR_MAX because if there is no encoding + * possible for the field it can't be a valid integer. */ + if (vencoding != UCHAR_MAX && ll == vll) { + return p; + } + } + + /* Reset skip count */ + skipcnt = skip; + p += entry_size; + } else { + /* Skip entry */ + skipcnt--; + + /* Move to next entry, avoid use `lpNext` due to `ASSERT_INTEGRITY` in + * `lpNext` will call `lpBytes`, will cause performance degradation */ + p = lpSkip(p); + } + + assert(p >= lp + LP_HDR_SIZE && p < lp + lp_bytes); + if (p[0] == LP_EOF) break; + } + + return NULL; +} + +/* Insert, delete or replace the specified string element 'elestr' of length + * 'size' or integer element 'eleint' at the specified position 'p', with 'p' + * being a listpack element pointer obtained with lpFirst(), lpLast(), lpNext(), + * lpPrev() or lpSeek(). * * The element is inserted before, after, or replaces the element pointed * by 'p' depending on the 'where' argument, that can be LP_BEFORE, LP_AFTER * or LP_REPLACE. - * - * If 'ele' is set to NULL, the function removes the element pointed by 'p' - * instead of inserting one. - * + * + * If both 'elestr' and `eleint` are NULL, the function removes the element + * pointed by 'p' instead of inserting one. + * If `eleint` is non-NULL, 'size' is the length of 'eleint', the function insert + * or replace with a 64 bit integer, which is stored in the 'eleint' buffer. + * If 'elestr` is non-NULL, 'size' is the length of 'elestr', the function insert + * or replace with a string, which is stored in the 'elestr' buffer. + * * Returns NULL on out of memory or when the listpack total length would exceed * the max allowed size of 2^32-1, otherwise the new pointer to the listpack * holding the new element is returned (and the old pointer passed is no longer @@ -646,19 +758,22 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { * to the address of the element just added, so that it will be possible to * continue an interaction with lpNext() and lpPrev(). * - * For deletion operations ('ele' set to NULL) 'newp' is set to the next - * element, on the right of the deleted one, or to NULL if the deleted element - * was the last one. */ -unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, unsigned char *p, int where, unsigned char **newp) { + * For deletion operations (both 'elestr' and 'eleint' set to NULL) 'newp' is + * set to the next element, on the right of the deleted one, or to NULL if the + * deleted element was the last one. */ +unsigned char *lpInsert(unsigned char *lp, unsigned char *elestr, unsigned char *eleint, + uint32_t size, unsigned char *p, int where, unsigned char **newp) +{ unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; unsigned char backlen[LP_MAX_BACKLEN_SIZE]; uint64_t enclen; /* The length of the encoded element. */ + int delete = (elestr == NULL && eleint == NULL); - /* An element pointer set to NULL means deletion, which is conceptually - * replacing the element with a zero-length element. So whatever we - * get passed as 'where', set it to LP_REPLACE. */ - if (ele == NULL) where = LP_REPLACE; + /* when deletion, it is conceptually replacing the element with a + * zero-length element. So whatever we get passed as 'where', set + * it to LP_REPLACE. */ + if (delete) where = LP_REPLACE; /* If we need to insert after the current element, we just jump to the * next element (that could be the EOF one) and handle the case of @@ -674,17 +789,21 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un * address again after a reallocation. */ unsigned long poff = p-lp; - /* Calling lpEncodeGetType() results into the encoded version of the - * element to be stored into 'intenc' in case it is representable as - * an integer: in that case, the function returns LP_ENCODING_INT. - * Otherwise if LP_ENCODING_STR is returned, we'll have to call - * lpEncodeString() to actually write the encoded string on place later. - * - * Whatever the returned encoding is, 'enclen' is populated with the - * length of the encoded element. */ int enctype; - if (ele) { - enctype = lpEncodeGetType(ele,size,intenc,&enclen); + if (elestr) { + /* Calling lpEncodeGetType() results into the encoded version of the + * element to be stored into 'intenc' in case it is representable as + * an integer: in that case, the function returns LP_ENCODING_INT. + * Otherwise if LP_ENCODING_STR is returned, we'll have to call + * lpEncodeString() to actually write the encoded string on place later. + * + * Whatever the returned encoding is, 'enclen' is populated with the + * length of the encoded element. */ + enctype = lpEncodeGetType(elestr,size,intenc,&enclen); + if (enctype == LP_ENCODING_INT) eleint = intenc; + } else if (eleint) { + enctype = LP_ENCODING_INT; + enclen = size; /* 'size' is the length of the encoded integer element. */ } else { enctype = -1; enclen = 0; @@ -693,7 +812,7 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un /* We need to also encode the backward-parsable length of the element * and append it to the end: this allows to traverse the listpack from * the end to the start. */ - unsigned long backlen_size = ele ? lpEncodeBacklen(backlen,enclen) : 0; + unsigned long backlen_size = (!delete) ? lpEncodeBacklen(backlen,enclen) : 0; uint64_t old_listpack_bytes = lpGetTotalBytes(lp); uint32_t replaced_len = 0; if (where == LP_REPLACE) { @@ -743,13 +862,13 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un *newp = dst; /* In case of deletion, set 'newp' to NULL if the next element is * the EOF element. */ - if (!ele && dst[0] == LP_EOF) *newp = NULL; + if (delete && dst[0] == LP_EOF) *newp = NULL; } - if (ele) { + if (!delete) { if (enctype == LP_ENCODING_INT) { - memcpy(dst,intenc,enclen); + memcpy(dst,eleint,enclen); } else { - lpEncodeString(dst,ele,size); + lpEncodeString(dst,elestr,size); } dst += enclen; memcpy(dst,backlen,backlen_size); @@ -757,10 +876,10 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un } /* Update header. */ - if (where != LP_REPLACE || ele == NULL) { + if (where != LP_REPLACE || delete) { uint32_t num_elements = lpGetNumElements(lp); if (num_elements != LP_HDR_NUMELE_UNKNOWN) { - if (ele) + if (!delete) lpSetNumElements(lp,num_elements+1); else lpSetNumElements(lp,num_elements-1); @@ -790,13 +909,59 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un return lp; } +/* This is just a wrapper for lpInsert() to directly use a 64 bit integer + * instead of a string. */ +unsigned char *lpInsertInteger(unsigned char *lp, long long lval, unsigned char *p, int where, unsigned char **newp) { + uint64_t enclen; /* The length of the encoded element. */ + unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; + + lpEncodeIntegerGetType(lval, intenc, &enclen); + return lpInsert(lp, NULL, intenc, enclen, p, where, newp); +} + +/* Append the specified element 's' of length 'slen' at the head of the listpack. */ +unsigned char *lpPrepend(unsigned char *lp, unsigned char *s, uint32_t slen) { + unsigned char *p = lpFirst(lp); + if (!p) return lpAppend(lp, s, slen); + return lpInsert(lp, s, NULL, slen, p, LP_BEFORE, NULL); +} + +/* Append the specified integer element 'lval' at the head of the listpack. */ +unsigned char *lpPrependInteger(unsigned char *lp, long long lval) { + unsigned char *p = lpFirst(lp); + if (!p) return lpAppendInteger(lp, lval); + return lpInsertInteger(lp, lval, p, LP_BEFORE, NULL); +} + /* Append the specified element 'ele' of length 'len' at the end of the * listpack. It is implemented in terms of lpInsert(), so the return value is * the same as lpInsert(). */ unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) { uint64_t listpack_bytes = lpGetTotalBytes(lp); unsigned char *eofptr = lp + listpack_bytes - 1; - return lpInsert(lp,ele,size,eofptr,LP_BEFORE,NULL); + return lpInsert(lp,ele,NULL,size,eofptr,LP_BEFORE,NULL); +} + +/* Append the specified integer element 'lval' at the end of the listpack. */ +unsigned char *lpAppendInteger(unsigned char *lp, long long lval) { + uint64_t listpack_bytes = lpGetTotalBytes(lp); + unsigned char *eofptr = lp + listpack_bytes - 1; + return lpInsertInteger(lp, lval, eofptr, LP_BEFORE, NULL); +} + +/* This is just a wrapper for lpInsert() to directly use a string to replace + * the current element. The function returns the new listpack as return + * value, and also updates the current cursor by updating '*p'. */ +unsigned char *lpReplace(unsigned char *lp, unsigned char **p, unsigned char *s, uint32_t slen) { + return lpInsert(lp, s, NULL, slen, *p, LP_REPLACE, p); +} + +/* This is just a wrapper for lpInsertInteger() to directly use a 64 bit integer + * instead of a string to replace the current element. The function returns + * the new listpack as return value, and also updates the current cursor + * by updating '*p'. */ +unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **p, long long lval) { + return lpInsertInteger(lp, lval, *p, LP_REPLACE, p); } /* Remove the element pointed by 'p', and return the resulting listpack. @@ -804,11 +969,11 @@ unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) { * deleted one) is returned by reference. If the deleted element was the * last one, '*newp' is set to NULL. */ unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp) { - return lpInsert(lp,NULL,0,p,LP_REPLACE,newp); + return lpInsert(lp,NULL,NULL,0,p,LP_REPLACE,newp); } /* Return the total number of bytes the listpack is composed of. */ -uint32_t lpBytes(unsigned char *lp) { +size_t lpBytes(unsigned char *lp) { return lpGetTotalBytes(lp); } @@ -927,7 +1092,8 @@ static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigne /* Validate the integrity of the data structure. * when `deep` is 0, only the integrity of the header is validated. * when `deep` is 1, we scan all the entries one by one. */ -int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ +int lpValidateIntegrity(unsigned char *lp, size_t size, int deep, + listpackValidateEntryCB entry_cb, void *cb_userdata) { /* Check that we can actually read the header. (and EOF) */ if (size < LP_HDR_SIZE + 1) return 0; @@ -948,8 +1114,17 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ uint32_t count = 0; unsigned char *p = lp + LP_HDR_SIZE; while(p && p[0] != LP_EOF) { + unsigned char *prev = p; + + /* Validate this entry and move to the next entry in advance + * to avoid callback crash due to corrupt listpack. */ if (!lpValidateNext(lp, &p, bytes)) return 0; + + /* Optionally let the caller validate the entry too. */ + if (entry_cb && !entry_cb(prev, cb_userdata)) + return 0; + count++; } @@ -960,3 +1135,826 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ return 1; } + +unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen) { + unsigned char buf[LP_INTBUF_SIZE]; + unsigned char *value; + int64_t sz; + + if (p[0] == LP_EOF) return 0; + value = lpGet(p, &sz, buf); + return (slen == sz) && memcmp(value,s,slen) == 0; +} + +/* uint compare for qsort */ +static int uintCompare(const void *a, const void *b) { + return (*(unsigned int *) a - *(unsigned int *) b); +} + +/* Helper method to store a string into from val or lval into dest */ +static inline void lpSaveValue(unsigned char *val, unsigned int len, int64_t lval, listpackEntry *dest) { + dest->sval = val; + dest->slen = len; + dest->lval = lval; +} + +/* Randomly select a pair of key and value. + * total_count is a pre-computed length/2 of the listpack (to avoid calls to lpLength) + * 'key' and 'val' are used to store the result key value pair. + * 'val' can be NULL if the value is not needed. */ +void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val) { + unsigned char *p; + + /* Avoid div by zero on corrupt listpack */ + assert(total_count); + + /* Generate even numbers, because listpack saved K-V pair */ + int r = (rand() % total_count) * 2; + assert((p = lpSeek(lp, r))); + key->sval = lpGetValue(p, &(key->slen), &(key->lval)); + + if (!val) + return; + assert((p = lpNext(lp, p))); + val->sval = lpGetValue(p, &(val->slen), &(val->lval)); +} + +/* Randomly select count of key value pairs and store into 'keys' and + * 'vals' args. The order of the picked entries is random, and the selections + * are non-unique (repetitions are possible). + * The 'vals' arg can be NULL in which case we skip these. */ +void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) { + unsigned char *p, *key, *value; + unsigned int klen = 0, vlen = 0; + long long klval = 0, vlval = 0; + + /* Notice: the index member must be first due to the use in uintCompare */ + typedef struct { + unsigned int index; + unsigned int order; + } rand_pick; + rand_pick *picks = zmalloc(sizeof(rand_pick)*count); + unsigned int total_size = lpLength(lp)/2; + + /* Avoid div by zero on corrupt listpack */ + assert(total_size); + + /* create a pool of random indexes (some may be duplicate). */ + for (unsigned int i = 0; i < count; i++) { + picks[i].index = (rand() % total_size) * 2; /* Generate even indexes */ + /* keep track of the order we picked them */ + picks[i].order = i; + } + + /* sort by indexes. */ + qsort(picks, count, sizeof(rand_pick), uintCompare); + + /* fetch the elements form the listpack into a output array respecting the original order. */ + unsigned int lpindex = picks[0].index, pickindex = 0; + p = lpSeek(lp, lpindex); + while (p && pickindex < count) { + key = lpGetValue(p, &klen, &klval); + assert((p = lpNext(lp, p))); + value = lpGetValue(p, &vlen, &vlval); + while (pickindex < count && lpindex == picks[pickindex].index) { + int storeorder = picks[pickindex].order; + lpSaveValue(key, klen, klval, &keys[storeorder]); + if (vals) + lpSaveValue(value, vlen, vlval, &vals[storeorder]); + pickindex++; + } + lpindex += 2; + p = lpNext(lp, p); + } + + zfree(picks); +} + +/* Randomly select count of key value pairs and store into 'keys' and + * 'vals' args. The selections are unique (no repetitions), and the order of + * the picked entries is NOT-random. + * The 'vals' arg can be NULL in which case we skip these. + * The return value is the number of items picked which can be lower than the + * requested count if the listpack doesn't hold enough pairs. */ +unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) { + unsigned char *p, *key; + unsigned int klen = 0; + long long klval = 0; + unsigned int total_size = lpLength(lp)/2; + unsigned int index = 0; + if (count > total_size) + count = total_size; + + /* To only iterate once, every time we try to pick a member, the probability + * we pick it is the quotient of the count left we want to pick and the + * count still we haven't visited in the dict, this way, we could make every + * member be equally picked.*/ + p = lpFirst(lp); + unsigned int picked = 0, remaining = count; + while (picked < count && p) { + double randomDouble = ((double)rand()) / RAND_MAX; + double threshold = ((double)remaining) / (total_size - index); + if (randomDouble <= threshold) { + key = lpGetValue(p, &klen, &klval); + lpSaveValue(key, klen, klval, &keys[picked]); + assert((p = lpNext(lp, p))); + if (vals) { + key = lpGetValue(p, &klen, &klval); + lpSaveValue(key, klen, klval, &vals[picked]); + } + remaining--; + picked++; + } else { + assert((p = lpNext(lp, p))); + } + p = lpNext(lp, p); + index++; + } + return picked; +} + +#ifdef REDIS_TEST + +#include +#include "adlist.h" +#include "sds.h" + +#define UNUSED(x) (void)(x) +#define TEST(name) printf("test — %s\n", name); + +char *mixlist[] = {"hello", "foo", "quux", "1024"}; +char *intlist[] = {"4294967296", "-100", "100", "128000", + "non integer", "much much longer non integer"}; + +static unsigned char *createList() { + unsigned char *lp = lpNew(0); + lp = lpAppend(lp, (unsigned char*)mixlist[1], strlen(mixlist[1])); + lp = lpAppend(lp, (unsigned char*)mixlist[2], strlen(mixlist[2])); + lp = lpPrepend(lp, (unsigned char*)mixlist[0], strlen(mixlist[0])); + lp = lpAppend(lp, (unsigned char*)mixlist[3], strlen(mixlist[3])); + return lp; +} + +static unsigned char *createIntList() { + unsigned char *lp = lpNew(0); + lp = lpAppend(lp, (unsigned char*)intlist[2], strlen(intlist[2])); + lp = lpAppend(lp, (unsigned char*)intlist[3], strlen(intlist[3])); + lp = lpPrepend(lp, (unsigned char*)intlist[1], strlen(intlist[1])); + lp = lpPrepend(lp, (unsigned char*)intlist[0], strlen(intlist[0])); + lp = lpAppend(lp, (unsigned char*)intlist[4], strlen(intlist[4])); + lp = lpAppend(lp, (unsigned char*)intlist[5], strlen(intlist[5])); + return lp; +} + +static long long usec(void) { + struct timeval tv; + gettimeofday(&tv, NULL); + return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; +} + +static void stress(int pos, int num, int maxsize, int dnum) { + int i, j, k; + unsigned char *lp; + char posstr[2][5] = { "HEAD", "TAIL" }; + long long start; + for (i = 0; i < maxsize; i+=dnum) { + lp = lpNew(0); + for (j = 0; j < i; j++) { + lp = lpAppend(lp, (unsigned char*)"quux", 4); + } + + /* Do num times a push+pop from pos */ + start = usec(); + for (k = 0; k < num; k++) { + if (pos == 0) { + lp = lpPrepend(lp, (unsigned char*)"quux", 4); + } else { + lp = lpAppend(lp, (unsigned char*)"quux", 4); + + } + lp = lpDelete(lp, lpFirst(lp), NULL); + } + printf("List size: %8d, bytes: %8zu, %dx push+pop (%s): %6lld usec\n", + i, lpBytes(lp), num, posstr[pos], usec()-start); + lpFree(lp); + } +} + +static unsigned char *pop(unsigned char *lp, int where) { + unsigned char *p, *vstr; + int64_t vlen; + + p = lpSeek(lp, where == 0 ? 0 : -1); + vstr = lpGet(p, &vlen, NULL); + if (where == 0) + printf("Pop head: "); + else + printf("Pop tail: "); + + if (vstr) { + if (vlen && fwrite(vstr, vlen, 1, stdout) == 0) perror("fwrite"); + } else { + printf("%lld", (long long)vlen); + } + + printf("\n"); + return lpDelete(lp, p, &p); +} + +static int randstring(char *target, unsigned int min, unsigned int max) { + int p = 0; + int len = min+rand()%(max-min+1); + int minval, maxval; + switch(rand() % 3) { + case 0: + minval = 0; + maxval = 255; + break; + case 1: + minval = 48; + maxval = 122; + break; + case 2: + minval = 48; + maxval = 52; + break; + default: + assert(NULL); + } + + while(p < len) + target[p++] = minval+rand()%(maxval-minval+1); + return len; +} + +static void verifyEntry(unsigned char *p, unsigned char *s, size_t slen) { + assert(lpCompare(p, s, slen)); +} + +static int lpValidation(unsigned char *p, void *userdata) { + UNUSED(p); + + int ret; + long *count = userdata; + ret = lpCompare(p, (unsigned char *)mixlist[*count], strlen(mixlist[*count])); + (*count)++; + return ret; +} + +int listpackTest(int argc, char *argv[], int accurate) { + UNUSED(argc); + UNUSED(argv); + UNUSED(accurate); + + int i; + unsigned char *lp, *p, *vstr; + int64_t vlen; + unsigned char intbuf[LP_INTBUF_SIZE]; + + TEST("Create int list") { + lp = createIntList(); + assert(lpLength(lp) == 6); + lpFree(lp); + } + + TEST("Create list") { + lp = createList(); + assert(lpLength(lp) == 4); + lpFree(lp); + } + + TEST("Test lpPrepend") { + lp = lpNew(0); + lp = lpPrepend(lp, (unsigned char*)"abc", 3); + lp = lpPrepend(lp, (unsigned char*)"1024", 4); + verifyEntry(lpSeek(lp, 0), (unsigned char*)"1024", 4); + verifyEntry(lpSeek(lp, 1), (unsigned char*)"abc", 3); + lpFree(lp); + } + + TEST("Test lpPrependInteger") { + lp = lpNew(0); + lp = lpPrependInteger(lp, 127); + lp = lpPrependInteger(lp, 4095); + lp = lpPrependInteger(lp, 32767); + lp = lpPrependInteger(lp, 8388607); + lp = lpPrependInteger(lp, 2147483647); + lp = lpPrependInteger(lp, 9223372036854775807); + verifyEntry(lpSeek(lp, 0), (unsigned char*)"9223372036854775807", 19); + verifyEntry(lpSeek(lp, -1), (unsigned char*)"127", 3); + lpFree(lp); + } + + TEST("Get element at index") { + lp = createList(); + verifyEntry(lpSeek(lp, 0), (unsigned char*)"hello", 5); + verifyEntry(lpSeek(lp, 3), (unsigned char*)"1024", 4); + verifyEntry(lpSeek(lp, -1), (unsigned char*)"1024", 4); + verifyEntry(lpSeek(lp, -4), (unsigned char*)"hello", 5); + assert(lpSeek(lp, 4) == NULL); + assert(lpSeek(lp, -5) == NULL); + lpFree(lp); + } + + TEST("Pop list") { + lp = createList(); + lp = pop(lp, 1); + lp = pop(lp, 0); + lp = pop(lp, 1); + lp = pop(lp, 1); + lpFree(lp); + } + + TEST("Get element at index") { + lp = createList(); + verifyEntry(lpSeek(lp, 0), (unsigned char*)"hello", 5); + verifyEntry(lpSeek(lp, 3), (unsigned char*)"1024", 4); + verifyEntry(lpSeek(lp, -1), (unsigned char*)"1024", 4); + verifyEntry(lpSeek(lp, -4), (unsigned char*)"hello", 5); + assert(lpSeek(lp, 4) == NULL); + assert(lpSeek(lp, -5) == NULL); + lpFree(lp); + } + + TEST("Iterate list from 0 to end") { + lp = createList(); + p = lpFirst(lp); + i = 0; + while (p) { + verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i])); + p = lpNext(lp, p); + i++; + } + lpFree(lp); + } + + TEST("Iterate list from 1 to end") { + lp = createList(); + i = 1; + p = lpSeek(lp, i); + while (p) { + verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i])); + p = lpNext(lp, p); + i++; + } + lpFree(lp); + } + + TEST("Iterate list from 2 to end") { + lp = createList(); + i = 2; + p = lpSeek(lp, i); + while (p) { + verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i])); + p = lpNext(lp, p); + i++; + } + lpFree(lp); + } + + TEST("Iterate from back to front") { + lp = createList(); + p = lpLast(lp); + i = 3; + while (p) { + verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i])); + p = lpPrev(lp, p); + i--; + } + lpFree(lp); + } + + TEST("Iterate from back to front, deleting all items") { + lp = createList(); + p = lpLast(lp); + i = 3; + while ((p = lpLast(lp))) { + verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i])); + lp = lpDelete(lp, p, &p); + assert(p == NULL); + i--; + } + lpFree(lp); + } + + TEST("Delete foo while iterating") { + lp = createList(); + p = lpFirst(lp); + while (p) { + if (lpCompare(p, (unsigned char*)"foo", 3)) { + lp = lpDelete(lp, p, &p); + } else { + p = lpNext(lp, p); + } + } + lpFree(lp); + } + + TEST("Replace with same size") { + lp = createList(); /* "hello", "foo", "quux", "1024" */ + unsigned char *orig_lp = lp; + p = lpSeek(lp, 0); + lp = lpReplace(lp, &p, (unsigned char*)"zoink", 5); + p = lpSeek(lp, 3); + lp = lpReplace(lp, &p, (unsigned char*)"y", 1); + p = lpSeek(lp, 1); + lp = lpReplace(lp, &p, (unsigned char*)"65536", 5); + p = lpSeek(lp, 0); + assert(!memcmp((char*)p, + "\x85zoink\x06" + "\xf2\x00\x00\x01\x04" /* 65536 as int24 */ + "\x84quux\05" "\x81y\x02" "\xff", + 22)); + assert(lp == orig_lp); /* no reallocations have happened */ + lpFree(lp); + } + + TEST("Replace with different size") { + lp = createList(); /* "hello", "foo", "quux", "1024" */ + p = lpSeek(lp, 1); + lp = lpReplace(lp, &p, (unsigned char*)"squirrel", 8); + p = lpSeek(lp, 0); + assert(!strncmp((char*)p, + "\x85hello\x06" "\x88squirrel\x09" "\x84quux\x05" + "\xc4\x00\x02" "\xff", + 27)); + lpFree(lp); + } + + TEST("Regression test for >255 byte strings") { + char v1[257] = {0}, v2[257] = {0}; + memset(v1,'x',256); + memset(v2,'y',256); + lp = lpNew(0); + lp = lpAppend(lp, (unsigned char*)v1 ,strlen(v1)); + lp = lpAppend(lp, (unsigned char*)v2 ,strlen(v2)); + + /* Pop values again and compare their value. */ + p = lpFirst(lp); + vstr = lpGet(p, &vlen, NULL); + assert(strncmp(v1, (char*)vstr, vlen) == 0); + p = lpSeek(lp, 1); + vstr = lpGet(p, &vlen, NULL); + assert(strncmp(v2, (char*)vstr, vlen) == 0); + lpFree(lp); + } + + TEST("Create long list and check indices") { + lp = lpNew(0); + char buf[32]; + int i,len; + for (i = 0; i < 1000; i++) { + len = sprintf(buf, "%d", i); + lp = lpAppend(lp, (unsigned char*)buf, len); + } + for (i = 0; i < 1000; i++) { + p = lpSeek(lp, i); + vstr = lpGet(p, &vlen, NULL); + assert(i == vlen); + + p = lpSeek(lp, -i-1); + vstr = lpGet(p, &vlen, NULL); + assert(999-i == vlen); + } + lpFree(lp); + } + + TEST("Compare strings with listpack entries") { + lp = createList(); + p = lpSeek(lp,0); + assert(lpCompare(p,(unsigned char*)"hello",5)); + assert(!lpCompare(p,(unsigned char*)"hella",5)); + + p = lpSeek(lp,3); + assert(lpCompare(p,(unsigned char*)"1024",4)); + assert(!lpCompare(p,(unsigned char*)"1025",4)); + lpFree(lp); + } + + TEST("Random pair with one element") { + listpackEntry key, val; + unsigned char *lp = lpNew(0); + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lpRandomPair(lp, 1, &key, &val); + assert(memcmp(key.sval, "abc", key.slen) == 0); + assert(val.lval == 123); + lpFree(lp); + } + + TEST("Random pair with many elements") { + listpackEntry key, val; + unsigned char *lp = lpNew(0); + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lp = lpAppend(lp, (unsigned char*)"456", 3); + lp = lpAppend(lp, (unsigned char*)"def", 3); + lpRandomPair(lp, 2, &key, &val); + if (key.sval) { + assert(!memcmp(key.sval, "abc", key.slen)); + assert(key.slen == 3); + assert(val.lval == 123); + } + if (!key.sval) { + assert(key.lval == 456); + assert(!memcmp(val.sval, "def", val.slen)); + } + lpFree(lp); + } + + TEST("Random pairs with one element") { + int count = 5; + unsigned char *lp = lpNew(0); + listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count); + listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count); + + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lpRandomPairs(lp, count, keys, vals); + assert(memcmp(keys[4].sval, "abc", keys[4].slen) == 0); + assert(vals[4].lval == 123); + zfree(keys); + zfree(vals); + lpFree(lp); + } + + TEST("Random pairs with many elements") { + int count = 5; + lp = lpNew(0); + listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count); + listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count); + + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lp = lpAppend(lp, (unsigned char*)"456", 3); + lp = lpAppend(lp, (unsigned char*)"def", 3); + lpRandomPairs(lp, count, keys, vals); + for (int i = 0; i < count; i++) { + if (keys[i].sval) { + assert(!memcmp(keys[i].sval, "abc", keys[i].slen)); + assert(keys[i].slen == 3); + assert(vals[i].lval == 123); + } + if (!keys[i].sval) { + assert(keys[i].lval == 456); + assert(!memcmp(vals[i].sval, "def", vals[i].slen)); + } + } + zfree(keys); + zfree(vals); + lpFree(lp); + } + + TEST("Random pairs unique with one element") { + unsigned picked; + int count = 5; + lp = lpNew(0); + listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count); + listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count); + + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + picked = lpRandomPairsUnique(lp, count, keys, vals); + assert(picked == 1); + assert(memcmp(keys[0].sval, "abc", keys[0].slen) == 0); + assert(vals[0].lval == 123); + zfree(keys); + zfree(vals); + lpFree(lp); + } + + TEST("Random pairs unique with many elements") { + unsigned picked; + int count = 5; + lp = lpNew(0); + listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count); + listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count); + + lp = lpAppend(lp, (unsigned char*)"abc", 3); + lp = lpAppend(lp, (unsigned char*)"123", 3); + lp = lpAppend(lp, (unsigned char*)"456", 3); + lp = lpAppend(lp, (unsigned char*)"def", 3); + picked = lpRandomPairsUnique(lp, count, keys, vals); + assert(picked == 2); + for (int i = 0; i < 2; i++) { + if (keys[i].sval) { + assert(!memcmp(keys[i].sval, "abc", keys[i].slen)); + assert(keys[i].slen == 3); + assert(vals[i].lval == 123); + } + if (!keys[i].sval) { + assert(keys[i].lval == 456); + assert(!memcmp(vals[i].sval, "def", vals[i].slen)); + } + } + zfree(keys); + zfree(vals); + lpFree(lp); + } + + TEST("push various encodings") { + lp = lpNew(0); + + /* Push integer encode element using lpAppend */ + lp = lpAppend(lp, (unsigned char*)"127", 3); + assert(LP_ENCODING_IS_7BIT_UINT(lpLast(lp)[0])); + lp = lpAppend(lp, (unsigned char*)"4095", 4); + assert(LP_ENCODING_IS_13BIT_INT(lpLast(lp)[0])); + lp = lpAppend(lp, (unsigned char*)"32767", 5); + assert(LP_ENCODING_IS_16BIT_INT(lpLast(lp)[0])); + lp = lpAppend(lp, (unsigned char*)"8388607", 7); + assert(LP_ENCODING_IS_24BIT_INT(lpLast(lp)[0])); + lp = lpAppend(lp, (unsigned char*)"2147483647", 10); + assert(LP_ENCODING_IS_32BIT_INT(lpLast(lp)[0])); + lp = lpAppend(lp, (unsigned char*)"9223372036854775807", 19); + assert(LP_ENCODING_IS_64BIT_INT(lpLast(lp)[0])); + + /* Push integer encode element using lpAppendInteger */ + lp = lpAppendInteger(lp, 127); + assert(LP_ENCODING_IS_7BIT_UINT(lpLast(lp)[0])); + verifyEntry(lpLast(lp), (unsigned char*)"127", 3); + lp = lpAppendInteger(lp, 4095); + verifyEntry(lpLast(lp), (unsigned char*)"4095", 4); + assert(LP_ENCODING_IS_13BIT_INT(lpLast(lp)[0])); + lp = lpAppendInteger(lp, 32767); + verifyEntry(lpLast(lp), (unsigned char*)"32767", 5); + assert(LP_ENCODING_IS_16BIT_INT(lpLast(lp)[0])); + lp = lpAppendInteger(lp, 8388607); + verifyEntry(lpLast(lp), (unsigned char*)"8388607", 7); + assert(LP_ENCODING_IS_24BIT_INT(lpLast(lp)[0])); + lp = lpAppendInteger(lp, 2147483647); + verifyEntry(lpLast(lp), (unsigned char*)"2147483647", 10); + assert(LP_ENCODING_IS_32BIT_INT(lpLast(lp)[0])); + lp = lpAppendInteger(lp, 9223372036854775807); + verifyEntry(lpLast(lp), (unsigned char*)"9223372036854775807", 19); + assert(LP_ENCODING_IS_64BIT_INT(lpLast(lp)[0])); + + /* string encode */ + unsigned char *str = zmalloc(65535); + memset(str, 0, 65535); + lp = lpAppend(lp, (unsigned char*)str, 63); + assert(LP_ENCODING_IS_6BIT_STR(lpLast(lp)[0])); + lp = lpAppend(lp, (unsigned char*)str, 4095); + assert(LP_ENCODING_IS_12BIT_STR(lpLast(lp)[0])); + lp = lpAppend(lp, (unsigned char*)str, 65535); + assert(LP_ENCODING_IS_32BIT_STR(lpLast(lp)[0])); + zfree(str); + lpFree(lp); + } + + TEST("Test lpFind") { + lp = createList(); + assert(lpFind(lp, lpFirst(lp), (unsigned char*)"abc", 3, 0) == NULL); + verifyEntry(lpFind(lp, lpFirst(lp), (unsigned char*)"hello", 5, 0), (unsigned char*)"hello", 5); + verifyEntry(lpFind(lp, lpFirst(lp), (unsigned char*)"1024", 4, 0), (unsigned char*)"1024", 4); + lpFree(lp); + } + + TEST("Test lpValidateIntegrity") { + lp = createList(); + long count = 0; + assert(lpValidateIntegrity(lp, lpBytes(lp), 1, lpValidation, &count) == 1); + lpFree(lp); + } + + TEST("Stress with random payloads of different encoding") { + unsigned long long start = usec(); + int i,j,len,where; + unsigned char *p; + char buf[1024]; + int buflen; + list *ref; + listNode *refnode; + + int iteration = accurate ? 20000 : 20; + for (i = 0; i < iteration; i++) { + lp = lpNew(0); + ref = listCreate(); + listSetFreeMethod(ref,(void (*)(void*))sdsfree); + len = rand() % 256; + + /* Create lists */ + for (j = 0; j < len; j++) { + where = (rand() & 1) ? 0 : 1; + if (rand() % 2) { + buflen = randstring(buf,1,sizeof(buf)-1); + } else { + switch(rand() % 3) { + case 0: + buflen = sprintf(buf,"%lld",(0LL + rand()) >> 20); + break; + case 1: + buflen = sprintf(buf,"%lld",(0LL + rand())); + break; + case 2: + buflen = sprintf(buf,"%lld",(0LL + rand()) << 20); + break; + default: + assert(NULL); + } + } + + /* Add to listpack */ + if (where == 0) { + lp = lpPrepend(lp, (unsigned char*)buf, buflen); + } else { + lp = lpAppend(lp, (unsigned char*)buf, buflen); + } + + /* Add to reference list */ + if (where == 0) { + listAddNodeHead(ref,sdsnewlen(buf, buflen)); + } else if (where == 1) { + listAddNodeTail(ref,sdsnewlen(buf, buflen)); + } else { + assert(NULL); + } + } + + assert(listLength(ref) == lpLength(lp)); + for (j = 0; j < len; j++) { + /* Naive way to get elements, but similar to the stresser + * executed from the Tcl test suite. */ + p = lpSeek(lp,j); + refnode = listIndex(ref,j); + + vstr = lpGet(p, &vlen, intbuf); + assert(memcmp(vstr,listNodeValue(refnode),vlen) == 0); + } + lpFree(lp); + listRelease(ref); + } + printf("Done. usec=%lld\n\n", usec()-start); + } + + TEST("Stress with variable listpack size") { + unsigned long long start = usec(); + int maxsize = accurate ? 16384 : 16; + stress(0,100000,maxsize,256); + stress(1,100000,maxsize,256); + printf("Done. usec=%lld\n\n", usec()-start); + } + + /* Benchmarks */ + { + int iteration = accurate ? 100000 : 100; + lp = lpNew(0); + TEST("Benchmark lpAppend") { + unsigned long long start = usec(); + for (int i=0; icursor = 1; cursor->done = 1; ret = 0; - } else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { + } else if (o->type == OBJ_ZSET) { unsigned char *p = ziplistIndex(o->ptr,0); unsigned char *vstr; unsigned int vlen; @@ -8013,6 +8013,25 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc cursor->cursor = 1; cursor->done = 1; ret = 0; + } else if (o->type == OBJ_HASH) { + unsigned char *p = lpFirst(o->ptr); + unsigned char *vstr; + int64_t vlen; + unsigned char intbuf[LP_INTBUF_SIZE]; + while(p) { + vstr = lpGet(p,&vlen,intbuf); + robj *field = createStringObject((char*)vstr,vlen); + p = lpNext(o->ptr,p); + vstr = lpGet(p,&vlen,intbuf); + robj *value = createStringObject((char*)vstr,vlen); + fn(key, field, value, privdata); + p = lpNext(o->ptr,p); + decrRefCount(field); + decrRefCount(value); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; } errno = 0; return ret; diff --git a/src/object.c b/src/object.c index b65430784a0b..90fa282544bc 100644 --- a/src/object.c +++ b/src/object.c @@ -255,9 +255,9 @@ robj *createIntsetObject(void) { } robj *createHashObject(void) { - unsigned char *zl = ziplistNew(); + unsigned char *zl = lpNew(0); robj *o = createObject(OBJ_HASH, zl); - o->encoding = OBJ_ENCODING_ZIPLIST; + o->encoding = OBJ_ENCODING_LISTPACK; return o; } @@ -342,8 +342,8 @@ void freeHashObject(robj *o) { case OBJ_ENCODING_HT: dictRelease((dict*) o->ptr); break; - case OBJ_ENCODING_ZIPLIST: - zfree(o->ptr); + case OBJ_ENCODING_LISTPACK: + lpFree(o->ptr); break; default: serverPanic("Unknown hash encoding type"); @@ -929,6 +929,7 @@ char *strEncoding(int encoding) { case OBJ_ENCODING_HT: return "hashtable"; case OBJ_ENCODING_QUICKLIST: return "quicklist"; case OBJ_ENCODING_ZIPLIST: return "ziplist"; + case OBJ_ENCODING_LISTPACK: return "listpack"; case OBJ_ENCODING_INTSET: return "intset"; case OBJ_ENCODING_SKIPLIST: return "skiplist"; case OBJ_ENCODING_EMBSTR: return "embstr"; @@ -1038,7 +1039,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { serverPanic("Unknown sorted set encoding"); } } else if (o->type == OBJ_HASH) { - if (o->encoding == OBJ_ENCODING_ZIPLIST) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { asize = sizeof(*o)+zmalloc_size(o->ptr); } else if (o->encoding == OBJ_ENCODING_HT) { d = o->ptr; diff --git a/src/rdb.c b/src/rdb.c index 111398b4ff33..be94da90c82a 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -676,8 +676,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) { else serverPanic("Unknown sorted set encoding"); case OBJ_HASH: - if (o->encoding == OBJ_ENCODING_ZIPLIST) - return rdbSaveType(rdb,RDB_TYPE_HASH_ZIPLIST); + if (o->encoding == OBJ_ENCODING_LISTPACK) + return rdbSaveType(rdb,RDB_TYPE_HASH_LISTPACK); else if (o->encoding == OBJ_ENCODING_HT) return rdbSaveType(rdb,RDB_TYPE_HASH); else @@ -897,12 +897,11 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { } } else if (o->type == OBJ_HASH) { /* Save a hash value */ - if (o->encoding == OBJ_ENCODING_ZIPLIST) { - size_t l = ziplistBlobLen((unsigned char*)o->ptr); + if (o->encoding == OBJ_ENCODING_LISTPACK) { + size_t l = lpBytes((unsigned char*)o->ptr); if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1; nwritten += n; - } else if (o->encoding == OBJ_ENCODING_HT) { dictIterator *di = dictGetIterator(o->ptr); dictEntry *de; @@ -1703,7 +1702,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { o = createHashObject(); /* Too many entries? Use a hash table right from the start. */ - if (len > server.hash_max_ziplist_entries) + if (len > server.hash_max_listpack_entries) hashTypeConvert(o, OBJ_ENCODING_HT); else if (deep_integrity_validation) { /* In this mode, we need to guarantee that the server won't crash @@ -1715,7 +1714,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { /* Load every field and value into the ziplist */ - while (o->encoding == OBJ_ENCODING_ZIPLIST && len > 0) { + while (o->encoding == OBJ_ENCODING_LISTPACK && len > 0) { len--; /* Load raw strings */ if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) { @@ -1743,15 +1742,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { } } - /* Add pair to ziplist */ - o->ptr = ziplistPush(o->ptr, (unsigned char*)field, - sdslen(field), ZIPLIST_TAIL); - o->ptr = ziplistPush(o->ptr, (unsigned char*)value, - sdslen(value), ZIPLIST_TAIL); + /* Add pair to listpack */ + o->ptr = lpAppend(o->ptr, (unsigned char*)field, sdslen(field)); + o->ptr = lpAppend(o->ptr, (unsigned char*)value, sdslen(value)); /* Convert to hash table if size threshold is exceeded */ - if (sdslen(field) > server.hash_max_ziplist_value || - sdslen(value) > server.hash_max_ziplist_value) + if (sdslen(field) > server.hash_max_listpack_value || + sdslen(value) > server.hash_max_listpack_value) { sdsfree(field); sdsfree(value); @@ -1845,7 +1842,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { rdbtype == RDB_TYPE_LIST_ZIPLIST || rdbtype == RDB_TYPE_SET_INTSET || rdbtype == RDB_TYPE_ZSET_ZIPLIST || - rdbtype == RDB_TYPE_HASH_ZIPLIST) + rdbtype == RDB_TYPE_HASH_ZIPLIST || + rdbtype == RDB_TYPE_HASH_LISTPACK) { size_t encoded_len; unsigned char *encoded = @@ -1874,7 +1872,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { /* Convert to ziplist encoded hash. This must be deprecated * when loading dumps created by Redis 2.4 gets deprecated. */ { - unsigned char *zl = ziplistNew(); + unsigned char *zl = lpNew(0); unsigned char *zi = zipmapRewind(o->ptr); unsigned char *fstr, *vstr; unsigned int flen, vlen; @@ -1884,8 +1882,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) { if (flen > maxlen) maxlen = flen; if (vlen > maxlen) maxlen = vlen; - zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL); - zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL); + zl = lpAppend(zl, fstr, flen); + zl = lpAppend(zl, vstr, vlen); /* search for duplicate records */ sds field = sdstrynewlen(fstr, flen); @@ -1904,10 +1902,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { zfree(o->ptr); o->ptr = zl; o->type = OBJ_HASH; - o->encoding = OBJ_ENCODING_ZIPLIST; + o->encoding = OBJ_ENCODING_LISTPACK; - if (hashTypeLength(o) > server.hash_max_ziplist_entries || - maxlen > server.hash_max_ziplist_value) + if (hashTypeLength(o) > server.hash_max_listpack_entries || + maxlen > server.hash_max_listpack_value) { hashTypeConvert(o, OBJ_ENCODING_HT); } @@ -1970,16 +1968,44 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { zsetConvert(o,OBJ_ENCODING_SKIPLIST); break; case RDB_TYPE_HASH_ZIPLIST: + { + unsigned char *lp = lpNew(encoded_len); + if (!hashZiplistConvertAndValidateIntegrity(encoded, encoded_len, &lp)) { + rdbReportCorruptRDB("Hash ziplist integrity check failed."); + zfree(lp); + zfree(encoded); + o->ptr = NULL; + decrRefCount(o); + return NULL; + } + + zfree(o->ptr); + o->ptr = lp; + o->type = OBJ_HASH; + o->encoding = OBJ_ENCODING_LISTPACK; + if (hashTypeLength(o) == 0) { + zfree(o->ptr); + o->ptr = NULL; + decrRefCount(o); + goto emptykey; + } + + if (hashTypeLength(o) > server.hash_max_listpack_entries) { + hashTypeConvert(o, OBJ_ENCODING_HT); + } + break; + } + case RDB_TYPE_HASH_LISTPACK: if (deep_integrity_validation) server.stat_dump_payload_sanitizations++; - if (!hashZiplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) { - rdbReportCorruptRDB("Hash ziplist integrity check failed."); + if (!hashListpackValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) { + rdbReportCorruptRDB("Hash listpack integrity check failed."); zfree(encoded); o->ptr = NULL; decrRefCount(o); return NULL; } o->type = OBJ_HASH; - o->encoding = OBJ_ENCODING_ZIPLIST; + o->encoding = OBJ_ENCODING_LISTPACK; if (hashTypeLength(o) == 0) { zfree(encoded); o->ptr = NULL; @@ -1987,7 +2013,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { goto emptykey; } - if (hashTypeLength(o) > server.hash_max_ziplist_entries) + if (hashTypeLength(o) > server.hash_max_listpack_entries) hashTypeConvert(o, OBJ_ENCODING_HT); break; default: diff --git a/src/rdb.h b/src/rdb.h index 0b495713564e..e3a0658f5dd1 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -38,7 +38,7 @@ /* The current RDB version. When the format changes in a way that is no longer * backward compatible this number gets incremented. */ -#define RDB_VERSION 9 +#define RDB_VERSION 10 /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of @@ -91,10 +91,11 @@ #define RDB_TYPE_HASH_ZIPLIST 13 #define RDB_TYPE_LIST_QUICKLIST 14 #define RDB_TYPE_STREAM_LISTPACKS 15 +#define RDB_TYPE_HASH_LISTPACK 16 /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Test if a type is an object type. */ -#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 15)) +#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 16)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ #define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */ diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 82429bb256a9..9e62e25db165 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -90,7 +90,8 @@ char *rdb_type_string[] = { "zset-ziplist", "hash-ziplist", "quicklist", - "stream" + "stream", + "hash-listpack" }; /* Show a few stats collected into 'rdbstate' */ diff --git a/src/server.c b/src/server.c index 756fa5be8b39..301e21e2898d 100644 --- a/src/server.c +++ b/src/server.c @@ -6241,7 +6241,8 @@ struct redisTest { {"crc64", crc64Test}, {"zmalloc", zmalloc_test}, {"sds", sdsTest}, - {"dict", dictTest} + {"dict", dictTest}, + {"listpack", listpackTest} }; redisTestProc *getTestProcByName(const char *name) { int numtests = sizeof(redisTests)/sizeof(struct redisTest); diff --git a/src/server.h b/src/server.h index 4f797d592143..f01ab4ba0e43 100644 --- a/src/server.h +++ b/src/server.h @@ -707,6 +707,7 @@ typedef struct RedisModuleDigest { #define OBJ_ENCODING_EMBSTR 8 /* Embedded sds string encoding */ #define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of ziplists */ #define OBJ_ENCODING_STREAM 10 /* Encoded as a radix tree of listpacks */ +#define OBJ_ENCODING_LISTPACK 11 /* Encoded as a listpack */ #define LRU_BITS 24 #define LRU_CLOCK_MAX ((1<lru */ @@ -1569,8 +1570,8 @@ struct redisServer { int sort_bypattern; int sort_store; /* Zip structure config, see redis.conf for more information */ - size_t hash_max_ziplist_entries; - size_t hash_max_ziplist_value; + size_t hash_max_listpack_entries; + size_t hash_max_listpack_value; size_t set_max_intset_entries; size_t zset_max_ziplist_entries; size_t zset_max_ziplist_value; @@ -2341,10 +2342,10 @@ unsigned long hashTypeLength(const robj *o); hashTypeIterator *hashTypeInitIterator(robj *subject); void hashTypeReleaseIterator(hashTypeIterator *hi); int hashTypeNext(hashTypeIterator *hi); -void hashTypeCurrentFromZiplist(hashTypeIterator *hi, int what, - unsigned char **vstr, - unsigned int *vlen, - long long *vll); +void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what, + unsigned char **vstr, + unsigned int *vlen, + long long *vll); sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what); void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll); sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what); @@ -2352,7 +2353,8 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key); robj *hashTypeGetValueObject(robj *o, sds field); int hashTypeSet(robj *o, sds field, sds value, int flags); robj *hashTypeDup(robj *o); -int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep); +int hashZiplistConvertAndValidateIntegrity(unsigned char *zl, size_t size, unsigned char **lp); +int hashListpackValidateIntegrity(unsigned char *lp, size_t size, int deep); /* Pub / Sub */ int pubsubUnsubscribeAllChannels(client *c, int notify); diff --git a/src/t_hash.c b/src/t_hash.c index 68d304c502d6..26089fa7283f 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -35,16 +35,16 @@ *----------------------------------------------------------------------------*/ /* Check the length of a number of objects to see if we need to convert a - * ziplist to a real hash. Note that we only check string encoded objects + * listpack to a real hash. Note that we only check string encoded objects * as their string length can be queried in constant time. */ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { int i; - if (o->encoding != OBJ_ENCODING_ZIPLIST) return; + if (o->encoding != OBJ_ENCODING_LISTPACK) return; for (i = start; i <= end; i++) { if (sdsEncodedObject(argv[i]) && - sdslen(argv[i]->ptr) > server.hash_max_ziplist_value) + sdslen(argv[i]->ptr) > server.hash_max_listpack_value) { hashTypeConvert(o, OBJ_ENCODING_HT); break; @@ -52,32 +52,30 @@ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { } } -/* Get the value from a ziplist encoded hash, identified by field. +/* Get the value from a listpack encoded hash, identified by field. * Returns -1 when the field cannot be found. */ -int hashTypeGetFromZiplist(robj *o, sds field, - unsigned char **vstr, - unsigned int *vlen, - long long *vll) +int hashTypeGetFromListpack(robj *o, sds field, + unsigned char **vstr, + unsigned int *vlen, + long long *vll) { unsigned char *zl, *fptr = NULL, *vptr = NULL; - int ret; - serverAssert(o->encoding == OBJ_ENCODING_ZIPLIST); + serverAssert(o->encoding == OBJ_ENCODING_LISTPACK); zl = o->ptr; - fptr = ziplistIndex(zl, ZIPLIST_HEAD); + fptr = lpFirst(zl); if (fptr != NULL) { - fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); + fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); if (fptr != NULL) { /* Grab pointer to the value (fptr points to the field) */ - vptr = ziplistNext(zl, fptr); + vptr = lpNext(zl, fptr); serverAssert(vptr != NULL); } } if (vptr != NULL) { - ret = ziplistGet(vptr, vstr, vlen, vll); - serverAssert(ret); + *vstr = lpGetValue(vptr, vlen, vll); return 0; } @@ -107,9 +105,9 @@ sds hashTypeGetFromHashTable(robj *o, sds field) { * can always check the function return by checking the return value * for C_OK and checking if vll (or vstr) is NULL. */ int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll) { - if (o->encoding == OBJ_ENCODING_ZIPLIST) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { *vstr = NULL; - if (hashTypeGetFromZiplist(o, field, vstr, vlen, vll) == 0) + if (hashTypeGetFromListpack(o, field, vstr, vlen, vll) == 0) return C_OK; } else if (o->encoding == OBJ_ENCODING_HT) { sds value; @@ -143,12 +141,12 @@ robj *hashTypeGetValueObject(robj *o, sds field) { * exist. */ size_t hashTypeGetValueLength(robj *o, sds field) { size_t len = 0; - if (o->encoding == OBJ_ENCODING_ZIPLIST) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *vstr = NULL; unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; - if (hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll) == 0) + if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0) len = vstr ? vlen : sdigits10(vll); } else if (o->encoding == OBJ_ENCODING_HT) { sds aux; @@ -164,12 +162,12 @@ size_t hashTypeGetValueLength(robj *o, sds field) { /* Test if the specified field exists in the given hash. Returns 1 if the field * exists, and 0 when it doesn't. */ int hashTypeExists(robj *o, sds field) { - if (o->encoding == OBJ_ENCODING_ZIPLIST) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *vstr = NULL; unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; - if (hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll) == 0) return 1; + if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0) return 1; } else if (o->encoding == OBJ_ENCODING_HT) { if (hashTypeGetFromHashTable(o, field) != NULL) return 1; } else { @@ -202,36 +200,33 @@ int hashTypeExists(robj *o, sds field) { int hashTypeSet(robj *o, sds field, sds value, int flags) { int update = 0; - if (o->encoding == OBJ_ENCODING_ZIPLIST) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl, *fptr, *vptr; zl = o->ptr; - fptr = ziplistIndex(zl, ZIPLIST_HEAD); + fptr = lpFirst(zl); if (fptr != NULL) { - fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); + fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); if (fptr != NULL) { /* Grab pointer to the value (fptr points to the field) */ - vptr = ziplistNext(zl, fptr); + vptr = lpNext(zl, fptr); serverAssert(vptr != NULL); update = 1; /* Replace value */ - zl = ziplistReplace(zl, vptr, (unsigned char*)value, - sdslen(value)); + zl = lpReplace(zl, &vptr, (unsigned char*)value, sdslen(value)); } } if (!update) { - /* Push new field/value pair onto the tail of the ziplist */ - zl = ziplistPush(zl, (unsigned char*)field, sdslen(field), - ZIPLIST_TAIL); - zl = ziplistPush(zl, (unsigned char*)value, sdslen(value), - ZIPLIST_TAIL); + /* Push new field/value pair onto the tail of the listpack */ + zl = lpAppend(zl, (unsigned char*)field, sdslen(field)); + zl = lpAppend(zl, (unsigned char*)value, sdslen(value)); } o->ptr = zl; - /* Check if the ziplist needs to be converted to a hash table */ - if (hashTypeLength(o) > server.hash_max_ziplist_entries) + /* Check if the listpack needs to be converted to a hash table */ + if (hashTypeLength(o) > server.hash_max_listpack_entries) hashTypeConvert(o, OBJ_ENCODING_HT); } else if (o->encoding == OBJ_ENCODING_HT) { dictEntry *de = dictFind(o->ptr,field); @@ -276,16 +271,16 @@ int hashTypeSet(robj *o, sds field, sds value, int flags) { int hashTypeDelete(robj *o, sds field) { int deleted = 0; - if (o->encoding == OBJ_ENCODING_ZIPLIST) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl, *fptr; zl = o->ptr; - fptr = ziplistIndex(zl, ZIPLIST_HEAD); + fptr = lpFirst(zl); if (fptr != NULL) { - fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); + fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); if (fptr != NULL) { - zl = ziplistDelete(zl,&fptr); /* Delete the key. */ - zl = ziplistDelete(zl,&fptr); /* Delete the value. */ + zl = lpDelete(zl,fptr,&fptr); /* Delete the key. */ + zl = lpDelete(zl,fptr,&fptr); /* Delete the value. */ o->ptr = zl; deleted = 1; } @@ -308,8 +303,8 @@ int hashTypeDelete(robj *o, sds field) { unsigned long hashTypeLength(const robj *o) { unsigned long length = ULONG_MAX; - if (o->encoding == OBJ_ENCODING_ZIPLIST) { - length = ziplistLen(o->ptr) / 2; + if (o->encoding == OBJ_ENCODING_LISTPACK) { + length = lpLength(o->ptr) / 2; } else if (o->encoding == OBJ_ENCODING_HT) { length = dictSize((const dict*)o->ptr); } else { @@ -323,7 +318,7 @@ hashTypeIterator *hashTypeInitIterator(robj *subject) { hi->subject = subject; hi->encoding = subject->encoding; - if (hi->encoding == OBJ_ENCODING_ZIPLIST) { + if (hi->encoding == OBJ_ENCODING_LISTPACK) { hi->fptr = NULL; hi->vptr = NULL; } else if (hi->encoding == OBJ_ENCODING_HT) { @@ -343,7 +338,7 @@ void hashTypeReleaseIterator(hashTypeIterator *hi) { /* Move to the next entry in the hash. Return C_OK when the next entry * could be found and C_ERR when the iterator reaches the end. */ int hashTypeNext(hashTypeIterator *hi) { - if (hi->encoding == OBJ_ENCODING_ZIPLIST) { + if (hi->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl; unsigned char *fptr, *vptr; @@ -354,16 +349,16 @@ int hashTypeNext(hashTypeIterator *hi) { if (fptr == NULL) { /* Initialize cursor */ serverAssert(vptr == NULL); - fptr = ziplistIndex(zl, 0); + fptr = lpFirst(zl); } else { /* Advance cursor */ serverAssert(vptr != NULL); - fptr = ziplistNext(zl, vptr); + fptr = lpNext(zl, vptr); } if (fptr == NULL) return C_ERR; /* Grab pointer to the value (fptr points to the field) */ - vptr = ziplistNext(zl, fptr); + vptr = lpNext(zl, fptr); serverAssert(vptr != NULL); /* fptr, vptr now point to the first or next pair */ @@ -378,22 +373,18 @@ int hashTypeNext(hashTypeIterator *hi) { } /* Get the field or value at iterator cursor, for an iterator on a hash value - * encoded as a ziplist. Prototype is similar to `hashTypeGetFromZiplist`. */ -void hashTypeCurrentFromZiplist(hashTypeIterator *hi, int what, - unsigned char **vstr, - unsigned int *vlen, - long long *vll) + * encoded as a listpack. Prototype is similar to `hashTypeGetFromListpack`. */ +void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what, + unsigned char **vstr, + unsigned int *vlen, + long long *vll) { - int ret; - - serverAssert(hi->encoding == OBJ_ENCODING_ZIPLIST); + serverAssert(hi->encoding == OBJ_ENCODING_LISTPACK); if (what & OBJ_HASH_KEY) { - ret = ziplistGet(hi->fptr, vstr, vlen, vll); - serverAssert(ret); + *vstr = lpGetValue(hi->fptr, vlen, vll); } else { - ret = ziplistGet(hi->vptr, vstr, vlen, vll); - serverAssert(ret); + *vstr = lpGetValue(hi->vptr, vlen, vll); } } @@ -421,9 +412,9 @@ sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what) { * can always check the function return by checking the return value * type checking if vstr == NULL. */ void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll) { - if (hi->encoding == OBJ_ENCODING_ZIPLIST) { + if (hi->encoding == OBJ_ENCODING_LISTPACK) { *vstr = NULL; - hashTypeCurrentFromZiplist(hi, what, vstr, vlen, vll); + hashTypeCurrentFromListpack(hi, what, vstr, vlen, vll); } else if (hi->encoding == OBJ_ENCODING_HT) { sds ele = hashTypeCurrentFromHashTable(hi, what); *vstr = (unsigned char*) ele; @@ -456,10 +447,11 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key) { return o; } -void hashTypeConvertZiplist(robj *o, int enc) { - serverAssert(o->encoding == OBJ_ENCODING_ZIPLIST); - if (enc == OBJ_ENCODING_ZIPLIST) { +void hashTypeConvertListpack(robj *o, int enc) { + serverAssert(o->encoding == OBJ_ENCODING_LISTPACK); + + if (enc == OBJ_ENCODING_LISTPACK) { /* Nothing to do... */ } else if (enc == OBJ_ENCODING_HT) { @@ -480,9 +472,9 @@ void hashTypeConvertZiplist(robj *o, int enc) { value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE); ret = dictAdd(dict, key, value); if (ret != DICT_OK) { - serverLogHexDump(LL_WARNING,"ziplist with dup elements dump", - o->ptr,ziplistBlobLen(o->ptr)); - serverPanic("Ziplist corruption detected"); + serverLogHexDump(LL_WARNING,"listpack with dup elements dump", + o->ptr,lpBytes(o->ptr)); + serverPanic("Listpack corruption detected"); } } hashTypeReleaseIterator(hi); @@ -495,8 +487,8 @@ void hashTypeConvertZiplist(robj *o, int enc) { } void hashTypeConvert(robj *o, int enc) { - if (o->encoding == OBJ_ENCODING_ZIPLIST) { - hashTypeConvertZiplist(o, enc); + if (o->encoding == OBJ_ENCODING_LISTPACK) { + hashTypeConvertListpack(o, enc); } else if (o->encoding == OBJ_ENCODING_HT) { serverPanic("Not implemented"); } else { @@ -515,13 +507,13 @@ robj *hashTypeDup(robj *o) { serverAssert(o->type == OBJ_HASH); - if(o->encoding == OBJ_ENCODING_ZIPLIST){ + if(o->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *zl = o->ptr; - size_t sz = ziplistBlobLen(zl); + size_t sz = lpBytes(zl); unsigned char *new_zl = zmalloc(sz); memcpy(new_zl, zl, sz); hobj = createObject(OBJ_HASH, new_zl); - hobj->encoding = OBJ_ENCODING_ZIPLIST; + hobj->encoding = OBJ_ENCODING_LISTPACK; } else if(o->encoding == OBJ_ENCODING_HT){ dict *d = dictCreate(&hashDictType); dictExpand(d, dictSize((const dict*)o->ptr)); @@ -549,20 +541,25 @@ robj *hashTypeDup(robj *o) { return hobj; } -/* callback for to check the ziplist doesn't have duplicate records */ -static int _hashZiplistEntryValidation(unsigned char *p, void *userdata) { +/* callback for hashZiplistConvertAndValidateIntegrity. + * Check that the ziplist doesn't have duplicate hash field names. + * The ziplist element pointed by 'p' will be converted and stored into listpack. */ +static int _hashZiplistEntryConvertAndValidation(unsigned char *p, void *userdata) { + unsigned char *str; + unsigned int slen; + long long vll; + struct { long count; dict *fields; + unsigned char **lp; } *data = userdata; + if (!ziplistGet(p, &str, &slen, &vll)) + return 0; + /* Even records are field names, add to dict and check that's not a dup */ if (((data->count) & 1) == 0) { - unsigned char *str; - unsigned int slen; - long long vll; - if (!ziplistGet(p, &str, &slen, &vll)) - return 0; sds field = str? sdsnewlen(str, slen): sdsfromlonglong(vll); if (dictAdd(data->fields, field, NULL) != DICT_OK) { /* Duplicate, return an error */ @@ -571,16 +568,70 @@ static int _hashZiplistEntryValidation(unsigned char *p, void *userdata) { } } + if (str) { + *(data->lp) = lpAppend(*(data->lp), (unsigned char*)str, slen); + } else { + *(data->lp) = lpAppendInteger(*(data->lp), vll); + } + + (data->count)++; + return 1; +} + +/* callback for to check the listpack doesn't have duplicate records */ +static int _hashListpackEntryValidation(unsigned char *p, void *userdata) { + struct { + long count; + dict *fields; + } *data = userdata; + + /* Even records are field names, add to dict and check that's not a dup */ + if (((data->count) & 1) == 0) { + unsigned char *str; + int64_t slen; + unsigned char buf[LP_INTBUF_SIZE]; + + str = lpGet(p, &slen, buf); + sds field = sdsnewlen(str, slen); + if (dictAdd(data->fields, field, NULL) != DICT_OK) { + /* Duplicate, return an error */ + sdsfree(field); + return 0; + } + } + (data->count)++; return 1; } -/* Validate the integrity of the data structure. +/* Validate the integrity of the data structure while converting it to + * listpack and storing it at 'lp'. + * The function is safe to call on non-validated ziplists, it returns 0 + * when encounter an integrity validation issue. */ +int hashZiplistConvertAndValidateIntegrity(unsigned char *zl, size_t size, unsigned char **lp) { + /* Keep track of the field names to locate duplicate ones */ + struct { + long count; + dict *fields; + unsigned char **lp; + } data = {0, dictCreate(&hashDictType), lp}; + + int ret = ziplistValidateIntegrity(zl, size, 1, _hashZiplistEntryConvertAndValidation, &data); + + /* make sure we have an even number of records. */ + if (data.count & 1) + ret = 0; + + dictRelease(data.fields); + return ret; +} + +/* Validate the integrity of the listpack structure. * when `deep` is 0, only the integrity of the header is validated. * when `deep` is 1, we scan all the entries one by one. */ -int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) { +int hashListpackValidateIntegrity(unsigned char *lp, size_t size, int deep) { if (!deep) - return ziplistValidateIntegrity(zl, size, 0, NULL, NULL); + return lpValidateIntegrity(lp, size, 0, NULL, NULL); /* Keep track of the field names to locate duplicate ones */ struct { @@ -588,7 +639,7 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) { dict *fields; } data = {0, dictCreate(&hashDictType)}; - int ret = ziplistValidateIntegrity(zl, size, 1, _hashZiplistEntryValidation, &data); + int ret = lpValidateIntegrity(lp, size, 1, _hashListpackEntryValidation, &data); /* make sure we have an even number of records. */ if (data.count & 1) @@ -598,13 +649,13 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) { return ret; } -/* Create a new sds string from the ziplist entry. */ -sds hashSdsFromZiplistEntry(ziplistEntry *e) { +/* Create a new sds string from the listpack entry. */ +sds hashSdsFromListpackEntry(listpackEntry *e) { return e->sval ? sdsnewlen(e->sval, e->slen) : sdsfromlonglong(e->lval); } -/* Reply with bulk string from the ziplist entry. */ -void hashReplyFromZiplistEntry(client *c, ziplistEntry *e) { +/* Reply with bulk string from the listpack entry. */ +void hashReplyFromListpackEntry(client *c, listpackEntry *e) { if (e->sval) addReplyBulkCBuffer(c, e->sval, e->slen); else @@ -615,7 +666,7 @@ void hashReplyFromZiplistEntry(client *c, ziplistEntry *e) { * 'key' and 'val' will be set to hold the element. * The memory in them is not to be freed or modified by the caller. * 'val' can be NULL in which case it's not extracted. */ -void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, ziplistEntry *key, ziplistEntry *val) { +void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpackEntry *key, listpackEntry *val) { if (hashobj->encoding == OBJ_ENCODING_HT) { dictEntry *de = dictGetFairRandomKey(hashobj->ptr); sds s = dictGetKey(de); @@ -626,8 +677,8 @@ void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, ziplistEntry * val->sval = (unsigned char*)s; val->slen = sdslen(s); } - } else if (hashobj->encoding == OBJ_ENCODING_ZIPLIST) { - ziplistRandomPair(hashobj->ptr, hashsize, key, val); + } else if (hashobj->encoding == OBJ_ENCODING_LISTPACK) { + lpRandomPair(hashobj->ptr, hashsize, key, val); } else { serverPanic("Unknown hash encoding"); } @@ -774,12 +825,12 @@ static void addHashFieldToReply(client *c, robj *o, sds field) { return; } - if (o->encoding == OBJ_ENCODING_ZIPLIST) { + if (o->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *vstr = NULL; unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; - ret = hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll); + ret = hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll); if (ret < 0) { addReplyNull(c); } else { @@ -871,12 +922,12 @@ void hstrlenCommand(client *c) { } static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int what) { - if (hi->encoding == OBJ_ENCODING_ZIPLIST) { + if (hi->encoding == OBJ_ENCODING_LISTPACK) { unsigned char *vstr = NULL; unsigned int vlen = UINT_MAX; long long vll = LLONG_MAX; - hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll); + hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll); if (vstr) addReplyBulkCBuffer(c, vstr, vlen); else @@ -957,7 +1008,7 @@ void hscanCommand(client *c) { scanGenericCommand(c,o,cursor); } -static void harndfieldReplyWithZiplist(client *c, unsigned int count, ziplistEntry *keys, ziplistEntry *vals) { +static void harndfieldReplyWithListpack(client *c, unsigned int count, listpackEntry *keys, listpackEntry *vals) { for (unsigned long i = 0; i < count; i++) { if (vals && c->resp > 2) addReplyArrayLen(c,2); @@ -1028,18 +1079,19 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { if (withvalues) addReplyBulkCBuffer(c, value, sdslen(value)); } - } else if (hash->encoding == OBJ_ENCODING_ZIPLIST) { - ziplistEntry *keys, *vals = NULL; + } else if (hash->encoding == OBJ_ENCODING_LISTPACK) { + listpackEntry *keys, *vals = NULL; unsigned long limit, sample_count; + limit = count > HRANDFIELD_RANDOM_SAMPLE_LIMIT ? HRANDFIELD_RANDOM_SAMPLE_LIMIT : count; - keys = zmalloc(sizeof(ziplistEntry)*limit); + keys = zmalloc(sizeof(listpackEntry)*limit); if (withvalues) - vals = zmalloc(sizeof(ziplistEntry)*limit); + vals = zmalloc(sizeof(listpackEntry)*limit); while (count) { sample_count = count > limit ? limit : count; count -= sample_count; - ziplistRandomPairs(hash->ptr, sample_count, keys, vals); - harndfieldReplyWithZiplist(c, sample_count, keys, vals); + lpRandomPairs(hash->ptr, sample_count, keys, vals); + harndfieldReplyWithListpack(c, sample_count, keys, vals); } zfree(keys); zfree(vals); @@ -1133,15 +1185,15 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { * to the temporary hash, trying to eventually get enough unique elements * to reach the specified count. */ else { - if (hash->encoding == OBJ_ENCODING_ZIPLIST) { + if (hash->encoding == OBJ_ENCODING_LISTPACK) { /* it is inefficient to repeatedly pick one random element from a - * ziplist. so we use this instead: */ - ziplistEntry *keys, *vals = NULL; - keys = zmalloc(sizeof(ziplistEntry)*count); + * listpack. so we use this instead: */ + listpackEntry *keys, *vals = NULL; + keys = zmalloc(sizeof(listpackEntry)*count); if (withvalues) - vals = zmalloc(sizeof(ziplistEntry)*count); - serverAssert(ziplistRandomPairsUnique(hash->ptr, count, keys, vals) == count); - harndfieldReplyWithZiplist(c, count, keys, vals); + vals = zmalloc(sizeof(listpackEntry)*count); + serverAssert(lpRandomPairsUnique(hash->ptr, count, keys, vals) == count); + harndfieldReplyWithListpack(c, count, keys, vals); zfree(keys); zfree(vals); return; @@ -1149,7 +1201,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { /* Hashtable encoding (generic implementation) */ unsigned long added = 0; - ziplistEntry key, value; + listpackEntry key, value; dict *d = dictCreate(&hashDictType); dictExpand(d, count); while(added < count) { @@ -1158,7 +1210,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { /* Try to add the object to the dictionary. If it already exists * free it, otherwise increment the number of objects we have * in the result dictionary. */ - sds skey = hashSdsFromZiplistEntry(&key); + sds skey = hashSdsFromListpackEntry(&key); if (dictAdd(d,skey,NULL) != DICT_OK) { sdsfree(skey); continue; @@ -1168,9 +1220,9 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { /* We can reply right away, so that we don't need to store the value in the dict. */ if (withvalues && c->resp > 2) addReplyArrayLen(c,2); - hashReplyFromZiplistEntry(c, &key); + hashReplyFromListpackEntry(c, &key); if (withvalues) - hashReplyFromZiplistEntry(c, &value); + hashReplyFromListpackEntry(c, &value); } /* Release memory */ @@ -1183,7 +1235,7 @@ void hrandfieldCommand(client *c) { long l; int withvalues = 0; robj *hash; - ziplistEntry ele; + listpackEntry ele; if (c->argc >= 3) { if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return; @@ -1203,5 +1255,5 @@ void hrandfieldCommand(client *c) { } hashTypeRandomElement(hash,hashTypeLength(hash),&ele,NULL); - hashReplyFromZiplistEntry(c, &ele); + hashReplyFromListpackEntry(c, &ele); } diff --git a/src/t_stream.c b/src/t_stream.c index 1b0d012e49e1..d1b144a8f704 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -241,24 +241,6 @@ robj *streamDup(robj *o) { return sobj; } -/* This is just a wrapper for lpAppend() to directly use a 64 bit integer - * instead of a string. */ -unsigned char *lpAppendInteger(unsigned char *lp, int64_t value) { - char buf[LONG_STR_SIZE]; - int slen = ll2string(buf,sizeof(buf),value); - return lpAppend(lp,(unsigned char*)buf,slen); -} - -/* This is just a wrapper for lpReplace() to directly use a 64 bit integer - * instead of a string to replace the current element. The function returns - * the new listpack as return value, and also updates the current cursor - * by updating '*pos'. */ -unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **pos, int64_t value) { - char buf[LONG_STR_SIZE]; - int slen = ll2string(buf,sizeof(buf),value); - return lpInsert(lp, (unsigned char*)buf, slen, *pos, LP_REPLACE, pos); -} - /* This is a wrapper function for lpGet() to directly get an integer value * from the listpack (that may store numbers as a string), converting * the string if needed. @@ -3577,7 +3559,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) { /* Since we don't want to run validation of all records twice, we'll * run the listpack validation of just the header and do the rest here. */ - if (!lpValidateIntegrity(lp, size, 0)) + if (!lpValidateIntegrity(lp, size, 0, NULL, NULL)) return 0; /* In non-deep mode we just validated the listpack header (encoded size) */ diff --git a/tests/assets/hash-ziplist.rdb b/tests/assets/hash-ziplist.rdb new file mode 100644 index 000000000000..bcc39a393da0 Binary files /dev/null and b/tests/assets/hash-ziplist.rdb differ diff --git a/tests/integration/convert-ziplist-hash-on-load.tcl b/tests/integration/convert-ziplist-hash-on-load.tcl new file mode 100644 index 000000000000..c8265b25ae9c --- /dev/null +++ b/tests/integration/convert-ziplist-hash-on-load.tcl @@ -0,0 +1,28 @@ +tags {"external:skip"} { + +# Copy RDB with ziplist encoded hash to server path +set server_path [tmpdir "server.convert-ziplist-hash-on-load"] + +exec cp -f tests/assets/hash-ziplist.rdb $server_path +start_server [list overrides [list "dir" $server_path "dbfilename" "hash-ziplist.rdb"]] { + test "RDB load ziplist hash: converts to listpack when RDB loading" { + r select 0 + + assert_encoding listpack hash + assert_equal 2 [r hlen hash] + assert_match {v1 v2} [r hmget hash f1 f2] + } +} + +exec cp -f tests/assets/hash-ziplist.rdb $server_path +start_server [list overrides [list "dir" $server_path "dbfilename" "hash-ziplist.rdb" "hash-max-ziplist-entries" 1]] { + test "RDB load ziplist hash: converts to hash table when hash-max-ziplist-entries is exceeded" { + r select 0 + + assert_encoding hashtable hash + assert_equal 2 [r hlen hash] + assert_match {v1 v2} [r hmget hash f1 f2] + } +} + +} diff --git a/tests/integration/convert-zipmap-hash-on-load.tcl b/tests/integration/convert-zipmap-hash-on-load.tcl index af3c704d7c8a..f7eda0e06ea2 100644 --- a/tests/integration/convert-zipmap-hash-on-load.tcl +++ b/tests/integration/convert-zipmap-hash-on-load.tcl @@ -5,10 +5,10 @@ set server_path [tmpdir "server.convert-zipmap-hash-on-load"] exec cp -f tests/assets/hash-zipmap.rdb $server_path start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb"]] { - test "RDB load zipmap hash: converts to ziplist" { + test "RDB load zipmap hash: converts to listpack" { r select 0 - assert_match "*ziplist*" [r debug object hash] + assert_match "*listpack*" [r debug object hash] assert_equal 2 [r hlen hash] assert_match {v1 v2} [r hmget hash f1 f2] } diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index 468082b913b3..cfd54478d1dd 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -43,32 +43,31 @@ test {corrupt payload: #7445 - without sanitize - 2} { test {corrupt payload: hash with valid zip list header, invalid entry len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { - r config set sanitize-dump-payload no - r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x14\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xD9\x10\x54\x92\x15\xF5\x5F\x52" - r config set hash-max-ziplist-entries 1 - catch {r hset key b b} - verify_log_message 0 "*zipEntrySafe*" 0 + catch { + r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x14\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xD9\x10\x54\x92\x15\xF5\x5F\x52" + } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 } } test {corrupt payload: invalid zlbytes header} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { - r config set sanitize-dump-payload no catch { r restore key 0 "\x0D\x1B\x25\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xB7\xF7\x6E\x9F\x43\x43\x14\xC6" } err assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 } } test {corrupt payload: valid zipped hash header, dup records} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { - r config set sanitize-dump-payload no - r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x61\x00\x04\x02\x64\x00\xFF\x09\x00\xA1\x98\x36\x78\xCC\x8E\x93\x2E" - r config set hash-max-ziplist-entries 1 - # cause an assertion when converting to hash table - catch {r hset key b b} - verify_log_message 0 "*ziplist with dup elements dump*" 0 + catch { + r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x61\x00\x04\x02\x64\x00\xFF\x09\x00\xA1\x98\x36\x78\xCC\x8E\x93\x2E" + } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 } } @@ -235,6 +234,16 @@ test {corrupt payload: hash ziplist with duplicate records} { } } +test {corrupt payload: hash listpack with duplicate records} { + # when we do perform full sanitization, we expect duplicate records to fail the restore + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch { r RESTORE _hash 0 "\x10\x17\x17\x00\x00\x00\x04\x00\x82a\x00\x03\x82b\x00\x03\x82a\x00\x03\x82d\x00\x03\xff\n\x00\xc0\xcf\xa6\x87\xe5\xa7\xc5\xbe" } err + assert_match "*Bad data format*" $err + } +} + test {corrupt payload: hash ziplist uneven record count} { # when we do perform full sanitization, we expect duplicate records to fail the restore start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { @@ -304,16 +313,14 @@ test {corrupt payload: fuzzer findings - NPD in quicklistIndex} { } } -test {corrupt payload: fuzzer findings - invalid read in ziplistFind} { +test {corrupt payload: fuzzer findings - encoded entry header reach outside the allocation} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { - r config set sanitize-dump-payload no r debug set-skip-checksum-validation 1 catch { r RESTORE key 0 "\x0D\x19\x19\x00\x00\x00\x16\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\xF2\x02\x02\x5F\x31\x04\x99\x02\xF3\xFF\x09\x00\xC5\xB8\x10\xC0\x8A\xF9\x16\xDF" - r HEXISTS key -688319650333 - } - assert_equal [count_log_message 0 "crashed by signal"] 0 - assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 } } @@ -342,12 +349,12 @@ test {corrupt payload: fuzzer findings - hash crash} { test {corrupt payload: fuzzer findings - uneven entry count in hash} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { - r config set sanitize-dump-payload no r debug set-skip-checksum-validation 1 - r RESTORE _hashbig 0 "\x0D\x3D\x3D\x00\x00\x00\x38\x00\x00\x00\x14\x00\x00\xF2\x02\x02\x5F\x31\x04\x1C\x02\xF7\x02\xF1\x02\xF1\x02\xF5\x02\xF5\x02\xF4\x02\x02\x5F\x33\x04\xF6\x02\x02\x5F\x35\x04\xF8\x02\x02\x5F\x37\x04\xF9\x02\xF9\x02\xF3\x02\xF3\x02\xFA\x02\x02\x5F\x39\xFF\x09\x00\x73\xB7\x68\xC8\x97\x24\x8E\x88" - catch { r HSCAN _hashbig -250 } - assert_equal [count_log_message 0 "crashed by signal"] 0 - assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + catch { + r RESTORE _hashbig 0 "\x0D\x3D\x3D\x00\x00\x00\x38\x00\x00\x00\x14\x00\x00\xF2\x02\x02\x5F\x31\x04\x1C\x02\xF7\x02\xF1\x02\xF1\x02\xF5\x02\xF5\x02\xF4\x02\x02\x5F\x33\x04\xF6\x02\x02\x5F\x35\x04\xF8\x02\x02\x5F\x37\x04\xF9\x02\xF9\x02\xF3\x02\xF3\x02\xFA\x02\x02\x5F\x39\xFF\x09\x00\x73\xB7\x68\xC8\x97\x24\x8E\x88" + } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 } } @@ -474,15 +481,14 @@ test {corrupt payload: fuzzer findings - infinite loop} { } } -test {corrupt payload: fuzzer findings - hash convert asserts on RESTORE with shallow sanitization} { - # if we don't perform full sanitization, and the next command can assert on converting - # a ziplist to hash records, then we're ok with that happning in RESTORE too +test {corrupt payload: fuzzer findings - hash ziplist too long entry len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { - r config set sanitize-dump-payload no r debug set-skip-checksum-validation 1 - catch { r RESTORE _hash 0 "\x0D\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x13\x00\xF5\x02\xF5\x02\xF2\x02\x53\x5F\x31\x04\xF3\x02\xF3\x02\xF7\x02\xF7\x02\xF8\x02\x02\x5F\x37\x04\xF1\x02\xF1\x02\xF6\x02\x02\x5F\x35\x04\xF4\x02\x02\x5F\x33\x04\xFA\x02\x02\x5F\x39\x04\xF9\x02\xF9\xFF\x09\x00\xB5\x48\xDE\x62\x31\xD0\xE5\x63" } - assert_equal [count_log_message 0 "crashed by signal"] 0 - assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + catch { + r RESTORE _hash 0 "\x0D\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x13\x00\xF5\x02\xF5\x02\xF2\x02\x53\x5F\x31\x04\xF3\x02\xF3\x02\xF7\x02\xF7\x02\xF8\x02\x02\x5F\x37\x04\xF1\x02\xF1\x02\xF6\x02\x02\x5F\x35\x04\xF4\x02\x02\x5F\x33\x04\xFA\x02\x02\x5F\x39\x04\xF9\x02\xF9\xFF\x09\x00\xB5\x48\xDE\x62\x31\xD0\xE5\x63" + } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 } } @@ -688,5 +694,15 @@ test {corrupt payload: fuzzer findings - hash with len of 0} { } } +test {corrupt payload: fuzzer findings - hash listpack first element too long entry len} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r debug set-skip-checksum-validation 1 + r config set sanitize-dump-payload yes + catch { r restore _hash 0 "\x10\x15\x15\x00\x00\x00\x06\x00\xF0\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x02\x01\x02\x01\xFF\x0A\x00\x94\x21\x0A\xFA\x06\x52\x9F\x44" replace } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 + } +} + } ;# tags diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 370d1190ab7d..ae4c46723076 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -48,6 +48,7 @@ set ::all_tests { integration/corrupt-dump integration/corrupt-dump-fuzzer integration/convert-zipmap-hash-on-load + integration/convert-ziplist-hash-on-load integration/logging integration/psync2 integration/psync2-reg diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index 31da1b8f1081..451966d038cf 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -127,10 +127,10 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}} } foreach d {string int} { - foreach e {ziplist hashtable} { + foreach e {listpack hashtable} { test "AOF rewrite of hash with $e encoding, $d data" { r flushall - if {$e eq {ziplist}} {set len 10} else {set len 1000} + if {$e eq {listpack}} {set len 10} else {set len 1000} for {set j 0} {$j < $len} {incr j} { if {$d eq {string}} { set data [randstring 0 16 alpha] diff --git a/tests/unit/keyspace.tcl b/tests/unit/keyspace.tcl index 7fa7ba127546..d39f2fc6e144 100644 --- a/tests/unit/keyspace.tcl +++ b/tests/unit/keyspace.tcl @@ -313,10 +313,10 @@ start_server {tags {"keyspace"}} { r config set zset-max-ziplist-entries $original_max } - test {COPY basic usage for ziplist hash} { + test {COPY basic usage for listpack hash} { r del hash1{t} newhash1{t} r hset hash1{t} tmp 17179869184 - assert_encoding ziplist hash1{t} + assert_encoding listpack hash1{t} r copy hash1{t} newhash1{t} set digest [debug_digest_value hash1{t}] assert_equal $digest [debug_digest_value newhash1{t}] diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 65d2f99a5cdb..870f1a145447 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -132,11 +132,11 @@ start_server {tags {"scan network"}} { } } - foreach enc {ziplist hashtable} { + foreach enc {listpack hashtable} { test "HSCAN with encoding $enc" { # Create the Hash r del hash - if {$enc eq {ziplist}} { + if {$enc eq {listpack}} { set count 30 } else { set count 1000 diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index 5f00a732e9d4..9161926ded33 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -14,8 +14,8 @@ start_server {tags {"hash"}} { list [r hlen smallhash] } {8} - test {Is the small hash encoded with a ziplist?} { - assert_encoding ziplist smallhash + test {Is the small hash encoded with a listpack?} { + assert_encoding listpack smallhash } proc create_hash {key entries} { @@ -34,12 +34,15 @@ start_server {tags {"hash"}} { return $res } - foreach {type contents} "ziplist {{a 1} {b 2} {c 3}} hashtable {{a 1} {b 2} {[randstring 70 90 alpha] 3}}" { + foreach {type contents} "listpack {{a 1} {b 2} {c 3}} hashtable {{a 1} {b 2} {[randstring 70 90 alpha] 3}}" { set original_max_value [lindex [r config get hash-max-ziplist-value] 1] r config set hash-max-ziplist-value 10 create_hash myhash $contents assert_encoding $type myhash + # coverage for objectComputeSize + assert_morethan [r memory usage myhash] 0 + test "HRANDFIELD - $type" { unset -nocomplain myhash array set myhash {} @@ -87,7 +90,7 @@ start_server {tags {"hash"}} { foreach {type contents} " hashtable {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {[randstring 70 90 alpha] 10}} - ziplist {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " { + listpack {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " { test "HRANDFIELD with - $type" { set original_max_value [lindex [r config get hash-max-ziplist-value] 1] r config set hash-max-ziplist-value 10