Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hfe serialization listpack #13243

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
770beaa
Add RDB de/serialization to metadata hashes (WIP)
ronen-kalish Apr 16, 2024
3cc8f9d
Add RDB de/serialization to metadata hashes in listpack encoding (WIP)
ronen-kalish Apr 30, 2024
3c00e30
Fix RDB reading of hash metadata to set expiration in addition to value
ronen-kalish May 2, 2024
dfb45ce
Modify hash-md RDB format; remove type and use key-value-ttl tuple fo…
ronen-kalish May 2, 2024
d6c3b24
rename
ronen-kalish May 2, 2024
60287f7
typos
ronen-kalish May 5, 2024
d3b4b46
return line accidentally removed during rebase
ronen-kalish May 5, 2024
52198ee
fix UTs for HFE RDB de/serialization
ronen-kalish May 5, 2024
4866704
remove redisDebug logs
ronen-kalish May 5, 2024
213f24f
HFE RDB de/serialization - PR comments
ronen-kalish May 7, 2024
4109d96
HFE RDB de/serialization - PR comments #2
ronen-kalish May 7, 2024
3650681
fix rebase glitch
ronen-kalish May 9, 2024
0bc645e
HFE RDB de/serialization - PR comments #3
ronen-kalish May 9, 2024
e600e9b
set hash objects and expiry directly
ronen-kalish May 9, 2024
0152265
remove listpack-encoded hash active expiry on load
ronen-kalish May 9, 2024
679b7f1
HFE RDB de/serialization - PR comments #4
ronen-kalish May 12, 2024
0399074
typos
ronen-kalish May 12, 2024
a065704
fix dict encoding initialization on RDB reading
ronen-kalish May 15, 2024
d1eb0c7
added fields w/o expiry to active-expiry test
ronen-kalish May 15, 2024
ae2c287
fix mem leak
ronen-kalish May 15, 2024
daaad1d
more tests
ronen-kalish May 15, 2024
06a182d
HFE RDB de/serialization - PR comments #5
ronen-kalish May 15, 2024
4101585
save dict-encoded hash with expiry in ttl-field-value order
ronen-kalish May 15, 2024
2fd4112
fix freeing uninitialized values
ronen-kalish May 15, 2024
2935658
HFE RDB de/serialization - PR comments #6
ronen-kalish May 15, 2024
237d19c
typo
ronen-kalish May 15, 2024
53f24df
fix dict->listpack, listpack->dict tests
ronen-kalish May 16, 2024
5eb92e0
more test fixes and fixing listpack to dict conversion not added to g…
ronen-kalish May 16, 2024
a5280ca
HFE RDB de/serialization - PR comments #7
ronen-kalish May 16, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/cluster.c
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ void restoreCommand(client *c) {

rioInitWithBuffer(&payload,c->argv[3]->ptr);
if (((type = rdbLoadObjectType(&payload)) == -1) ||
((obj = rdbLoadObject(type,&payload,key->ptr,c->db->id,NULL)) == NULL))
((obj = rdbLoadObject(type,&payload,key->ptr,c->db,0,NULL)) == NULL))
{
addReplyError(c,"Bad data format");
return;
Expand Down
3 changes: 2 additions & 1 deletion src/ebuckets.c
Original file line number Diff line number Diff line change
Expand Up @@ -1403,7 +1403,8 @@ int ebRemove(ebuckets *eb, EbucketsType *type, eItem item) {
* @param item - The eItem to be added to the ebucket.
* @param expireTime - The expiration time of the item.
*
* @return 1 if the item was successfully added; Otherwise, return 0 on failure.
* @return 0 (C_OK) if the item was successfully added;
* Otherwise, return -1 (C_ERR) on failure.
*/
int ebAdd(ebuckets *eb, EbucketsType *type, eItem item, uint64_t expireTime) {
int res;
Expand Down
84 changes: 49 additions & 35 deletions src/listpack.c
Original file line number Diff line number Diff line change
Expand Up @@ -245,51 +245,61 @@ unsigned char* lpShrinkToFit(unsigned char *lp) {
static inline void lpEncodeIntegerGetType(int64_t v, unsigned char *intenc, uint64_t *enclen) {
if (v >= 0 && v <= 127) {
/* Single byte 0-127 integer. */
intenc[0] = v;
*enclen = 1;
if (intenc != NULL) intenc[0] = v;
if (enclen != NULL) *enclen = 1;
} else if (v >= -4096 && v <= 4095) {
/* 13 bit integer. */
if (v < 0) v = ((int64_t)1<<13)+v;
intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT;
intenc[1] = v&0xff;
*enclen = 2;
if (intenc != NULL) {
intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT;
intenc[1] = v&0xff;
}
if (enclen != NULL) *enclen = 2;
} else if (v >= -32768 && v <= 32767) {
/* 16 bit integer. */
if (v < 0) v = ((int64_t)1<<16)+v;
intenc[0] = LP_ENCODING_16BIT_INT;
intenc[1] = v&0xff;
intenc[2] = v>>8;
*enclen = 3;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_16BIT_INT;
intenc[1] = v&0xff;
intenc[2] = v>>8;
}
if (enclen != NULL) *enclen = 3;
} else if (v >= -8388608 && v <= 8388607) {
/* 24 bit integer. */
if (v < 0) v = ((int64_t)1<<24)+v;
intenc[0] = LP_ENCODING_24BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = v>>16;
*enclen = 4;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_24BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = v>>16;
}
if (enclen != NULL) *enclen = 4;
} else if (v >= -2147483648 && v <= 2147483647) {
/* 32 bit integer. */
if (v < 0) v = ((int64_t)1<<32)+v;
intenc[0] = LP_ENCODING_32BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = (v>>16)&0xff;
intenc[4] = v>>24;
*enclen = 5;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_32BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = (v>>16)&0xff;
intenc[4] = v>>24;
}
if (enclen != NULL) *enclen = 5;
} else {
/* 64 bit integer. */
uint64_t uv = v;
intenc[0] = LP_ENCODING_64BIT_INT;
intenc[1] = uv&0xff;
intenc[2] = (uv>>8)&0xff;
intenc[3] = (uv>>16)&0xff;
intenc[4] = (uv>>24)&0xff;
intenc[5] = (uv>>32)&0xff;
intenc[6] = (uv>>40)&0xff;
intenc[7] = (uv>>48)&0xff;
intenc[8] = uv>>56;
*enclen = 9;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_64BIT_INT;
intenc[1] = uv&0xff;
intenc[2] = (uv>>8)&0xff;
intenc[3] = (uv>>16)&0xff;
intenc[4] = (uv>>24)&0xff;
intenc[5] = (uv>>32)&0xff;
intenc[6] = (uv>>40)&0xff;
intenc[7] = (uv>>48)&0xff;
intenc[8] = uv>>56;
}
if (enclen != NULL) *enclen = 9;
}
}

Expand Down Expand Up @@ -1199,13 +1209,17 @@ size_t lpBytes(unsigned char *lp) {
return lpGetTotalBytes(lp);
}

/* Returns the size of a listpack consisting of an integer repeated 'rep' times. */
size_t lpEstimateBytesRepeatedInteger(long long lval, unsigned long rep) {
/* Returns the size 'lval' will require when encoded, in bytes */
size_t lpEntrySizeInteger(long long lval) {
ronen-kalish marked this conversation as resolved.
Show resolved Hide resolved
uint64_t enclen;
unsigned char intenc[LP_MAX_INT_ENCODING_LEN];
lpEncodeIntegerGetType(lval, intenc, &enclen);
lpEncodeIntegerGetType(lval, NULL, &enclen);
unsigned long backlen = lpEncodeBacklen(NULL, enclen);
return LP_HDR_SIZE + (enclen + backlen) * rep + 1;
return enclen + backlen;
}

/* Returns the size of a listpack consisting of an integer repeated 'rep' times. */
size_t lpEstimateBytesRepeatedInteger(long long lval, unsigned long rep) {
return LP_HDR_SIZE + lpEntrySizeInteger(lval) * rep + 1;
}

/* Seek the specified element and returns the pointer to the seeked element.
Expand Down
1 change: 1 addition & 0 deletions src/listpack.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ unsigned char *lpLast(unsigned char *lp);
unsigned char *lpNext(unsigned char *lp, unsigned char *p);
unsigned char *lpPrev(unsigned char *lp, unsigned char *p);
size_t lpBytes(unsigned char *lp);
size_t lpEntrySizeInteger(long long lval);
size_t lpEstimateBytesRepeatedInteger(long long lval, unsigned long rep);
unsigned char *lpSeek(unsigned char *lp, long index);
typedef int (*listpackValidateEntryCB)(unsigned char *p, unsigned int head_count, void *userdata);
Expand Down