Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Added ZSET command ZDIFFSTORE #448

Open
wants to merge 3 commits into from

3 participants

@parrish

I've posted to an existing topic on the google group (pending approval) to explain our use case, but I'll go ahead and duplicate it here.

We use Redis to select the next object to be shown to a user for classification.

Two things are important in this selection:

  1. Users are shown objects only once (unique selection)
  2. Objects are scored based upon prior classifications (the score is updated in real time) and users should be shown the most highly scored object they haven't seen.

Currently we do the following:

  • Store many keys for object scores (objects_score_<id>)
  • Store a set of object ids seen by a user (seen_objects_for_<user_id>)
  • Store a set of object ids available for classification (objects)
  • Remove the diff set after selection is done (since the stored order from the score is invalidated shortly after creation)

Then perform:

sdiffstore user_<id>_unseen_objects objects seen_objects_for_user_<id>
sort user_<id>_unseen_objects DESC BY objects_score_* LIMIT 0 1
del user_<id>_unseen_objects

Given we have over 600,000 users and well over 10,000,000 objects between our projects, staying responsive while handling rates above 100 requests/second is non-trivial. Optimizing this specific use-case is pretty critical for us.

Please feel free to suggest changes, revisions, etc.

Thanks!

-Michael

@parrish parrish referenced this pull request
Open

zdiffstore #446

@Plasma

This would be appreciated; I have a use case for this as well:

  1. Maintain a sorted set of "New Item Ids", capped at 100 items, sorted/scored by insertion timestamp
  2. Maintain a set/sorted set of "Recently Seen Item Ids", capped at 100 items, sorted/scored by insertion timestamp

To answer the query of "What New Item Ids have not been seen recently?" I would like to ZDIFF the sets.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Apr 10, 2012
  1. @parrish

    Added ZSET command ZDIFFSTORE

    parrish authored
Commits on Feb 8, 2013
  1. @arfon

    Binaries

    arfon authored
Commits on May 7, 2013
  1. @arfon

    Redis 64bit binaries

    arfon authored
This page is out of date. Refresh to see the latest.
View
10 .gitignore
@@ -2,11 +2,11 @@
*.o
*.rdb
*.log
-redis-cli
-redis-server
-redis-benchmark
-redis-check-dump
-redis-check-aof
+src/redis-cli
+src/redis-server
+src/redis-benchmark
+src/redis-check-dump
+src/redis-check-aof
doc-tools
release
misc/*
View
BIN  binaries/redis-benchmark
Binary file not shown
View
BIN  binaries/redis-check-aof
Binary file not shown
View
BIN  binaries/redis-check-dump
Binary file not shown
View
BIN  binaries/redis-cli
Binary file not shown
View
BIN  binaries/redis-server
Binary file not shown
View
1  src/redis.c
@@ -166,6 +166,7 @@ struct redisCommand redisCommandTable[] = {
{"zremrangebyrank",zremrangebyrankCommand,4,"w",0,NULL,1,1,1,0,0},
{"zunionstore",zunionstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
{"zinterstore",zinterstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
+ {"zdiffstore",zdiffstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
{"zrange",zrangeCommand,-4,"r",0,NULL,1,1,1,0,0},
{"zrangebyscore",zrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
{"zrevrangebyscore",zrevrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
View
1  src/redis.h
@@ -1219,6 +1219,7 @@ void hlenCommand(redisClient *c);
void zremrangebyrankCommand(redisClient *c);
void zunionstoreCommand(redisClient *c);
void zinterstoreCommand(redisClient *c);
+void zdiffstoreCommand(redisClient *c);
void hkeysCommand(redisClient *c);
void hvalsCommand(redisClient *c);
void hgetallCommand(redisClient *c);
View
44 src/t_zset.c
@@ -1460,7 +1460,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
if (setnum < 1) {
addReplyError(c,
- "at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE");
+ "at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE/ZDIFFSTORE");
return;
}
@@ -1507,7 +1507,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
return;
}
}
- } else if (remaining >= 2 && !strcasecmp(c->argv[j]->ptr,"aggregate")) {
+ } else if ( op != REDIS_OP_DIFF && remaining >= 2 && !strcasecmp(c->argv[j]->ptr,"aggregate")) {
j++; remaining--;
if (!strcasecmp(c->argv[j]->ptr,"sum")) {
aggregate = REDIS_AGGR_SUM;
@@ -1534,7 +1534,9 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
/* sort sets from the smallest to largest, this will improve our
* algorithm's performance */
- qsort(src,setnum,sizeof(zsetopsrc),zuiCompareByCardinality);
+ if ( op != REDIS_OP_DIFF ) {
+ qsort(src, setnum, sizeof(zsetopsrc), zuiCompareByCardinality);
+ }
dstobj = createZsetObject();
dstzset = dstobj->ptr;
@@ -1620,6 +1622,38 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
maxelelen = sdslen(tmp->ptr);
}
}
+ } else if (op == REDIS_OP_DIFF) {
+ /* Skip everything if the smallest input is empty. */
+ if (zuiLength(&src[0]) > 0) {
+ while (zuiNext(&src[0],&zval)) {
+ double score, value;
+ int exists = 0;
+
+ // weighting doesn't make sense here
+ score = zval.score;
+ if (isnan(score)) score = 0;
+
+ // if this key exists in any other input set, stop looking for it
+ for (j = 1; j < setnum; j++) {
+ if (zuiFind(&src[j],&zval,&value)) {
+ exists = 1;
+ break;
+ }
+ }
+
+ if (!exists) {
+ tmp = zuiObjectFromValue(&zval);
+ znode = zslInsert(dstzset->zsl,score,tmp);
+ incrRefCount(zval.ele); /* added to skiplist */
+ dictAdd(dstzset->dict,tmp,&znode->score);
+ incrRefCount(zval.ele); /* added to dictionary */
+
+ if (tmp->encoding == REDIS_ENCODING_RAW)
+ if (sdslen(tmp->ptr) > maxelelen)
+ maxelelen = sdslen(tmp->ptr);
+ }
+ }
+ }
} else {
redisPanic("Unknown operator");
}
@@ -1657,6 +1691,10 @@ void zinterstoreCommand(redisClient *c) {
zunionInterGenericCommand(c,c->argv[1], REDIS_OP_INTER);
}
+void zdiffstoreCommand(redisClient *c) {
+ zunionInterGenericCommand(c, c->argv[1], REDIS_OP_DIFF);
+}
+
void zrangeGenericCommand(redisClient *c, int reverse) {
robj *key = c->argv[1];
robj *zobj;
View
15 tests/unit/type/zset.tcl
@@ -471,6 +471,21 @@ start_server {tags {"zset"}} {
assert_equal {b 2 c 3} [r zrange zsetc 0 -1 withscores]
}
+ test "ZDIFFSTORE basics - $encoding" {
+ assert_equal 1 [r zdiffstore zsetc 2 zseta zsetb]
+ assert_equal {a 1} [r zrange zsetc 0 -1 withscores]
+ }
+
+ test "ZDIFFSTORE with WITHSCORES - $encoding" {
+ r del zsetc
+ r zadd zseta 4 e
+ r zadd zseta 5 f
+ r zadd zsetc 4 e
+
+ assert_equal 2 [r zdiffstore zsetd 3 zseta zsetb zsetc]
+ assert_equal {a 1 f 5} [r zrange zsetd 0 -1 withscores]
+ }
+
foreach cmd {ZUNIONSTORE ZINTERSTORE} {
test "$cmd with +inf/-inf scores - $encoding" {
r del zsetinf1 zsetinf2
Something went wrong with that request. Please try again.