Skip to content
This repository

Added ZSET command ZDIFFSTORE #448

Open
wants to merge 3 commits into from

3 participants

Michael Parrish Andrew Armstrong Arfon Smith
Michael Parrish

I've posted to an existing topic on the google group (pending approval) to explain our use case, but I'll go ahead and duplicate it here.

We use Redis to select the next object to be shown to a user for classification.

Two things are important in this selection:

  1. Users are shown objects only once (unique selection)
  2. Objects are scored based upon prior classifications (the score is updated in real time) and users should be shown the most highly scored object they haven't seen.

Currently we do the following:

  • Store many keys for object scores (objects_score_<id>)
  • Store a set of object ids seen by a user (seen_objects_for_<user_id>)
  • Store a set of object ids available for classification (objects)
  • Remove the diff set after selection is done (since the stored order from the score is invalidated shortly after creation)

Then perform:

sdiffstore user_<id>_unseen_objects objects seen_objects_for_user_<id>
sort user_<id>_unseen_objects DESC BY objects_score_* LIMIT 0 1
del user_<id>_unseen_objects

Given we have over 600,000 users and well over 10,000,000 objects between our projects, staying responsive while handling rates above 100 requests/second is non-trivial. Optimizing this specific use-case is pretty critical for us.

Please feel free to suggest changes, revisions, etc.

Thanks!

-Michael

Michael Parrish parrish referenced this pull request April 19, 2012
Open

zdiffstore #446

Andrew Armstrong

This would be appreciated; I have a use case for this as well:

  1. Maintain a sorted set of "New Item Ids", capped at 100 items, sorted/scored by insertion timestamp
  2. Maintain a set/sorted set of "Recently Seen Item Ids", capped at 100 items, sorted/scored by insertion timestamp

To answer the query of "What New Item Ids have not been seen recently?" I would like to ZDIFF the sets.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Showing 3 unique commits by 2 authors.

Apr 10, 2012
Michael Parrish Added ZSET command ZDIFFSTORE 6e78744
Feb 08, 2013
Arfon Smith Binaries 5023ca2
May 07, 2013
Arfon Smith Redis 64bit binaries e9f324a
This page is out of date. Refresh to see the latest.
10  .gitignore
@@ -2,11 +2,11 @@
2 2
 *.o
3 3
 *.rdb
4 4
 *.log
5  
-redis-cli
6  
-redis-server
7  
-redis-benchmark
8  
-redis-check-dump
9  
-redis-check-aof
  5
+src/redis-cli
  6
+src/redis-server
  7
+src/redis-benchmark
  8
+src/redis-check-dump
  9
+src/redis-check-aof
10 10
 doc-tools
11 11
 release
12 12
 misc/*
BIN  binaries/redis-benchmark
Binary file not shown
BIN  binaries/redis-check-aof
Binary file not shown
BIN  binaries/redis-check-dump
Binary file not shown
BIN  binaries/redis-cli
Binary file not shown
BIN  binaries/redis-server
Binary file not shown
1  src/redis.c
@@ -166,6 +166,7 @@ struct redisCommand redisCommandTable[] = {
166 166
     {"zremrangebyrank",zremrangebyrankCommand,4,"w",0,NULL,1,1,1,0,0},
167 167
     {"zunionstore",zunionstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
168 168
     {"zinterstore",zinterstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
  169
+    {"zdiffstore",zdiffstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
169 170
     {"zrange",zrangeCommand,-4,"r",0,NULL,1,1,1,0,0},
170 171
     {"zrangebyscore",zrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
171 172
     {"zrevrangebyscore",zrevrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
1  src/redis.h
@@ -1219,6 +1219,7 @@ void hlenCommand(redisClient *c);
1219 1219
 void zremrangebyrankCommand(redisClient *c);
1220 1220
 void zunionstoreCommand(redisClient *c);
1221 1221
 void zinterstoreCommand(redisClient *c);
  1222
+void zdiffstoreCommand(redisClient *c);
1222 1223
 void hkeysCommand(redisClient *c);
1223 1224
 void hvalsCommand(redisClient *c);
1224 1225
 void hgetallCommand(redisClient *c);
44  src/t_zset.c
@@ -1460,7 +1460,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
1460 1460
 
1461 1461
     if (setnum < 1) {
1462 1462
         addReplyError(c,
1463  
-            "at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE");
  1463
+            "at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE/ZDIFFSTORE");
1464 1464
         return;
1465 1465
     }
1466 1466
 
@@ -1507,7 +1507,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
1507 1507
                         return;
1508 1508
                     }
1509 1509
                 }
1510  
-            } else if (remaining >= 2 && !strcasecmp(c->argv[j]->ptr,"aggregate")) {
  1510
+            } else if ( op != REDIS_OP_DIFF && remaining >= 2 && !strcasecmp(c->argv[j]->ptr,"aggregate")) {
1511 1511
                 j++; remaining--;
1512 1512
                 if (!strcasecmp(c->argv[j]->ptr,"sum")) {
1513 1513
                     aggregate = REDIS_AGGR_SUM;
@@ -1534,7 +1534,9 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
1534 1534
 
1535 1535
     /* sort sets from the smallest to largest, this will improve our
1536 1536
      * algorithm's performance */
1537  
-    qsort(src,setnum,sizeof(zsetopsrc),zuiCompareByCardinality);
  1537
+    if ( op != REDIS_OP_DIFF ) {
  1538
+        qsort(src, setnum, sizeof(zsetopsrc), zuiCompareByCardinality);
  1539
+    }
1538 1540
 
1539 1541
     dstobj = createZsetObject();
1540 1542
     dstzset = dstobj->ptr;
@@ -1620,6 +1622,38 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) {
1620 1622
                         maxelelen = sdslen(tmp->ptr);
1621 1623
             }
1622 1624
         }
  1625
+    } else if (op == REDIS_OP_DIFF) {
  1626
+        /* Skip everything if the smallest input is empty. */
  1627
+        if (zuiLength(&src[0]) > 0) {
  1628
+            while (zuiNext(&src[0],&zval)) {
  1629
+                double score, value;
  1630
+                int exists = 0;
  1631
+
  1632
+                // weighting doesn't make sense here
  1633
+                score = zval.score;
  1634
+                if (isnan(score)) score = 0;
  1635
+
  1636
+                // if this key exists in any other input set, stop looking for it
  1637
+                for (j = 1; j < setnum; j++) {
  1638
+                    if (zuiFind(&src[j],&zval,&value)) {
  1639
+                        exists = 1;
  1640
+                        break;
  1641
+                    }
  1642
+                }
  1643
+
  1644
+                if (!exists) {
  1645
+                    tmp = zuiObjectFromValue(&zval);
  1646
+                    znode = zslInsert(dstzset->zsl,score,tmp);
  1647
+                    incrRefCount(zval.ele); /* added to skiplist */
  1648
+                    dictAdd(dstzset->dict,tmp,&znode->score);
  1649
+                    incrRefCount(zval.ele); /* added to dictionary */
  1650
+
  1651
+                    if (tmp->encoding == REDIS_ENCODING_RAW)
  1652
+                        if (sdslen(tmp->ptr) > maxelelen)
  1653
+                            maxelelen = sdslen(tmp->ptr);
  1654
+                }
  1655
+            }
  1656
+        }
1623 1657
     } else {
1624 1658
         redisPanic("Unknown operator");
1625 1659
     }
@@ -1657,6 +1691,10 @@ void zinterstoreCommand(redisClient *c) {
1657 1691
     zunionInterGenericCommand(c,c->argv[1], REDIS_OP_INTER);
1658 1692
 }
1659 1693
 
  1694
+void zdiffstoreCommand(redisClient *c) {
  1695
+    zunionInterGenericCommand(c, c->argv[1], REDIS_OP_DIFF);
  1696
+}
  1697
+
1660 1698
 void zrangeGenericCommand(redisClient *c, int reverse) {
1661 1699
     robj *key = c->argv[1];
1662 1700
     robj *zobj;
15  tests/unit/type/zset.tcl
@@ -471,6 +471,21 @@ start_server {tags {"zset"}} {
471 471
             assert_equal {b 2 c 3} [r zrange zsetc 0 -1 withscores]
472 472
         }
473 473
 
  474
+        test "ZDIFFSTORE basics - $encoding" {
  475
+            assert_equal 1 [r zdiffstore zsetc 2 zseta zsetb]
  476
+            assert_equal {a 1} [r zrange zsetc 0 -1 withscores]
  477
+        }
  478
+
  479
+        test "ZDIFFSTORE with WITHSCORES - $encoding" {
  480
+            r del zsetc
  481
+            r zadd zseta 4 e
  482
+            r zadd zseta 5 f
  483
+            r zadd zsetc 4 e
  484
+
  485
+            assert_equal 2 [r zdiffstore zsetd 3 zseta zsetb zsetc]
  486
+            assert_equal {a 1 f 5} [r zrange zsetd 0 -1 withscores]
  487
+        }
  488
+
474 489
         foreach cmd {ZUNIONSTORE ZINTERSTORE} {
475 490
             test "$cmd with +inf/-inf scores - $encoding" {
476 491
                 r del zsetinf1 zsetinf2
Commit_comment_tip

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.