Merge branch 'expires'

Conflicts: src/test/scala/com/twitter/haplocheirus/RedisShardSpec.scala
twitter-archive · Sep 16, 2011 · 2c97671 · 2c97671
2 parents 9eb4112 + 62191ad
commit 2c97671
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 8 deletions.
diff --git a/src/main/scala/com/twitter/haplocheirus/PipelinedRedisClient.scala b/src/main/scala/com/twitter/haplocheirus/PipelinedRedisClient.scala
@@ -273,7 +273,11 @@ class PipelinedRedisClient(hostname: String, pipelineMaxSize: Int, batchSize: In
     val start = -1 - offset
     val end = if (length > 0) (start - length + 1) else 0
     Stats.timeMicros("redis-get-usec") {
-      redisClient.lrange(timeline, end, start).get(timeout.inMillis, TimeUnit.MILLISECONDS).toSeq.reverse
+      val rv = redisClient.lrange(timeline, end, start).get(timeout.inMillis, TimeUnit.MILLISECONDS).toSeq.reverse
+      if (rv.size > 0) {
+        redisClient.expire(timeline, expiration.inSeconds)
+      }
+      rv
     }
   }
 
@@ -298,7 +302,8 @@ class PipelinedRedisClient(hostname: String, pipelineMaxSize: Int, batchSize: In
           redisClient.rpushx(tempName, slice: _*)
         }
 
-        redisClient.rename(tempName, timeline).get(timeout.inMillis, TimeUnit.MILLISECONDS)
+        redisClient.rename(tempName, timeline)
+        redisClient.expire(timeline, expiration.inSeconds).get(timeout.inMillis, TimeUnit.MILLISECONDS)
       }
     }
   }
@@ -320,7 +325,8 @@ class PipelinedRedisClient(hostname: String, pipelineMaxSize: Int, batchSize: In
   def setLive(timeline: String, entries: Seq[Array[Byte]]) {
     Stats.timeMicros("redis-setlive-usec") {
       if (entries.length > 0) {
-        redisClient.lpushx(timeline, entries.toArray: _*).get(timeout.inMillis, TimeUnit.MILLISECONDS)
+        redisClient.lpushx(timeline, entries.toArray: _*)
+        redisClient.expire(timeline, expiration.inSeconds).get(timeout.inMillis, TimeUnit.MILLISECONDS)
       }
       0
     }

diff --git a/src/test/scala/com/twitter/haplocheirus/IntegrationSpec.scala b/src/test/scala/com/twitter/haplocheirus/IntegrationSpec.scala
@@ -156,13 +156,16 @@ object IntegrationSpec extends ConfiguredSpecification with JMocker with ClassMo
         one(future).get(200L, TimeUnit.MILLISECONDS) willReturn 2L
         one(jredisClient).lrange(timeline1, -3, -1) willReturn timelineFuture
         one(timelineFuture).get(200L, TimeUnit.MILLISECONDS) willReturn List("a", "b").map { _.getBytes }.toJavaList
+        one(jredisClient).expire(timeline1, 86400)
 
         one(jredisClient).lrange(timeline1, 0, -1) willReturn timelineFuture
         one(timelineFuture).get(200L, TimeUnit.MILLISECONDS) willReturn List("a", "b").map { _.getBytes }.toJavaList
+        one(jredisClient).expire(timeline1, 86400)
 
         one(jredisClient).del(timeline1)
         one(jredisClient).rpush(timeline1, TimelineEntry.EmptySentinel)
         one(jredisClient).lpushx(timeline1, Array("b", "a").map(_.getBytes): _*)
+        one(jredisClient).expire(timeline1, 86400)
         allowing(jredisClient).quit()
       }
 
@@ -178,6 +181,7 @@ object IntegrationSpec extends ConfiguredSpecification with JMocker with ClassMo
         one(future).get(200L, TimeUnit.MILLISECONDS) willReturn 2L
         one(jredisClient).lrange(timeline1, -3, -1) willReturn timelineFuture
         one(timelineFuture).get(200L, TimeUnit.MILLISECONDS) willReturn List("a", "b").map { _.getBytes }.toJavaList
+        one(jredisClient).expire(timeline1, 86400)
       }
 
       val cmd = new thrift.TimelineGet(timeline1, 0, 2)

diff --git a/src/test/scala/com/twitter/haplocheirus/PipelinedRedisClientSpec.scala b/src/test/scala/com/twitter/haplocheirus/PipelinedRedisClientSpec.scala
@@ -138,6 +138,7 @@ object PipelinedRedisClientSpec extends ConfiguredSpecification with JMocker wit
       expect {
         one(jredis).lrange(timeline, -15, -6) willReturn future2
         one(future2).get(1000, TimeUnit.MILLISECONDS) willReturn result.reverse.toJavaList
+        one(jredis).expire(timeline, 86400)
       }
 
       client.get(timeline, 5, 10).toList mustEqual result
@@ -149,9 +150,10 @@ object PipelinedRedisClientSpec extends ConfiguredSpecification with JMocker wit
       val entry3 = List(19L).pack.array
 
       expect {
-        one(jredis).rpush(timeline + "~1", entry3) willReturn longFuture
-        one(jredis).rpushx(timeline + "~1", entry2, entry1) willReturn longFuture
-        one(jredis).rename(timeline + "~1", timeline) willReturn future
+        one(jredis).rpush(timeline + "~1", entry3)
+        one(jredis).rpushx(timeline + "~1", entry2, entry1)
+        one(jredis).rename(timeline + "~1", timeline)
+        one(jredis).expire(timeline, 86400) willReturn future
         one(future).get(1000, TimeUnit.MILLISECONDS) willReturn ResponseStatus.STATUS_OK
       }
 
@@ -174,6 +176,8 @@ object PipelinedRedisClientSpec extends ConfiguredSpecification with JMocker wit
 
       expect {
         one(jredis).lpushx(timeline, entry1, entry2, entry3)
+        one(jredis).expire(timeline, 86400) willReturn future
+        one(future).get(1000, TimeUnit.MILLISECONDS) willReturn ResponseStatus.STATUS_OK
       }
 
       client.setLive(timeline, List(entry1, entry2, entry3))

diff --git a/src/test/scala/com/twitter/haplocheirus/RedisShardSpec.scala b/src/test/scala/com/twitter/haplocheirus/RedisShardSpec.scala
@@ -132,6 +132,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         one(shardInfo).hostname willReturn "host1"
         lrange(timeline, 0, -1, List(entry23a, entry19a).reverse)
         one(jredis).lrem(timeline, entry23a, 0)
+        one(jredis).expire(timeline, 1)
       }
 
       val oldOperationCount = operationCount
@@ -151,6 +152,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         expect {
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, 0, -1, List(entry20, entry22).reverse)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.filter(timeline, List(20L, 21L), -1).get.toList mustEqual List(entry20)
@@ -160,6 +162,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         expect {
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -3, -1, List(entry21, entry22, entry23).reverse)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.filter(timeline, List(20L, 21L), 3).get.toList mustEqual List(entry21)
@@ -172,6 +175,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -11, -1, List(entry23, entry20, entry19).reverse)
           llen(timeline, 3L)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.get(timeline, 0, 10, false).get.entries.toList mustEqual List(entry23, entry20, entry19)
@@ -184,6 +188,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
             one(shardInfo).hostname willReturn "host1"
             lrange(timeline, -21, -11, List(entry23, entry20, entry19).reverse)
             llen(timeline, 23L)
+            one(jredis).expire(timeline, 1)
           }
 
           redisShard.get(timeline, 10, 10, false).get.entries.toList mustEqual List(entry23, entry20, entry19)
@@ -195,6 +200,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
             one(shardInfo).hostname willReturn "host1"
             lrange(timeline, -21, -11, List())
             llen(timeline, 3L)
+            one(jredis).expire(timeline, 1)
           }
 
           redisShard.get(timeline, 10, 10, false).get.entries.toList mustEqual List()
@@ -217,6 +223,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -11, -1, List(entry23, entry23a, entry19).reverse)
           llen(timeline, 3L)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.get(timeline, 0, 10, false).get.entries.toList mustEqual List(entry23, entry19)
@@ -229,8 +236,10 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
             allowing(shardInfo).hostname willReturn "host1"
             lrange(timeline, -11, -1, List(entry23a, entry20, entry19a).reverse)
             llen(timeline, 3L)
+            one(jredis).expire(timeline, 1)
             lrange(timeline, -11, -1, List(entry23a, entry20, entry19a).reverse)
             llen(timeline, 3L)
+            one(jredis).expire(timeline, 1)
           }
 
           redisShard.get(timeline, 0, 10, false).get.entries.toList mustEqual List(entry23a, entry20, entry19a)
@@ -243,6 +252,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
             allowing(shardInfo).hostname willReturn "host1"
             lrange(timeline, -11, -1, List(entry23a, entry20, entry19uniq).reverse)
             llen(timeline, 3L)
+            one(jredis).expire(timeline, 1)
           }
 
           redisShard.get(timeline, 0, 10, true).get.entries.toList mustEqual List(entry23a, entry20, entry19uniq)
@@ -255,6 +265,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -11, -1, List(entry23share, entry23, entry20).reverse)
           llen(timeline, 3L)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.get(timeline, 0, 10, true).get.entries.toList mustEqual List(entry23, entry20)
@@ -270,6 +281,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -11, -1, List(entry1, entry2, entry3).reverse)
           llen(timeline, 3L)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.get(timeline, 0, 10, true).get.entries.toList mustEqual List(entry2, entry3, entry1)
@@ -281,6 +293,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
       expect {
         one(shardInfo).hostname willReturn "host1"
         lrange(timeline, 0, -1, List(entry23, entry20, entry19).reverse)
+        one(jredis).expire(timeline, 1)
       }
 
       redisShard.getRaw(timeline) mustEqual Some(List(entry23, entry20, entry19))
@@ -292,6 +305,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         expect {
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -600, -1, List(entry23, entry20, entry19, entry10).reverse)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.getRange(timeline, 11L, 0L, false).get.entries.toList mustEqual List(entry23, entry20, entry19)
@@ -302,6 +316,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         expect {
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -600, -1, List(entry23, entry20, entry19, entry10).reverse)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.getRange(timeline, 19L, 0L, false).get.entries.toList mustEqual List(entry23, entry20)
@@ -312,8 +327,11 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         expect {
           allowing(shardInfo).hostname willReturn "host1"
           lrange(timeline, -600, -1, List(entry23, entry20, entry19).reverse)
+          one(jredis).expire(timeline, 1)
           lrange(timeline, -600, -1, List(entry23, entry20, entry19).reverse)
+          one(jredis).expire(timeline, 1)
           lrange(timeline, -600, -1, List(entry23, entry20, entry19).reverse)
+          one(jredis).expire(timeline, 1)
           allowing(jredis).expire(timeline, 1)
         }
 
@@ -327,6 +345,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         expect {
           one(shardInfo).hostname willReturn "host1"
           lrange(timeline, -600, -1, List(entry23, entry20, entry20,entry20, entry17, entry13).reverse)
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.getRange(timeline, 13L, 0L, false).get.entries.toList mustEqual List(entry23, entry20, entry17)
@@ -355,6 +374,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           one(jredis).lrange(timeline, 0, -1) willReturn future
           one(future).get(1000, TimeUnit.MILLISECONDS) willReturn existing.map { List(_).pack.array }.reverse.toJavaList
+          one(jredis).expire(timeline, 1)
           one(jredis).linsertBefore(timeline, List(7L).pack.array, insert(0)) willReturn longFuture
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
@@ -371,6 +391,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           one(jredis).lrange(timeline, 0, -1) willReturn future
           one(future).get(1000, TimeUnit.MILLISECONDS) willReturn existing.map { List(_).pack.array }.reverse.toJavaList
+          one(jredis).expire(timeline, 1)
         }
 
         redisShard.merge(timeline, List[Array[Byte]](), None)
@@ -384,6 +405,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           one(jredis).lrange(timeline, 0, -1) willReturn future
           one(future).get(1000, TimeUnit.MILLISECONDS) willReturn existing.map { List(_).pack.array }.reverse.toJavaList
+          one(jredis).expire(timeline, 1)
           one(jredis).rpushx(timeline, Array(List(29L).pack.array): _*) willReturn longFuture
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
@@ -405,6 +427,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           one(jredis).lrange(timeline, 0, -1) willReturn future
           one(future).get(1000, TimeUnit.MILLISECONDS) willReturn existing.map { List(_).pack.array }.reverse.toJavaList
+          one(jredis).expire(timeline, 1)
           one(jredis).linsertBefore(timeline, List(7L).pack.array, List(5L).pack.array) willReturn longFuture
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
@@ -423,6 +446,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           one(jredis).lrange(timeline, 0, -1) willReturn future
           one(future).get(1000, TimeUnit.MILLISECONDS) willReturn existing.map { List(_).pack.array }.reverse.toJavaList
+          one(jredis).expire(timeline, 1)
           one(jredis).linsertBefore(timeline, List(20L).pack.array, List(19L).pack.array) willReturn longFuture
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
           one(longFuture).get(1000, TimeUnit.MILLISECONDS)
@@ -444,6 +468,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
           one(shardInfo).hostname willReturn "host1"
           one(jredis).lrange(timeline, 0, -1) willReturn future
           one(future).get(1000, TimeUnit.MILLISECONDS) willReturn existing.map { List(_).pack.array }.reverse.toJavaList
+          one(jredis).expire(timeline, 1)
           one(jredis).linsertBefore(timeline, List(16L).pack.array, List(15L).pack.array)
         }
 
@@ -463,6 +488,7 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
         one(jredis).rpush("generated-name", TimelineEntry.EmptySentinel)
         one(jredis).rpushx("generated-name", entry3, entry2, entry1)
         one(jredis).rename("generated-name", timeline)
+        one(jredis).expire(timeline, 1)
       }
 
       redisShard.store(timeline, List(entry1, entry2, entry3))
@@ -544,8 +570,9 @@ object RedisShardSpec extends ConfiguredSpecification with JMocker with ClassMoc
 
       expect {
         one(shardInfo).hostname willReturn "host1"
-        one(jredis).lpushx(timeline, entry1, entry2) willReturn longFuture
-        one(longFuture).get(1000, TimeUnit.MILLISECONDS) willReturn 1L
+        one(jredis).lpushx(timeline, entry1, entry2)
+        one(jredis).expire(timeline, 1) willReturn future
+        one(future).get(1000, TimeUnit.MILLISECONDS) willReturn 1L
       }
 
       redisShard.doCopy(timeline, List(entry1, entry2))