Give each thread a chunk of 100 keys at a time to truncate

noahhl · Sep 3, 2012 · cd9fbd5 · cd9fbd5
1 parent 676ea23
commit cd9fbd5
Showing 1 changed file with 6 additions and 4 deletions.
diff --git a/lib/batsd/truncator.rb b/lib/batsd/truncator.rb
@@ -45,10 +45,12 @@ def run(retention)
         keys.each { |key| @redis.truncate_zset(key, min_ts) }
       else
         # Stored on disk
-        keys.each do |key|
-          key = "#{key}:#{retention}"
-          @threadpool.queue @diskstore, key, min_ts do |diskstore, key, min_ts|
-            diskstore.truncate(diskstore.build_filename(key), min_ts)
+        keys.each_slice(100) do |keys|
+          @threadpool.queue @diskstore, keys, retention, min_ts do |diskstore, keys, retention, min_ts|
+            keys.each do |key|
+              key = "#{key}:#{retention}"
+              diskstore.truncate(diskstore.build_filename(key), min_ts)
+            end
           end
         end
         while @threadpool.size > 0