Skip to content
Browse files

fork resque to use mongo instead of redis

  • Loading branch information...
1 parent db3bdf6 commit 9fbe0d99700343e19b416cd66b32c8bb018d70ac @mediocretes mediocretes committed
View
4 Rakefile
@@ -1,8 +1,6 @@
#
# Setup
#
-
-load 'tasks/redis.rake'
require 'rake/testtask'
$LOAD_PATH.unshift 'lib'
@@ -40,7 +38,7 @@ end
# Install
#
-task :install => [ 'redis:install', 'dtach:install' ]
+task :install => [ 'dtach:install' ]
#
View
6 bin/resque-web
@@ -16,8 +16,8 @@ Vegas::Runner.new(Resque::Server, 'resque-web', {
load path.to_s.strip if path
}
}) do |runner, opts, app|
- opts.on('-N NAMESPACE', "--namespace NAMESPACE", "set the Redis namespace") {|namespace|
- runner.logger.info "Using Redis namespace '#{namespace}'"
- Resque.redis.namespace = namespace
+ opts.on('-N NAMESPACE', "--namespace NAMESPACE", "set the Mongo database") {|namespace|
+ runner.logger.info "Using Mongo database '#{namespace}'"
+ Resque.mongo_db= namespace
}
end
View
132 lib/resque.rb
@@ -1,4 +1,3 @@
-require 'redis/namespace'
begin
require 'yajl'
@@ -6,6 +5,8 @@
require 'json'
end
+require 'mongo'
+
require 'resque/version'
require 'resque/errors'
@@ -23,48 +24,57 @@ module Resque
include Helpers
extend self
- # Accepts:
- # 1. A 'hostname:port' string
- # 2. A 'hostname:port:db' string (to select the Redis db)
- # 3. A 'hostname:port/namespace' string (to set the Redis namespace)
- # 4. A redis URL string 'redis://host:port'
- # 5. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
- # or `Redis::Namespace`.
- def redis=(server)
- if server.respond_to? :split
- if server =~ /redis\:\/\//
- redis = Redis.connect(:url => server)
+ def mongo=(server)
+ if server.is_a? String
+ opts = server.split(':')
+ host = opts[0]
+ if opts[1] =~ /\//
+ opts = opts[1].split('/')
+ port = opts[0]
+ queuedb = opts[1]
else
- server, namespace = server.split('/', 2)
- host, port, db = server.split(':')
- redis = Redis.new(:host => host, :port => port,
- :thread_safe => true, :db => db)
+ port = opts[1]
end
- namespace ||= :resque
-
- @redis = Redis::Namespace.new(namespace, :redis => redis)
- elsif server.respond_to? :namespace=
- @redis = server
- else
- @redis = Redis::Namespace.new(:resque, :redis => server)
+ conn = Mongo::Connection.new host, port
+ elsif server.is_a? Hash
+ conn = Mongo::Connection.new(options[:server], options[:port], options)
+ queuedb = options[:queuedb] || 'resque'
+ elsif server.is_a? Mongo::Connection
+ conn = server
end
+ queuedb ||= 'resque'
+ @mongo = conn.db queuedb
+ add_indexes
end
- # Returns the current Redis connection. If none has been created, will
+ # Returns the current Mongo connection. If none has been created, will
# create a new one.
- def redis
- return @redis if @redis
- self.redis = 'localhost:6379'
- self.redis
+ def mongo
+ return @mongo if @mongo
+ self.mongo = 'localhost:27017/resque'
+ self.mongo
end
- def redis_id
- # support 1.x versions of redis-rb
- if redis.respond_to?(:server)
- redis.server
- else
- redis.client.id
- end
+ def mongo_db=(db)
+ @mongo.conn.db = db
+ end
+
+
+ def add_indexes
+ mongo_workers.create_index :worker
+ mongo_stats.create_index :stat
+ end
+
+ def mongo_workers
+ mongo['resque.workers']
+ end
+
+ def mongo_stats
+ mongo['resque.stats']
+ end
+
+ def mongo_failures
+ mongo['resque.failures']
end
# The `before_first_fork` hook will be run in the **parent** process
@@ -115,7 +125,7 @@ def after_fork=(after_fork)
end
def to_s
- "Resque Client connected to #{redis_id}"
+ "Resque Client connected to #{mongo.connection.host}:#{mongo.connection.port}/#{mongo.name}"
end
@@ -126,21 +136,23 @@ def to_s
# Pushes a job onto a queue. Queue name should be a string and the
# item should be any JSON-able Ruby object.
def push(queue, item)
- watch_queue(queue)
- redis.rpush "queue:#{queue}", encode(item)
+ mongo[queue] << item
end
# Pops a job off a queue. Queue name should be a string.
#
# Returns a Ruby object.
def pop(queue)
- decode redis.lpop("queue:#{queue}")
+ mongo[queue].find_and_modify(:sort => [:natural, :desc], :remove => true )
+ rescue Mongo::OperationFailure => e
+ return nil if e.message =~ /No matching object/
+ raise e
end
# Returns an integer representing the size of a queue.
# Queue name should be a string.
def size(queue)
- redis.llen("queue:#{queue}").to_i
+ mongo[queue].count
end
# Returns an array of items currently queued. Queue name should be
@@ -152,39 +164,27 @@ def size(queue)
# To get the 3rd page of a 30 item, paginatied list one would use:
# Resque.peek('my_list', 59, 30)
def peek(queue, start = 0, count = 1)
- list_range("queue:#{queue}", start, count)
+ list_range(queue, start, count)
end
# Does the dirty work of fetching a range of items from a Redis list
# and converting them into Ruby objects.
def list_range(key, start = 0, count = 1)
- if count == 1
- decode redis.lindex(key, start)
- else
- Array(redis.lrange(key, start, start+count-1)).map do |item|
- decode item
- end
- end
+ items = mongo[key].find({ }, { :limit => count, :skip => start}).to_a.map{ |i| i}
+ count > 1 ? items : items.first
end
# Returns an array of all known Resque queues as strings.
- def queues
- Array(redis.smembers(:queues))
+ def queues
+ names = mongo.collection_names
+ names.delete_if{ |name| name == 'system.indexes' || name =~ /resque\./ }
end
# Given a queue name, completely deletes the queue.
def remove_queue(queue)
- redis.srem(:queues, queue.to_s)
- redis.del("queue:#{queue}")
- end
-
- # Used internally to keep track of which queues we've created.
- # Don't call this directly.
- def watch_queue(queue)
- redis.sadd(:queues, queue.to_s)
+ mongo[queue].drop
end
-
#
# job shortcuts
#
@@ -286,9 +286,9 @@ def info
:processed => Stat[:processed],
:queues => queues.size,
:workers => workers.size.to_i,
- :working => working.size,
+ :working => working.count,
:failed => Stat[:failed],
- :servers => [redis_id],
+ :servers => to_s,
:environment => ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
}
end
@@ -296,8 +296,12 @@ def info
# Returns an array of all known Resque keys in Redis. Redis' KEYS operation
# is O(N) for the keyspace, so be careful - this can be slow for big databases.
def keys
- redis.keys("*").map do |key|
- key.sub("#{redis.namespace}:", '')
- end
+ names = mongo.collection_names
+ #names.delete_if{ |name| name == 'system.indexes' || name =~ /resque\./ }
+ end
+
+ def drop
+ mongo.collections.each{ |collection| collection.drop unless collection.name =~ /^system./ }
+ @mongo = nil
end
end
View
12 lib/resque/failure/redis.rb
@@ -1,6 +1,6 @@
module Resque
module Failure
- # A Failure backend that stores exceptions in Redis. Very simple but
+ # A Failure backend that stores exceptions in Mongo. Very simple but
# works out of the box, along with support in the Resque web app.
class Redis < Base
def save
@@ -13,20 +13,20 @@ def save
:worker => worker.to_s,
:queue => queue
}
- data = Resque.encode(data)
- Resque.redis.rpush(:failed, data)
+ Resque.mongo_failures << data
end
def self.count
- Resque.redis.llen(:failed).to_i
+ Resque.mongo_failures.count
end
def self.all(start = 0, count = 1)
- Resque.list_range(:failed, start, count)
+ all_failures = Resque.mongo_failures.find().sort([:natural, :desc]).skip(start).limit(count).to_a
+ all_failures.size == 1 ? all_failures.first : all_failures
end
def self.clear
- Resque.redis.del(:failed)
+ Resque.mongo_failures.remove
end
def self.requeue(index)
View
12 lib/resque/helpers.rb
@@ -2,8 +2,16 @@ module Resque
# Methods used by various classes in Resque.
module Helpers
# Direct access to the Redis instance.
- def redis
- Resque.redis
+ #def mongo
+ # Resque.mongo
+ #end
+
+ def mongo_workers
+ Resque.mongo_workers
+ end
+
+ def mongo_stats
+ Resque.mongo_stats
end
# Given a Ruby object, returns a string suitable for storage in a
View
19 lib/resque/job.rb
@@ -80,20 +80,11 @@ def self.create(queue, klass, *args)
# depending on the size of your queue, as it loads all jobs into
# a Ruby array before processing.
def self.destroy(queue, klass, *args)
- klass = klass.to_s
- queue = "queue:#{queue}"
- destroyed = 0
-
- if args.empty?
- redis.lrange(queue, 0, -1).each do |string|
- if decode(string)['class'] == klass
- destroyed += redis.lrem(queue, 0, string).to_i
- end
- end
- else
- destroyed += redis.lrem(queue, 0, encode(:class => klass, :args => args))
- end
-
+ collection = Resque.mongo[queue]
+ selector = {'class' => klass.to_s}
+ selector['args'] = args unless args.empty?
+ destroyed = collection.find(selector).count
+ collection.remove(selector, :safe => true)
destroyed
end
View
12 lib/resque/server/views/layout.erb
@@ -16,11 +16,9 @@
<%= tab tab_name %>
<% end %>
</ul>
- <% if Resque.redis.namespace != :resque %>
- <abbr class="namespace" title="Resque's Redis Namespace">
- <%= Resque.redis.namespace %>
- </abbr>
- <% end %>
+ <abbr class="namespace" title="Mongo Connection">
+ <%= Resque.to_s %>
+ </abbr>
</div>
<% if @subtabs %>
@@ -37,8 +35,8 @@
<div id="footer">
<p>Powered by <a href="http://github.com/defunkt/resque">Resque</a> v<%=Resque::Version%></p>
- <p>Connected to Redis namespace <%= Resque.redis.namespace %> on <%=Resque.redis_id%></p>
+ <p>Connected to Mongo at <%= Resque.to_s %></p>
</div>
</body>
-</html>
+</html>
View
17 lib/resque/server/views/stats.erb
@@ -1,4 +1,4 @@
-<% @subtabs = %w( resque redis keys ) %>
+<% @subtabs = %w( resque mongo keys ) %>
<% if params[:key] %>
@@ -20,11 +20,11 @@
<% end %>
</table>
-<% elsif params[:id] == 'redis' %>
+<% elsif params[:id] == 'mongo' %>
- <h1><%= resque.redis_id %></h1>
+ <h1><%= resque.to_s %></h1>
<table class='stats'>
- <% for key, value in resque.redis.info.to_a.sort_by { |i| i[0].to_s } %>
+ <% resque.mongo.stats.each_pair do |key, value| %>
<tr>
<th>
<%= key %>
@@ -38,12 +38,10 @@
<% elsif params[:id] == 'keys' %>
- <h1>Keys owned by <%= resque %></h1>
- <p class='sub'>(All keys are actually prefixed with "<%= Resque.redis.namespace %>:")</p>
+ <h1>Collections in <%= resque.to_s %></h1>
<table class='stats'>
<tr>
- <th>key</th>
- <th>type</th>
+ <th>collection</th>
<th>size</th>
</tr>
<% for key in resque.keys.sort %>
@@ -51,8 +49,7 @@
<th>
<a href="<%=u "/stats/keys/#{key}" %>"><%= key %></a>
</th>
- <td><%= resque.redis.type key %></td>
- <td><%= redis_get_size key %></td>
+ <td><%= resque.mongo[key].count %></td>
</tr>
<% end %>
</table>
View
9 lib/resque/stat.rb
@@ -11,7 +11,8 @@ module Stat
# Returns the int value of a stat, given a string stat name.
def get(stat)
- redis.get("stat:#{stat}").to_i
+ value = mongo_stats.find_one :stat => stat
+ value.nil? ? 0 : value['value']
end
# Alias of `get`
@@ -24,7 +25,7 @@ def [](stat)
# Can optionally accept a second int parameter. The stat is then
# incremented by that amount.
def incr(stat, by = 1)
- redis.incrby("stat:#{stat}", by)
+ mongo_stats.update({:stat => stat}, {'$inc' => {:value => by}}, :upsert => true)
end
# Increments a stat by one.
@@ -37,7 +38,7 @@ def <<(stat)
# Can optionally accept a second int parameter. The stat is then
# decremented by that amount.
def decr(stat, by = 1)
- redis.decrby("stat:#{stat}", by)
+ mongo_stats.update({ :stat => stat}, { '$inc' => { :value => -by}})
end
# Decrements a stat by one.
@@ -47,7 +48,7 @@ def >>(stat)
# Removes a stat from Redis, effectively setting it to 0.
def clear(stat)
- redis.del("stat:#{stat}")
+ mongo_stats.remove({:stat => stat})
end
end
end
View
55 lib/resque/worker.rb
@@ -24,18 +24,13 @@ class Worker
# Returns an array of all worker objects.
def self.all
- Array(redis.smembers(:workers)).map { |id| find(id) }.compact
+ mongo_workers.distinct(:worker).map{|worker| find(worker)}.compact
end
# Returns an array of all worker objects currently processing
# jobs.
def self.working
- names = all
- return [] unless names.any?
- names.map! { |name| "worker:#{name}" }
- redis.mapped_mget(*names).keys.map do |key|
- find key.sub("worker:", '')
- end.compact
+ working = mongo_workers.find({ 'working_on' => { '$exists' => true}}).to_a.map{|w| find(w['worker'])}
end
# Returns a single worker object. Accepts a string id.
@@ -58,7 +53,7 @@ def self.attach(worker_id)
# Given a string worker id, return a boolean indicating whether the
# worker exists
def self.exists?(worker_id)
- redis.sismember(:workers, worker_id)
+ mongo_workers.find({ :worker => worker_id.to_s}).count > 0
end
# Workers should be initialized with an array of string queue
@@ -298,15 +293,15 @@ def unpause_processing
end
# Looks for any workers which should be running on this server
- # and, if they're not, removes them from Redis.
+ # and, if they're not, removes them from Mongo.
#
# This is a form of garbage collection. If a server is killed by a
# hard shutdown, power failure, or something else beyond our
# control, the Resque workers will not die gracefully and therefore
- # will leave stale state information in Redis.
+ # will leave stale state information in Mongo.
#
- # By checking the current Redis state against the actual
- # environment, we can determine if Redis is old and clean it up a bit.
+ # By checking the current Mongo state against the actual
+ # environment, we can determine if Mongo is old and clean it up a bit.
def prune_dead_workers
all_workers = Worker.all
known_workers = worker_pids unless all_workers.empty?
@@ -322,7 +317,7 @@ def prune_dead_workers
# Registers ourself as a worker. Useful when entering the worker
# lifecycle on startup.
def register_worker
- redis.sadd(:workers, self)
+ mongo_workers << { :worker => self.to_s}
started!
end
@@ -348,30 +343,28 @@ def unregister_worker
job.fail(DirtyExit.new)
end
- redis.srem(:workers, self)
- redis.del("worker:#{self}")
- redis.del("worker:#{self}:started")
+ mongo_workers.remove :worker => self.to_s
Stat.clear("processed:#{self}")
Stat.clear("failed:#{self}")
end
- # Given a job, tells Redis we're working on it. Useful for seeing
+ # Given a job, tells Mongo we're working on it. Useful for seeing
# what workers are doing and when.
def working_on(job)
job.worker = self
data = encode \
- :queue => job.queue,
- :run_at => Time.now.to_s,
- :payload => job.payload
- redis.set("worker:#{self}", data)
+ :queue => job.queue,
+ :run_at => Time.now.to_s,
+ :payload => job.payload
+ mongo_workers.update({:worker => self.to_s}, { '$set' => { 'working_on' => data}}, :upsert => true)
end
# Called when we are done working - clears our `working_on` state
- # and tells Redis we processed a job.
+ # and tells Mongo we processed a job.
def done_working
processed!
- redis.del("worker:#{self}")
+ mongo_workers.remove({ :worker => self.to_s})
end
# How many jobs has this worker processed? Returns an int.
@@ -379,7 +372,7 @@ def processed
Stat["processed:#{self}"]
end
- # Tell Redis we've processed a job.
+ # Tell Mongo we've processed a job.
def processed!
Stat << "processed"
Stat << "processed:#{self}"
@@ -390,7 +383,7 @@ def failed
Stat["failed:#{self}"]
end
- # Tells Redis we've failed a job.
+ # Tells Mongo we've failed a job.
def failed!
Stat << "failed"
Stat << "failed:#{self}"
@@ -398,17 +391,19 @@ def failed!
# What time did this worker start? Returns an instance of `Time`
def started
- redis.get "worker:#{self}:started"
+ worker = mongo_workers.find_one(:worker => self.to_s)
+ worker.nil? ? nil : worker['started']
end
- # Tell Redis we've started
+ # Tell Mongo we've started
def started!
- redis.set("worker:#{self}:started", Time.now.to_s)
+ mongo_workers.update({ :worker => self.to_s}, { '$set' => { :started => Time.now.to_s}})
end
# Returns a hash explaining the Job we're currently processing, if any.
def job
- decode(redis.get("worker:#{self}")) || {}
+ worker = mongo_workers.find_one :worker => self.to_s
+ worker.nil? ? { } : decode(worker['working_on'])
end
alias_method :processing, :job
@@ -425,7 +420,7 @@ def idle?
# Returns a symbol representing the current worker state,
# which can be either :working or :idle
def state
- redis.exists("worker:#{self}") ? :working : :idle
+ mongo_workers.find_one(:worker => self.to_s) ? :working : :idle
end
# Is this worker the same as another worker?
View
159 tasks/redis.rake
@@ -1,159 +0,0 @@
-# Inspired by rabbitmq.rake the Redbox project at http://github.com/rick/redbox/tree/master
-require 'fileutils'
-require 'open-uri'
-require 'pathname'
-
-class RedisRunner
-
- def self.redis_dir
- @redis_dir ||= if ENV['PREFIX']
- Pathname.new(ENV['PREFIX'])
- else
- Pathname.new(`which redis-server`) + '..' + '..'
- end
- end
-
- def self.bin_dir
- redis_dir + 'bin'
- end
-
- def self.config
- @config ||= if File.exists?(redis_dir + 'etc/redis.conf')
- redis_dir + 'etc/redis.conf'
- else
- redis_dir + '../etc/redis.conf'
- end
- end
-
- def self.dtach_socket
- '/tmp/redis.dtach'
- end
-
- # Just check for existance of dtach socket
- def self.running?
- File.exists? dtach_socket
- end
-
- def self.start
- puts 'Detach with Ctrl+\ Re-attach with rake redis:attach'
- sleep 1
- command = "#{bin_dir}/dtach -A #{dtach_socket} #{bin_dir}/redis-server #{config}"
- sh command
- end
-
- def self.attach
- exec "#{bin_dir}/dtach -a #{dtach_socket}"
- end
-
- def self.stop
- sh 'echo "SHUTDOWN" | nc localhost 6379'
- end
-
-end
-
-namespace :redis do
-
- desc 'About redis'
- task :about do
- puts "\nSee http://code.google.com/p/redis/ for information about redis.\n\n"
- end
-
- desc 'Start redis'
- task :start do
- RedisRunner.start
- end
-
- desc 'Stop redis'
- task :stop do
- RedisRunner.stop
- end
-
- desc 'Restart redis'
- task :restart do
- RedisRunner.stop
- RedisRunner.start
- end
-
- desc 'Attach to redis dtach socket'
- task :attach do
- RedisRunner.attach
- end
-
- desc 'Install the latest verison of Redis from Github (requires git, duh)'
- task :install => [:about, :download, :make] do
- bin_dir = '/usr/bin'
- conf_dir = '/etc'
-
- if ENV['PREFIX']
- bin_dir = "#{ENV['PREFIX']}/bin"
- sh "mkdir -p #{bin_dir}" unless File.exists?("#{bin_dir}")
-
- conf_dir = "#{ENV['PREFIX']}/etc"
- sh "mkdir -p #{conf_dir}" unless File.exists?("#{conf_dir}")
- end
-
- %w(redis-benchmark redis-cli redis-server).each do |bin|
- sh "cp /tmp/redis/src/#{bin} #{bin_dir}"
- end
-
- puts "Installed redis-benchmark, redis-cli and redis-server to #{bin_dir}"
-
- unless File.exists?("#{conf_dir}/redis.conf")
- sh "cp /tmp/redis/redis.conf #{conf_dir}/redis.conf"
- puts "Installed redis.conf to #{conf_dir} \n You should look at this file!"
- end
- end
-
- task :make do
- sh "cd /tmp/redis/src && make clean"
- sh "cd /tmp/redis/src && make"
- end
-
- desc "Download package"
- task :download do
- sh 'rm -rf /tmp/redis/' if File.exists?("/tmp/redis/.svn")
- sh 'git clone git://github.com/antirez/redis.git /tmp/redis' unless File.exists?('/tmp/redis')
- sh "cd /tmp/redis && git pull" if File.exists?("/tmp/redis/.git")
- end
-
-end
-
-namespace :dtach do
-
- desc 'About dtach'
- task :about do
- puts "\nSee http://dtach.sourceforge.net/ for information about dtach.\n\n"
- end
-
- desc 'Install dtach 0.8 from source'
- task :install => [:about, :download, :make] do
-
- bin_dir = "/usr/bin"
-
- if ENV['PREFIX']
- bin_dir = "#{ENV['PREFIX']}/bin"
- sh "mkdir -p #{bin_dir}" unless File.exists?("#{bin_dir}")
- end
-
- sh "cp /tmp/dtach-0.8/dtach #{bin_dir}"
- end
-
- task :make do
- sh 'cd /tmp/dtach-0.8/ && ./configure && make'
- end
-
- desc "Download package"
- task :download do
- unless File.exists?('/tmp/dtach-0.8.tar.gz')
- require 'net/http'
-
- url = 'http://downloads.sourceforge.net/project/dtach/dtach/0.8/dtach-0.8.tar.gz'
- open('/tmp/dtach-0.8.tar.gz', 'wb') do |file| file.write(open(url).read) end
- end
-
- unless File.directory?('/tmp/dtach-0.8')
- sh 'cd /tmp && tar xzf dtach-0.8.tar.gz'
- end
- end
-end
-
View
115 test/redis-test.conf
@@ -1,115 +0,0 @@
-# Redis configuration file example
-
-# By default Redis does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
-daemonize yes
-
-# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
-# You can specify a custom pid file location here.
-pidfile ./test/redis-test.pid
-
-# Accept connections on the specified port, default is 6379
-port 9736
-
-# If you want you can bind a single interface, if the bind option is not
-# specified all the interfaces will listen for connections.
-#
-# bind 127.0.0.1
-
-# Close the connection after a client is idle for N seconds (0 to disable)
-timeout 300
-
-# Save the DB on disk:
-#
-# save <seconds> <changes>
-#
-# Will save the DB if both the given number of seconds and the given
-# number of write operations against the DB occurred.
-#
-# In the example below the behaviour will be to save:
-# after 900 sec (15 min) if at least 1 key changed
-# after 300 sec (5 min) if at least 10 keys changed
-# after 60 sec if at least 10000 keys changed
-save 900 1
-save 300 10
-save 60 10000
-
-# The filename where to dump the DB
-dbfilename dump.rdb
-
-# For default save/load DB in/from the working directory
-# Note that you must specify a directory not a file name.
-dir ./test/
-
-# Set server verbosity to 'debug'
-# it can be one of:
-# debug (a lot of information, useful for development/testing)
-# notice (moderately verbose, what you want in production probably)
-# warning (only very important / critical messages are logged)
-loglevel debug
-
-# Specify the log file name. Also 'stdout' can be used to force
-# the demon to log on the standard output. Note that if you use standard
-# output for logging but daemonize, logs will be sent to /dev/null
-logfile stdout
-
-# Set the number of databases. The default database is DB 0, you can select
-# a different one on a per-connection basis using SELECT <dbid> where
-# dbid is a number between 0 and 'databases'-1
-databases 16
-
-################################# REPLICATION #################################
-
-# Master-Slave replication. Use slaveof to make a Redis instance a copy of
-# another Redis server. Note that the configuration is local to the slave
-# so for example it is possible to configure the slave to save the DB with a
-# different interval, or to listen to another port, and so on.
-
-# slaveof <masterip> <masterport>
-
-################################## SECURITY ###################################
-
-# Require clients to issue AUTH <PASSWORD> before processing any other
-# commands. This might be useful in environments in which you do not trust
-# others with access to the host running redis-server.
-#
-# This should stay commented out for backward compatibility and because most
-# people do not need auth (e.g. they run their own servers).
-
-# requirepass foobared
-
-################################### LIMITS ####################################
-
-# Set the max number of connected clients at the same time. By default there
-# is no limit, and it's up to the number of file descriptors the Redis process
-# is able to open. The special value '0' means no limts.
-# Once the limit is reached Redis will close all the new connections sending
-# an error 'max number of clients reached'.
-
-# maxclients 128
-
-# Don't use more memory than the specified amount of bytes.
-# When the memory limit is reached Redis will try to remove keys with an
-# EXPIRE set. It will try to start freeing keys that are going to expire
-# in little time and preserve keys with a longer time to live.
-# Redis will also try to remove objects from free lists if possible.
-#
-# If all this fails, Redis will start to reply with errors to commands
-# that will use more memory, like SET, LPUSH, and so on, and will continue
-# to reply to most read-only commands like GET.
-#
-# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
-# 'state' server or cache, not as a real DB. When Redis is used as a real
-# database the memory usage will grow over the weeks, it will be obvious if
-# it is going to use too much memory in the long run, and you'll have the time
-# to upgrade. With maxmemory after the limit is reached you'll start to get
-# errors for write operations, and this may even lead to DB inconsistency.
-
-# maxmemory <bytes>
-
-############################### ADVANCED CONFIG ###############################
-
-# Glue small output buffers together in order to send small replies in a
-# single TCP packet. Uses a bit more CPU but most of the times it is a win
-# in terms of number of queries per second. Use 'yes' if unsure.
-glueoutputbuf yes
View
74 test/resque_test.rb
@@ -2,7 +2,7 @@
context "Resque" do
setup do
- Resque.redis.flushall
+ Resque.drop
Resque.push(:people, { 'name' => 'chris' })
Resque.push(:people, { 'name' => 'bob' })
@@ -10,10 +10,10 @@
end
test "can set a namespace through a url-like string" do
- assert Resque.redis
- assert_equal :resque, Resque.redis.namespace
- Resque.redis = 'localhost:9736/namespace'
- assert_equal 'namespace', Resque.redis.namespace
+ assert Resque.mongo
+ assert_equal 'resque', Resque.mongo.name
+ Resque.mongo = 'localhost:27017/namespace'
+ assert_equal 'namespace', Resque.mongo.name
end
test "can put jobs on a queue" do
@@ -132,36 +132,48 @@
assert Resque.push(:people, { 'name' => 'jon' })
end
+ def pop_no_id(queue)
+ item = Resque.pop(queue)
+ item.delete("_id")
+ item
+ end
+
+
test "can pull items off a queue" do
- assert_equal({ 'name' => 'chris' }, Resque.pop(:people))
- assert_equal({ 'name' => 'bob' }, Resque.pop(:people))
- assert_equal({ 'name' => 'mark' }, Resque.pop(:people))
+ assert_equal({ 'name' => 'chris' }, pop_no_id(:people))
+ assert_equal({ 'name' => 'bob' }, pop_no_id(:people))
+ assert_equal({ 'name' => 'mark' }, pop_no_id(:people))
assert_equal nil, Resque.pop(:people)
end
test "knows how big a queue is" do
assert_equal 3, Resque.size(:people)
- assert_equal({ 'name' => 'chris' }, Resque.pop(:people))
+ assert_equal({ 'name' => 'chris' }, pop_no_id(:people))
assert_equal 2, Resque.size(:people)
- assert_equal({ 'name' => 'bob' }, Resque.pop(:people))
- assert_equal({ 'name' => 'mark' }, Resque.pop(:people))
+ assert_equal({ 'name' => 'bob' }, pop_no_id(:people))
+ assert_equal({ 'name' => 'mark' }, pop_no_id(:people))
assert_equal 0, Resque.size(:people)
end
test "can peek at a queue" do
- assert_equal({ 'name' => 'chris' }, Resque.peek(:people))
+ peek = Resque.peek(:people)
+ peek.delete "_id"
+ assert_equal({ 'name' => 'chris' }, peek)
assert_equal 3, Resque.size(:people)
end
test "can peek multiple items on a queue" do
- assert_equal({ 'name' => 'bob' }, Resque.peek(:people, 1, 1))
-
- assert_equal([{ 'name' => 'bob' }, { 'name' => 'mark' }], Resque.peek(:people, 1, 2))
- assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }], Resque.peek(:people, 0, 2))
- assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }, { 'name' => 'mark' }], Resque.peek(:people, 0, 3))
- assert_equal({ 'name' => 'mark' }, Resque.peek(:people, 2, 1))
+ assert_equal('bob', Resque.peek(:people, 1, 1)['name'])
+ peek = Resque.peek(:people, 1, 2).map { |hash| { 'name' => hash['name']}}
+ assert_equal([{ 'name' => 'bob' }, { 'name' => 'mark' }], peek)
+ peek = Resque.peek(:people, 0, 2).map { |hash| { 'name' => hash['name']} }
+ assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }], peek)
+ peek = Resque.peek(:people, 0, 3).map { |hash| { 'name' => hash['name']} }
+ assert_equal([{ 'name' => 'chris' }, { 'name' => 'bob' }, { 'name' => 'mark' }], peek)
+ peek = Resque.peek(:people, 2, 1)
+ assert_equal('mark', peek['name'])
assert_equal nil, Resque.peek(:people, 3)
assert_equal [], Resque.peek(:people, 3, 2)
end
@@ -169,24 +181,24 @@
test "knows what queues it is managing" do
assert_equal %w( people ), Resque.queues
Resque.push(:cars, { 'make' => 'bmw' })
- assert_equal %w( cars people ), Resque.queues
+ assert_equal %w( cars people ), Resque.queues.sort
end
test "queues are always a list" do
- Resque.redis.flushall
+ Resque.drop
assert_equal [], Resque.queues
end
test "can delete a queue" do
Resque.push(:cars, { 'make' => 'bmw' })
- assert_equal %w( cars people ), Resque.queues
+ assert_equal %w( cars people ), Resque.queues.sort
Resque.remove_queue(:people)
assert_equal %w( cars ), Resque.queues
assert_equal nil, Resque.pop(:people)
end
test "keeps track of resque keys" do
- assert_equal ["queue:people", "queues"], Resque.keys
+ assert Resque.keys.include? 'people'
end
test "badly wants a class name, too" do
@@ -223,26 +235,12 @@
assert_equal 3, stats[:queues]
assert_equal 3, stats[:processed]
assert_equal 1, stats[:failed]
- assert_equal [Resque.redis.respond_to?(:server) ? 'localhost:9736' : 'redis://localhost:9736/0'], stats[:servers]
+ # assert_equal [Resque.redis.respond_to?(:server) ? 'localhost:9736' : 'redis://localhost:9736/0'], stats[:servers]
end
test "decode bad json" do
assert_nil Resque.decode("{\"error\":\"Module not found \\u002\"}")
end
-=begin
- class UniqueJob
- @queue = :unique
- @unique_jobs = true
- end
-
- class NonUnique
- @queue = :unique
- end
-
- class OtherUnique
- @queue = :unique2
- @unique_jobs = true
- end
test "unique jobs are unique" do
#does uniqueness work?
@@ -277,5 +275,5 @@ class OtherUnique
assert_equal(6, Resque.size(:unique))
assert_equal('my args4', Resque.peek(:unique, 5)['args'][0]['arg1'])
end
-=end
+
end
View
97 test/test_helper.rb
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
dir = File.dirname(File.expand_path(__FILE__))
$LOAD_PATH.unshift dir + '/../lib'
$TESTING = true
@@ -10,44 +11,11 @@
rescue LoadError
end
-
-#
-# make sure we can run redis
-#
-
-if !system("which redis-server")
- puts '', "** can't find `redis-server` in your path"
- puts "** try running `sudo rake install`"
- abort ''
-end
-
-
#
# start our own redis when the tests start,
# kill it when they end
#
-at_exit do
- next if $!
-
- if defined?(MiniTest)
- exit_code = MiniTest::Unit.new.run(ARGV)
- else
- exit_code = Test::Unit::AutoRunner.run
- end
-
- pid = `ps -A -o pid,command | grep [r]edis-test`.split(" ")[0]
- puts "Killing test redis server..."
- `rm -f #{dir}/dump.rdb`
- Process.kill("KILL", pid.to_i)
- exit exit_code
-end
-
-puts "Starting redis for testing at localhost:9736..."
-`redis-server #{dir}/redis-test.conf`
-Resque.redis = 'localhost:9736'
-
-
##
# test/spec/mini 3
# http://gist.github.com/25455
@@ -114,3 +82,66 @@ def self.perform
raise SyntaxError, "Extra Bad job!"
end
end
+
+class UniqueJob
+ @queue = :unique
+ @unique_jobs = true
+end
+
+class NonUnique
+ @queue = :unique
+end
+
+class OtherUnique
+ @queue = :unique2
+ @unique_jobs = true
+end
+
+
+#some redgreen fun
+# -*- coding: utf-8 -*-
+begin
+ require 'redgreen'
+ module Test
+ module Unit
+ module UI
+ module Console
+ class TestRunner
+ def test_started(name)
+ @individual_test_start_time = Time.now
+ output_single(name + ": ", VERBOSE)
+ end
+
+ def test_finished(name)
+ elapsed_test_time = Time.now - @individual_test_start_time
+ char_to_output = elapsed_test_time > 1 ? "" : "."
+ output_single(char_to_output, PROGRESS_ONLY) unless (@already_outputted)
+ nl(VERBOSE)
+ @already_outputted = false
+ end
+ end
+ end
+ end
+ end
+ end
+
+ # -*- coding: utf-8 -*-
+ class Test::Unit::UI::Console::RedGreenTestRunner < Test::Unit::UI::Console::TestRunner
+ def output_single(something, level=NORMAL)
+ return unless (output?(level))
+ something = case something
+ when '.' then Color.green('.')
+ when '' then Color.green('')
+ when 'F' then Color.red("F")
+ when 'E' then Color.yellow("E")
+ when '+' then Color.green('+')
+ else something
+ end
+ @io.write(something)
+ @io.flush
+ end
+ end
+rescue LoadError
+ puts "consider gem install redgreen"
+end
+
View
8 test/worker_test.rb
@@ -2,7 +2,7 @@
context "Resque::Worker" do
setup do
- Resque.redis.flushall
+ Resque.drop
Resque.before_first_fork = nil
Resque.before_fork = nil
@@ -260,7 +260,7 @@
end
test "Will call a before_first_fork hook only once" do
- Resque.redis.flushall
+ Resque.drop
$BEFORE_FORK_CALLED = 0
Resque.before_first_fork = Proc.new { $BEFORE_FORK_CALLED += 1 }
workerA = Resque::Worker.new(:jobs)
@@ -277,7 +277,7 @@
end
test "Will call a before_fork hook before forking" do
- Resque.redis.flushall
+ Resque.drop
$BEFORE_FORK_CALLED = false
Resque.before_fork = Proc.new { $BEFORE_FORK_CALLED = true }
workerA = Resque::Worker.new(:jobs)
@@ -289,7 +289,7 @@
end
test "Will call an after_fork hook after forking" do
- Resque.redis.flushall
+ Resque.drop
$AFTER_FORK_CALLED = false
Resque.after_fork = Proc.new { $AFTER_FORK_CALLED = true }
workerA = Resque::Worker.new(:jobs)

0 comments on commit 9fbe0d9

Please sign in to comment.
Something went wrong with that request. Please try again.