Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

move index operation to resque job.

  • Loading branch information...
commit 06c526cc191dac27c5526327478902043a7c6975 1 parent aba2f69
@saberma saberma authored
View
1  Gemfile
@@ -47,6 +47,7 @@ gem "activemerchant" # 支付
gem "activemerchant_patch_for_china"
gem "httparty"
gem "resque" # 后台任务
+gem "resque-retry" # 任务失败后自动重试(支持时间指数递增)
gem "chinese_pinyin" # 汉字转拼音
gem "nokogiri" # 解释模板config/settings.html
gem "uuid" # 生成36位(或32位)唯一序列号
View
10 Gemfile.lock
@@ -255,6 +255,13 @@ GEM
redis-namespace (~> 1.0.2)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
+ resque-retry (0.1.0)
+ resque (>= 1.8.0)
+ resque-scheduler (>= 1.8.0)
+ resque-scheduler (1.9.9)
+ redis (>= 2.0.1)
+ resque (>= 1.8.0)
+ rufus-scheduler
resque_spec (0.7.0)
resque (>= 1.15.0)
rspec (>= 2.5.0)
@@ -274,6 +281,8 @@ GEM
railties (~> 3.0)
rspec (~> 2.6.0)
rubyzip (0.9.4)
+ rufus-scheduler (2.0.11)
+ tzinfo (>= 0.3.23)
sanitize (2.0.3)
nokogiri (< 1.6, >= 1.4.4)
sass (3.1.7)
@@ -385,6 +394,7 @@ DEPENDENCIES
rails-dev-boost!
rails3-generators
resque
+ resque-retry
resque_spec
rspec-rails
rubyzip
View
9 Procfile
@@ -1,4 +1,5 @@
-redis: /opt/redis/redis-server
-web: bundle exec unicorn_rails -p 4000 -c config/development.unicorn.conf.rb
-resque: QUEUE=* INTERVAL=1 bundle exec rake resque:work
-sunspot: bundle exec rake sunspot:solr:start
+redis: /opt/redis/redis-server
+web: bundle exec unicorn_rails -p 4000 -c config/development.unicorn.conf.rb
+resque: QUEUE=* INTERVAL=1 bundle exec rake resque:work
+resque-retry: bundle exec rake resque:scheduler
+sunspot: bundle exec rake sunspot:solr:start
View
2  Rakefile
@@ -4,8 +4,8 @@
require File.expand_path('../config/application', __FILE__)
require 'rake'
-#require 'thinking_sphinx/deltas/resque_delta/tasks'
require 'resque/tasks'
+require 'resque_scheduler/tasks'
task "resque:setup" => :environment do
Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } # 第二次执行resque任务时失败 PGError: ERROR: prepared statement "a3" already exists 参考 http://j.mp/rpVqhc http://j.mp/rqj9CQ ;Rails已经打了补丁,未发布版本 http://j.mp/ntPQMr
end
View
46 app/jobs/sunspot_worker.rb
@@ -0,0 +1,46 @@
+require 'resque-retry'
+
+class SunspotWorker # 参考http://j.mp/r64ADZ
+ extend Resque::Plugins::ExponentialBackoff
+ @queue = :solr_index
+
+ def self.perform(sunspot_method, object = nil)
+ sunspot_method = sunspot_method.to_sym
+ object = object.with_indifferent_access if object.is_a? Hash
+
+ session = Sunspot.session
+ Sunspot.session = Sunspot::Rails.build_session
+ case sunspot_method
+ when :index
+ self.index( object[:class].constantize.find(object[:id]) )
+ when :remove
+ self.remove_by_id(object[:class], object[:id])
+ when :remove_all
+ self.remove_all(object)
+ when :commit
+ self.commit
+ else
+ raise "Error: undefined protocol for SunspotWorker: #{sunspot_method} (#{objects})"
+ end
+ Sunspot.session = session
+ end
+
+ def self.index(object)
+ Sunspot.index(object)
+ end
+
+ def self.remove_by_id(klass, id)
+ Sunspot.remove_by_id(klass, id)
+ end
+
+ def self.remove_all(klass = nil)
+ klass = klass.constantize unless klass.nil?
+ Sunspot.remove_all(klass)
+ end
+
+ def self.commit
+ # on production, use autocommit in solrconfig.xml
+ # or commitWithin whenever sunspot supports it
+ Sunspot.commit unless Rails.env == 'production'
+ end
+end
View
2  app/jobs/theme_extracter.rb
@@ -3,7 +3,7 @@ module ThemeExtracter
@queue = "theme_extracter"
- def self.perform(shop_id, theme_id, zip_path, addition_root_dir) # 解压
+ def self.perform(shop_id, theme_id, zip_path, addition_root_dir) # 用户上传主题文件后,转入后台解压
shop = Shop.find(shop_id)
shop_theme = shop.themes.find(theme_id)
path = shop_theme.public_path
View
66 config/initializers/sunspot.rb
@@ -0,0 +1,66 @@
+module Sunspot
+ module SessionProxy
+ class ResqueSessionProxy < AbstractSessionProxy # 参考http://j.mp/r64ADZ
+ attr_reader :search_session
+
+ delegate :new_search, :search, :config,
+ :new_more_like_this, :more_like_this,
+ :delete_dirty, :delete_dirty?, :dirty?,
+ :to => :search_session
+
+ def initialize(search_session = Sunspot.session)
+ @search_session = search_session
+ end
+
+ def rescued_exception(method, exception)
+ $stderr.puts("Exception in SunspotSessionProxy\##{method}: #{exception.message}")
+ end
+
+ [:index!, :index, :remove!, :remove].each do |method|
+ module_eval(<<-RUBY)
+ def #{method}(*objects)
+ missed_objects = []
+ objects.each do |object|
+ if(object.is_a? ActiveRecord::Base)
+ Resque.enqueue SunspotWorker, :#{method}, {:class => object.class.name, :id => object.id }
+ else
+ missed_objects << object
+ end
+ end
+ begin
+ @search_session.#{method}(missed_objects) unless missed_objects.empty?
+ rescue => e
+ self.rescued_exception(:#{method}, e)
+ end
+ end
+ RUBY
+ end
+
+ [:remove_by_id, :remove_by_id!].each do |method|
+ module_eval(<<-RUBY)
+ def #{method}(clazz, id)
+ Resque.enqueue SunspotWorker, :remove, {:class => clazz, :id => id}
+ end
+ RUBY
+ end
+
+ def remove_all(clazz = nil)
+ Resque.enqueue SunspotWorker, :remove_all, clazz.to_s
+ end
+
+ def remove_all!(clazz = nil)
+ Resque.enqueue SunspotWorker, :remove_all, clazz.to_s
+ end
+
+ [:commit_if_dirty, :commit_if_delete_dirty, :commit].each do |method|
+ module_eval(<<-RUBY)
+ def #{method}
+ Resque.enqueue(SunspotWorker, :commit) unless ::Rails.env == 'production'
+ end
+ RUBY
+ end
+ end
+ end
+end
+
+Sunspot.session = Sunspot::SessionProxy::ResqueSessionProxy.new(Sunspot.session)
View
3  config/sunspot.yml
@@ -3,15 +3,18 @@ production:
hostname: localhost
port: 8983
log_level: WARNING
+ auto_commit_after_request: false # 索引切换至后台操作后,不能auto_commit
development:
solr:
hostname: localhost
port: 8982
log_level: INFO
+ auto_commit_after_request: false
test:
solr:
hostname: localhost
port: 8981
log_level: WARNING
+ auto_commit_after_request: false
Please sign in to comment.
Something went wrong with that request. Please try again.