@@ -1,4 +1,6 @@ +v1.9.1. Add ultrasphinx:index:merge task for index merging, and a note in DEPLOYMENT_NOTES about how to use it. + v1.9. Delta indexing. ERb now supported in .base files. Allow setting the searched indexes at runtime. v1.8.1. Use multifind/multiget for record loading; avoid using HashWithIndifferentAccess internally for speed; other minor performance improvements. CHANGELOG @@ -24,6 +24,8 @@ It's easy to keep the search daemon and the indexer running in a production envi The first line reindexes the delta index every 10 minutes. The second line reindexes the main index once a day at 4am. The third line will try to restart the search daemon every three minutes. If it's already running, nothing happens. +Note that you can use <tt>ultrasphinx:index:merge</tt> in place of <tt>ultrasphinx:index:main</tt>. It will be faster, but will not remove deleted records from the index. + Of course if you don't have any models with deltas, don't include the <tt>ultrasphinx:index:delta</tt> task. If you are under severe memory limitations you might want to manage the daemon with Monit instead, so you can keep a closer eye on it. The search daemon is extremely reliable, so don't bother with fancy monitoring infrastructure unless you're sure you need it. DEPLOYMENT_NOTES @@ -7,6 +7,7 @@ These Rake tasks are made available to your Rails app: <tt>ultrasphinx:index</tt>:: Reindex and rotate all indexes. <tt>ultrasphinx:index:delta</tt>:: Reindex and rotate the delta index. <tt>ultrasphinx:index:main</tt>:: Reindex and rotate the main index. +<tt>ultrasphinx:index:merge</tt>:: Merge the delta index into the main index. <tt>ultrasphinx:daemon:restart</tt>:: Restart the search daemon. <tt>ultrasphinx:daemon:start</tt>:: Start the search daemon. <tt>ultrasphinx:daemon:stop</tt>:: Stop the search daemon. @@ -14,4 +15,4 @@ These Rake tasks are made available to your Rails app: <tt>ultrasphinx:spelling:build</tt>:: Rebuild the custom spelling dictionary. You may need to use <tt>sudo</tt> if your Aspell folder is not writable by the app user. <tt>ultrasphinx:bootstrap</tt>:: Bootstrap a full Sphinx environment by running configure, index, then daemon:start. -All tasks have shortcuts. Use <tt>us:conf</tt>, <tt>us:index</tt>, <tt>us:main</tt>, <tt>us:delta</tt>, <tt>us:restart</tt>, <tt>us:start</tt>, <tt>us:stop</tt>, <tt>us:stat</tt>, <tt>us:spell</tt>, and <tt>us:boot</tt>. \ No newline at end of file +All tasks have shortcuts. Use <tt>us:conf</tt>, <tt>us:index</tt>, <tt>us:main</tt>, <tt>us:delta</tt>, <tt>us:merge</tt>, <tt>us:restart</tt>, <tt>us:start</tt>, <tt>us:stop</tt>, <tt>us:stat</tt>, <tt>us:spell</tt>, and <tt>us:boot</tt>. \ No newline at end of file RAKE_TASKS @@ -31,6 +31,11 @@ namespace :ultrasphinx do ultrasphinx_index(Ultrasphinx::DELTA_INDEX) end + desc "Merge the delta index into the main index." + task :merge => [:_environment] do + ultrasphinx_merge + end + end desc "Reindex and rotate all indexes." @@ -121,6 +126,7 @@ namespace :us do task :in => ["ultrasphinx:index"] task :main => ["ultrasphinx:index:main"] task :delta => ["ultrasphinx:index:delta"] + task :merge => ["ultrasphinx:index:merge"] task :spell => ["ultrasphinx:spelling:build"] task :conf => ["ultrasphinx:configure"] task :boot => ["ultrasphinx:bootstrap"] @@ -144,8 +150,7 @@ end def ultrasphinx_index(index) rotate = ultrasphinx_daemon_running? - index_path = Ultrasphinx::INDEX_SETTINGS['path'] - mkdir_p index_path unless File.directory? index_path + ultrasphinx_create_index_path cmd = "indexer --config '#{Ultrasphinx::CONF_PATH}'" cmd << " #{ENV['OPTS']} " if ENV['OPTS'] @@ -154,16 +159,43 @@ def ultrasphinx_index(index) say "$ #{cmd}" system cmd + + ultrasphinx_check_rotate if rotate +end + +def ultrasphinx_merge + rotate = ultrasphinx_daemon_running? + + indexes = [Ultrasphinx::MAIN_INDEX, Ultrasphinx::DELTA_INDEX] + indexes.each do |index| + raise "#{index} index is missing" unless File.exist? "#{Ultrasphinx::INDEX_SETTINGS['path']}/sphinx_index_#{index}.spa" + end + + cmd = "indexer --config '#{Ultrasphinx::CONF_PATH}'" + cmd << " #{ENV['OPTS']} " if ENV['OPTS'] + cmd << " --rotate" if rotate + cmd << " --merge #{indexes.join(' ')}" + + say "$ #{cmd}" + system cmd - if rotate - sleep(4) - failed = Dir[index_path + "/*.new.*"] - if failed.any? - say "warning; index failed to rotate! Deleting new indexes" - failed.each {|f| File.delete f } - else - say "index rotated ok" - end + ultrasphinx_check_rotate +end + +def ultrasphinx_check_rotate + sleep(4) + failed = Dir[Ultrasphinx::INDEX_SETTINGS['path'] + "/*.new.*"] + if failed.any? + say "warning; index failed to rotate! Deleting new indexes" + failed.each {|f| File.delete f } + else + say "index rotated ok" + end +end + +def ultrasphinx_create_index_path + unless File.directory? Ultrasphinx::INDEX_SETTINGS['path'] + mkdir_p Ultrasphinx::INDEX_SETTINGS['path'] end end tasks/ultrasphinx.rake @@ -36,4 +36,17 @@ class DeltaTest < Test::Unit::TestCase assert_equal @count, S.new.total_entries end + + def test_merge + Dir.chdir "#{HERE}/integration/app" do + Echoe.silence do + system("rake ultrasphinx:daemon:start") + end + + output = `rake ultrasphinx:index:merge 2>&1` + assert_match /merged 0.1 Kwords/, output + assert_match /Index rotated ok/, output + end + end + end test/integration/delta_test.rb d2f828dfb803dc73c8dcf17755574bb8e900d50f evanweaver evanweaver@c1ad287d-65d5-445d-b84c-e64f8492f1e6 http://github.com/DrMark/ultrasphinx/commit/fbbde7fc8d3fe13039349fa3f63766b80dcbd8cf fbbde7fc8d3fe13039349fa3f63766b80dcbd8cf 2008-03-09T13:48:26-07:00 2008-03-09T13:48:26-07:00 Add merge rake task; add test and documentation for same; DRY up some other rake configuration. git-svn-id: svn://rubyforge.org/var/svn/fauna/ultrasphinx/trunk@1723 c1ad287d-65d5-445d-b84c-e64f8492f1e6 a030a8e3c60bd875cb7186b78de265c28cb48b3f evanweaver evanweaver@c1ad287d-65d5-445d-b84c-e64f8492f1e6