public
Description: a maintained fork of Evan Weaver's Ultrasphinx code -- see the escape_sql branch
Homepage: http://blog.evanweaver.com/files/doc/fauna/ultrasphinx/files/README.html
Clone URL: git://github.com/DrMark/ultrasphinx.git
ultrasphinx / tasks / ultrasphinx.rake
100644 206 lines (172 sloc) 6.125 kb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
ENV['RAILS_ENV'] ||= "development"
 
namespace :ultrasphinx do
 
  task :_environment => [:environment] do
    # We can't just chain :environment because we want to make
    # sure it's set only for known Sphinx tasks
    Ultrasphinx.with_rake = true
  end
  
  desc "Bootstrap a full Sphinx environment"
  task :bootstrap => [:_environment, :configure, :index, :"daemon:restart"] do
    say "done"
    say "please restart your application containers"
  end
  
  desc "Rebuild the configuration file for this particular environment."
  task :configure => [:_environment] do
    Ultrasphinx::Configure.run
  end
  
  namespace :index do
    desc "Reindex and rotate the main index."
    task :main => [:_environment] do
      ultrasphinx_index(Ultrasphinx::MAIN_INDEX)
    end
 
    desc "Reindex and rotate the delta index."
    task :delta => [:_environment] do
      ultrasphinx_index(Ultrasphinx::DELTA_INDEX)
    end
    
    desc "Merge the delta index into the main index."
    task :merge => [:_environment] do
      ultrasphinx_merge
    end
    
  end
 
  desc "Reindex and rotate all indexes."
  task :index => [:_environment] do
    ultrasphinx_index("--all")
  end
  
  namespace :daemon do
    desc "Start the search daemon"
    task :start => [:_environment] do
      FileUtils.mkdir_p File.dirname(Ultrasphinx::DAEMON_SETTINGS["log"]) rescue nil
      raise Ultrasphinx::DaemonError, "Already running" if ultrasphinx_daemon_running?
      system "searchd --config '#{Ultrasphinx::CONF_PATH}'"
      sleep(4) # give daemon a chance to write the pid file
      if ultrasphinx_daemon_running?
        say "started successfully"
      else
        say "failed to start"
      end
    end
    
    desc "Stop the search daemon"
    task :stop => [:_environment] do
      raise Ultrasphinx::DaemonError, "Doesn't seem to be running" unless ultrasphinx_daemon_running?
      system "kill #{pid = ultrasphinx_daemon_pid}"
      sleep(1)
      if ultrasphinx_daemon_running?
        system "kill -9 #{pid}"
        sleep(1)
      end
      if ultrasphinx_daemon_running?
        say "#{pid} could not be stopped"
      else
        say "stopped #{pid}"
      end
    end
 
    desc "Restart the search daemon"
    task :restart => [:_environment] do
      Rake::Task["ultrasphinx:daemon:stop"].invoke if ultrasphinx_daemon_running?
      sleep(3)
      Rake::Task["ultrasphinx:daemon:start"].invoke
    end
    
    desc "Check if the search daemon is running"
    task :status => [:_environment] do
      if ultrasphinx_daemon_running?
        say "daemon is running."
      else
        say "daemon is stopped."
      end
    end
  end
          
    
  namespace :spelling do
    desc "Rebuild the custom spelling dictionary. You may need to use 'sudo' if your Aspell folder is not writable by the app user."
    task :build => [:_environment] do
      ENV['OPTS'] = "--buildstops #{Ultrasphinx::STOPWORDS_PATH} #{Ultrasphinx::MAX_WORDS} --buildfreqs"
      Rake::Task["ultrasphinx:index"].invoke
      tmpfile = "/tmp/ultrasphinx-stopwords.txt"
      words = []
      say "filtering"
      File.open(Ultrasphinx::STOPWORDS_PATH).each do |line|
        if line =~ /^([^\s\d_]{4,}) (\d+)/
          # XXX should be configurable
          words << $1 if $2.to_i > 40
          # ideally we would also skip words within X edit distance of a correction
          # by aspell-en, in order to not add typos to the dictionary
        end
      end
      say "writing #{words.size} words"
      File.open(tmpfile, 'w').write(words.join("\n"))
      say "loading dictionary '#{Ultrasphinx::DICTIONARY}' into aspell"
      system("aspell --lang=en create master #{Ultrasphinx::DICTIONARY}.rws < #{tmpfile}")
    end
  end
  
end
 
# task shortcuts
namespace :us do
  task :start => ["ultrasphinx:daemon:start"]
  task :restart => ["ultrasphinx:daemon:restart"]
  task :stop => ["ultrasphinx:daemon:stop"]
  task :stat => ["ultrasphinx:daemon:status"]
  task :index => ["ultrasphinx:index"]
  task :in => ["ultrasphinx:index"]
  task :main => ["ultrasphinx:index:main"]
  task :delta => ["ultrasphinx:index:delta"]
  task :merge => ["ultrasphinx:index:merge"]
  task :spell => ["ultrasphinx:spelling:build"]
  task :conf => ["ultrasphinx:configure"]
  task :boot => ["ultrasphinx:bootstrap"]
end
 
# Support methods
 
def ultrasphinx_daemon_pid
  open(Ultrasphinx::DAEMON_SETTINGS['pid_file']).readline.chomp rescue nil
end
 
def ultrasphinx_daemon_running?
  if ultrasphinx_daemon_pid and `ps -p#{ultrasphinx_daemon_pid} | wc`.to_i > 1
    true
  else
    # Remove bogus lockfiles.
    Dir[Ultrasphinx::INDEX_SETTINGS["path"] + "*spl"].each {|file| File.delete(file)}
    false
  end
end
 
def ultrasphinx_index(index)
  rotate = ultrasphinx_daemon_running?
  ultrasphinx_create_index_path
  
  cmd = "indexer --config '#{Ultrasphinx::CONF_PATH}'"
  cmd << " #{ENV['OPTS']} " if ENV['OPTS']
  cmd << " --rotate" if rotate
  cmd << " #{index}"
  
  say "$ #{cmd}"
  system cmd
    
  ultrasphinx_check_rotate if rotate
end
 
def ultrasphinx_merge
  rotate = ultrasphinx_daemon_running?
 
  indexes = [Ultrasphinx::MAIN_INDEX, Ultrasphinx::DELTA_INDEX]
  indexes.each do |index|
    raise "#{index} index is missing" unless File.exist? "#{Ultrasphinx::INDEX_SETTINGS['path']}/sphinx_index_#{index}.spa"
  end
  
  cmd = "indexer --config '#{Ultrasphinx::CONF_PATH}'"
  cmd << " #{ENV['OPTS']} " if ENV['OPTS']
  cmd << " --rotate" if rotate
  cmd << " --merge #{indexes.join(' ')}"
  
  say "$ #{cmd}"
  system cmd
      
  ultrasphinx_check_rotate if rotate
end
 
def ultrasphinx_check_rotate
  sleep(4)
  failed = Dir[Ultrasphinx::INDEX_SETTINGS['path'] + "/*.new.*"]
  if failed.any?
    say "warning; index failed to rotate! Deleting new indexes"
    say "try 'killall searchd' and then 'rake ultrasphinx:daemon:start'"
    failed.each {|f| File.delete f }
  else
    say "index rotated ok"
  end
end
 
def ultrasphinx_create_index_path
  unless File.directory? Ultrasphinx::INDEX_SETTINGS['path']
    mkdir_p Ultrasphinx::INDEX_SETTINGS['path']
  end
end
 
def say msg
  Ultrasphinx.say msg
end