@@ -7,25 +7,47 @@ Rakefile bin/capitate config/hoe.rb config/requirements.rb +docs/nginx.README +docs/recipes/centos.txt +docs/recipes/deploy-pending.txt +docs/recipes/deploy-web.txt +docs/recipes/deploy.txt +docs/recipes/docs.txt +docs/recipes/imagemagick-centos.txt +docs/recipes/imagemagick.txt +docs/recipes/index.txt +docs/recipes/memcached-centos.txt +docs/recipes/memcached.txt +docs/recipes/mongrel_cluster-centos.txt +docs/recipes/mongrel_cluster.txt +docs/recipes/monit-centos.txt +docs/recipes/monit.txt +docs/recipes/mysql-centos.txt +docs/recipes/mysql.txt +docs/recipes/nginx-centos.txt +docs/recipes/nginx.txt +docs/recipes/rails.txt +docs/recipes/ruby-centos.txt +docs/recipes/ruby.txt +docs/recipes/sphinx-centos.txt +docs/recipes/sphinx.txt lib/capitate.rb lib/capitate/cap_ext/connections.rb lib/capitate/cap_ext/extension_proxy.rb +lib/capitate/cap_ext/run_via.rb lib/capitate/cap_ext/variables.rb lib/capitate/plugins/base.rb lib/capitate/plugins/gem.rb -lib/capitate/plugins/package.rb -lib/capitate/plugins/profiles.rb +lib/capitate/plugins/prompt.rb lib/capitate/plugins/script.rb lib/capitate/plugins/templates.rb lib/capitate/plugins/upload.rb -lib/capitate/plugins/wget.rb lib/capitate/plugins/yum.rb lib/capitate/recipes.rb lib/capitate/task_node.rb lib/capitate/version.rb -lib/profiles/centos-sick.rb -lib/recipes/README -lib/recipes/centos/README +lib/deployment/deploy.rb +lib/deployment/install-centos-rubyweb.rb lib/recipes/centos/centos.rb lib/recipes/centos/imagemagick.rb lib/recipes/centos/memcached.rb @@ -37,18 +59,13 @@ lib/recipes/centos/ruby.rb lib/recipes/centos/sphinx.rb lib/recipes/deploy.rb lib/recipes/docs.rb -lib/recipes/gems.rb lib/recipes/memcached.rb lib/recipes/mongrel_cluster.rb lib/recipes/mysql.rb lib/recipes/nginx.rb -lib/recipes/packages.rb lib/recipes/rails.rb -lib/recipes/recipes.rb lib/recipes/sphinx.rb lib/templates/capistrano/Capfile -lib/templates/centos/setup_for_web.sh -lib/templates/centos/sudoers lib/templates/memcached/memcached.initd.centos.erb lib/templates/memcached/memcached.monitrc.erb lib/templates/mongrel/mongrel_cluster.initd.erb Manifest.txt @@ -1,69 +1,30 @@ #!/usr/bin/env ruby -require 'optparse' - -OptionParser.new do |opts| - opts.banner = "Usage: #{File.basename($0)} [path]" - - opts.on("-h", "--help", "Displays this help info") do - puts opts - exit 0 - end - - begin - opts.parse!(ARGV) - rescue OptionParser::ParseError => e - warn e.message - puts opts - exit 1 - end -end - begin require 'capitate' rescue LoadError require 'rubygems' - require 'capitate' + require 'capitate' end +require 'capistrano/cli' -class CapitateBin - class << self - include Capitate::Plugins::Templates - end -end - -if ARGV.empty? - abort "Please specify the directory to capify, e.g. `#{File.basename($0)} .'" -elsif !File.exists?(ARGV.first) - abort "`#{ARGV.first}' does not exist." -elsif !File.directory?(ARGV.first) - abort "`#{ARGV.first}' is not a directory." -elsif ARGV.length > 1 - abort "Too many arguments; please specify only the directory to capitate." -end +cli = Capistrano::CLI.new(ARGV) +cli.parse_options! -base = ARGV.shift +config = cli.instantiate_configuration +config.logger.level = cli.options[:verbose] -puts <<-EOS +cli.set_pre_vars(config) +cli.load_recipes(config) - Generating capistrano configuration... +config.load do + require 'capitate/recipes' +end -EOS +config.trigger(:load) +cli.execute_requested_actions(config) +config.trigger(:exit) -files = { "Capfile" => "capistrano/Capfile" } -files.each do |file, template| - file = File.join(base, file) - if File.exists?(file) - warn "`#{file}' already exists" - elsif File.exists?(file.downcase) - warn "`#{file.downcase}' exists, which could conflict with `#{file}'" - elsif !File.exists?(File.dirname(file)) - warn "Directory `#{File.dirname(file)}' does not exist" - else - # Write Capfile - CapitateBin.write(template, file) - end -end bin/capitate @@ -55,7 +55,7 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p| p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT p.test_globs = ["test/**/test_*.rb"] p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean. - p.clean_globs += ['website/recipes/**', 'docs/recipes/**'] + p.clean_globs += ['website/recipes/**'] # == Optional p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n") @@ -69,4 +69,4 @@ end CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n") PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}" hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc') -hoe.rsync_args = '-av --delete --ignore-errors' \ No newline at end of file +hoe.rsync_args = '-av --delete --ignore-errors' config/hoe.rb @@ -16,9 +16,12 @@ module Capitate::Plugins::Script def make_install(name, options) install(name, options) do |dir| configure_options = options[:configure_options] || "" - run_via "echo 'Configuring #{name}...' && cd #{dir} && ./configure #{configure_options} > configure.log" - run_via "echo 'Compiling #{name}...' && cd #{dir} && make > make.log" - run_via "echo 'Installing #{name}...' && cd #{dir} && make install > make_install.log" + + run_all <<-CMDS + echo 'Configuring #{name}...' && cd #{dir} && ./configure #{configure_options} > configure.log + echo 'Compiling #{name}...' && cd #{dir} && make > make.log + echo 'Installing #{name}...' && cd #{dir} && make install > make_install.log + CMDS end end @@ -100,8 +103,10 @@ module Capitate::Plugins::Script unpack_dir ||= file.gsub(/\.tar\.gz|\.tgz/, "") - run_via "echo 'Getting #{url}...' && mkdir -p #{dest} && cd #{dest} && wget -nv #{url}" - run_via "echo 'Unpacking...' && cd #{dest} && tar zxf #{file}" + run_all <<-CMDS + echo 'Getting #{url}...' && mkdir -p #{dest} && cd #{dest} && wget -nv #{url} + run_via "echo 'Unpacking...' && cd #{dest} && tar zxf #{file} + CMDS if block_given? yield("#{dest}/#{unpack_dir}") @@ -110,6 +115,18 @@ module Capitate::Plugins::Script end end + # Run all commands (separated by newlines) + # + # ==== Options + # +cmds+:: Commands (separated by newlines) + # +options+:: See invoke_command options + # + def run_all(cmds, options = {}, &block) + cmds.split("\n").each do |cmd| + run_via(cmd, options, &block) + end + end + end Capistrano.plugin :script, Capitate::Plugins::Script \ No newline at end of file lib/capitate/plugins/script.rb @@ -1,8 +1,8 @@ module Capitate #:nodoc: module VERSION #:nodoc: MAJOR = 0 - MINOR = 1 - TINY = 9 + MINOR = 2 + TINY = 1 STRING = [MAJOR, MINOR, TINY].join('.') end lib/capitate/version.rb @@ -10,10 +10,8 @@ set :deploy_to, "/var/www/apps/sick" set :web_host, "192.168.1.111" set :db_host, "192.168.1.111" set :db_user, "sick" -set :db_pass, "sick" +set :db_pass, prompt.password("DB Password: ") set :db_name, "sick" -set :sphinx_host, "127.0.0.1" -set :sphinx_port, 3312 set :repository, "http://svn.ducktyper.com/scratch/testapp/trunk" set :mongrel_port, 12000 set :mongrel_size, 3 lib/deployment/deploy.rb @@ -19,12 +19,17 @@ task :install do role :install, prompt.ask("Server: ") if find_servers_for_task(current_task).blank? # Setup for web - script.sh("setup_for_web.sh") - - # Install sudoers - put template.load("sudoers"), "/tmp/sudoers" - run "install -o root -m 440 /tmp/sudoers /etc/sudoers && rm -f /tmp/sudoers" - + # * Add admin group + # * Change inittab to runlevel 3 + # * Create web apps directory + # * Add admin group to suders ALL=(ALL) ALL + script.run_all <<-CMDS + egrep "^admin" /etc/group || /usr/sbin/groupadd admin + sed -i -e 's/^id:5:initdefault:/id:3:initdefault:/g' /etc/inittab + mkdir -p /var/www/apps + egrep "^%admin" /etc/sudoers || echo "%admin ALL=(ALL) ALL" > /etc/sudoers + CMDS + # Package installs yum.remove [ "openoffice.org-*", "ImageMagick" ] yum.update @@ -43,11 +48,11 @@ task :install do # Install monit hooks nginx.install_monit mysql.install_monit - + # Gem installs gems.install([ "rake", "mysql -- --with-mysql-include=/usr/include/mysql --with-mysql-lib=/usr/lib/mysql --with-mysql-config", "raspell", "rmagick", "mongrel", "mongrel_cluster","json" ]) - + # Cleanup yum.clean end lib/deployment/install-centos-rubyweb.rb @@ -3,12 +3,8 @@ namespace :sphinx do namespace :centos do desc <<-DESC - Install sphinx. - - *sphinx_prefix*: Sphinx install prefix. _Defaults to "/usr/local/sphinx"_ - - @set :sphinx_prefix, "/usr/local/sphinx"@ - + Install sphinx.\n + *sphinx_prefix*: Sphinx install prefix. _Defaults to "/usr/local/sphinx"_\n DESC task :install do @@ -30,22 +26,17 @@ namespace :sphinx do end desc <<-DESC - Setup sphinx for application. - - *sphinx_prefix*: Sphinx install prefix. _Defaults to "/usr/local/sphinx"_ - - @set :sphinx_prefix, "/usr/local/sphinx"@ - + Setup sphinx for application.\n + *sphinx_prefix*: Sphinx install prefix. _Defaults to "/usr/local/sphinx"_\n + *sphinx_pid_path*: Directory to sphinx pid. _Defaults to "[shared_path]/pids/searchd.pid"_\n + *sphinx_conf_path*: Path to sphinx.conf. _Defaults to "[shared_path]/config/sphinx.conf"_\n DESC task :setup do # Settings fetch_or_default(:sphinx_prefix, "/usr/local/sphinx") - - # Derived settings - set :sphinx_bin_path, "#{sphinx_prefix}/bin" - set :sphinx_conf_path, "#{shared_path}/config/sphinx.conf" - set :sphinx_pid_path, "#{shared_path}/pids/searchd.pid" + fetch_or_default(:sphinx_pid_path, "#{shared_path}/pids/searchd.pid") + fetch_or_default(:sphinx_conf_path, "#{shared_path}/config/sphinx.conf") # Install initscript put template.load("sphinx/sphinx_app.initd.centos.erb"), "/tmp/sphinx.initd" lib/recipes/centos/sphinx.rb @@ -4,18 +4,11 @@ namespace :mysql do desc <<-DESC Install mysql monit hooks. - *db_port*: Mysql port. _Defaults to 3306_ - - @set :db_port, 3306@ - - *mysql_pid_path*: Path to mysql pid file. _Defaults to /var/run/mysqld/mysqld.pid_ - - @set :mysql_pid_path, "/var/run/mysqld/mysqld.pid"@ - - *monit_conf_dir*: Destination for monitrc. _Defaults to "/etc/monit"_ - - @set :monit_conf_dir, "/etc/monit"@ - + *db_port*: Mysql port. _Defaults to 3306_\n + *mysql_pid_path*: Path to mysql pid file. _Defaults to /var/run/mysqld/mysqld.pid_\n + @set :mysql_pid_path, "/var/run/mysqld/mysqld.pid"@\n + *monit_conf_dir*: Destination for monitrc. _Defaults to "/etc/monit"_\n + @set :monit_conf_dir, "/etc/monit"@\n DESC task :install_monit do @@ -29,33 +22,17 @@ namespace :mysql do end desc <<-DESC - Create database user, and database with appropriate permissions. - - *db_name*: Database name (application). - - @set :db_name, "app_db_name"@ - - *db_user*: Database user (application). - - @set :db_user, "app_db_user"@ - - *db_pass*: Database password (application). - - @set :db_pass, "the_password"@ - - *web_host*: Web host to provide access privileges to (if recipe used in context of web app). - _Defaults to nil_ (TODO: Support multiple web hosts) - - @set :web_host, 10.0.6.100@ - - *db_host*: Database host (to provide itself with access). - - @set :db_host, 10.0.6.101@ - - *mysql_admin_password*: Mysql admin password (to use to connect). Defaults to password prompt. - - @set :mysql_admin_password, prompt.password('Mysql admin password: '))@ - + Create database, database user, and set grant permissions. + + *db_name*: Database name (application).\n + *db_user*: Database user (application).\n + *db_pass*: Database password (application).\n + *grant_locations*: Grant locations. _Defaults to localhost_\n + @set :grant_locations, [ "localhost", "192.168.1.111" ]@\n + *grant_priv_type*: Grant privilege types. _Defaults to ALL_\n + @set :grant_priv_type, "ALL"@\n + *mysql_admin_password*: Mysql admin password (to use to connect). Defaults to password prompt.\n + @set :mysql_admin_password, prompt.password('Mysql admin password: '))@ DESC task :setup do @@ -63,9 +40,9 @@ namespace :mysql do fetch(:db_name) fetch(:db_user) fetch(:db_pass) - fetch(:db_host) fetch_or_default(:mysql_admin_password, prompt.password('Mysql admin password: ')) - fetch_or_default(:web_host, nil) + fetch_or_default(:grant_locations, [ "localhost" ]) + fetch_or_default(:grant_priv_type, "ALL") # Add localhost to grant locations set :locations_for_grant, [ "localhost", web_host, db_host ].compact lib/recipes/mysql.rb @@ -2,48 +2,106 @@ namespace :sphinx do desc <<-DESC - Create monit configuration for sphinx. - - *monit_conf_dir*: Destination for monitrc. _Defaults to "/etc/monit"_ - - @set :monit_conf_dir, "/etc/monit"@ - + Create monit configuration for sphinx.\n + *monit_conf_dir*: Destination for monitrc. _Defaults to "/etc/monit"_\n + *sphinx_pid_path*: Location for sphinx pid. _Defaults to "[shared_path]/pids/searchd.pid"_\n DESC task :setup_monit do # Settings fetch_or_default(:monit_conf_dir, "/etc/monit") - - set :sphinx_pid_path, "#{shared_path}/pids/searchd.pid" + fetch_or_default(:sphinx_pid_path, "#{shared_path}/pids/searchd.pid") put template.load("sphinx/sphinx.monitrc.erb"), "/tmp/sphinx_#{application}.monitrc" sudo "install -o root /tmp/sphinx_#{application}.monitrc #{monit_conf_dir}/sphinx_#{application}.monitrc" end - desc "Update sphinx for application" - task :update_code do + desc <<-DESC + Update sphinx for application. + + *sphinx_conf_template*: Path to sphinx.conf.erb. _Defaults to "config/templates/sphinx.conf.erb"_\n + *sphinx_conf_path*: Path to sphinx.conf. _Defaults to "[shared_path]/config/sphinx.conf"_\n + *sphinx_port*: Sphinx port. _Defaults to 3312_\n + *sphinx_conf_root*: Directory for sphinx configuration, like stopwords.txt. _Defaults to [current_path]/config_\n + *sphinx_index_root*: Directory for sphinx indexes. _Defaults to "[shared_path]/var/index"_\n + *sphinx_log_root*: Directory for sphinx logs. _Defaults to "[shared_path]/log"_\n + *sphinx_pid_root*: Directory for sphinx pids. _Defaults to "[shared_path]/pids"_\n + + *sphinx_db_user*: Sphinx DB user. _Defaults to db_user_\n + *sphinx_db_pass*: Sphinx DB password. _Defaults to db_pass_\n + *sphinx_db_name*: Sphinx DB name. _Defaults to db_name_\n + + *sphinx_db_host*: Sphinx DB host. _Defaults to location for primary :db role_\n + *sphinx_host*: Sphinx DB host. _Defaults to location for :search role_\n + + DESC + task :update_conf do + + fetch_or_default(:sphinx_conf_template, "config/templates/sphinx.conf.erb") + fetch_or_default(:sphinx_port, 3312) + fetch_or_default(:sphinx_conf_path, "#{shared_path}/config/sphinx.conf") + fetch_or_default(:sphinx_conf_root, "#{current_path}/config") + fetch_or_default(:sphinx_index_root, "#{shared_path}/var/index") + fetch_or_default(:sphinx_log_root, "#{shared_path}/log") + fetch_or_default(:sphinx_pid_path, "#{shared_path}/pids/searchd.pid") + + fetch_or_default(:sphinx_db_user, db_user) + fetch_or_default(:sphinx_db_pass, db_pass) + fetch_or_default(:sphinx_db_name, db_name) - set :rails_root, current_path - set :index_root, "#{shared_path}/var/index"; - set :log_root, "#{shared_path}/log" - set :pid_root, "#{shared_path}/pids" + unless exists?(:sphinx_db_host) + db_servers = roles[:db] + unless db_servers.empty? + set :sphinx_db_host, db_servers.first.host + else + raise "No :db roles, and no :sphinx_db_host setting specified" + end + end - put template.load("config/templates/sphinx.conf.erb"), "#{shared_path}/config/sphinx.conf" + unless exists?(:sphinx_host) + search_servers = roles[:search] + unless search_servers.empty? + set :sphinx_host, search_servers.first.host + else + raise "No :search roles, and no :sphinx_host setting specified" + end + end + + put template.load(sphinx_conf_template), sphinx_conf_path end - desc "Rotate sphinx index for application" + desc <<-DESC + Rotate sphinx index for application. + + *sphinx_prefix*: Location to sphinx install. _Defaults to nil_\n + *sphinx_conf*: Location to sphinx conf. _Defaults to "[shared_path]/config/sphinx.conf"_\n + DESC task :rotate_all do - run "#{sphinx_prefix}/bin/indexer --config #{shared_path}/config/sphinx.conf --rotate --all" + fetch_or_default(:sphinx_prefix, nil) + fetch_or_default(:sphinx_conf, "#{shared_path}/config/sphinx.conf") + + indexer_path = sphinx_prefix ? "#{sphinx_prefix}/bin/indexer" : "indexer" + + run "#{indexer_path} --config #{sphinx_conf} --rotate --all" end - desc "Build sphinx indexes for application" + desc <<-DESC + Build sphinx indexes for application. + + *sphinx_prefix*: Location to sphinx install. _Defaults to nil_\n + *sphinx_conf*: Location to sphinx conf. _Defaults to "[shared_path]/config/sphinx.conf"_\n + DESC task :index_all do - run "#{sphinx_prefix}/bin/indexer --config #{shared_path}/config/sphinx.conf --all" + fetch_or_default(:sphinx_prefix, nil) + fetch_or_default(:sphinx_conf, "#{shared_path}/config/sphinx.conf") + + indexer_path = sphinx_prefix ? "#{sphinx_prefix}/bin/indexer" : "indexer" + + run "#{indexer_path} --config #{sphinx_conf} --all" end - desc "Start sphinx" - task :start do - # TODO: Monit + desc "Restart sphinx" + task :restart do sudo "/sbin/service monit restart sphinx_#{application}" end end \ No newline at end of file lib/recipes/sphinx.rb @@ -1,5 +1,5 @@ -<% locations_for_grant.each do |location| %> -GRANT ALL PRIVILEGES ON <%= db_name %>.* TO '<%= db_user %>'@'<%= location %>' IDENTIFIED BY '<%= db_pass %>'; +<% grant_locations.each do |location| %> +GRANT <%= grant %> ON <%= db_name %>.* TO '<%= db_user %>'@'<%= location %>' IDENTIFIED BY '<%= db_pass %>'; <% end %> CREATE DATABASE IF NOT EXISTS <%= db_name %>; lib/templates/mysql/install_db.sql.erb @@ -1,422 +1,51 @@ -# -# Sphinx configuration file sample -# TODO: This isn't currently used by any recipe. The current setup recipe uses sphinx conf on per application -# basis. -# - -############################################################################# -## data source definition -############################################################################# - -source <%= application %> +# sphinx config +source pages { - # data source type - # for now, known types are 'mysql', 'pgsql' and 'xmlpipe' - # MUST be defined type = mysql - # whether to strip HTML # values can be 0 (don't strip) or 1 (do strip) # WARNING, only works with mysql source for now # WARNING, should work ok for PERFECTLY formed XHTML for now # WARNING, POSSIBLE TO BUG on malformed everday HTML # optional, default is 0 - strip_html = 0 + strip_html = 1 # what HTML attributes to index if stripping HTML # format is as follows: # - # index_html_attrs = img=alt,title; a=title; - # - # optional, default is to not index anything - index_html_attrs = - - ##################################################################### - - # some straightforward parameters for 'mysql' source type - sql_host = 127.0.0.1 - sql_user = <%= db_user %> - sql_pass = <%= db_pass %> - sql_db = <%= db_name %> - sql_port = 3306 # optional, default is 3306 - - # sql_sock = /tmp/mysql.sock - # - # optional - # usually '/var/lib/mysql/mysql.sock' on Linux - # usually '/tmp/mysql.sock' on FreeBSD - - # pre-query, executed before the main fetch query - # useful eg. to setup encoding or mark records - # optional, default is empty - # + index_html_attrs = img=alt,title; a=title; + + sql_host = <%= sphinx_db_host %> + sql_user = <%= sphinx_db_user %> + sql_pass = <%= sphinx_db_pass %> + sql_db = <%= sphinx_db_name %> + sql_port = <%= sphinx_db_port %> # optional, default is 3306 # sql_query_pre = SET CHARACTER_SET_RESULTS=cp1251 sql_query_pre = SET NAMES UTF8 - - # main document fetch query - # - # you can specify up to 32 (formally SPH_MAX_FIELDS in sphinx.h) fields; - # all of the fields which are not document_id or attributes (see below) - # will be full-text indexed - # - # document_id MUST be the very first field - # document_id MUST be positive (non-zero, non-negative) - # document_id MUST fit into 32 bits - # document_id MUST be unique - # # mandatory - sql_query = \ - SELECT id, UNIX_TIMESTAMP(created_at) AS created_at, body FROM todo - - # query range setup - # - # useful to avoid MyISAM table locks and big result sets - # when indexing lots of data - # - # to use query ranges, you should - # 1) provide a query to fetch min/max id (ie. id range) from data set; - # 2) configure step size in which this range will be walked; - # 3) use $start and $end macros somewhere in the main fetch query. - # - # 'sql_query_range' must return exactly two integer fields - # in exactly min_id, max_id order - # - # 'sql_range_step' must be a positive integer - # optional, default is 1024 - # - # 'sql_query' must contain both '$start' and '$end' macros - # if you are using query ranges (because it obviously would be an - # error to index the whole table many times) - # - # note that the intervals specified by $start/$end do not - # overlap, so you should NOT remove document ids which are exactly - # equal to $start or $end in your query - # - # here's an example which will index 'documents' table - # fetching (at most) one thousand entries at a time: - # - # sql_query_range = SELECT MIN(id),MAX(id) FROM documents - # sql_range_step = 1000 - # sql_query = \ - # SELECT doc.id, doc.id AS group, doc.title, doc.data \ - # FROM documents doc \ - # WHERE id>=$start AND id<=$end - - - # attribute columns - # - # attribute values MUST be positive (non-zero, non-negative) integers - # attribute values MUST fit into 32 bits - # - # attributes are additional values associated with each document which - # may be used to perform additional filtering and sorting during search. - # attributes are NOT full-text indexed; they are stored in the full text - # index as is. - # - # a good example would be a forum posts table. one might need to search - # through 'title' and 'content' fields but to limit search to specific - # values of 'author_id', or 'forum_id', or to sort by 'post_date', or to - # group matches by 'thread_id', or to group posts by month of the - # 'post_date' and provide statistics. - # - # this all can be achieved by specifying all the mentioned columns - # (excluding 'title' and 'content' which are full-text fields) as - # attributes and then using API calls to setup filtering, sorting, - # and grouping. - # - # sql_group_column is used to declare integer attributes. - # - # sql_date_column is used to declare UNIX timestamp attributes. - # - # sql_str2ordinal_column is used to declare integer attributes which - # values are computed as ordinal numbers of corresponding column value - # in sorted list of column values. WARNING, all such strings values - # are going to be stored in RAM while indexing, and "C" locale will - # be used when sorting! - # - # starting with 0.9.7, there may be multiple attribute columns specified. - # here's an example for that mentioned posts table: - # - # sql_group_column = author_id - # sql_group_column = forum_id - # sql_group_column = thread_id - # sql_date_column = post_unix_timestamp - # sql_date_column = last_edit_unix_timestamp - # - # optional, default is empty - ##sql_group_column = group_id - sql_date_column = created_at - # sql_str2ordinal_column = author_name - - # post-query, executed on the end of main fetch query - # - # note that indexing is NOT completed at the point when post-query - # gets executed and might very well fail - # - # optional, default is empty - ##sql_query_post = - - # post-index-query, executed on succsefully completed indexing - # - # $maxid macro is the max document ID which was actually - # fetched from the database - # - # optional, default is empty - # - # sql_query_post_index = REPLACE INTO counters ( id, val ) \ - # VALUES ( 'max_indexed_id', $maxid ) - - - # document info query - # - # ONLY used by search utility to display document information - # MUST be able to fetch document info by its id, therefore - # MUST contain '$id' macro - # - # optional, default is empty - ##sql_query_info = SELECT * FROM documents WHERE id=$id - - ##################################################################### - - # demo config for 'xmlpipe' source type is a little below - # - # with xmlpipe, indexer opens a pipe to a given command, - # and then reads documents from stdin - # - # indexer expects one or more documents from xmlpipe stdin - # each document must be formatted exactly as follows: - # - # <document> - # <id>123</id> - # <group>45</group> - # <timestamp>1132223498</timestamp> - # <title>test title</title> - # <body> - # this is my document body - # </body> - # </document> - # - # timestamp element is optional, its default value is 1 - # all the other elements are mandatory - - # type = xmlpipe - # xmlpipe_command = cat /var/test.xml + sql_query = QUERY + sql_query_range = SELECT MIN(id),MAX(id) FROM TABLE_NAME + sql_range_step = 1000 + sql_group_column = user_id + sql_group_column = language + sql_date_column = published_date + sql_date_column = last_modified } -# inherited source example -# -# all the parameters are copied from the parent source, -# and may then be overridden in this source definition -##source src1stripped : src1 -##{ -## strip_html = 1 -##} - -############################################################################# -## index definition -############################################################################# - -# local index example -# -# this is an index which is stored locally in the filesystem -# -# all indexing-time options (such as morphology and charsets) -# are configured per local index -index <%= application %> +index pages { - # which document source to index - # at least one MUST be defined - # - # multiple sources MAY be specified; to do so, just add more - # "source = NAME" lines. in this case, ALL the document IDs - # in ALL the specified sources MUST be unique - source = <%= application %> - - # this is path and index file name without extension - # - # indexer will append different extensions to this path to - # generate names for both permanent and temporary index files - # - # .tmp* files are temporary and can be safely removed - # if indexer fails to remove them automatically - # - # .sp* files are fulltext index data files. specifically, - # .spa contains attribute values attached to each document id - # .spd contains doclists and hitlists - # .sph contains index header (schema and other settings) - # .spi contains wordlists - # - # MUST be defined - path = /var/sphinx/<%= application %> - - # docinfo (ie. per-document attribute values) storage strategy - # defines how docinfo will be stored - # - # available values are "none", "inline" and "extern" - # - # "none" means there'll be no docinfo at all (no groups/dates) - # - # "inline" means that the docinfo will be stored in the .spd - # file along with the document ID lists (doclists) - # - # "extern" means that the docinfo will be stored in the .spa - # file separately - # - # externally stored docinfo should (basically) be kept in RAM - # when querying; therefore, "inline" may be the only viable option - # for really huge (50-100+ million docs) datasets. however, for - # smaller datasets "extern" storage makes both indexing and - # searching MUCH more efficient. - # - # additional search-time memory requirements for extern storage are - # - # ( 1 + number_of_attrs )*number_of_docs*4 bytes - # - # so 10 million docs with 2 groups and 1 timestamp will take - # (1+2+1)*10M*4 = 160 MB of RAM. this is PER DAEMON, ie. searchd - # will alloc 160 MB on startup, read the data and keep it shared - # between queries; the children will NOT allocate additional - # copies of this data. - # - # default is "extern" (as most collections are smaller than 100M docs) + source = pages + path = <%= sphinx_index_root %>/pages docinfo = extern - - # morphology - # - # currently supported morphology preprocessors are Porter stemmers - # for English and Russian, and Soundex. more stemmers could be added - # at users request. - # - # available values are "none", "stem_en", "stem_ru", "stem_enru", - # and "soundex" - # - # optional, default is "none" - # - # morphology = none - # morphology = stem_en - # morphology = stem_ru - # morphology = stem_enru - # morphology = soundex - morphology = none - - # stopwords file - # - # format is plain text in whatever encoding you use - # optional, default is empty - # - # stopwords = /var/data/stopwords.txt - #stopwords = - - # minimum word length - # - # only the words that are of this length and above will be indexed; - # for example, if min_word_len is 4, "the" won't be indexed, - # but "they" will be. - # - # default is 1, which (obviously) means to index everything + morphology = stem_en + stopwords = <%= sphinx_conf_path %>/stopwords.txt min_word_len = 1 - - # charset encoding type - # - # known types are 'sbcs' (Single Byte CharSet) and 'utf-8' - # - # optional, default is sbcs charset_type = utf-8 - - # charset definition and case folding rules "table" - # - # optional, default value depends on charset_type - # - # for now, defaults are configured to support English and Russian - # this behavior MAY change in future versions - # - # 'sbcs' default value is - # charset_table = 0..9, A..Z->a..z, _, a..z, U+A8->U+B8, U+B8, U+C0..U+DF->U+E0..U+FF, U+E0..U+FF - # - # 'utf-8' default value is - # charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F - - # minimum prefix length - # - # if prefix length is positive, indexer will not only index all words, - # but all the possible prefixes (ie. word beginnings) as well - # - # for instance, "exam" query against such index will match documents - # which contain "example" word, even if they do not contain "exam" - # - # indexing prefixes will make the index grow significantly - # and could degrade search times - # - # currently there's no way to rank perfect word matches higher - # than prefix matches using only one index; you could setup two - # indexes for that - # - # default is 0, which means NOT to index prefixes min_prefix_len = 0 - - # minimum infix length - # - # if infix length is positive, indexer will not only index all words, - # but all the possible infixes (ie. characters subsequences starting - # anywhere inside the word) as well - # - # for instance, "amp" query against such index will match documents - # which contain "example" word, even if they do not contain "amp" - # - # indexing prefixes will make the index grow significantly - # and could degrade search times - # - # currently there's no way to rank perfect word matches higher - # than infix matches using only one index; you could setup two - # indexes for that - # - # default is 0, which means NOT to index infixes min_infix_len = 0 - - # n-grams length - # - # n-grams provide basic CJK support for unsegmented texts. if using - # n-grams, streams of CJK characters are indexed as n-grams. for example, - # if incoming stream is ABCDEF and n is 2, this text would be indexed - # as if it was AB BC CD DE EF. - # - # this feature is in alpha version state and only n=1 is currently - # supported; this is going to be improved. - # - # note that if search query is segmented (ie. words are separated with - # whitespace), words are in quotes and extended matching mode is used, - # then all matching documents will be returned even if their text was - # *not* segmented. in the example above, ABCDEF text will be indexed as - # A B C D E F, and "BCD" query will be transformed to "B C D" (where - # quotes is phrase matching operator), so the document will match. - # - # optional, default is 0, which means NOT to use n-grams - # - # ngram_len = 1 - - # n-gram characters table - # - # specifies what specific characters are subject to n-gram - # extraction. format is similar to charset_table. - # - # optional, default is empty - # - # ngrams_chars = U+3000..U+2FA1F } - -# inherited index example -# -# all the parameters are copied from the parent index, -# and may then be overridden in this index definition -##index test1stemmed : test1 -##{ -## path = /var/data/test1stemmed -## morphology = stem_en -##} - - ############################################################################# ## indexer settings ############################################################################# @@ -432,7 +61,7 @@ indexer # will warn if set too low and potentially hurting the performance # # optional, default is 32M - mem_limit = 32M + mem_limit = 64M } ############################################################################# @@ -447,22 +76,22 @@ searchd # optional, default is to listen on all addresses, # ie. address = 0.0.0.0 # - address = 127.0.0.1 + address = <%= sphinx_host %> # address = 192.168.0.1 # port on which search daemon will listen - port = 3312 + port = <%= sphinx_port %> # log file # searchd run info is logged here - log = /var/sphinx/searchd.log + log = <%= sphinx_log_root %>/searchd.log # query log file # all the search queries are logged here - query_log = /var/sphinx/query.log + query_log = <%= sphinx_log_root %>/query.log # client read timeout, seconds @@ -477,7 +106,7 @@ searchd # a file which will contain searchd process ID # used for different external automation scripts # MUST be present - pid_file = /var/sphinx/searchd.pid + pid_file = <%= sphinx_pid_path %> # maximum amount of matches this daemon would ever retrieve lib/templates/sphinx/sphinx.conf.erb @@ -14,7 +14,7 @@ RETVAL=0 DESC="sphinx daemon (<%= application %>)" NAME=searchd -DAEMON=<%= sphinx_bin_path %>/searchd +DAEMON=<%= sphinx_prefix %>/bin/searchd CONFIGFILE=<%= sphinx_conf_path %> PIDFILE=<%= sphinx_pid_path %> lib/templates/sphinx/sphinx_app.initd.centos.erb @@ -38,7 +38,7 @@ <div id="version" class="clickable box" onclick='document.location = "http://rubyforge.org/projects/capitate"; return false'> <p>Get Version</p> - <a href="http://rubyforge.org/projects/capitate" class="numbers">0.1.9</a> + <a href="http://rubyforge.org/projects/capitate" class="numbers">0.2.1</a> </div> <div id="recipes"> website/index.html lib/deployment/templates/setup_for_web.sh lib/deployment/templates/sudoers f4757bdcd00ad72bd9a729388f75cc24a8e3f8d9 Gabriel Handford gabrielh@gmail.com http://github.com/gabriel/capitate/commit/f2df06ba495fa5feb7dc02fe7b14180fbdddb541 f2df06ba495fa5feb7dc02fe7b14180fbdddb541 2008-02-24T13:06:20-08:00 2008-02-24T13:06:20-08:00 fixes for sphinx 7a2c58d4fa62eba7ec44caadc4a2e9969b7e0cef Gabriel Handford gabrielh@gmail.com