Browse files

move solr querying logic from chef-solr into chef

  • Loading branch information...
1 parent b83409f commit 67adee437910d89bee34ead3663d84d2fd4a3d63 @danielsdeleo danielsdeleo committed Feb 3, 2011
View
6 chef-server-api/app/controllers/search.rb
@@ -17,7 +17,7 @@
# limitations under the License.
-require 'chef/solr/query'
+require 'chef/solr_query'
class Search < Application
provides :json
@@ -40,7 +40,7 @@ def show
raise NotFound, "I don't know how to search for #{params[:id]} data objects."
end
- query = Chef::Solr::Query.new(Chef::Config[:solr_url])
+ query = Chef::SolrQuery.new(Chef::Config[:solr_url])
params[:sort] ||= nil
params[:rows] ||= 20
@@ -55,7 +55,7 @@ def show
end
def reindex
- display(Chef::Solr.new.rebuild_index)
+ display(Chef::SolrQuery.new.rebuild_index)
end
end
View
28 chef-solr/bin/chef-solr-indexer
@@ -1,28 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Author:: Adam Jacob (<adam@opscode.com>)
-# Copyright:: Copyright (c) 2009 Opscode, Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require 'rubygems'
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "chef", "lib")))
-
-require 'chef/solr/application/indexer'
-
-Chef::Solr::Application::Indexer.new.run
-
View
148 chef-solr/lib/chef/solr/application/indexer.rb
@@ -1,148 +0,0 @@
-#
-# Author:: AJ Christensen (<aj@opscode.com)
-# Copyright:: Copyright (c) 2008 Opscode, Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-require 'chef'
-require 'chef/log'
-require 'chef/config'
-require 'chef/application'
-require 'chef/solr'
-require 'chef/solr/index'
-require 'chef/solr/index_queue_consumer'
-require 'chef/daemon'
-require 'chef/webui_user'
-
-class Chef
- class Solr
- class Application
- class Indexer < Chef::Application
-
- option :config_file,
- :short => "-c CONFIG",
- :long => "--config CONFIG",
- :default => "/etc/chef/solr.rb",
- :description => "The configuration file to use"
-
- option :log_level,
- :short => "-l LEVEL",
- :long => "--log_level LEVEL",
- :description => "Set the log level (debug, info, warn, error, fatal)",
- :proc => lambda { |l| l.to_sym }
-
- option :log_location,
- :short => "-L LOGLOCATION",
- :long => "--logfile LOGLOCATION",
- :description => "Set the log file location, defaults to STDOUT - recommended for daemonizing",
- :proc => nil
-
- option :pid_file,
- :short => "-P PID_FILE",
- :long => "--pid PIDFILE",
- :description => "Set the PID file location, defaults to /tmp/chef-solr-indexer.pid",
- :proc => nil
-
-
- option :help,
- :short => "-h",
- :long => "--help",
- :description => "Show this message",
- :on => :tail,
- :boolean => true,
- :show_options => true,
- :exit => 0
-
- option :user,
- :short => "-u USER",
- :long => "--user USER",
- :description => "User to set privilege to",
- :proc => nil
-
- option :group,
- :short => "-g GROUP",
- :long => "--group GROUP",
- :description => "Group to set privilege to",
- :proc => nil
-
- option :daemonize,
- :short => "-d",
- :long => "--daemonize",
- :description => "Daemonize the process",
- :proc => lambda { |p| true }
-
- option :amqp_host,
- :long => "--amqp-host HOST",
- :description => "The amqp host"
-
- option :amqp_port,
- :long => "--amqp-port PORT",
- :description => "The amqp port"
-
- option :amqp_user,
- :long => "--amqp-user USER",
- :description => "The amqp user"
-
- option :amqp_pass,
- :long => "--amqp-pass PASS",
- :description => "The amqp password"
-
- option :amqp_vhost,
- :long => "--amqp-vhost VHOST",
- :description => "The amqp vhost"
-
- option :version,
- :short => "-v",
- :long => "--version",
- :description => "Show chef-solr-indexer version",
- :boolean => true,
- :proc => lambda {|v| puts "chef-solr-indexer: #{::Chef::Solr::VERSION}"},
- :exit => 0
-
- Signal.trap("INT") do
- begin
- AmqpClient.instance.stop
- rescue Bunny::ProtocolError, Bunny::ConnectionError, Bunny::UnsubscribeError
- end
- fatal!("SIGINT received, stopping", 2)
- end
-
- Kernel.trap("TERM") do
- begin
- AmqpClient.instance.stop
- rescue Bunny::ProtocolError, Bunny::ConnectionError, Bunny::UnsubscribeError
- end
- fatal!("SIGTERM received, stopping", 1)
- end
-
- def initialize
- super
-
- @index = Chef::Solr::Index.new
- @consumer = Chef::Solr::IndexQueueConsumer.new
- end
-
- def setup_application
- Chef::Daemon.change_privilege
- Chef::Log.level = Chef::Config[:log_level]
- end
-
- def run_application
- Chef::Daemon.daemonize("chef-solr-indexer") if Chef::Config[:daemonize]
- @consumer.start
- end
- end
- end
- end
-end
View
103 chef-solr/lib/chef/solr/index.rb
@@ -1,103 +0,0 @@
-#
-# Author:: Adam Jacob (<adam@opscode.com>)
-# Copyright:: Copyright (c) 2009 Opscode, Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require 'chef/log'
-require 'chef/config'
-require 'chef/solr'
-require 'libxml'
-require 'net/http'
-
-class Chef
- class Solr
- class Index < Solr
-
- UNDERSCORE = '_'
- X = 'X'
-
- X_CHEF_id_CHEF_X = 'X_CHEF_id_CHEF_X'
- X_CHEF_database_CHEF_X = 'X_CHEF_database_CHEF_X'
- X_CHEF_type_CHEF_X = 'X_CHEF_type_CHEF_X'
-
- def add(id, database, type, item)
- unless item.respond_to?(:keys)
- raise ArgumentError, "#{self.class.name} can only index Hash-like objects. You gave #{item.inspect}"
- end
-
- to_index = flatten_and_expand(item)
-
- to_index[X_CHEF_id_CHEF_X] = [id]
- to_index[X_CHEF_database_CHEF_X] = [database]
- to_index[X_CHEF_type_CHEF_X] = [type]
-
- solr_add(to_index)
- to_index
- end
-
- def delete(id)
- solr_delete_by_id(id)
- end
-
- def delete_by_query(query)
- solr_delete_by_query(query)
- end
-
- def flatten_and_expand(item)
- @flattened_item = Hash.new {|hash, key| hash[key] = []}
-
- item.each do |key, value|
- flatten_each([key.to_s], value)
- end
-
- @flattened_item.each_value { |values| values.uniq! }
- @flattened_item
- end
-
- def flatten_each(keys, values)
- case values
- when Hash
- values.each do |child_key, child_value|
- add_field_value(keys, child_key)
- flatten_each(keys + [child_key.to_s], child_value)
- end
- when Array
- values.each { |child_value| flatten_each(keys, child_value) }
- else
- add_field_value(keys, values)
- end
- end
-
- def add_field_value(keys, value)
- value = value.to_s
- each_expando_field(keys) { |expando_field| @flattened_item[expando_field] << value }
- @flattened_item[keys.join(UNDERSCORE)] << value
- @flattened_item[keys.last] << value
- end
-
- def each_expando_field(keys)
- return if keys.size == 1
- 0.upto(keys.size - 1) do |index|
- original = keys[index]
- keys[index] = X
- yield keys.join(UNDERSCORE)
- keys[index] = original
- end
- end
-
- end
- end
-end
View
82 chef-solr/lib/chef/solr/index_queue_consumer.rb
@@ -1,82 +0,0 @@
-#
-# Author:: Adam Jacob (<adam@opscode.com>)
-# Copyright:: Copyright (c) 2009 Opscode, Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require 'chef/log'
-require 'chef/config'
-require 'chef/solr'
-require 'chef/solr/index'
-require 'chef/couchdb'
-require 'chef/index_queue'
-
-class Chef
- class Solr
- class IndexQueueConsumer
- include Chef::IndexQueue::Consumer
-
- expose :add, :delete
-
- def add(payload)
- index = Chef::Solr::Index.new
- Chef::Log.debug("Dequeued item for indexing: #{payload.inspect}")
-
- begin
- # older producers will send the raw item, and we no longer inflate it
- # to an object.
- pitem = payload["item"].to_hash
- pitem.delete("json_class")
- response = generate_response { index.add(payload["id"], payload["database"], payload["type"], pitem) }
- rescue NoMethodError
- response = generate_response() { raise ArgumentError, "Payload item does not respond to :keys or :to_hash, cannot index!" }
- end
-
- msg = "Indexing #{payload["type"]} #{payload["id"]} from #{payload["database"]} status #{status_message(response)}}"
- Chef::Log.info(msg)
- response
- end
-
- def delete(payload)
- response = generate_response { Chef::Solr::Index.new.delete(payload["id"]) }
- Chef::Log.info("Removed #{payload["id"]} from the index")
- response
- end
-
- private
-
- def generate_response(&block)
- response = {}
- begin
- block.call
- rescue => e
- response[:status] = :error
- response[:error] = e
- else
- response[:status] = :ok
- end
- response
- end
-
- def status_message(response)
- msg = response[:status].to_s
- msg << ' ' + response[:error].to_s if response[:status] == :error
- msg
- end
-
- end
- end
-end
-
View
164 chef-solr/lib/chef/solr/query.rb
@@ -1,164 +0,0 @@
-#
-# Author:: Adam Jacob (<adam@opscode.com>)
-# Author:: Nuo Yan (<nuo@opscode.com>)
-# Author:: Chris Walters (<cw@opscode.com>)
-# Author:: Seth Falcon (<seth@opscode.com>)
-# Copyright:: Copyright (c) 2010 Opscode, Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require 'chef/couchdb'
-require 'chef/node'
-require 'chef/role'
-require 'chef/data_bag'
-require 'chef/data_bag_item'
-require 'chef/solr'
-require 'chef/log'
-require 'chef/config'
-
-class Chef
- class Solr
- class Query < Chef::Solr
- ID_KEY = "X_CHEF_id_CHEF_X"
-
- # Create a new Query object - takes the solr_url and optional
- # Chef::CouchDB object to inflate objects into.
- def initialize(solr_url=Chef::Config[:solr_url], couchdb = nil)
- super(solr_url)
- if couchdb.nil?
- @database = Chef::Config[:couchdb_database]
- @couchdb = Chef::CouchDB.new(nil, Chef::Config[:couchdb_database])
- else
- unless couchdb.kind_of?(Chef::CouchDB)
- Chef::Log.warn("Passing the database name to Chef::Solr::Query initialization is deprecated. Please pass in the Chef::CouchDB object instead.")
- @database = couchdb
- @couchdb = Chef::CouchDB.new(nil, couchdb)
- else
- @database = couchdb.couchdb_database
- @couchdb = couchdb
- end
- end
- end
-
- # A raw query against CouchDB - takes the type of object to find, and raw
- # Solr options.
- #
- # You'll wind up having to page things yourself.
- def raw(type, options={})
- qtype = case options[:type].to_s
- when "role","node","client","environment"
- options[:type]
- else
- [ "data_bag_item", options[:type] ]
- end
- results = solr_select(@database, qtype, options)
- Chef::Log.debug("Searching #{@database} #{qtype.inspect} for #{options.inspect} with results:\n#{results.inspect}")
- objects = if results["response"]["docs"].length > 0
- bulk_objects = @couchdb.bulk_get( results["response"]["docs"].collect { |d| d[ID_KEY] } )
- Chef::Log.debug("bulk get of objects: #{bulk_objects.inspect}")
- bulk_objects
- else
- []
- end
- [ objects, results["response"]["start"], results["response"]["numFound"], results["responseHeader"] ]
- end
-
- # Search Solr for objects of a given type, for a given query. If
- # you give it a block, it will handle the paging for you
- # dynamically.
- def search(params, &block)
- defaults = Mash.new({:q => "*:*", :start => 0, :rows => 1000})
- options = defaults.merge(params)
- options[:sort] = "#{ID_KEY} asc" if options[:sort].nil? || options[:sort].empty?
- options[:q] = transform_search_query(options[:q])
- objects, start, total, response_header = raw(options)
- if block
- objects.each { |o| block.call(o) }
- unless (start + objects.length) >= total
- nstart = start + rows
- search(type, query, sort, nstart, rows, &block)
- end
- true
- else
- [ objects, start, total ]
- end
- end
-
- # Constants used for search query transformation
- FLD_SEP = "\001"
- SPC_SEP = "\002"
- QUO_SEP = "\003"
- QUO_KEY = "\004"
-
- def transform_search_query(q)
- return q if q == "*:*"
-
- # handled escaped quotes
- q = q.gsub(/\\"/, QUO_SEP)
-
- # handle quoted strings
- i = 1
- quotes = {}
- q = q.gsub(/([^ \\+()]+):"([^"]+)"/) do |m|
- key = QUO_KEY + i.to_s
- quotes[key] = "content#{FLD_SEP}\"#{$1}__=__#{$2}\""
- i += 1
- key
- end
-
- # a:[* TO *] => a*
- q = q.gsub(/\[\*[+ ]TO[+ ]\*\]/, '*')
-
- keyp = '[^ \\+()]+'
- lbrak = '[\[{]'
- rbrak = '[\]}]'
-
- # a:[blah TO zah] =>
- # content\001[a__=__blah\002TO\002a__=__zah]
- # includes the cases a:[* TO zah] and a:[blah TO *], but not
- # [* TO *]; that is caught above
- q = q.gsub(/(#{keyp}):(#{lbrak})([^\]}]+)[+ ]TO[+ ]([^\]}]+)(#{rbrak})/) do |m|
- if $3 == "*"
- "content#{FLD_SEP}#{$2}#{$1}__=__#{SPC_SEP}TO#{SPC_SEP}#{$1}__=__#{$4}#{$5}"
- elsif $4 == "*"
- "content#{FLD_SEP}#{$2}#{$1}__=__#{$3}#{SPC_SEP}TO#{SPC_SEP}#{$1}__=__\\ufff0#{$5}"
- else
- "content#{FLD_SEP}#{$2}#{$1}__=__#{$3}#{SPC_SEP}TO#{SPC_SEP}#{$1}__=__#{$4}#{$5}"
- end
- end
-
- # foo:bar => content:foo__=__bar
- q = q.gsub(/([^ \\+()]+):([^ +]+)/) { |m| "content:#{$1}__=__#{$2}" }
-
- # /002 => ' '
- q = q.gsub(/#{SPC_SEP}/, ' ')
-
- # replace quoted query chunks
- quotes.keys.each do |key|
- q = q.gsub(key, quotes[key])
- end
-
- # replace escaped quotes
- q = q.gsub(QUO_SEP, '\"')
-
- # /001 => ':'
- q = q.gsub(/#{FLD_SEP}/, ':')
- q
- end
-
- end
- end
-end
-
View
187 chef-solr/spec/chef/solr/index_spec.rb
@@ -1,187 +0,0 @@
-require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', '..', 'spec_helper'))
-
-
-
-describe Chef::Solr::Index do
- before(:each) do
- @index = Chef::Solr::Index.new
- end
-
- describe "initialize" do
- it "should return a Chef::Solr::Index" do
- @index.should be_a_kind_of(Chef::Solr::Index)
- end
- end
-
- describe "add" do
- before(:each) do
- @index.stub!(:solr_add).and_return(true)
- @index.stub!(:solr_commit).and_return(true)
- end
-
- it "should take an object that responds to .keys as it's argument" do
- lambda { @index.add(1, "chef_opscode", "node", { :one => :two }) }.should_not raise_error(ArgumentError)
- lambda { @index.add(1, "chef_opscode", "node", "SOUP") }.should raise_error(ArgumentError)
- lambda { @index.add(2, "chef_opscode", "node", mock("Foo", :keys => true)) }.should_not raise_error(ArgumentError)
- end
-
- it "should index the object as a single flat hash, with only strings or arrays as values" do
- validate = {
- "X_CHEF_id_CHEF_X" => [1],
- "X_CHEF_database_CHEF_X" => ["monkey"],
- "X_CHEF_type_CHEF_X" => ["snakes"],
- "foo" => ["bar"],
- "battles" => [ "often", "but", "for" ],
- "battles_often" => ["sings like smurfs"],
- "often" => ["sings like smurfs"],
- "battles_but" => ["still has good records"],
- "but" => ["still has good records"],
- "battles_for" => [ "all", "of", "that" ],
- "for" => [ "all", "of", "that" ],
- "snoopy" => ["sits-in-a-barn"],
- "battles_X" => [ "sings like smurfs", "still has good records", "all", "of", "that" ],
- "X_often" =>[ "sings like smurfs"],
- "X_but" => ["still has good records"],
- "X_for" => [ "all", "of", "that" ]
- }
- to_index = @index.add(1, "monkey", "snakes", {
- "foo" => :bar,
- "battles" => {
- "often" => "sings like smurfs",
- "but" => "still has good records",
- "for" => [ "all", "of", "that" ]
- },
- "snoopy" => "sits-in-a-barn"
- })
-
- validate.each do |k, v|
- if v.kind_of?(Array)
- to_index[k].sort.should == v.sort
- else
- to_index[k].should == v
- end
- end
- end
-
- it "should send the document to solr" do
- @index.should_receive(:solr_add)
- @index.add(1, "monkey", "snakes", { "foo" => "bar" })
- end
- end
-
- describe "delete" do
- it "should delete by id" do
- @index.should_receive(:solr_delete_by_id).with(1)
- @index.delete(1)
- end
- end
-
- describe "delete_by_query" do
- it "should delete by query" do
- @index.should_receive(:solr_delete_by_query).with("foo:bar")
- @index.delete_by_query("foo:bar")
- end
- end
-
- describe "flatten_and_expand" do
- before(:each) do
- @fields = Hash.new
- end
-
- it "should set a value for the parent as key, with the key as the value" do
- @fields = @index.flatten_and_expand("omerta" => { "one" => "woot" })
- @fields["omerta"].should == ["one"]
- end
-
- it "should call itself recursively for values that are hashes" do
- @fields = @index.flatten_and_expand({ "one" => { "two" => "three", "four" => { "five" => "six" } }})
- expected = {"one" => [ "two", "four" ],
- "one_two" => ["three"],
- "X_two" => ["three"],
- "two" => ["three"],
- "one_four" => ["five"],
- "X_four" => ["five"],
- "one_X" => [ "three", "five" ],
- "one_four_five" => ["six"],
- "X_four_five" => ["six"],
- "one_X_five" => ["six"],
- "one_four_X" => ["six"],
- "five" => ["six"]}
- expected.each do |k, v|
- @fields[k].should == v
- end
- end
-
- it "should call itself recursively for hashes nested in arrays" do
- @fields = @index.flatten_and_expand({ :one => [ { :two => "three" }, { :four => { :five => "six" } } ] })
- expected = {"one_X_five" => ["six"],
- "one_four" => ["five"],
- "one_X" => [ "three", "five" ],
- "two" => ["three"],
- "one_four_X" => ["six"],
- "X_four" => ["five"],
- "X_four_five" => ["six"],
- "one" => [ "two", "four" ],
- "one_four_five" => ["six"],
- "five" => ["six"],
- "X_two" => ["three"],
- "one_two" => ["three"]}
-
- expected.each do |key, expected_value|
- @fields[key].should == expected_value
- end
- end
-
- it "generates unlimited levels of expando fields when expanding" do
- expected_keys = ["one",
- "one_two",
- "X_two",
- "one_X",
- "one_two_three",
- "X_two_three",
- "one_X_three",
- "one_two_X",
- "one_two_three_four",
- "X_two_three_four",
- "one_X_three_four",
- "one_two_X_four",
- "one_two_three_X",
- "one_two_three_four_five",
- "X_two_three_four_five",
- "one_X_three_four_five",
- "one_two_X_four_five",
- "one_two_three_X_five",
- "one_two_three_four_X",
- "six",
- "one_two_three_four_five_six",
- "X_two_three_four_five_six",
- "one_X_three_four_five_six",
- "one_two_X_four_five_six",
- "one_two_three_X_five_six",
- "one_two_three_four_X_six",
- "one_two_three_four_five_X"].sort
-
- nested = {:one => {:two => {:three => {:four => {:five => {:six => :end}}}}}}
- @fields = @index.flatten_and_expand(nested)
-
- @fields.keys.sort.should include(*expected_keys)
- end
-
- end
-
- describe "creating expando fields" do
- def make_expando_fields(parts)
- expando_fields = []
- @index.each_expando_field(parts) { |ex| expando_fields << ex }
- expando_fields
- end
-
- it "joins the fields with a big X" do
- make_expando_fields(%w{foo bar baz qux}).should == ["X_bar_baz_qux", "foo_X_baz_qux", "foo_bar_X_qux", "foo_bar_baz_X"]
- make_expando_fields(%w{foo bar baz}).should == ["X_bar_baz", "foo_X_baz", "foo_bar_X"]
- make_expando_fields(%w{foo bar}).should == ["X_bar", "foo_X"]
- make_expando_fields(%w{foo}).should == []
- end
- end
-
-end
View
16 chef-solr/spec/chef/solr/query_spec.rb
@@ -1,16 +0,0 @@
-require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', '..', 'spec_helper'))
-
-describe Chef::Solr::Query do
- before(:each) do
- @query = Chef::Solr::Query.new
- end
-
- it "should transform queries correctly" do
- testcases = Hash[*(File.readlines("#{CHEF_SOLR_SPEC_DATA}/search_queries_to_transform.txt").select{|line| line !~ /^\s*$/}.map{|line| line.chomp})]
- testcases.each do |input, expected|
- @query.transform_search_query(input).should == expected
- end
- end
-
-end
-
View
1 chef-solr/spec/spec.opts
@@ -1 +0,0 @@
--cbfs
View
16 chef-solr/spec/spec_helper.rb
@@ -1,16 +0,0 @@
-require 'rubygems'
-require 'spec'
-
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', '..', 'chef', 'lib'))
-require 'chef'
-require 'chef/solr'
-require 'chef/solr/index'
-require 'chef/solr/query'
-
-CHEF_SOLR_SPEC_DATA = File.expand_path(File.dirname(__FILE__) + "/data/")
-
-Spec::Runner.configure do |config|
-
-end
View
130 chef-solr/lib/chef/solr.rb → chef/lib/chef/solr_query.rb
@@ -25,18 +25,143 @@
require 'uri'
class Chef
- class Solr
+ class SolrQuery
include Chef::Mixin::XMLEscape
attr_accessor :solr_url, :http
- def initialize(solr_url=Chef::Config[:solr_url])
+ ID_KEY = "X_CHEF_id_CHEF_X"
+
+ # Create a new Query object - takes the solr_url and optional
+ # Chef::CouchDB object to inflate objects into.
+ def initialize(solr_url=Chef::Config[:solr_url], couchdb = nil)
@solr_url = solr_url
uri = URI.parse(@solr_url)
@http = Net::HTTP.new(uri.host, uri.port)
+
+ if couchdb.nil?
+ @database = Chef::Config[:couchdb_database]
+ @couchdb = Chef::CouchDB.new(nil, Chef::Config[:couchdb_database])
+ else
+ unless couchdb.kind_of?(Chef::CouchDB)
+ Chef::Log.warn("Passing the database name to Chef::Solr::Query initialization is deprecated. Please pass in the Chef::CouchDB object instead.")
+ @database = couchdb
+ @couchdb = Chef::CouchDB.new(nil, couchdb)
+ else
+ @database = couchdb.couchdb_database
+ @couchdb = couchdb
+ end
+ end
end
+ # A raw query against CouchDB - takes the type of object to find, and raw
+ # Solr options.
+ #
+ # You'll wind up having to page things yourself.
+ def raw(type, options={})
+ qtype = case options[:type].to_s
+ when "role","node","client","environment"
+ options[:type]
+ else
+ [ "data_bag_item", options[:type] ]
+ end
+ results = solr_select(@database, qtype, options)
+ Chef::Log.debug("Searching #{@database} #{qtype.inspect} for #{options.inspect} with results:\n#{results.inspect}")
+ objects = if results["response"]["docs"].length > 0
+ bulk_objects = @couchdb.bulk_get( results["response"]["docs"].collect { |d| d[ID_KEY] } )
+ Chef::Log.debug("bulk get of objects: #{bulk_objects.inspect}")
+ bulk_objects
+ else
+ []
+ end
+ [ objects, results["response"]["start"], results["response"]["numFound"], results["responseHeader"] ]
+ end
+
+ # Search Solr for objects of a given type, for a given query. If
+ # you give it a block, it will handle the paging for you
+ # dynamically.
+ def search(params, &block)
+ defaults = Mash.new({:q => "*:*", :start => 0, :rows => 1000})
+ options = defaults.merge(params)
+ options[:sort] = "#{ID_KEY} asc" if options[:sort].nil? || options[:sort].empty?
+ options[:q] = transform_search_query(options[:q])
+ objects, start, total, response_header = raw(options)
+ if block
+ objects.each { |o| block.call(o) }
+ unless (start + objects.length) >= total
+ nstart = start + rows
+ search(type, query, sort, nstart, rows, &block)
+ end
+ true
+ else
+ [ objects, start, total ]
+ end
+ end
+
+ # Constants used for search query transformation
+ FLD_SEP = "\001"
+ SPC_SEP = "\002"
+ QUO_SEP = "\003"
+ QUO_KEY = "\004"
+
+ def transform_search_query(q)
+ return q if q == "*:*"
+
+ # handled escaped quotes
+ q = q.gsub(/\\"/, QUO_SEP)
+
+ # handle quoted strings
+ i = 1
+ quotes = {}
+ q = q.gsub(/([^ \\+()]+):"([^"]+)"/) do |m|
+ key = QUO_KEY + i.to_s
+ quotes[key] = "content#{FLD_SEP}\"#{$1}__=__#{$2}\""
+ i += 1
+ key
+ end
+
+ # a:[* TO *] => a*
+ q = q.gsub(/\[\*[+ ]TO[+ ]\*\]/, '*')
+
+ keyp = '[^ \\+()]+'
+ lbrak = '[\[{]'
+ rbrak = '[\]}]'
+
+ # a:[blah TO zah] =>
+ # content\001[a__=__blah\002TO\002a__=__zah]
+ # includes the cases a:[* TO zah] and a:[blah TO *], but not
+ # [* TO *]; that is caught above
+ q = q.gsub(/(#{keyp}):(#{lbrak})([^\]}]+)[+ ]TO[+ ]([^\]}]+)(#{rbrak})/) do |m|
+ if $3 == "*"
+ "content#{FLD_SEP}#{$2}#{$1}__=__#{SPC_SEP}TO#{SPC_SEP}#{$1}__=__#{$4}#{$5}"
+ elsif $4 == "*"
+ "content#{FLD_SEP}#{$2}#{$1}__=__#{$3}#{SPC_SEP}TO#{SPC_SEP}#{$1}__=__\\ufff0#{$5}"
+ else
+ "content#{FLD_SEP}#{$2}#{$1}__=__#{$3}#{SPC_SEP}TO#{SPC_SEP}#{$1}__=__#{$4}#{$5}"
+ end
+ end
+
+ # foo:bar => content:foo__=__bar
+ q = q.gsub(/([^ \\+()]+):([^ +]+)/) { |m| "content:#{$1}__=__#{$2}" }
+
+ # /002 => ' '
+ q = q.gsub(/#{SPC_SEP}/, ' ')
+
+ # replace quoted query chunks
+ quotes.keys.each do |key|
+ q = q.gsub(key, quotes[key])
+ end
+
+ # replace escaped quotes
+ q = q.gsub(QUO_SEP, '\"')
+
+ # /001 => ':'
+ q = q.gsub(/#{FLD_SEP}/, ':')
+ q
+ end
+
+
def solr_select(database, type, options={})
options[:wt] = :ruby
options[:indent] = "off"
@@ -230,5 +355,6 @@ def http_request_handler(req, description='HTTP call')
raise Chef::Exceptions::SolrConnectionError, "#{e.class.name}: #{e.to_s}"
end
+
end
end
View
0 ...spec/data/search_queries_to_transform.txt → ...spec/data/search_queries_to_transform.txt
File renamed without changes.
View
44 chef/spec/unit/couchdb_spec.rb
@@ -20,6 +20,7 @@
describe Chef::CouchDB do
before(:each) do
+ Chef::Config[:couchdb_database] = "chef"
@mock_rest = mock("Chef::REST", :null_object => true)
@mock_rest.stub!(:run_request).and_return({"couchdb" => "Welcome", "version" =>"0.9.0"})
@mock_rest.stub!(:url).and_return("http://localhost:5984")
@@ -235,39 +236,28 @@ def do_delete(rev=nil)
end
end
-end
-
-
-
+ describe "get_view" do
+ it "should construct a call to the view for the proper design document" do
+ @mock_rest.should_receive(:get_rest).with("chef/_design/nodes/_view/mastodon")
+ @couchdb.get_view("nodes", "mastodon")
+ end
-describe Chef::CouchDB, "get_view" do
- before do
- @mock_rest = mock("Chef::REST", :null_object => true, :url => "http://monkeypants")
- Chef::REST.stub!(:new).and_return(@mock_rest)
- @couchdb = Chef::CouchDB.new("http://localhost")
- end
+ it "should allow arguments to the view" do
+ @mock_rest.should_receive(:get_rest).with("chef/_design/nodes/_view/mastodon?startkey=%22dont%20stay%22")
+ @couchdb.get_view("nodes", "mastodon", :startkey => "dont stay")
+ end
- it "should construct a call to the view for the proper design document" do
- @mock_rest.should_receive(:get_rest).with("chef/_design/nodes/_view/mastodon")
- @couchdb.get_view("nodes", "mastodon")
end
- it "should allow arguments to the view" do
- @mock_rest.should_receive(:get_rest).with("chef/_design/nodes/_view/mastodon?startkey=%22dont%20stay%22")
- @couchdb.get_view("nodes", "mastodon", :startkey => "dont stay")
+ describe "view_uri" do
+ it "should output an appropriately formed view URI" do
+ @couchdb.should_receive(:view_uri).with("nodes", "all").and_return("chef/_design/nodes/_view/all")
+ @couchdb.view_uri("nodes", "all")
+ end
end
end
-describe Chef::CouchDB, "view_uri" do
- before do
- @mock_rest = mock("Chef::REST", :null_object => true, :url => "http://monkeypants")
- Chef::REST.stub!(:new).and_return(@mock_rest)
- @couchdb = Chef::CouchDB.new("http://localhost")
- end
- it "should output an appropriately formed view URI" do
- @couchdb.should_receive(:view_uri).with("nodes", "all").and_return("chef/_design/nodes/_view/all")
- @couchdb.view_uri("nodes", "all")
- end
-end
+
+
View
29 chef-solr/spec/chef/solr_spec.rb → chef/spec/unit/solr_query_spec.rb
@@ -1,18 +1,17 @@
require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', 'spec_helper'))
+
+require 'chef/solr_query'
require 'net/http'
-describe Chef::Solr do
+describe Chef::SolrQuery do
before(:each) do
- @solr = Chef::Solr.new
+ @solr = Chef::SolrQuery.new
end
- describe "initialize" do
- it "should create a new Chef::Solr object" do
- @solr.should be_a_kind_of(Chef::Solr)
- end
+ describe "when first created" do
it "should accept an alternate solr url" do
- solr = Chef::Solr.new("http://example.com")
+ solr = Chef::SolrQuery .new("http://example.com")
solr.solr_url.should eql("http://example.com")
end
end
@@ -290,4 +289,20 @@
@solr.rebuild_index["Chef::DataBag"].should == "success"
end
end
+
+ describe "when transforming queries to match to support backwards compatibility with the old solr schema" do
+ before(:each) do
+ @query = Chef::SolrQuery.new
+ end
+
+ it "should transform queries correctly" do
+ testcases = Hash[*(File.readlines("#{CHEF_SPEC_DATA}/search_queries_to_transform.txt").select{|line| line !~ /^\s*$/}.map{|line| line.chomp})]
+ testcases.each do |input, expected|
+ @query.transform_search_query(input).should == expected
+ end
+ end
+
+ end
+
+
end

0 comments on commit 67adee4

Please sign in to comment.