Merge 78aa05c into bdde17d

.travis.yml

@@ -3,6 +3,8 @@ addons:
   chrome: stable
 language: ruby
 sudo: false
+services:
+  - elasticsearch
 
 notifications:
   email: false
@@ -13,6 +15,8 @@ matrix:
       env: "RAILS_VERSION=5.1.6"
     - rvm: 2.4.4
       env: "RAILS_VERSION=5.2.0"
+    - rvm: 2.5.1
+      env: "RAILS_VERSION=5.2.0 BLACKLIGHT_INDEX=elasticsearch ENGINE_CART_RAILS_OPTIONS=\"--api --skip-git --skip-bundle --skip-listen --skip-spring --skip-yarn --skip-keeps --skip-action-cable --skip-coffee --skip-test\""
     - rvm: 2.5.1
       env: "RAILS_VERSION=5.2.0 BLACKLIGHT_API_TEST=true ENGINE_CART_RAILS_OPTIONS=\"--api --skip-git --skip-bundle --skip-listen --skip-spring --skip-yarn --skip-keeps --skip-action-cable --skip-coffee --skip-test\""
     - rvm: 2.4.4
@@ -30,6 +34,10 @@ before_install:
   - gem install bundler
   - google-chrome-stable --headless --disable-gpu --no-sandbox --remote-debugging-port=9222 http://localhost &
 
+before_script:
+  - until curl --silent -XGET --fail http://localhost:9200; do printf '.'; sleep 1; done
+  - curl -XPUT http://localhost:9200/blacklight-core
+
 notifications:
   irc: "irc.freenode.org#blacklight"
   email:

app/controllers/concerns/blacklight/search_context.rb

@@ -118,5 +118,8 @@ def setup_next_and_previous_documents
   rescue Blacklight::Exceptions::InvalidRequest => e
     logger.warn "Unable to setup next and previous documents: #{e}"
     nil
+  rescue Elasticsearch::Transport::Transport::Errors::BadRequest => e
+    logger.warn "Unable to setup next and previous documents: #{e}"
+    nil
   end
 end

app/services/blacklight/search_service.rb

@@ -113,7 +113,8 @@ def solr_opensearch_params(field)
     def previous_and_next_document_params(index, window = 1)
       solr_params = blacklight_config.document_pagination_params.dup
 
-      if solr_params.empty?
+      # This is a Solr specific parameter, so we should move it elsewhere
+      if solr_params.empty? && blacklight_config.document_model == ::SolrDocument
         solr_params[:fl] = blacklight_config.document_model.unique_key
       end
 
@@ -125,7 +126,10 @@ def previous_and_next_document_params(index, window = 1)
         solr_params[:rows] = 2 * window # but there should be one after
       end
 
-      solr_params[:facet] = false
+      # This is a Solr specific parameter, so we should move it elsewhere
+      if blacklight_config.document_model == ::SolrDocument
+        solr_params[:facet] = false
+      end
       solr_params
     end
 

lib/blacklight.rb

@@ -15,6 +15,7 @@ module Blacklight
   autoload :SearchBuilder, 'blacklight/search_builder'
   autoload :SearchState, 'blacklight/search_state'
   autoload :Solr, 'blacklight/solr'
+  autoload :Elasticsearch, 'blacklight/elasticsearch'
 
   extend Deprecation
 

lib/blacklight/elasticsearch.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Blacklight
+  module Elasticsearch
+    autoload :Repository, "blacklight/elasticsearch/repository"
+    autoload :Response, "blacklight/elasticsearch/response/facets"
+    autoload :SearchBuilderBehavior, "blacklight/elasticsearch/search_builder_behavior"
+  end
+end

lib/blacklight/elasticsearch/repository.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Blacklight::Elasticsearch
+  class Repository < Blacklight::AbstractRepository
+    class SingleDocumentResponse
+      def initialize doc
+        @doc = doc
+      end
+
+      def documents
+        [@doc]
+      end
+    end
+
+    # TODO: remove?
+    class FacetResponse
+      attr_reader :name, :aggregation
+
+      def initialize name, aggregation
+        @name = name
+        @aggregation = aggregation
+      end
+
+      def items
+        aggregation.buckets.map do |b|
+          OpenStruct.new(name: name, value: b['key'], hits: b['doc_count'])
+        end
+      end
+
+      def sort; end
+
+      def offset; end
+
+      def limit; end
+    end
+
+    class NullSpelling
+      def words
+        []
+      end
+    end
+
+    class SearchResponse
+      attr_reader :response, :params
+
+      include Kaminari::PageScopeMethods
+      include Kaminari::ConfigurationMethods::ClassMethods
+
+      def initialize response, params
+        @response = response
+        @params = params
+      end
+
+      def inspect
+        "#<#{self.class.name} results=#{response.results}>"
+      end
+
+      def grouped?
+        false
+      end
+
+      delegate :results, :total, to: :response
+
+      alias documents results
+
+      delegate :empty?, to: :results
+
+      # TODO: remove?
+      def facet_by_field_name field_name
+        agg = response.response.aggregations[field_name]
+        return unless agg
+        FacetResponse.new(field_name, agg)
+      end
+
+      def docs
+        response.results
+      end
+
+      # Get the aggregations from ES and transform them to look like Solr facets
+      def aggregations
+        raw = response.response.aggregations
+        raw.each_with_object({}) do |(k, v), o|
+          items = v.buckets.to_a.map { |f| Blacklight::Elasticsearch::Response::Facets::FacetItem.new(value: f['key'], hits: f['doc_count']) }
+          o[k] = Blacklight::Elasticsearch::Response::Facets::FacetField.new(k, items)
+        end
+        # response.response.aggregations.each_with_object({}) do |(field_name, agg), o|
+        #   o[field_name] = FacetResponse.new(field_name, agg)
+        # end
+      end
+
+      def facet_pivot *_args
+        {}
+      end
+
+      def facet_queries *_args
+        []
+      end
+
+      alias total_count total
+
+      def start
+        params['from'] || 0
+      end
+      alias offset_value start
+
+      def limit_value
+        params['size'] || response.size
+      end
+      alias rows limit_value
+
+      def sort
+        nil
+      end
+
+      def spelling
+        NullSpelling.new
+      end
+    end
+
+    ##
+    # Find a single document result (by id) using the document configuration
+    # @param [String] document's unique key value
+    def find id, _params = {}
+      SingleDocumentResponse.new(connection.find(id))
+    rescue Elasticsearch::Persistence::Repository::DocumentNotFound
+      raise Blacklight::Exceptions::RecordNotFound
+    end
+
+    ##
+    # Execute a search query
+    # @param [Hash] elastic search query parameters
+    def search params = {}
+      Rails.logger.info "ES parameters: #{params.to_h}"
+      SearchResponse.new(connection.search(params.to_h), params)
+    end
+
+    def suggestions(request_params)
+      key = 'resp'
+      Blacklight::Suggest::Response.new({}, request_params, key)
+    end
+
+    private
+
+    def build_connection
+      c = Elasticsearch::Client.new connection_config.except(:adapter, :index)
+      idx = connection_config[:index]
+      Elasticsearch::Persistence::Repository.new do
+        # Configure the Elasticsearch client
+        client c
+
+        # Set a custom index name
+        index idx
+
+        type :document
+
+        klass ElasticsearchDocument
+
+        settings number_of_shards: 1 do
+          mapping do
+            indexes :text, analyzer: 'snowball'
+          end
+        end
+      end
+    end
+  end
+end

lib/blacklight/elasticsearch/response/facets.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Blacklight::Elasticsearch
+  module Response
+    module Facets
+      # represents a facet; which is a field and its values
+      class FacetField
+        attr_reader :name, :items
+
+        def initialize name, items
+          @name = name
+          @items = items
+        end
+
+        def sort; end
+
+        def offset; end
+
+        def limit; end
+
+        def prefix; end
+      end
+
+      # represents a facet value; which is a field value and its hit count
+      class FacetItem < OpenStruct
+        def label
+          super || value
+        end
+
+        def as_json(props = nil)
+          table.as_json(props)
+        end
+      end
+    end
+  end
+end

lib/blacklight/elasticsearch/search_builder_behavior.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Blacklight::Elasticsearch
+  module SearchBuilderBehavior
+    extend ActiveSupport::Concern
+    included do
+      self.default_processor_chain = [:default_parameters, :build_query, :add_filters, :add_aggregations, :add_pagination, :add_sort]
+    end
+    ####
+    # Start with general defaults from BL config. Need to use custom
+    # merge to dup values, to avoid later mutating the original by mistake.
+    def default_parameters(es_parameters)
+      return unless blacklight_config.default_elasticsearch_params
+
+      blacklight_config.default_elasticsearch_params.each do |key, value|
+        es_parameters[key] = if value.respond_to? :deep_dup
+                               value.deep_dup
+                             elsif value.respond_to?(:dup) && value.duplicable?
+                               value.dup
+                             else
+                               value
+                             end
+      end
+    end
+
+    def build_query(es_parameters)
+      return unless blacklight_params[:q]
+      es_parameters[:query] ||= {}
+
+      if blacklight_params[:q].is_a? Hash
+        es_parameters[:query][:match] ||= {}
+
+        blacklight_params[:q].each do |k, v|
+          es_parameters[:query][:match][k] = v
+        end
+        return
+      end
+
+      if search_field && search_field.template
+        es_parameters[:query][:template] = search_field.template.dup
+        es_parameters[:query][:template][:params] ||= {}
+        es_parameters[:query][:template][:params][:q] = blacklight_params[:q]
+      else
+        es_parameters[:query][:match] ||= {}
+        es_parameters[:query][:match][:_all] = blacklight_params[:q]
+      end
+    end
+
+    def add_filters(es_parameters)
+      return unless blacklight_params[:f]
+      es_parameters[:query] ||= {}
+      es_parameters[:query][:filtered] ||= {}
+      es_parameters[:query][:filtered][:filter] ||= {}
+      es_parameters[:query][:filtered][:filter][:term] ||= {}
+
+      blacklight_params[:f].each_pair do |facet_field, value_list|
+        facet_config = blacklight_config.facet_fields[facet_field]
+
+        field = facet_config.field if facet_config
+        field ||= facet_field
+
+        es_parameters[:query][:filtered][:filter][:term][field] ||= value_list
+      end
+    end
+
+    def add_aggregations(es_parameters)
+      es_parameters[:aggregations] ||= {}
+
+      results = blacklight_config.facet_fields.select do |_field_name, facet|
+        facet.include_in_request || (facet.include_in_request.nil? && blacklight_config.add_facet_fields_to_solr_request)
+      end
+
+      results.each do |_key, facet_config|
+        es_parameters[:aggregations][facet_config.field] = { terms: { field: facet_config.field } }
+      end
+    end
+
+    def add_pagination(es_parameters)
+      es_parameters[:from] = start
+      es_parameters[:size] = rows
+    end
+
+    def add_sort(es_parameters)
+      es_parameters[:sort] = sort if sort.present?
+    end
+
+    def query(*args)
+      super.except(:fl, :facet)
+    end
+  end
+end

lib/blacklight/solr/repository.rb

@@ -18,6 +18,7 @@ def find id, params = {}
     ##
     # Execute a search query against solr
     # @param [Hash] params solr query parameters
+    # @return [Blacklight::Solr::Response] the solr response object
     def search params = {}
       send_and_receive blacklight_config.solr_path, params.reverse_merge(qt: blacklight_config.qt)
     end

lib/generators/blacklight/controller_generator.rb

@@ -4,6 +4,8 @@ class ControllerGenerator < Rails::Generators::Base
     source_root File.expand_path('../templates', __FILE__)
 
     argument :controller_name, type: :string, default: "catalog"
+    argument :document_name, type: :string, default: "solr_document"
+    argument :index, type: :string, default: 'solr'
 
     desc <<-EOS
       This generator makes the following changes to your application:
@@ -23,7 +25,7 @@ def inject_blacklight_controller_behavior
 
     # Generate blacklight catalog controller
     def create_blacklight_catalog
-      template "catalog_controller.rb", "app/controllers/#{controller_name}_controller.rb"
+      template "catalog_controller.tt", "app/controllers/#{controller_name}_controller.rb"
     end
 
     def inject_blacklight_routes