diff --git a/Gemfile b/Gemfile index 35483a8..d05048e 100644 --- a/Gemfile +++ b/Gemfile @@ -10,5 +10,10 @@ gem 'aleph', gem 'spectrum-config', git: 'https://github.com/mlibrary/spectrum-config', branch: 'master' + +gem 'mlibrary_search_parser', + git: 'https://github.com/mlibrary/mlibrary_search_parser', + branch: 'master' + # Specify your gem's dependencies in spectrum-json.gemspec gemspec diff --git a/app/controllers/json_controller.rb b/app/controllers/json_controller.rb index 2ce457f..82ffb94 100644 --- a/app/controllers/json_controller.rb +++ b/app/controllers/json_controller.rb @@ -285,7 +285,7 @@ def search_response { request: @request.spectrum, response: @response.spectrum, - messages: @messages.map(&:spectrum), + messages: (@messages + @request.messages).map(&:spectrum), total_available: @response.total_available, specialists: @specialists.spectrum, datastore: @datastore.spectrum, diff --git a/lib/spectrum/facet_list.rb b/lib/spectrum/facet_list.rb index 56c5102..1ccd269 100644 --- a/lib/spectrum/facet_list.rb +++ b/lib/spectrum/facet_list.rb @@ -46,13 +46,14 @@ def query(filter_map = {}, value_map = {}) end.join(' AND ') if key == 'date_of_publication' || original_key == 'date_of_publication' - value.match(/^before(\d+)$/) do |m| + raw_value = value.gsub(/\\/, '') + raw_value.match(/^before\s*(\d+)$/) do |m| value = "[* TO #{m[1]}]" end - value.match(/^after(\d+)$/) do |m| + raw_value.match(/^after\s*(\d+)$/) do |m| value = "[#{m[1]} TO *]" end - value.match(/^(\d+)to(\d+)$/) do |m| + raw_value.match(/^(\d+)\s*to\s*(\d+)$/) do |m| value = "[#{m[1]} TO #{m[2]}]" end end diff --git a/lib/spectrum/json/railtie.rb b/lib/spectrum/json/railtie.rb index 31b68f8..e7b240e 100644 --- a/lib/spectrum/json/railtie.rb +++ b/lib/spectrum/json/railtie.rb @@ -14,7 +14,7 @@ class Railtie < Rails::Railtie end config_data = if File.exist?(aleph_config_file = File.join(Rails.root, 'config', 'aleph.yml')) - YAML.load_file(aleph_config_file) + YAML.load(ERB.new(File.read(aleph_config_file)).result) else {} end diff --git a/lib/spectrum/policy/get_this.rb b/lib/spectrum/policy/get_this.rb index a5a5f65..8e9da88 100644 --- a/lib/spectrum/policy/get_this.rb +++ b/lib/spectrum/policy/get_this.rb @@ -102,7 +102,7 @@ class << self attr_reader :options def load_config(config_file) - @options = YAML.load_file(config_file).map do |option| + @options = YAML.load(ERB.new(File.read(config_file)).result).map do |option| Option.new(option) end end diff --git a/lib/spectrum/request/null.rb b/lib/spectrum/request/null.rb index 3922d5f..9ae5678 100644 --- a/lib/spectrum/request/null.rb +++ b/lib/spectrum/request/null.rb @@ -7,6 +7,10 @@ def proxy_prefix 'https://proxy.lib.umich.edu/login?url=' end + def messages + [] + end + def spectrum nil end diff --git a/lib/spectrum/request/record.rb b/lib/spectrum/request/record.rb index b496554..72a592a 100644 --- a/lib/spectrum/request/record.rb +++ b/lib/spectrum/request/record.rb @@ -14,6 +14,10 @@ def proxy_prefix DEFAULT_PROXY_PREFIX end + def messages + [] + end + def initialize(request) @request = request if request.params[:source] == 'summon' diff --git a/lib/spectrum/request/requesty.rb b/lib/spectrum/request/requesty.rb index 02771f4..bedf8e5 100644 --- a/lib/spectrum/request/requesty.rb +++ b/lib/spectrum/request/requesty.rb @@ -1,17 +1,19 @@ # frozen_string_literal: true +require 'mlibrary_search_parser' + module Spectrum module Request module Requesty extend ActiveSupport::Concern - FLINT = 'Flint' - FLINT_PROXY_PREFIX = 'http://libproxy.umflint.edu:2048/login?url=' + FLINT = 'Flint' + FLINT_PROXY_PREFIX = 'http://libproxy.umflint.edu:2048/login?url=' DEFAULT_PROXY_PREFIX = 'https://proxy.lib.umich.edu/login?url=' - INSTITUTION_KEY = 'dlpsInstitutionId' + INSTITUTION_KEY = 'dlpsInstitutionId' included do - attr_accessor :request_id, :slice, :sort, :start + attr_accessor :request_id, :slice, :sort, :start, :messages end def can_sort? @@ -23,55 +25,86 @@ def proxy_prefix DEFAULT_PROXY_PREFIX end - def initialize(request = nil, focus = nil) - @request = request - @focus = focus - if (@data = get_data(@request)) + def is_new_parser?(focus, data) + focus and + focus.new_parser? and + data.has_key?('raw_query') and + data['raw_query'].match(/\S/) + end + def initialize(request = nil, focus = nil) + @request = request + @focus = focus + @data = get_data(@request) + @messages = [] + if (@data) bad_request 'Request json did not validate' unless Spectrum::Json::Schema.validate(:request, @data) - if @data - @uid = @data['uid'] - @start = @data['start'].to_i - @count = @data['count'].to_i - @page = @data['page'] - @tree = Spectrum::FieldTree.new(@data['field_tree']) - @facets = Spectrum::FacetList.new(@focus.default_facets.merge(@focus.filter_facets(@data['facets'] || {}))) - @sort = @data['sort'] - @settings = @data['settings'] - @request_id = @data['request_id'] - - if @page || @count == 0 - @page_size = @count - elsif @start < @count - @slice = [@start, @count] - @page_size = @start + @count - @page_number = 0 - else - last_record = @start + @count - @page_size = @count - @page_number = (@start / @page_size).floor + @is_new_parser = is_new_parser?(@focus, @data) + if @is_new_parser + @psearch = build_psearch + @psearch.errors.each do |msg| + @messages << Spectrum::Response::Message.error( + summary: "Query Parse Error", + details: msg.details, + data: msg.to_h + ) + end + @psearch.warnings.each do |msg| + @messages << Spectrum::Response::Message.warn( + summary: "Query Parse Warning", + details: msg.details, + data: msg.to_h, + ) + end + end - while @page_number > 0 && @page_size * (@page_number + 1) < last_record - first_record = @page_size * @page_number - if @start - first_record < @page_number - @page_size = (last_record / @page_number).ceil - else - @page_size += (@start - first_record).floor - end - @page_number = (@start / @page_size).floor + ############################## + ############################## + + + @uid = @data['uid'] + @start = @data['start'].to_i + @count = @data['count'].to_i + @page = @data['page'] + @tree = Spectrum::FieldTree.new(@data['field_tree']) + @facets = Spectrum::FacetList.new(@focus.default_facets.merge(@focus.filter_facets(@data['facets'] || {}))) + @sort = @data['sort'] + @settings = @data['settings'] + @request_id = @data['request_id'] + + + if @page || @count == 0 + @page_size = @count + elsif @start < @count + @slice = [@start, @count] + @page_size = @start + @count + @page_number = 0 + else + last_record = @start + @count + @page_size = @count + @page_number = (@start / @page_size).floor + + while @page_number > 0 && @page_size * (@page_number + 1) < last_record + first_record = @page_size * @page_number + if @start - first_record < @page_number + @page_size = (last_record / @page_number).ceil + else + @page_size += (@start - first_record).floor end - @slice = [@start - @page_size * @page_number, @count] + @page_number = (@start / @page_size).floor end - - validate! + @slice = [@start - @page_size * @page_number, @count] end + + validate! end @page = (@page_number || 0) + 1 - @start ||= 0 - @count ||= 0 - @slice ||= [0, @count] + @start ||= 0 + @count ||= 0 + @slice ||= [0, @count] + @tree ||= Spectrum::FieldTree::Empty.new @facets ||= Spectrum::FacetList.new(nil) @page_size ||= 0 @@ -152,34 +185,55 @@ def fvf(_filter_map = {}) @focus ? @focus.fvf(@facets) : [] end - def query(query_map = {}, filter_map = {}) + + def new_parser_query(query_map = {}, filter_map = {}, built_search = @psearch) + lp = MLibrarySearchParser::Transformer::Solr::LocalParams.new(built_search) + defaults = lp.config['search_attr_defaults'] || {} + base_query(query_map, filter_map).merge(defaults).merge(lp.params) + end + + def base_query(query_map = {}, filter_map = {}) { - q: @tree.query(query_map), - page: @page, - start: @start, - rows: @page_size, - fq: @facets.query(filter_map, (@focus&.value_map) || {}), - per_page: @page_size - }.merge(@tree.params(query_map)).tap do |ret| + page: @page, + start: @start, + rows: @page_size, + fq: @facets.query(filter_map, (@focus&.value_map) || {}), + per_page: @page_size + } + end + + # Query derived from pride-based parser + def tree_query(query_map = {}, filter_map = {}) + base_query(query_map, filter_map).merge({ + q: @tree.query(query_map), + }).merge(@tree.params(query_map)).tap do |ret| if ret[:q].match(/ (AND|OR|NOT) /) ret[:q] = '+(' + ret[:q] + ')' end end end + def query(query_map = {}, filter_map = {}) + if @is_new_parser + new_parser_query(query_map, filter_map) + else + tree_query(query_map, filter_map) + end + end + def facets @facets end def spectrum ret = { - uid: @uid, - start: @start, - count: @count, - field_tree: @tree.spectrum, - facets: @facets.spectrum, - sort: @sort, - settings: @settings + uid: @uid, + start: @start, + count: @count, + field_tree: @tree.spectrum, + facets: @facets.spectrum, + sort: @sort, + settings: @settings } if @request_id ret.merge(request_id: @request_id) @@ -188,6 +242,11 @@ def spectrum end end + def build_psearch(focus = @focus) + @builder = MLibrarySearchParser::SearchBuilder.new(focus.raw_config) + @builder.build(@data['raw_query']) + end + private def bad_request(message) diff --git a/lib/spectrum/response/message.rb b/lib/spectrum/response/message.rb index 0dc97e1..97c0a03 100644 --- a/lib/spectrum/response/message.rb +++ b/lib/spectrum/response/message.rb @@ -9,6 +9,7 @@ def initialize(args = {}) @class = TYPES.include?(args[:class]) ? args[:class] : DEFAULT_TYPE @summary = args[:summary] @details = args[:details] + @data = args[:data] end class << self @@ -21,7 +22,7 @@ class << self end def spectrum - { class: @class, summary: @summary, details: @details } + { class: @class, summary: @summary, details: @details, data: @data } end end end diff --git a/lib/spectrum/response/specialists.rb b/lib/spectrum/response/specialists.rb index 694a6a7..5f7142f 100644 --- a/lib/spectrum/response/specialists.rb +++ b/lib/spectrum/response/specialists.rb @@ -87,10 +87,7 @@ def client end def fetch_records(query) - params = focus.first.solr_params.merge( - q: query[:q], - fq: query[:fq], - qq: '"' + RSolr.solr_escape(query[:q]) + '"', + params = focus.first.solr_params.merge(query).merge( rows: rows.first, fl: fields.first ) @@ -195,7 +192,7 @@ class Specialists class << self attr_reader :config, :logger, :cache def configure(file) - @config = YAML.load_file(file).map do |key, value| + @config = YAML.load(ERB.new(::File.read(file)).result).map do |key, value| if key == 'logger' @logger = value nil @@ -231,9 +228,14 @@ def engines def spectrum return [] if data[:request].instance_eval { @request&.env&.fetch('dlpsInstitutionId', nil)&.include?('Flint') } specialist_focus = Spectrum::Json.foci['mirlyn'] - query = data[:request].query( + # catalog/mirlyn is on new parser + # because diff datastores have diff search fields, + # we re-parse the original search under catalog config + psearch = data[:request].build_psearch(specialist_focus) + query = data[:request].new_parser_query( specialist_focus.fields, - specialist_focus.facet_map + specialist_focus.facet_map, + psearch ) return [] if query[:q] == '*:*' begin diff --git a/spectrum-json.gemspec b/spectrum-json.gemspec index 1ac704b..061c19b 100644 --- a/spectrum-json.gemspec +++ b/spectrum-json.gemspec @@ -47,4 +47,6 @@ Gem::Specification.new do |spec| spec.add_dependency 'execjs' spec.add_dependency 'rsolr' spec.add_dependency 'httparty' + spec.add_dependency 'dotenv' + end