Parser test 1

Gemfile

@@ -10,5 +10,10 @@ gem 'aleph',
 gem 'spectrum-config',
   git: 'https://github.com/mlibrary/spectrum-config',
   branch: 'master'
+
+gem 'mlibrary_search_parser',
+  git: 'https://github.com/mlibrary/mlibrary_search_parser',
+  branch: 'master'
+
 # Specify your gem's dependencies in spectrum-json.gemspec
 gemspec

app/controllers/json_controller.rb

@@ -285,7 +285,7 @@ def search_response
     {
       request:  @request.spectrum,
       response: @response.spectrum,
-      messages: @messages.map(&:spectrum),
+      messages: (@messages + @request.messages).map(&:spectrum),
       total_available: @response.total_available,
       specialists: @specialists.spectrum,
       datastore: @datastore.spectrum,

lib/spectrum/facet_list.rb

@@ -46,13 +46,14 @@ def query(filter_map = {}, value_map = {})
         end.join(' AND ')
 
         if key == 'date_of_publication' || original_key == 'date_of_publication'
-          value.match(/^before(\d+)$/) do |m|
+          raw_value = value.gsub(/\\/, '')
+          raw_value.match(/^before\s*(\d+)$/) do |m|
             value = "[* TO #{m[1]}]"
           end
-          value.match(/^after(\d+)$/) do |m|
+          raw_value.match(/^after\s*(\d+)$/) do |m|
             value = "[#{m[1]} TO *]"
           end
-          value.match(/^(\d+)to(\d+)$/) do |m|
+          raw_value.match(/^(\d+)\s*to\s*(\d+)$/) do |m|
             value = "[#{m[1]} TO #{m[2]}]"
           end
         end

lib/spectrum/json/railtie.rb

@@ -14,7 +14,7 @@ class Railtie < Rails::Railtie
         end
 
         config_data = if File.exist?(aleph_config_file = File.join(Rails.root, 'config', 'aleph.yml'))
-          YAML.load_file(aleph_config_file)
+          YAML.load(ERB.new(File.read(aleph_config_file)).result)
         else
           {}
         end

lib/spectrum/policy/get_this.rb

@@ -102,7 +102,7 @@ class << self
         attr_reader :options
 
         def load_config(config_file)
-          @options = YAML.load_file(config_file).map do |option|
+          @options = YAML.load(ERB.new(File.read(config_file)).result).map do |option|
             Option.new(option)
           end
         end

lib/spectrum/request/null.rb

@@ -7,6 +7,10 @@ def proxy_prefix
         'https://proxy.lib.umich.edu/login?url='
       end
 
+      def messages
+        []
+      end
+
       def spectrum
         nil
       end

lib/spectrum/request/record.rb

@@ -14,6 +14,10 @@ def proxy_prefix
         DEFAULT_PROXY_PREFIX
       end
 
+      def messages
+        []
+      end
+
       def initialize(request)
         @request = request
         if request.params[:source] == 'summon'

lib/spectrum/request/requesty.rb

@@ -1,17 +1,19 @@
 # frozen_string_literal: true
 
+require 'mlibrary_search_parser'
+
 module Spectrum
   module Request
     module Requesty
       extend ActiveSupport::Concern
 
-      FLINT = 'Flint'
-      FLINT_PROXY_PREFIX = 'http://libproxy.umflint.edu:2048/login?url='
+      FLINT                = 'Flint'
+      FLINT_PROXY_PREFIX   = 'http://libproxy.umflint.edu:2048/login?url='
       DEFAULT_PROXY_PREFIX = 'https://proxy.lib.umich.edu/login?url='
-      INSTITUTION_KEY = 'dlpsInstitutionId'
+      INSTITUTION_KEY      = 'dlpsInstitutionId'
 
       included do
-        attr_accessor :request_id, :slice, :sort, :start
+        attr_accessor :request_id, :slice, :sort, :start, :messages
       end
 
       def can_sort?
@@ -23,55 +25,86 @@ def proxy_prefix
         DEFAULT_PROXY_PREFIX
       end
 
-      def initialize(request = nil, focus = nil)
-        @request = request
-        @focus   = focus
-        if (@data = get_data(@request))
+      def is_new_parser?(focus, data)
+        focus and
+            focus.new_parser? and
+            data.has_key?('raw_query') and
+            data['raw_query'].match(/\S/)
+      end
 
+      def initialize(request = nil, focus = nil)
+        @request  = request
+        @focus    = focus
+        @data     = get_data(@request)
+        @messages = []
+        if (@data)
           bad_request 'Request json did not validate' unless Spectrum::Json::Schema.validate(:request, @data)
 
-          if @data
-            @uid        = @data['uid']
-            @start      = @data['start'].to_i
-            @count      = @data['count'].to_i
-            @page       = @data['page']
-            @tree       = Spectrum::FieldTree.new(@data['field_tree'])
-            @facets     = Spectrum::FacetList.new(@focus.default_facets.merge(@focus.filter_facets(@data['facets'] || {})))
-            @sort       = @data['sort']
-            @settings   = @data['settings']
-            @request_id = @data['request_id']
-
-            if @page || @count == 0
-              @page_size = @count
-            elsif @start < @count
-              @slice = [@start, @count]
-              @page_size = @start + @count
-              @page_number = 0
-            else
-              last_record = @start + @count
-              @page_size = @count
-              @page_number = (@start / @page_size).floor
+          @is_new_parser = is_new_parser?(@focus, @data)
+          if @is_new_parser
+            @psearch = build_psearch
+            @psearch.errors.each do |msg|
+              @messages << Spectrum::Response::Message.error(
+                summary: "Query Parse Error",
+                details: msg.details,
+                data: msg.to_h
+              )
+            end
+            @psearch.warnings.each do |msg|
+              @messages << Spectrum::Response::Message.warn(
+                summary: "Query Parse Warning",
+                details: msg.details,
+                data: msg.to_h,
+              )
+            end
+          end
 
-              while @page_number > 0 && @page_size * (@page_number + 1) < last_record
-                first_record = @page_size * @page_number
-                if @start - first_record < @page_number
-                  @page_size = (last_record / @page_number).ceil
-                else
-                  @page_size += (@start - first_record).floor
-                end
-                @page_number = (@start / @page_size).floor
+          ##############################
+          ##############################
+
+
+          @uid        = @data['uid']
+          @start      = @data['start'].to_i
+          @count      = @data['count'].to_i
+          @page       = @data['page']
+          @tree       = Spectrum::FieldTree.new(@data['field_tree'])
+          @facets     = Spectrum::FacetList.new(@focus.default_facets.merge(@focus.filter_facets(@data['facets'] || {})))
+          @sort       = @data['sort']
+          @settings   = @data['settings']
+          @request_id = @data['request_id']
+
+
+          if @page || @count == 0
+            @page_size = @count
+          elsif @start < @count
+            @slice       = [@start, @count]
+            @page_size   = @start + @count
+            @page_number = 0
+          else
+            last_record  = @start + @count
+            @page_size   = @count
+            @page_number = (@start / @page_size).floor
+
+            while @page_number > 0 && @page_size * (@page_number + 1) < last_record
+              first_record = @page_size * @page_number
+              if @start - first_record < @page_number
+                @page_size = (last_record / @page_number).ceil
+              else
+                @page_size += (@start - first_record).floor
               end
-              @slice = [@start - @page_size * @page_number, @count]
+              @page_number = (@start / @page_size).floor
             end
-
-            validate!
+            @slice = [@start - @page_size * @page_number, @count]
           end
+
+          validate!
         end
         @page = (@page_number || 0) + 1
 
-        @start     ||= 0
-        @count     ||= 0
-        @slice     ||= [0, @count]
+        @start ||= 0
+        @count ||= 0
+        @slice ||= [0, @count]
+
         @tree      ||= Spectrum::FieldTree::Empty.new
         @facets    ||= Spectrum::FacetList.new(nil)
         @page_size ||= 0
@@ -152,34 +185,55 @@ def fvf(_filter_map = {})
         @focus ? @focus.fvf(@facets) : []
       end
 
-      def query(query_map = {}, filter_map = {})
+
+      def new_parser_query(query_map = {}, filter_map = {}, built_search = @psearch)
+        lp = MLibrarySearchParser::Transformer::Solr::LocalParams.new(built_search)
+        defaults = lp.config['search_attr_defaults'] || {}
+        base_query(query_map, filter_map).merge(defaults).merge(lp.params)
+      end
+
+      def base_query(query_map = {}, filter_map = {})
         {
-          q: @tree.query(query_map),
-          page: @page,
-          start: @start,
-          rows: @page_size,
-          fq: @facets.query(filter_map, (@focus&.value_map) || {}),
-          per_page: @page_size
-        }.merge(@tree.params(query_map)).tap do |ret|
+            page:     @page,
+            start:    @start,
+            rows:     @page_size,
+            fq:       @facets.query(filter_map, (@focus&.value_map) || {}),
+            per_page: @page_size
+        }
+      end
+
+      # Query derived from pride-based parser
+      def tree_query(query_map = {}, filter_map = {})
+        base_query(query_map, filter_map).merge({
+                                                    q: @tree.query(query_map),
+                                                }).merge(@tree.params(query_map)).tap do |ret|
           if ret[:q].match(/ (AND|OR|NOT) /)
             ret[:q] = '+(' + ret[:q] + ')'
           end
         end
       end
 
+      def query(query_map = {}, filter_map = {})
+        if @is_new_parser
+          new_parser_query(query_map, filter_map)
+        else
+          tree_query(query_map, filter_map)
+        end
+      end
+
       def facets
         @facets
       end
 
       def spectrum
         ret = {
-          uid: @uid,
-          start: @start,
-          count: @count,
-          field_tree: @tree.spectrum,
-          facets: @facets.spectrum,
-          sort: @sort,
-          settings: @settings
+            uid:        @uid,
+            start:      @start,
+            count:      @count,
+            field_tree: @tree.spectrum,
+            facets:     @facets.spectrum,
+            sort:       @sort,
+            settings:   @settings
         }
         if @request_id
           ret.merge(request_id: @request_id)
@@ -188,6 +242,11 @@ def spectrum
         end
       end
 
+      def build_psearch(focus = @focus)
+        @builder = MLibrarySearchParser::SearchBuilder.new(focus.raw_config)
+        @builder.build(@data['raw_query'])
+      end
+
       private
 
       def bad_request(message)

lib/spectrum/response/message.rb

@@ -9,6 +9,7 @@ def initialize(args = {})
         @class = TYPES.include?(args[:class]) ? args[:class] : DEFAULT_TYPE
         @summary = args[:summary]
         @details = args[:details]
+        @data    = args[:data]
       end
 
       class << self
@@ -21,7 +22,7 @@ class << self
       end
 
       def spectrum
-        { class: @class, summary: @summary, details: @details }
+        { class: @class, summary: @summary, details: @details, data: @data }
       end
     end
   end

lib/spectrum/response/specialists.rb

@@ -87,10 +87,7 @@ def client
       end
 
       def fetch_records(query)
-        params = focus.first.solr_params.merge(
-          q: query[:q],
-          fq: query[:fq],
-          qq: '"' + RSolr.solr_escape(query[:q]) + '"',
+        params = focus.first.solr_params.merge(query).merge(
           rows: rows.first,
           fl: fields.first
         )
@@ -195,7 +192,7 @@ class Specialists
       class << self
         attr_reader :config, :logger, :cache
         def configure(file)
-          @config = YAML.load_file(file).map do |key, value|
+          @config = YAML.load(ERB.new(::File.read(file)).result).map do |key, value|
             if key == 'logger'
               @logger = value
               nil
@@ -231,9 +228,14 @@ def engines
       def spectrum
         return [] if data[:request].instance_eval { @request&.env&.fetch('dlpsInstitutionId', nil)&.include?('Flint') }
         specialist_focus = Spectrum::Json.foci['mirlyn']
-        query = data[:request].query(
+        # catalog/mirlyn is on new parser
+        # because diff datastores have diff search fields,
+        # we re-parse the original search under catalog config
+        psearch = data[:request].build_psearch(specialist_focus)
+        query = data[:request].new_parser_query(
           specialist_focus.fields,
-          specialist_focus.facet_map
+          specialist_focus.facet_map,
+          psearch
         )
         return [] if query[:q] == '*:*'
         begin

spectrum-json.gemspec

@@ -47,4 +47,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'execjs'
   spec.add_dependency 'rsolr'
   spec.add_dependency 'httparty'
+  spec.add_dependency 'dotenv'
+
 end