Merge 62f1752 into ae00aff

Gemfile

@@ -51,6 +51,8 @@ gem 'warning'
 
 gem 'rails-i18n', '~> 6.0.0'
 
+gem 'forty_facets'
+
 group :assets do
   gem 'uglifier'
 end

Gemfile.lock

@@ -177,6 +177,7 @@ GEM
       multipart-post (>= 1.2, < 3)
     ffi (1.13.1)
     flamegraph (0.9.5)
+    forty_facets (0.1.9.1)
     friendly_id (5.3.0)
       activerecord (>= 4.0.0)
     geocoder (1.6.3)
@@ -464,6 +465,7 @@ DEPENDENCIES
   devise_masquerade
   factory_bot_rails
   flamegraph
+  forty_facets
   friendly_id
   gravatar_image_tag
   iiif-image-api!

app/assets/stylesheets/facets.scss

@@ -0,0 +1,3 @@
+// Place all the styles related to the facets controller here.
+// They will automatically be included in application.css.
+// You can use Sass (SCSS) here: https://sass-lang.com/

app/assets/stylesheets/metadata.scss

@@ -0,0 +1,3 @@
+// Place all the styles related to the metadata controller here.
+// They will automatically be included in application.css.
+// You can use Sass (SCSS) here: https://sass-lang.com/

app/controllers/collection_controller.rb

@@ -39,6 +39,18 @@ def disable_document_sets
     redirect_to edit_collection_path(@collection.owner, @collection)
   end
 
+  def facets
+    collection = Collection.find(params[:collection_id])
+    @metadata_coverages = collection.metadata_coverages
+  end
+
+  def search
+    @works = @collection.works.includes(:work_statistic).paginate(page: params[:page], per_page: 10)
+    @work = WorkFacet.find(params['work_facet']).work
+    @search = WorkSearch.new(params[:page])
+    render :show
+  end
+
   def load_settings
     @main_owner = @collection.owner
     @owners = [@main_owner] + @collection.owners
@@ -55,6 +67,8 @@ def show
         ajax_redirect_to dashboard_path unless user_signed_in? && @collection.show_to?(current_user)
       end
 
+      # TODO change this parameter to reflect the distinction between the old work_search form
+      # and the `:search` parameter used by forty_facets
       if params[:search]
         @works = @collection.search_works(params[:search]).includes(:work_statistic).paginate(page: params[:page], per_page: 10)
       elsif (params[:works] == 'untranscribed')
@@ -82,6 +96,11 @@ def show
       else
         @works = @collection.works.includes(:work_statistic).paginate(page: params[:page], per_page: 10)
       end
+      # construct the search object from the parameters
+      @search = WorkSearch.new(params)
+      # the search results are WorkFacets, not works, so we need to fetch the works themselves
+      facet_ids = @search.result.pluck(:id)
+      @works = Work.joins(:work_facet).where('work_facets.id in (?)', facet_ids).paginate(page: params[:page], :per_page => @per_page)
     else
       redirect_to "/404"
     end

app/controllers/facets_controller.rb

@@ -0,0 +1,74 @@
+class FacetsController < ApplicationController
+  def enable
+    @collection = Collection.find(params[:collection_id])
+    @collection.facets_enabled = true
+    @collection.save
+
+    # create metadata_coverages for collections that don't have any.
+    if @collection.metadata_coverages.empty?
+      @collection.works.each do |w|
+        unless w.original_metadata.nil?
+          om = JSON.parse(w.original_metadata)
+
+          om.each do |m|
+            unless m['label'].blank?
+              label = m['label']
+
+              collection = w.collection
+
+              unless w.collection.nil?
+                mc = collection.metadata_coverages.build
+
+                # check that record exist
+                test = collection.metadata_coverages.where(key: label).first
+
+                # increment count field if a record is returned
+                if test
+                  test.count = test.count + 1
+                  test.save
+                end
+
+                if test.nil?
+                  mc.key = label.to_sym
+                  mc.save
+                  mc.create_facet_config(metadata_coverage_id: mc.collection_id)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    redirect_to edit_collection_path(@collection.owner, @collection)
+  end
+
+  def disable
+    @collection = Collection.find(params[:collection_id])
+    @collection.facets_enabled = false
+    @collection.save
+    redirect_to edit_collection_path(@collection.owner, @collection)
+  end
+
+  def update
+    collection = Collection.find(params[:collection_id])
+    errors = []
+
+    collection.metadata_coverages.each do |m|
+      metadata = params[:metadata][m[:key]]
+      m.facet_config.update(label: metadata['label'],
+                            input_type: metadata['input_type'],
+                            order: metadata['order'])
+
+      if m.facet_config.errors.any?
+        errors << m.facet_config.errors.full_messages.first
+      end
+    end
+
+    if errors.empty?
+      redirect_to collection_facets_path(collection.owner, collection), notice: "Collection facets updated successfully"
+    else
+      render('collection/facets', :locals => { :@metadata_coverages => collection.metadata_coverages, :@errors => errors })
+    end
+  end
+end

app/controllers/metadata_controller.rb

@@ -0,0 +1,28 @@
+class MetadataController < ApplicationController
+  layout false
+
+  def example
+    collection = Collection.find(params[:id])
+    example = Metadata.create_example(collection)
+    send_data example, filename: "example.csv"
+  end
+
+  def create
+    metadata_file = params[:metadata]['file'].tempfile
+    collection = Collection.find(params[:metadata][:collection_id])
+    metadata = Metadata.new(metadata_file: metadata_file, collection: collection)
+    result = metadata.process_csv
+    rows = result[:content].count
+    row_errors = result[:errors].count
+    link = helpers.link_to 'link', collection_metadata_csv_error_path
+
+    flash[:alert] = "Your upload has finished processing. #{rows} rows were updated successfully, #{row_errors} rows encountered errors. Download the error file here: #{link}"
+
+    ajax_redirect_to edit_collection_path(collection.owner, collection)
+  end
+
+  def csv_error
+    csv_string = Metadata.retrieve_error
+    send_data csv_string, filename: "error.csv"
+  end
+end

app/helpers/facets_helper.rb

@@ -0,0 +1,2 @@
+module FacetsHelper
+end

app/helpers/metadata_helper.rb

@@ -0,0 +1,2 @@
+module MetadataHelper
+end

app/models/collection.rb

@@ -17,6 +17,7 @@ class Collection < ApplicationRecord
 
   belongs_to :next_untranscribed_page, foreign_key: 'next_untranscribed_page_id', class_name: "Page", optional: true
   has_many :pages, through: :works
+  has_many :metadata_coverages, :dependent => :destroy
 
   belongs_to :owner, :class_name => 'User', :foreign_key => 'owner_user_id', optional: true
   has_and_belongs_to_many :owners, :class_name => 'User', :join_table => :collection_owners

app/models/facet_config.rb

@@ -0,0 +1,50 @@
+class FacetConfig < ApplicationRecord
+  belongs_to :metadata_coverage
+  after_commit :populate_facets, on: :update
+  validates :order, numericality: true, allow_blank: true
+  validates :order, inclusion: { in: 0..9, if: :type_text? }, allow_blank: true
+  validates :order, inclusion: { in: 0..2, if: :type_date? }, allow_blank: true
+
+  def populate_facets
+    works = self.metadata_coverage.collection.works
+
+    works.each do |w|
+      unless w.original_metadata.nil?
+        om = JSON.parse(w.original_metadata)
+
+        unless w.work_facet
+          w.create_work_facet
+        end
+
+        om.each do |o|
+          label = o['label']
+          value = o['value']
+
+
+          if label == self.metadata_coverage.key
+            input_type = self['input_type']
+
+            case input_type
+            when "text"
+              unless self['order'].nil?
+                w.work_facet.update("s#{self['order']}".to_sym => value)
+              end
+            when "date"
+              unless self['order'].nil?
+                w.work_facet.update("d#{self['order']}".to_sym => value)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def type_text?
+    self.input_type == "text"
+  end
+
+  def type_date?
+    self.input_type == "date"
+  end
+end

app/models/metadata.rb

@@ -0,0 +1,104 @@
+class Metadata
+  def initialize(metadata_file:, collection:)
+    @rowset_errors = []
+    @new_metadata = []
+    @canonical_metadata = []
+    @metadata_file = metadata_file
+    @collection = collection
+  end
+
+  def process_csv
+    begin
+      csv = CSV.open(@metadata_file)
+      headers = csv.shift
+    rescue CSV::MalformedCSVError
+      csv = CSV.open(@metadata_file, :encoding => "ISO8859-1")
+      headers = csv.shift
+    end
+
+    rows = CSV.parse(@metadata_file).map { |a| Hash[ headers.zip(a) ] }
+    rows.shift
+
+    # process rows.
+    rows.each do |row|
+      if row.include?('work_id')
+        row.each do |r|
+          @new_metadata << { label: r[0],  value: r[1] }
+        end
+
+        begin
+          work = Work.find(row['work_id'].to_i)
+          work.update(original_metadata: @new_metadata.to_json)
+
+          unless @collection.works.include?(work)
+            @rowset_errors << { error: "No work with ID #{row['work_id']} is in collection #{@collection.title}",
+                                work_id: row['work_id'],
+                                title: row['title'] }
+          end
+        rescue ActiveRecord::RecordNotFound
+          @rowset_errors << { error: "No work exists with ID #{row['work_id']}",
+                              work_id: row['work_id'],
+                              title: row['title'] }
+
+          # write the error.csv to the filesystem.
+          output_file(@rowset_errors)
+        end
+      elsif row.include?('filename')
+        work = Work.where(uploaded_filename: row['filename']).first
+
+        if work.nil?
+          @rowset_errors << { error: "No work exists with filename #{row['filename']}" }
+          output_file(@rowset_errors)
+        else
+          row.each do |k, v|
+            encval = v.force_encoding('ISO-8859-1') unless v.nil?
+            @new_metadata << { label: k, value: encval }
+          end
+
+          work.update(original_metadata: @new_metadata.to_json)
+        end
+      end
+    end
+
+    result = { content: rows, errors: @rowset_errors }
+    result
+  end
+
+  def output_file(rowset_errors)
+    CSV.open('/tmp/error.csv', 'wb') do |csv|
+
+      rowset_errors.each do |r|
+        csv << [r[:error], r[:work_id], r[:title]]
+      end
+    end
+  end
+
+  def self.retrieve_error
+    rows = CSV.parse(File.open('/tmp/error.csv'))
+
+    # delete the file after we are done reading it.
+    File.delete('/tmp/error.csv')
+
+    csv_string = CSV.generate(headers: true) do |csv|
+      csv << ['error', 'work_id', 'title']
+
+      rows.each do |r|
+        csv << r
+      end
+    end
+
+    csv_string
+  end
+
+  def self.create_example(collection)
+    csv_string = CSV.generate(headers: true) do |csv|
+      csv << ['work_id', 'title', 'your metadata_field_one', 'your_metadata_field_two']
+
+      collection.works.each do |work|
+        csv << [work.id, work.title]
+      end
+    end
+
+    csv_string
+  end
+end

app/models/metadata_coverage.rb

@@ -0,0 +1,6 @@
+class MetadataCoverage < ApplicationRecord
+  belongs_to :collection
+  has_one :facet_config, :dependent => :destroy
+  validates :key, uniqueness: { case_sensitive: true, scope: :collection_id }
+  validates :key, presence: true
+end