Create JP2 Derivatives on Ingest (#34)

* Create JP2 Derivatives on Ingest
* Install Kakadu in CI.

Gemfile.lock

@@ -18,7 +18,7 @@ GIT
 
 GIT
   remote: https://github.com/samvera-labs/valkyrie.git
-  revision: 40ddfc6871b8e4cd7af459ad0f149a5be711035b
+  revision: e5371268d53b0dd8cbd7b0fba70147d133d199d9
   specs:
     valkyrie (0.1.0)
       active-fedora

app/change_sets/file_set_change_set.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 class FileSetChangeSet < Valkyrie::ChangeSet
   self.fields = [:title]
+  property :files, virtual: true, multiple: true, required: false
 
   def primary_terms
     [:title]

app/derivative-services/jp2_derivative_service.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+class Jp2DerivativeService
+  class Factory
+    attr_reader :change_set_persister
+    delegate :metadata_adapter, to: :change_set_persister
+    delegate :query_service, to: :metadata_adapter
+    def initialize(change_set_persister:)
+      @change_set_persister = change_set_persister
+    end
+
+    def new(change_set)
+      Jp2DerivativeService.new(change_set: change_set, change_set_persister: change_set_persister, original_file: original_file(change_set.resource))
+    end
+
+    def original_file(resource)
+      members(resource).find { |x| x.use.include?(Valkyrie::Vocab::PCDMUse.OriginalFile) }
+    end
+
+    def members(resource)
+      metadata_adapter.query_service.find_members(resource: resource)
+    end
+  end
+
+  attr_reader :change_set, :change_set_persister, :original_file
+  delegate :mime_type, to: :original_file
+  def initialize(change_set:, change_set_persister:, original_file:)
+    @change_set = change_set
+    @change_set_persister = change_set_persister
+    @original_file = original_file
+  end
+
+  def valid?
+    mime_type == ['image/tiff']
+  end
+
+  def create_derivatives
+    Hydra::Derivatives::Jpeg2kImageDerivatives.create(
+      filename,
+      outputs: [
+        label: 'intermediate_file',
+        recipe: :default,
+        service: {
+          datastream: 'intermediate_file'
+        },
+        url: URI("file://#{temporary_output.path}")
+      ]
+    )
+    change_set.files = [build_file]
+    change_set_persister.save(change_set: change_set)
+  end
+
+  def cleanup_derivatives; end
+
+  class IoDecorator < SimpleDelegator
+    attr_reader :original_filename, :content_type, :use
+    def initialize(io, original_filename, content_type, use)
+      @original_filename = original_filename
+      @content_type = content_type
+      @use = use
+      super(io)
+    end
+  end
+
+  def build_file
+    IoDecorator.new(temporary_output, "intermediate_file.jp2", mime_type, use)
+  end
+
+  def use
+    [Valkyrie::Vocab::PCDMUse.ServiceFile]
+  end
+
+  def filename
+    return Pathname.new(file_object.io.path) if file_object.io.respond_to?(:path) && File.exist?(file_object.io.path)
+  end
+
+  def file_object
+    @file_object ||= Valkyrie::StorageAdapter.find_by(id: original_file.file_identifiers[0])
+  end
+
+  def temporary_output
+    @temporary_file ||= Tempfile.new
+  end
+end

app/jobs/create_derivatives_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+class CreateDerivativesJob < ApplicationJob
+  delegate :query_service, to: :metadata_adapter
+
+  def perform(file_set_id)
+    file_set = query_service.find_by(id: Valkyrie::ID.new(file_set_id))
+    Valkyrie::DerivativeService.for(FileSetChangeSet.new(file_set)).create_derivatives
+  end
+
+  def metadata_adapter
+    Valkyrie.config.metadata_adapter
+  end
+end

app/models/file_set.rb

@@ -3,6 +3,5 @@ class FileSet < Valhalla::Resource
   include Valkyrie::Resource::AccessControls
   attribute :id, Valkyrie::Types::ID.optional
   attribute :title, Valkyrie::Types::Set
-  attribute :file_identifiers, Valkyrie::Types::Set
   attribute :member_ids, Valkyrie::Types::Array
 end

app/services/file_appender.rb

@@ -11,17 +11,23 @@ def append_to(resource)
     return resource if files.blank?
     file_sets = build_file_sets || file_nodes
     resource.member_ids = resource.member_ids + file_sets.map(&:id)
-    resource.pending_uploads = (resource.pending_uploads || []) - files
+    adjust_pending_uploads(resource)
   end
 
   def build_file_sets
     return if processing_derivatives?
     file_nodes.map do |node|
       file_set = create_file_set(node)
+      CreateDerivativesJob.perform_later(file_set.id.to_s)
       file_set
     end
   end
 
+  def adjust_pending_uploads(resource)
+    return unless resource.respond_to?(:pending_uploads)
+    resource.pending_uploads = (resource.pending_uploads || []) - files
+  end
+
   def processing_derivatives?
     !file_nodes.first.use.include?(Valkyrie::Vocab::PCDMUse.OriginalFile)
   end

bin/ci_kakadu_install.sh

@@ -0,0 +1,8 @@
+if [ ! -d "kakadu" ]; then
+  mkdir ~/downloads
+  wget http://kakadusoftware.com/wp-content/uploads/2014/06/KDU77_Demo_Apps_for_Linux-x86-64_150710.zip -O ~/downloads/kakadu.zip
+  unzip ~/downloads/kakadu.zip
+  mv KDU77_Demo_Apps_for_Linux-x86-64_150710 kakadu
+fi
+sudo cp kakadu/*.so /usr/lib
+sudo cp kakadu/* /usr/bin

circle.yml

@@ -4,7 +4,10 @@ machine:
   services:
     - redis
 dependencies:
+  cache_directories:
+    - kakadu
   post:
+    - sudo sh bin/ci_kakadu_install.sh
     - bundle exec rake rubocop
     - bundle exec rake server:test:
         background: true

config/config.yml

@@ -0,0 +1,77 @@
+defaults: &defaults
+  derivative_path: <%= Rails.root.join("tmp", "derivatives") %>
+  jp2_recipes:
+    default_color: >
+      -rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171 
+      -jp2_space sRGB 
+      -double_buffering 10 
+      -num_threads 1 
+      -no_weights 
+      Clevels=6 
+      Clayers=8 
+      Cblk=\{64,64\} 
+      Cuse_sop=yes 
+      Cuse_eph=yes  
+      Corder=RPCL 
+      ORGgen_plt=yes 
+      ORGtparts=R 
+      Stiles=\{1024,1024\}
+    default_gray: >
+      -rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171 
+      -jp2_space sLUM 
+      -double_buffering 10 
+      -num_threads 1
+      -no_weights 
+      Clevels=6 
+      Clayers=8 
+      Cblk=\{64,64\} 
+      Cuse_sop=yes 
+      Cuse_eph=yes  
+      Corder=RPCL 
+      ORGgen_plt=yes 
+      ORGtparts=R 
+      Stiles=\{1024,1024\}
+    geo_color: >
+      -no_palette
+      -rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171 
+      -jp2_space sRGB 
+      -double_buffering 10 
+      -num_threads 1 
+      -no_weights 
+      Clevels=6 
+      Clayers=8 
+      Cblk=\{64,64\} 
+      Cuse_sop=yes 
+      Cuse_eph=yes  
+      Corder=RPCL 
+      ORGgen_plt=yes 
+      ORGtparts=R 
+      Stiles=\{1024,1024\}
+    geo_gray: >
+      -no_palette
+      -rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171 
+      -jp2_space sLUM 
+      -double_buffering 10 
+      -num_threads 1
+      -no_weights 
+      Clevels=6 
+      Clayers=8 
+      Cblk=\{64,64\} 
+      Cuse_sop=yes 
+      Cuse_eph=yes  
+      Corder=RPCL 
+      ORGgen_plt=yes 
+      ORGtparts=R 
+      Stiles=\{1024,1024\}
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults
+
+production:
+  <<: *defaults
+
+staging:
+  <<: *defaults

config/initializers/figgy.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+module Figgy
+  def config
+    @config ||= config_yaml.with_indifferent_access
+  end
+
+  private
+
+    def config_yaml
+      YAML.safe_load(ERB.new(File.read(Rails.root.join("config", "config.yml"))).result, [], [], true)[Rails.env]
+    end
+
+    module_function :config, :config_yaml
+end

config/initializers/valkyrie.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require_relative 'figgy'
 Rails.application.config.to_prepare do
   Valkyrie::StorageAdapter.register(
     Valkyrie::Storage::Disk.new(base_path: Rails.root.join("tmp", "files")),
@@ -10,6 +11,11 @@
     :disk
   )
 
+  Valkyrie::StorageAdapter.register(
+    Valkyrie::Storage::Disk.new(base_path: Figgy.config['derivative_path']),
+    :derivatives
+  )
+
   Valkyrie::MetadataAdapter.register(
     Valkyrie::Persistence::Postgres::MetadataAdapter.new,
     :postgres
@@ -38,4 +44,15 @@
     ),
     :indexing_persister
   )
+
+  Hydra::Derivatives.kdu_compress_recipes = Figgy.config['jp2_recipes']
+
+  # Jp2DerivativeService needs its own change_set_persister because the
+  # derivatives may not be in the primary metadata/file storage.
+  Valkyrie::DerivativeService.services << Jp2DerivativeService::Factory.new(
+    change_set_persister: PlumChangeSetPersister.new(
+      metadata_adapter: Valkyrie::MetadataAdapter.find(:indexing_persister),
+      storage_adapter: Valkyrie::StorageAdapter.find(:derivatives)
+    )
+  )
 end

spec/change_set_persisters/plum_change_set_persister_spec.rb

@@ -108,14 +108,21 @@
       expect(members.first).to be_kind_of FileSet
 
       file_metadata_nodes = query_service.find_members(resource: members.first)
-      expect(file_metadata_nodes.to_a.length).to eq 1
+      expect(file_metadata_nodes.to_a.length).to eq 2
       expect(file_metadata_nodes.first).to be_kind_of FileMetadata
 
-      original_file_node = file_metadata_nodes.first
+      original_file_node = file_metadata_nodes.find { |x| x.use == [Valkyrie::Vocab::PCDMUse.OriginalFile] }
 
       expect(original_file_node.file_identifiers.length).to eq 1
       original_file = Valkyrie::StorageAdapter.find_by(id: original_file_node.file_identifiers.first)
       expect(original_file).to respond_to(:read)
+
+      derivative_file_node = file_metadata_nodes.find { |x| x.use == [Valkyrie::Vocab::PCDMUse.ServiceFile] }
+
+      expect(derivative_file_node).not_to be_blank
+      derivative_file = Valkyrie::StorageAdapter.find_by(id: derivative_file_node.file_identifiers.first)
+      expect(derivative_file).not_to be_blank
+      expect(derivative_file.io.path).to start_with(Rails.root.join("tmp", "derivatives").to_s)
     end
   end
 

spec/derivative_services/jp2_derivative_service_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+require 'rails_helper'
+require 'valkyrie/specs/shared_specs'
+include ActionDispatch::TestProcess
+
+RSpec.describe Jp2DerivativeService do
+  it_behaves_like "a Valkyrie::DerivativeService"
+
+  let(:thumbnail) { Valkyrie::Vocab::PCDMUse.ThumbnailImage }
+  let(:derivative_service) do
+    Jp2DerivativeService::Factory.new(change_set_persister: change_set_persister)
+  end
+  let(:adapter) { Valkyrie::MetadataAdapter.find(:indexing_persister) }
+  let(:storage_adapter) { Valkyrie.config.storage_adapter }
+  let(:persister) { adapter.persister }
+  let(:query_service) { adapter.query_service }
+  let(:file) { fixture_file_upload('files/example.tif', 'image/tiff') }
+  let(:change_set_persister) { PlumChangeSetPersister.new(metadata_adapter: adapter, storage_adapter: storage_adapter) }
+  let(:scanned_resource) do
+    change_set_persister.save(change_set: ScannedResourceChangeSet.new(ScannedResource.new, files: [file]))
+  end
+  let(:book_members) { query_service.find_members(resource: scanned_resource) }
+  let(:valid_resource) { book_members.first }
+  let(:valid_change_set) { DynamicChangeSet.new(valid_resource) }
+
+  describe '#valid?' do
+    subject(:valid_file) { derivative_service.new(valid_change_set) }
+
+    context 'when given a valid mime_type' do
+      it { is_expected.to be_valid }
+    end
+
+    context 'when given an invalid mime_type' do
+      it 'does not validate' do
+        # rubocop:disable RSpec/SubjectStub
+        allow(valid_file).to receive(:mime_type).and_return(['image/jpeg'])
+        # rubocop:enable RSpec/SubjectStub
+        is_expected.not_to be_valid
+      end
+    end
+  end
+
+  it "creates a JP2 and attaches it to the fileset" do
+    derivative_service.new(valid_change_set).create_derivatives
+
+    reloaded = query_service.find_by(id: valid_resource.id)
+    members = query_service.find_members(resource: reloaded)
+    derivative = members.find { |x| x.use.include?(Valkyrie::Vocab::PCDMUse.ServiceFile) }
+
+    expect(derivative).to be_present
+    derivative_file = Valkyrie::StorageAdapter.find_by(id: derivative.file_identifiers.first)
+    expect(derivative_file.read).not_to be_blank
+  end
+end