From b276bca19000641329d579c33dcc98d573a59742 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 11 Mar 2024 14:59:57 +0100 Subject: [PATCH] Change image processing from ImageMagick to libvips --- Gemfile | 1 + Gemfile.lock | 7 ++++ app/models/concerns/account/avatar.rb | 4 +-- app/models/concerns/account/header.rb | 4 +-- app/models/media_attachment.rb | 10 ++---- app/models/preview_card.rb | 2 +- app/models/site_upload.rb | 2 +- lib/paperclip/blurhash_transcoder.rb | 5 ++- lib/paperclip/color_extractor.rb | 48 +++++++++++++++---------- lib/paperclip/lazy_thumbnail.rb | 52 +++++++++++++++++++++------ 10 files changed, 89 insertions(+), 46 deletions(-) diff --git a/Gemfile b/Gemfile index a10613b30b9fc9..d651e8928272a7 100644 --- a/Gemfile +++ b/Gemfile @@ -21,6 +21,7 @@ gem 'aws-sdk-s3', '~> 1.123', require: false gem 'blurhash', '~> 0.1' gem 'fog-core', '<= 2.4.0' gem 'fog-openstack', '~> 1.0', require: false +gem 'image_processing', '~> 1.0' gem 'kt-paperclip', '~> 7.2' gem 'md-paperclip-azure', '~> 2.2', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 7068f5dd55d344..2269f82ff58d4c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -346,6 +346,9 @@ GEM rainbow (>= 2.2.2, < 4.0) terminal-table (>= 1.5.1) idn-ruby (0.1.5) + image_processing (1.12.2) + mini_magick (>= 4.9.5, < 5) + ruby-vips (>= 2.0.17, < 3) inline_svg (1.9.0) activesupport (>= 3.0) nokogiri (>= 1.6) @@ -432,6 +435,7 @@ GEM mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) + mini_magick (4.12.0) mini_mime (1.1.5) mini_portile2 (2.8.6) minitest (5.22.3) @@ -681,6 +685,8 @@ GEM ruby-saml (1.16.0) nokogiri (>= 1.13.10) rexml + ruby-vips (2.2.1) + ffi (~> 1.12) ruby2_keywords (0.0.5) rubyzip (2.3.2) rufus-scheduler (3.9.1) @@ -867,6 +873,7 @@ DEPENDENCIES i18n (= 1.14.1) i18n-tasks (~> 1.0) idn-ruby + image_processing (~> 1.0) inline_svg irb (~> 1.8) json-ld diff --git a/app/models/concerns/account/avatar.rb b/app/models/concerns/account/avatar.rb index 39f599db182275..ebf8b976926a2e 100644 --- a/app/models/concerns/account/avatar.rb +++ b/app/models/concerns/account/avatar.rb @@ -9,7 +9,7 @@ module Account::Avatar class_methods do def avatar_styles(file) styles = { original: { geometry: '400x400#', file_geometry_parser: FastGeometryParser } } - styles[:static] = { geometry: '400x400#', format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' + styles[:static] = { geometry: '400x400#', format: 'png', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' styles end @@ -18,7 +18,7 @@ def avatar_styles(file) included do # Avatar upload - has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail] + has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, processors: [:lazy_thumbnail] validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES validates_attachment_size :avatar, less_than: LIMIT remotable_attachment :avatar, LIMIT, suppress_errors: false diff --git a/app/models/concerns/account/header.rb b/app/models/concerns/account/header.rb index 44ae774e94d034..aae19abbf3e5c6 100644 --- a/app/models/concerns/account/header.rb +++ b/app/models/concerns/account/header.rb @@ -10,7 +10,7 @@ module Account::Header class_methods do def header_styles(file) styles = { original: { pixels: MAX_PIXELS, file_geometry_parser: FastGeometryParser } } - styles[:static] = { format: 'png', convert_options: '-coalesce', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' + styles[:static] = { format: 'png', file_geometry_parser: FastGeometryParser } if file.content_type == 'image/gif' styles end @@ -19,7 +19,7 @@ def header_styles(file) included do # Header upload - has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail] + has_attached_file :header, styles: ->(f) { header_styles(f) }, processors: [:lazy_thumbnail] validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES validates_attachment_size :header, less_than: LIMIT remotable_attachment :header, LIMIT, suppress_errors: false diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index f53da04a9755be..d573ed40a6d7cb 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -170,18 +170,13 @@ class MediaAttachment < ApplicationRecord DEFAULT_STYLES = [:original].freeze - GLOBAL_CONVERT_OPTIONS = { - all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp -define jpeg:dct-method=float', - }.freeze - belongs_to :account, inverse_of: :media_attachments, optional: true belongs_to :status, inverse_of: :media_attachments, optional: true belongs_to :scheduled_status, inverse_of: :media_attachments, optional: true has_attached_file :file, styles: ->(f) { file_styles f }, - processors: ->(f) { file_processors f }, - convert_options: GLOBAL_CONVERT_OPTIONS + processors: ->(f) { file_processors f } before_file_validate :set_type_and_extension before_file_validate :check_video_dimensions @@ -192,8 +187,7 @@ class MediaAttachment < ApplicationRecord has_attached_file :thumbnail, styles: THUMBNAIL_STYLES, - processors: [:lazy_thumbnail, :blurhash_transcoder, :color_extractor], - convert_options: GLOBAL_CONVERT_OPTIONS + processors: [:lazy_thumbnail, :blurhash_transcoder, :color_extractor] validates_attachment_content_type :thumbnail, content_type: IMAGE_MIME_TYPES validates_attachment_size :thumbnail, less_than: IMAGE_LIMIT diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index 9fe02bd1681014..af412f3c75a647 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -55,7 +55,7 @@ class PreviewCard < ApplicationRecord has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy - has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, validate_media_type: false + has_attached_file :image, processors: [:lazy_thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, validate_media_type: false validates :url, presence: true, uniqueness: true, url: true validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb index 03d472cdb2e737..75d2b837160f98 100644 --- a/app/models/site_upload.rb +++ b/app/models/site_upload.rb @@ -41,7 +41,7 @@ class SiteUpload < ApplicationRecord mascot: {}.freeze, }.freeze - has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] + has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] validates_attachment_content_type :file, content_type: %r{\Aimage/.*\z} validates :file, presence: true diff --git a/lib/paperclip/blurhash_transcoder.rb b/lib/paperclip/blurhash_transcoder.rb index c22c20c57ad19c..682e0e890f6b95 100644 --- a/lib/paperclip/blurhash_transcoder.rb +++ b/lib/paperclip/blurhash_transcoder.rb @@ -5,10 +5,9 @@ class BlurhashTranscoder < Paperclip::Processor def make return @file unless options[:style] == :small || options[:blurhash] - pixels = convert(':source -depth 8 RGB:-', source: File.expand_path(@file.path)).unpack('C*') - geometry = options.fetch(:file_geometry_parser).from_file(@file) + image = Vips::Image.new_from_file(@file.path, access: :sequential) - attachment.instance.blurhash = Blurhash.encode(geometry.width, geometry.height, pixels, **(options[:blurhash] || {})) + attachment.instance.blurhash = Blurhash.encode(image.width, image.height, image.to_a.flatten, **(options[:blurhash] || {})) @file end diff --git a/lib/paperclip/color_extractor.rb b/lib/paperclip/color_extractor.rb index d2f7e7c6025bc8..3745536a41fad2 100644 --- a/lib/paperclip/color_extractor.rb +++ b/lib/paperclip/color_extractor.rb @@ -7,15 +7,18 @@ class ColorExtractor < Paperclip::Processor MIN_CONTRAST = 3.0 ACCENT_MIN_CONTRAST = 2.0 FREQUENCY_THRESHOLD = 0.01 + BINS = 10 def make - depth = 8 + image = Vips::Image.new_from_file(@file.path) - # Determine background palette by getting colors close to the image's edge only - background_palette = palette_from_histogram(convert(':source -alpha set -gravity Center -region 75%x75% -fill None -colorize 100% -alpha transparent +region -format %c -colors :quantity -depth :depth histogram:info:', source: File.expand_path(@file.path), quantity: 10, depth: depth), 10) + edge_image = begin + transparent = Vips::Image.black(image.width * 0.75, image.height * 0.75) + image.insert(transparent, (image.width * 0.25) / 2, (image.height * 0.25) / 2) + end - # Determine foreground palette from the whole image - foreground_palette = palette_from_histogram(convert(':source -format %c -colors :quantity -depth :depth histogram:info:', source: File.expand_path(@file.path), quantity: 10, depth: depth), 10) + background_palette = palette_from_image(edge_image) + foreground_palette = palette_from_image(image) background_color = background_palette.first || foreground_palette.first foreground_colors = [] @@ -78,6 +81,28 @@ def make private + def palette_from_image(image) + histogram = image.hist_find_ndim(bins: BINS) + _, colors = histogram.max(size: 10, out_array: true, x_array: true, y_array: true) + + colors['out_array'].map.with_index do |v, i| + x = colors['x_array'][i] + y = colors['y_array'][i] + + rgb_from_xyv(histogram, x, y, v) + end + end + + # rubocop:disable Naming/MethodParameterName + def rgb_from_xyv(image, x, y, v) + pixel = image.getpoint(x, y) + z = pixel.find_index(v) + r = (x + 0.5) * 256 / BINS + g = (y + 0.5) * 256 / BINS + b = (z + 0.5) * 256 / BINS + ColorDiff::Color::RGB.new(r, g, b) + end + def w3c_contrast(color1, color2) luminance1 = (color1.to_xyz.y * 0.01) + 0.05 luminance2 = (color2.to_xyz.y * 0.01) + 0.05 @@ -89,7 +114,6 @@ def w3c_contrast(color1, color2) end end - # rubocop:disable Naming/MethodParameterName def rgb_to_hsl(r, g, b) r /= 255.0 g /= 255.0 @@ -170,18 +194,6 @@ def lighten_or_darken(color, by) ColorDiff::Color::RGB.new(*hsl_to_rgb(hue, saturation, light)) end - def palette_from_histogram(result, quantity) - frequencies = result.scan(/([0-9]+):/).flatten.map(&:to_f) - hex_values = result.scan(/\#([0-9A-Fa-f]{6,8})/).flatten - total_frequencies = frequencies.sum.to_f - - frequencies.map.with_index { |f, i| [f / total_frequencies, hex_values[i]] } - .sort_by { |r| -r[0] } - .reject { |r| r[1].size == 8 && r[1].end_with?('00') } - .map { |r| ColorDiff::Color::RGB.new(*r[1][0..5].scan(/../).map { |c| c.to_i(16) }) } - .slice(0, quantity) - end - def rgb_to_hex(rgb) format('#%02x%02x%02x', rgb.r, rgb.g, rgb.b) end diff --git a/lib/paperclip/lazy_thumbnail.rb b/lib/paperclip/lazy_thumbnail.rb index 10b14860c4a8fe..4598d2fc706d2a 100644 --- a/lib/paperclip/lazy_thumbnail.rb +++ b/lib/paperclip/lazy_thumbnail.rb @@ -1,20 +1,50 @@ # frozen_string_literal: true module Paperclip - class LazyThumbnail < Paperclip::Thumbnail + class LazyThumbnail < Paperclip::Processor + class PixelGeometryParser + def self.parse(current_geometry, pixels) + width = Math.sqrt(pixels * (current_geometry.width.to_f / current_geometry.height)).round.to_i + height = Math.sqrt(pixels * (current_geometry.height.to_f / current_geometry.width)).round.to_i + + Paperclip::Geometry.new(width, height) + end + end + + def initialize(file, options = {}, attachment = nil) + super + + @crop = options[:geometry].to_s[-1, 1] == '#' + @current_geometry = options.fetch(:file_geometry_parser, Geometry).from_file(@file) + @target_geometry = options[:pixels] ? PixelGeometryParser.parse(@current_geometry, options[:pixels]) : options.fetch(:string_geometry_parser, Geometry).parse(options[:geometry].to_s) + @format = options[:format] + @current_format = File.extname(@file.path) + end + def make - return File.open(@file.path) unless needs_convert? - - if options[:geometry] - min_side = [@current_geometry.width, @current_geometry.height].min.to_i - options[:geometry] = "#{min_side}x#{min_side}#" if @target_geometry.square? && min_side < @target_geometry.width - elsif options[:pixels] - width = Math.sqrt(options[:pixels] * (@current_geometry.width.to_f / @current_geometry.height)).round.to_i - height = Math.sqrt(options[:pixels] * (@current_geometry.height.to_f / @current_geometry.width)).round.to_i - options[:geometry] = "#{width}x#{height}>" + source = File.open(@file.path) + + return source unless needs_convert? + + vips = ImageProcessing::Vips.source(source) + + vips = if @crop + vips.resize_to_fill(@target_geometry.width, @target_geometry.height, sharpen: false, crop: :attention) + else + vips.resize_to_limit(@target_geometry.width, @target_geometry.height, sharpen: false) + end + + vips = vips.custom do |image| + image.mutate do |mutable| + image.get_fields.each do |field| + mutable.remove!(field) unless field == 'icc-profile-data' + end + end end - Paperclip::Thumbnail.make(file, options, attachment) + vips.convert(@format) + .saver(interlace: true, quality: 90) + .call end private