Merge pull request mastodon#1252 from ThibG/glitch-soc/merge-upstream

Merge upstream changes

Gemfile

@@ -15,7 +15,7 @@ gem 'makara', '~> 0.4'
 gem 'pghero', '~> 2.4'
 gem 'dotenv-rails', '~> 2.7'
 
-gem 'aws-sdk-s3', '~> 1.57', require: false
+gem 'aws-sdk-s3', '~> 1.59', require: false
 gem 'fog-core', '<= 2.1.0'
 gem 'fog-openstack', '~> 0.3', require: false
 gem 'paperclip', '~> 6.0'
@@ -89,7 +89,7 @@ gem 'simple-navigation', '~> 4.1'
 gem 'simple_form', '~> 5.0'
 gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie'
 gem 'stoplight', '~> 2.2.0'
-gem 'strong_migrations', '~> 0.4'
+gem 'strong_migrations', '~> 0.5'
 gem 'tty-command', '~> 0.9', require: false
 gem 'tty-prompt', '~> 0.20', require: false
 gem 'twitter-text', '~> 1.14'

Gemfile.lock

@@ -105,17 +105,17 @@ GEM
     av (0.9.0)
       cocaine (~> 0.5.3)
     aws-eventstream (1.0.3)
-    aws-partitions (1.246.0)
-    aws-sdk-core (3.82.0)
+    aws-partitions (1.251.0)
+    aws-sdk-core (3.84.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
     aws-sdk-kms (1.26.0)
       aws-sdk-core (~> 3, >= 3.71.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.57.0)
-      aws-sdk-core (~> 3, >= 3.77.0)
+    aws-sdk-s3 (1.59.0)
+      aws-sdk-core (~> 3, >= 3.83.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.1.0)
@@ -594,7 +594,7 @@ GEM
       rufus-scheduler (~> 3.2)
       sidekiq (>= 3)
       tilt (>= 1.4.0)
-    sidekiq-unique-jobs (6.0.15)
+    sidekiq-unique-jobs (6.0.18)
       concurrent-ruby (~> 1.0, >= 1.0.5)
       sidekiq (>= 4.0, < 7.0)
       thor (~> 0)
@@ -623,7 +623,7 @@ GEM
     stoplight (2.2.0)
     streamio-ffmpeg (3.0.2)
       multi_json (~> 1.8)
-    strong_migrations (0.4.2)
+    strong_migrations (0.5.0)
       activerecord (>= 5)
     temple (0.8.1)
     terminal-table (1.8.0)
@@ -685,7 +685,7 @@ DEPENDENCIES
   active_record_query_trace (~> 1.7)
   addressable (~> 2.7)
   annotate (~> 3.0)
-  aws-sdk-s3 (~> 1.57)
+  aws-sdk-s3 (~> 1.59)
   better_errors (~> 2.5)
   binding_of_caller (~> 0.7)
   blurhash (~> 0.1)
@@ -798,7 +798,7 @@ DEPENDENCIES
   stackprof
   stoplight (~> 2.2.0)
   streamio-ffmpeg (~> 3.0)
-  strong_migrations (~> 0.4)
+  strong_migrations (~> 0.5)
   thor (~> 0.20)
   tty-command (~> 0.9)
   tty-prompt (~> 0.20)

app/controllers/application_controller.rb

@@ -212,6 +212,6 @@ def current_skin
 
   def respond_with_error(code)
     use_pack 'error'
-    render "errors/#{code}", layout: 'error', status: code
+    render "errors/#{code}", layout: 'error', status: code, formats: [:html]
   end
 end

app/javascript/mastodon/features/compose/components/poll_form.js

@@ -155,7 +155,7 @@ class PollForm extends ImmutablePureComponent {
         <div className='poll__footer'>
           <button disabled={options.size >= 5} className='button button-secondary' onClick={this.handleAddOption}><Icon id='plus' /> <FormattedMessage {...messages.add_option} /></button>
 
-          <select value={expiresIn} onBlur={this.handleSelectDuration}>
+          <select value={expiresIn} onChange={this.handleSelectDuration}>
             <option value={300}>{intl.formatMessage(messages.minutes, { number: 5 })}</option>
             <option value={1800}>{intl.formatMessage(messages.minutes, { number: 30 })}</option>
             <option value={3600}>{intl.formatMessage(messages.hours, { number: 1 })}</option>

app/models/media_attachment.rb

@@ -167,6 +167,18 @@ def audio_or_video?
     audio? || video?
   end
 
+  def variant?(other_file_name)
+    return true if file_file_name == other_file_name
+
+    formats = file.styles.values.map(&:format).compact
+
+    return false if formats.empty?
+
+    extension = File.extname(other_file_name)
+
+    formats.include?(extension.delete('.')) && File.basename(other_file_name, extension) == File.basename(file_file_name, File.extname(file_file_name))
+  end
+
   def to_param
     shortcode
   end

app/views/settings/applications/_fields.html.haml

@@ -7,7 +7,7 @@
 .fields-group
   = f.input :redirect_uri, wrapper: :with_block_label, label: t('activerecord.attributes.doorkeeper/application.redirect_uri'), hint: t('doorkeeper.applications.help.redirect_uri')
 
-  %p.hint= t('doorkeeper.applications.help.native_redirect_uri', native_redirect_uri: Doorkeeper.configuration.native_redirect_uri)
+  %p.hint= t('doorkeeper.applications.help.native_redirect_uri', native_redirect_uri: content_tag(:code, Doorkeeper.configuration.native_redirect_uri)).html_safe
 
 .field-group
   .input.with_block_label

config/initializers/paperclip.rb

@@ -52,7 +52,7 @@
   if ENV.has_key?('S3_ENDPOINT')
     Paperclip::Attachment.default_options[:s3_options].merge!(
       endpoint: ENV['S3_ENDPOINT'],
-      force_path_style: true
+      force_path_style: ENV['S3_OVERRIDE_PATH_STYLE'] != 'true',
     )
 
     Paperclip::Attachment.default_options[:url] = ':s3_path_url'
@@ -89,7 +89,7 @@
   Paperclip::Attachment.default_options.merge!(
     storage: :filesystem,
     use_timestamp: true,
-    path: ENV.fetch('PAPERCLIP_ROOT_PATH', ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':class', ':attachment', ':id_partition', ':style', ':filename'),
     url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename',
   )
 end

lib/mastodon/media_cli.rb

@@ -44,6 +44,105 @@ def remove
       say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true)
     end
 
+    option :start_after
+    option :dry_run, type: :boolean, default: false
+    desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments'
+    long_desc <<~LONG_DESC
+      Scans file storage for files that do not belong to existing media attachments. Because this operation
+      requires iterating over every single file individually, it will be slow.
+
+      Please mind that some storage providers charge for the necessary API requests to list objects.
+    LONG_DESC
+    def remove_orphans
+      progress        = create_progress_bar(nil)
+      reclaimed_bytes = 0
+      removed         = 0
+      dry_run         = options[:dry_run] ? ' (DRY RUN)' : ''
+
+      case Paperclip::Attachment.default_options[:storage]
+      when :s3
+        paperclip_instance = MediaAttachment.new.file
+        s3_interface       = paperclip_instance.s3_interface
+        bucket             = s3_interface.bucket(Paperclip::Attachment.default_options[:s3_credentials][:bucket])
+        last_key           = options[:start_after]
+
+        loop do
+          objects = begin
+            begin
+              bucket.objects(start_after: last_key, prefix: 'media_attachments/files/').limit(1000).map { |x| x }
+            rescue => e
+              progress.log(pastel.red("Error fetching list of files: #{e}"))
+              progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
+              break
+            end
+          end
+
+          break if objects.empty?
+
+          last_key        = objects.last.key
+          attachments_map = MediaAttachment.where(id: objects.map { |object| object.key.split('/')[2..-2].join.to_i }).each_with_object({}) { |attachment, map| map[attachment.id] = attachment }
+
+          objects.each do |object|
+            attachment_id = object.key.split('/')[2..-2].join.to_i
+            filename      = object.key.split('/').last
+
+            progress.increment
+
+            next unless attachments_map[attachment_id].nil? || !attachments_map[attachment_id].variant?(filename)
+
+            begin
+              object.delete unless options[:dry_run]
+
+              reclaimed_bytes += object.size
+              removed += 1
+
+              progress.log("Found and removed orphan: #{object.key}")
+            rescue => e
+              progress.log(pastel.red("Error processing #{object.key}: #{e}"))
+            end
+          end
+        end
+      when :fog
+        say('The fog storage driver is not supported for this operation at this time', :red)
+        exit(1)
+      when :filesystem
+        require 'find'
+
+        root_path = ENV.fetch('RAILS_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
+
+        Find.find(File.join(root_path, 'media_attachments', 'files')) do |path|
+          next if File.directory?(path)
+
+          key           = path.gsub("#{root_path}#{File::SEPARATOR}", '')
+          attachment_id = key.split(File::SEPARATOR)[2..-2].join.to_i
+          filename      = key.split(File::SEPARATOR).last
+          attachment    = MediaAttachment.find_by(id: attachment_id)
+
+          progress.increment
+
+          next unless attachment.nil? || !attachment.variant?(filename)
+
+          begin
+            size = File.size(path)
+
+            File.delete(path) unless options[:dry_run]
+
+            reclaimed_bytes += size
+            removed += 1
+
+            progress.log("Found and removed orphan: #{key}")
+          rescue => e
+            progress.log(pastel.red("Error processing #{key}: #{e}"))
+          end
+        end
+      end
+
+      progress.total = progress.progress
+      progress.finish
+
+      say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true)
+    end
+
     option :account, type: :string
     option :domain, type: :string
     option :status, type: :numeric

package.json

@@ -60,14 +60,14 @@
   },
   "private": true,
   "dependencies": {
-    "@babel/core": "^7.7.4",
+    "@babel/core": "^7.7.5",
     "@babel/plugin-proposal-class-properties": "^7.7.4",
     "@babel/plugin-proposal-decorators": "^7.7.4",
     "@babel/plugin-transform-react-inline-elements": "^7.7.4",
-    "@babel/plugin-transform-runtime": "^7.7.4",
+    "@babel/plugin-transform-runtime": "^7.7.6",
     "@babel/preset-env": "^7.7.4",
     "@babel/preset-react": "^7.7.4",
-    "@babel/runtime": "^7.7.4",
+    "@babel/runtime": "^7.7.6",
     "@gamestdio/websocket": "^0.3.2",
     "@clusterws/cws": "^0.16.0",
     "array-includes": "^3.0.3",
@@ -91,7 +91,7 @@
     "detect-passive-events": "^1.0.2",
     "dotenv": "^8.2.0",
     "emoji-mart": "Gargron/emoji-mart#build",
-    "es6-symbol": "^3.1.2",
+    "es6-symbol": "^3.1.3",
     "escape-html": "^1.0.3",
     "exif-js": "^2.3.0",
     "express": "^4.17.1",
@@ -126,7 +126,7 @@
     "prop-types": "^15.5.10",
     "punycode": "^2.1.0",
     "rails-ujs": "^5.2.4",
-    "react": "^16.10.2",
+    "react": "^16.12.0",
     "react-dom": "^16.12.0",
     "react-hotkeys": "^1.1.4",
     "react-immutable-proptypes": "^2.1.0",
@@ -157,7 +157,7 @@
     "sass-loader": "^8.0.0",
     "stringz": "^2.0.0",
     "substring-trie": "^1.0.2",
-    "terser-webpack-plugin": "^2.2.1",
+    "terser-webpack-plugin": "^2.2.2",
     "tesseract.js": "^2.0.0-alpha.16",
     "throng": "^4.0.0",
     "tiny-queue": "^0.2.1",
@@ -176,7 +176,7 @@
     "enzyme": "^3.10.0",
     "enzyme-adapter-react-16": "^1.15.1",
     "eslint": "^6.7.2",
-    "eslint-plugin-import": "~2.18.2",
+    "eslint-plugin-import": "~2.19.0",
     "eslint-plugin-jsx-a11y": "~6.2.3",
     "eslint-plugin-promise": "~4.2.1",
     "eslint-plugin-react": "~7.17.0",
@@ -186,6 +186,6 @@
     "react-test-renderer": "^16.11.0",
     "sass-lint": "^1.13.1",
     "webpack-dev-server": "^3.9.0",
-    "yargs": "^13.3.0"
+    "yargs": "^15.0.2"
   }
 }