Remove invalid upload files

RELEASE_NOTES.md

@@ -55,7 +55,19 @@ bundle remove spring spring-watcher-listen
 
 You can read more about this change on PR [#13235](https://github.com/decidim/decidim/pull/13235).
 
-### 3.2. [[TITLE OF THE ACTION]]
+### 3.2. Clean up orphaned attachment blobs
+
+We have added a new task that helps you clean the orphaned attachment blobs. This task will remove all the attachment blobs that have been created for more than 1 hour and are not yet referenced by any attachment record. This helps cleaning your filesystem of unused files.
+
+You can run the task with the following command:
+
+```bash
+bundle exec decidim:attachments:cleanup
+```
+
+You can see more details about this change on PR [\#11851](https://github.com/decidim/decidim/pull/11851)
+
+### 3.3. [[TITLE OF THE ACTION]]
 
 You can read more about this change on PR [#XXXX](https://github.com/decidim/decidim/pull/XXXX).
 

decidim-core/app/commands/decidim/validate_upload.rb

@@ -7,9 +7,18 @@ def initialize(form)
     end
 
     def call
-      return broadcast(:invalid, @form.errors) if @form.invalid?
+      if @form.invalid?
+        remove_invalid_file
+        return broadcast(:invalid, @form.errors)
+      end
 
       broadcast(:ok)
     end
+
+    private
+
+    def remove_invalid_file
+      @form.blob.purge_later if @form.blob.present?
+    end
   end
 end

decidim-core/app/controllers/concerns/decidim/direct_upload.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Decidim
+  module DirectUpload
+    extend ActiveSupport::Concern
+
+    included do
+      include Decidim::NeedsOrganization
+      skip_before_action :verify_organization
+
+      before_action :check_organization!,
+                    :check_authenticated!,
+                    :check_user_belongs_to_organization,
+                    :validate_direct_upload
+    end
+
+    protected
+
+    def verify_organization; end
+
+    def validate_direct_upload
+      return if current_admin.present?
+
+      head :unprocessable_entity unless [
+        maximum_allowed_size.try(:to_i) >= blob_args[:byte_size].try(:to_i),
+        content_types.any? { |pattern| pattern.match?(blob_args[:content_type]) },
+        content_types.any? { |pattern| pattern.match?(MiniMime.lookup_by_extension(extension)&.content_type) },
+        allowed_extensions.any? { |pattern| pattern.match?(extension) }
+      ].all?
+    rescue NoMethodError
+      head :unprocessable_entity
+    end
+
+    def extension
+      File.extname(blob_args[:filename]).delete(".")
+    end
+
+    def maximum_allowed_size
+      current_organization.settings.upload_maximum_file_size
+    end
+
+    def check_organization!
+      head :unauthorized if current_organization.blank? && current_admin.blank?
+    end
+
+    def check_authenticated!
+      head :unauthorized if current_user.blank? && current_admin.blank?
+    end
+
+    def check_user_belongs_to_organization
+      return if current_admin.present?
+
+      head :unauthorized unless current_organization == current_user.organization
+    end
+
+    def allowed_extensions
+      if user_has_elevated_role?
+        current_organization.settings.upload_allowed_file_extensions_admin
+      else
+        current_organization.settings.upload_allowed_file_extensions
+      end
+    end
+
+    def content_types
+      if user_has_elevated_role?
+        current_organization.settings.upload_allowed_content_types_admin
+      else
+        current_organization.settings.upload_allowed_content_types
+      end
+    end
+
+    private
+
+    def user_has_elevated_role?
+      [
+        current_user&.admin?,
+        defined?(Decidim::Assemblies::AssembliesWithUserRole) && Decidim::Assemblies::AssembliesWithUserRole.for(current_user).any?,
+        defined?(Decidim::Conferences::ConferencesWithUserRole) && Decidim::Conferences::ConferencesWithUserRole.for(current_user).any?,
+        defined?(Decidim::ParticipatoryProcessesWithUserRole) && Decidim::ParticipatoryProcessesWithUserRole.for(current_user).any?
+      ].any?
+    end
+  end
+end

decidim-core/app/forms/decidim/upload_validation_form.rb

@@ -10,7 +10,7 @@ class UploadValidationForm < Decidim::Form
     # Property is named as attribute in upload modal and passthru validator, but
     # it cannot be named as attribute here.
     attribute :property, String
-    attribute :blob, String
+    attribute :blob, Decidim::Attributes::Blob
     attribute :form_class, String
 
     validates :resource_class, presence: true

decidim-core/app/packs/src/decidim/direct_uploads/upload_modal.js

@@ -77,6 +77,9 @@ export default class UploadModal {
       uploader.upload.create((error, blob) => {
         if (error) {
           uploader.errors = [error]
+          this.uploadItems.replaceChild(this.createUploadItem(file, [error], { value: 100 }), item);
+          this.updateDropZone();
+
         } else {
           // attach the file hash to submit the form, when the file has been uploaded
           file.hiddenField = blob.signed_id

decidim-core/lib/decidim/core.rb

@@ -573,6 +573,12 @@ def self.reset_all_column_information
     {}
   end
 
+  # This config parameter is used to set the default amount of time after which the unattached blobs
+  # are cleaned up. The default value is 60 minutes.
+  config_accessor :clean_up_unattached_blobs_after do
+    60.minutes
+  end
+
   # Public: Registers a global engine. This method is intended to be used
   # by component engines that also offer unscoped functionality
   #

decidim-core/lib/decidim/core/engine.rb

@@ -279,6 +279,12 @@ class Engine < ::Rails::Engine
         app.config.exceptions_app = Decidim::Core::Engine.routes
       end
 
+      initializer "decidim_core.direct_uploader_paths", after: "decidim_core.exceptions_app" do |_app|
+        config.to_prepare do
+          ActiveStorage::DirectUploadsController.include Decidim::DirectUpload
+        end
+      end
+
       initializer "decidim_core.locales" do |app|
         app.config.i18n.fallbacks = true
       end

decidim-core/lib/decidim/organization_settings.rb

@@ -104,15 +104,16 @@ def defaults_hash
         {
           "upload" => {
             "allowed_file_extensions" => {
-              "default" => %w(jpg jpeg png webp pdf rtf txt),
-              "admin" => %w(jpg jpeg png webp pdf doc docx xls xlsx ppt pptx ppx rtf txt odt ott odf otg ods ots),
+              "default" => %w(jpg jpeg png webp pdf rtf txt md),
+              "admin" => %w(jpg jpeg png webp pdf doc docx xls xlsx ppt pptx ppx rtf txt odt ott odf otg ods ots csv json md),
               "image" => %w(jpg jpeg png webp)
             },
             "allowed_content_types" => {
               "default" => %w(
                 image/*
                 application/pdf
                 application/rtf
+                text/markdown
                 text/plain
               ),
               "admin" => %w(
@@ -125,7 +126,10 @@ def defaults_hash
                 application/vnd.oasis.opendocument
                 application/pdf
                 application/rtf
+                application/json
+                text/markdown
                 text/plain
+                text/csv
               )
             },
             "maximum_file_size" => {

decidim-core/lib/tasks/decidim_attachments.rake

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+namespace :decidim do
+  namespace :attachments do
+    desc "Cleanup the orphaned blobs attachments"
+    task cleanup: :environment do
+      return if Decidim.clean_up_unattached_blobs_after.to_i.zero?
+
+      ActiveStorage::Blob.unattached.where(created_at: ..Decidim.clean_up_unattached_blobs_after.ago).find_each(batch_size: 100, &:purge_later)
+    end
+  end
+end

decidim-core/spec/commands/decidim/validate_upload_spec.rb

@@ -9,16 +9,24 @@ module Decidim
       let(:form) do
         double(
           invalid?: invalid,
+          blob:,
           errors:
         )
       end
       let(:invalid) { false }
       let(:errors) { [] }
+      let(:io) { Decidim::Dev.test_file("city.jpeg", "image/jpeg") }
+      let(:blob) { ActiveStorage::Blob.create_and_upload!(io:, filename: "city.jpeg") }
 
       describe "when form is valid" do
         it "broadcasts ok" do
           expect { command.call }.to broadcast(:ok)
         end
+
+        it "ignores file removal" do
+          expect(ActiveStorage::Blob).not_to receive(:find_signed).with(blob)
+          command.call
+        end
       end
 
       describe "when the form is not valid" do
@@ -28,6 +36,11 @@ module Decidim
         it "broadcasts invalid" do
           expect { command.call }.to broadcast(:invalid, errors)
         end
+
+        it "removes the invalid file" do
+          expect(blob).to receive(:purge_later)
+          command.call
+        end
       end
     end
   end