Purge static files on delete

CHANGELOG.md

@@ -2,7 +2,11 @@
 
 ## Current (in progress)
 
-- Nothing yet
+- :warning: Deletion workflow changes [#2488](https://github.com/opendatateam/udata/pull/2488):
+  - Deleting a resource now triggers the deletion of the corresponding static file
+  - Deleting a dataset now triggers the deletion of the corresponding resources (including community resources) and their static files
+  - Adding a celery job `purge-orphan-community-resources` to remove community resources not linked to a dataset. This should be scheduled regularly.
+  - ⚠️ Adding a migration file to populate resources fs_filename new field, and to delete orphaned resources files
 
 ## 2.1.3 (2020-06-29)
 

udata/core/dataset/api.py

@@ -361,6 +361,10 @@ def delete(self, dataset, rid):
         '''Delete a given resource on a given dataset'''
         ResourceEditPermission(dataset).test()
         resource = self.get_resource_or_404(dataset, rid)
+        # Deletes resource's file from file storage
+        if resource.fs_filename is not None:
+            storages.resources.delete(resource.fs_filename)
+
         dataset.resources.remove(resource)
         dataset.last_modified = datetime.now()
         dataset.save()
@@ -437,6 +441,9 @@ def put(self, community):
     def delete(self, community):
         '''Delete a given community resource'''
         ResourceEditPermission(community).test()
+        # Deletes community resource's file from file storage
+        if community.fs_filename is not None:
+            storages.resources.delete(community.fs_filename)
         community.delete()
         return '', 204
 

udata/core/dataset/models.py

@@ -222,6 +222,7 @@ class ResourceMixin(object):
     format = db.StringField()
     mime = db.StringField()
     filesize = db.IntField()  # `size` is a reserved keyword for mongoengine.
+    fs_filename = db.StringField()
     extras = db.ExtrasField()
 
     created_at = db.DateTimeField(default=datetime.now, required=True)

udata/core/dataset/tasks.py

@@ -17,7 +17,7 @@
                           Organization)
 from udata.tasks import job
 
-from .models import Dataset, Resource, UPDATE_FREQUENCIES, Checksum
+from .models import Dataset, Resource, CommunityResource, UPDATE_FREQUENCIES, Checksum
 
 log = get_task_logger(__name__)
 
@@ -33,7 +33,7 @@ def flatten(iterable):
 @job('purge-datasets')
 def purge_datasets(self):
     for dataset in Dataset.objects(deleted__ne=None):
-        log.info('Purging dataset "{0}"'.format(dataset))
+        log.info(f'Purging dataset {dataset}')
         # Remove followers
         Follow.objects(following=dataset).delete()
         # Remove issues
@@ -49,10 +49,34 @@ def purge_datasets(self):
             topic.update(datasets=datasets)
         # Remove HarvestItem references
         HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
-        # Remove
+        # Remove each dataset's resource's file
+        storage = storages.resources
+        for resource in dataset.resources:
+            if resource.fs_filename is not None:
+                storage.delete(resource.fs_filename)
+        # Remove each dataset related community resource and it's file
+        community_resources = CommunityResource.objects(dataset=dataset)
+        for community_resource in community_resources:
+            if community_resource.fs_filename is not None:
+                storage.delete(community_resource.fs_filename)
+            community_resource.delete()
+        # Remove dataset
         dataset.delete()
 
 
+@job('purge-orphan-community-resources')
+def purge_orphan_community_resources(self):
+    '''
+    Gets community resources not linked with a dataset
+    and deletes them along with their files.
+    '''
+    community_resources = CommunityResource.objects(dataset=None)
+    for community_resource in community_resources:
+        if community_resource.fs_filename is not None:
+            storages.resources.delete(community_resource.fs_filename)
+        community_resource.delete()
+
+
 @job('send-frequency-reminder')
 def send_frequency_reminder(self):
     # We exclude irrelevant frequencies.

udata/core/organization/tasks.py

@@ -1,5 +1,6 @@
 from udata import mail
 from udata.i18n import lazy_gettext as _
+from udata.core import storages
 from udata.models import Follow, Activity, Dataset
 from udata.search import reindex
 from udata.tasks import job, task, get_logger
@@ -14,14 +15,21 @@
 @job('purge-organizations')
 def purge_organizations(self):
     for organization in Organization.objects(deleted__ne=None):
-        log.info('Purging organization "{0}"'.format(organization))
+        log.info(f'Purging organization {organization}')
         # Remove followers
         Follow.objects(following=organization).delete()
         # Remove activity
         Activity.objects(related_to=organization).delete()
         Activity.objects(organization=organization).delete()
         # Store datasets for later reindexation
         d_ids = [d.id for d in Dataset.objects(organization=organization)]
+        # Remove organization's logo in all sizes
+        if organization.logo.filename is not None:
+            storage = storages.avatars
+            storage.delete(organization.logo.filename)
+            storage.delete(organization.logo.original)
+            for key, value in organization.logo.thumbnails.items():
+                storage.delete(value)
         # Remove
         organization.delete()
         # Reindex the datasets that were linked to the organization

udata/core/reuse/tasks.py

@@ -1,5 +1,6 @@
 from udata import mail
 from udata.i18n import lazy_gettext as _
+from udata.core import storages
 from udata.models import Activity, Issue, Discussion, Follow
 from udata.tasks import get_logger, job, task
 
@@ -11,7 +12,7 @@
 @job('purge-reuses')
 def purge_reuses(self):
     for reuse in Reuse.objects(deleted__ne=None):
-        log.info('Purging reuse "{0}"'.format(reuse))
+        log.info(f'Purging reuse {reuse}')
         # Remove followers
         Follow.objects(following=reuse).delete()
         # Remove issues
@@ -20,6 +21,13 @@ def purge_reuses(self):
         Discussion.objects(subject=reuse).delete()
         # Remove activity
         Activity.objects(related_to=reuse).delete()
+        # Remove reuse's logo in all sizes
+        if reuse.image.filename is not None:
+            storage = storages.images
+            storage.delete(reuse.image.filename)
+            storage.delete(reuse.image.original)
+            for key, value in reuse.image.thumbnails.items():
+                storage.delete(value)
         reuse.delete()
 
 

udata/core/storages/api.py

@@ -135,20 +135,24 @@ def handle_upload(storage, prefix=None):
         if uploaded_file:
             save_chunk(uploaded_file, args)
         else:
-            filename = combine_chunks(storage, args, prefix=prefix)
+            fs_filename = combine_chunks(storage, args, prefix=prefix)
     elif not uploaded_file:
         raise UploadError('Missing file parameter')
     else:
         # Normalize filename including extension
         filename = utils.normalize(uploaded_file.filename)
-        filename = storage.save(uploaded_file, prefix=prefix,
-                                filename=filename)
-
-    metadata = storage.metadata(filename)
+        fs_filename = storage.save(
+            uploaded_file,
+            prefix=prefix,
+            filename=filename
+        )
+
+    metadata = storage.metadata(fs_filename)
+    metadata['fs_filename'] = fs_filename
     checksum = metadata.pop('checksum')
     algo, checksum = checksum.split(':', 1)
     metadata[algo] = checksum
-    metadata['format'] = utils.extension(filename)
+    metadata['format'] = utils.extension(fs_filename)
     return metadata
 
 

udata/core/user/api.py

@@ -3,6 +3,7 @@
 
 from udata import search
 from udata.api import api, API
+from udata.core import storages
 from udata.auth import admin_permission
 from udata.models import CommunityResource, Dataset, Reuse, User
 
@@ -298,6 +299,12 @@ def delete(self, user):
         if user == current_user._get_current_object():
             api.abort(403, 'You cannot delete yourself with this API. ' +
                       'Use the "me" API instead.')
+        if user.avatar.filename is not None:
+            storage = storages.avatars
+            storage.delete(user.avatar.filename)
+            storage.delete(user.avatar.original)
+            for key, value in user.avatar.thumbnails.items():
+                storage.delete(value)
         user.mark_as_deleted()
         return '', 204
 

udata/migrations/2020-06-11-add-resource-fs-filename.py

@@ -0,0 +1,41 @@
+'''
+The purpose here is to fill every resource with a fs_filename string field.
+'''
+import logging
+from urllib.parse import urlparse
+
+from udata.models import Dataset, CommunityResource
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info('Processing resources resources.')
+
+    datasets = Dataset.objects()
+    for dataset in datasets:
+        for resource in dataset.resources:
+            if resource.url.startswith('https://static.data.gouv.fr'):
+                parsed = urlparse(resource.url)
+                fs_name = parsed.path.strip('/resource/')
+                resource.fs_filename = fs_name
+                try:
+                    resource.save()
+                except Exception as e:
+                    log.warning(e)
+                    pass
+
+    log.info('Processing community resources.')
+
+    community_resources = CommunityResource.objects()
+    for community_resource in community_resources:
+        parsed = urlparse(community_resource.url)
+        fs_name = parsed.path.strip('/resource/')
+        community_resource.fs_filename = fs_name
+        try:
+            community_resource.save()
+        except Exception as e:
+            log.warning(e)
+            pass
+
+    log.info('Completed.')