Merge pull request GeoNode#820 from geosolutions-it/issue_190

[Fixes 190] Extend the ResourceBase model with FILES

geonode/base/migrations/0062_resourcebase_files.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2 on 2021-05-28 15:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0061_merge_0060_auto_20210512_1658_0060_resourcebase_state'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resourcebase',
+            name='files',
+            field=models.JSONField(default=dict),
+        ),
+    ]

geonode/base/models.py

@@ -29,6 +29,7 @@
 from django.db import models
 from django.conf import settings
 from django.core import serializers
+from django.db.models.fields.json import JSONField
 from django.utils.functional import cached_property
 from django.utils.html import escape
 from django.utils.timezone import now
@@ -85,6 +86,7 @@
 
 from urllib.parse import urlparse, urlsplit, urljoin
 from imagekit.cachefiles.backends import Simple
+from geonode.storage.manager import storage_manager
 
 logger = logging.getLogger(__name__)
 
@@ -590,6 +592,27 @@ def get_queryset(self):
     def polymorphic_queryset(self):
         return super(ResourceBaseManager, self).get_queryset()
 
+    @staticmethod
+    def upload_files(resource_id, files):
+        try:
+            out = {}
+            for f in files:
+                _, ext = os.path.splitext(f)
+                if os.path.isfile(f) and os.path.exists(f):
+
+                    with open(f, 'rb') as ff:
+                        folder = os.path.basename(os.path.dirname(f))
+                        filename = os.path.basename(f)
+                        file_uploaded_path = storage_manager.save(f'{folder}/{filename}', ff)
+                        out[ext] = storage_manager.path(file_uploaded_path)
+
+            # making an update instead of save in order to avoid others
+            # signal like post_save and commiunication with geoserver
+            ResourceBase.objects.filter(id=resource_id).update(files=out)
+            return out
+        except Exception as e:
+            logger.exception(e)
+
 
 class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase):
     """
@@ -915,6 +938,8 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase):
         default=False,
         help_text=_('If true, will be excluded from search'))
 
+    files = JSONField(null=False, default=dict)
+
     __is_approved = False
     __is_published = False
 

geonode/layers/models.py

@@ -647,7 +647,7 @@ def pre_delete_layer(instance, sender, **kwargs):
         from geonode.upload.models import Upload
         # Need to call delete one by one in ordee to invoke the
         #  'delete' overridden method
-        for upload in Upload.objects.filter(layer_id=instance.id):
+        for upload in Upload.objects.filter(resource_id=instance.id):
             upload.delete()
 
     # Delete object permissions

geonode/layers/utils.py

@@ -624,8 +624,8 @@ def gs_handle_layer(layer, base_files, user, action_type="append"):
         #  opening upload session for the selected layer
         from geonode.upload.models import Upload
 
-        upload_session, _ = Upload.objects.get_or_create(layer=layer, user=user)
-        upload_session.resource = layer
+        upload_session, _ = Upload.objects.get_or_create(resource=layer.resourcebase_ptr, user=user)
+        upload_session.resource = layer.resourcebase_ptr
         upload_session.processed = False
         upload_session.save()
 

geonode/layers/views.py

@@ -1079,7 +1079,7 @@ def layer_metadata(
 
         from geonode.upload.models import Upload
 
-        up_sessions = Upload.objects.filter(layer=layer)
+        up_sessions = Upload.objects.filter(resource_id=layer.resourcebase_ptr_id)
         if up_sessions.count() > 0 and up_sessions[0].user != layer.owner:
             up_sessions.update(user=layer.owner)
 

geonode/proxy/templatetags/proxy_lib_tags.py

@@ -18,6 +18,7 @@
 #
 #########################################################################
 
+from geonode.base.models import ResourceBase
 import traceback
 
 from django import template
@@ -28,8 +29,6 @@
 
 from urllib.parse import urlsplit, urljoin
 
-from geonode.layers.models import Layer
-from geonode.upload.models import Upload
 from geonode.utils import resolve_object
 
 register = template.Library()
@@ -41,7 +40,7 @@ def original_link_available(context, resourceid, url):
     request = context['request']
     instance = resolve_object(
         request,
-        Layer,
+        ResourceBase,
         {'pk': resourceid},
         permission='base.download_resourcebase',
         permission_msg=_not_permitted)
@@ -52,12 +51,11 @@ def original_link_available(context, resourceid, url):
         return True
 
     layer_files = []
-    if isinstance(instance, Layer):
+    if isinstance(instance, ResourceBase):
         try:
-            upload_session = Upload.objects.get(layer=instance)
-            for lyr in upload_session.uploadfile_set.all():
-                layer_files.append(lyr)
-                if not storage_manager.exists(lyr.file):
+            for ext, file in instance.files.items():
+                layer_files.append(file)
+                if not storage_manager.exists(file):
                     return False
         except Exception:
             traceback.print_exc()

geonode/proxy/tests.py

@@ -30,7 +30,7 @@
 from geonode.proxy.templatetags.proxy_lib_tags import original_link_available
 from django.test.client import RequestFactory
 from mock import patch
-from geonode.upload.models import Upload, UploadFile
+from geonode.upload.models import Upload
 import json
 from django.core.files.uploadedfile import SimpleUploadedFile
 
@@ -154,20 +154,25 @@ def test_download_url_with_not_existing_file(self):
     @on_ogc_backend(geoserver.BACKEND_PACKAGE)
     def test_download_url_with_existing_files(self, fopen, fexists):
         fexists.return_value = True
-        fopen.return_value = SimpleUploadedFile('file.shp', b'scc')
+        fopen.return_value = SimpleUploadedFile('foo_file.shp', b'scc')
         layer = Layer.objects.all().first()
 
+        layer.files = {
+            ".dbf": "/tmpe1exb9e9/foo_file.dbf",
+            ".prj": "/tmpe1exb9e9/foo_file.prj",
+            ".shp": "/tmpe1exb9e9/foo_file.shp",
+            ".shx": "/tmpe1exb9e9/foo_file.shx"
+        }
+
+        layer.save()
+
+        layer.refresh_from_db()
+
         upload = Upload.objects.create(
             state='RUNNING',
-            layer=layer
+            resource=layer
         )
 
-        _ = UploadFile.objects.create(
-            upload=upload,
-            file='/file.shp',
-            slug='foo_slug',
-            name="foo_name"
-        )
         self.client.login(username='admin', password='admin')
         # ... all should be good
         response = self.client.get(reverse('download', args=(layer.id,)))
@@ -218,25 +223,27 @@ def test_tag_original_link_available_with_different_netlock_should_return_true(s
     def test_should_return_false_if_no_files_are_available(self):
         _ = Upload.objects.create(
             state='RUNNING',
-            layer=self.resource
+            resource=self.resource
         )
 
         actual = original_link_available(self.context, self.resource.resourcebase_ptr_id, self.url)
         self.assertFalse(actual)
 
     @patch('geonode.storage.manager.storage_manager.exists', return_value=True)
-    def test_should_return_true_if_no_files_are_available(self, fexists):
+    def test_should_return_true_if_files_are_available(self, fexists):
         upload = Upload.objects.create(
             state='RUNNING',
-            layer=self.resource
-        )
-
-        _ = UploadFile.objects.create(
-            upload=upload,
-            file='/file.shp',
-            slug='foo_slug',
-            name="foo_name"
+            resource=self.resource
         )
+        self.resource.files = {
+            ".dbf": "/tmpe1exb9e9/foo_file.dbf",
+            ".prj": "/tmpe1exb9e9/foo_file.prj",
+            ".shp": "/tmpe1exb9e9/foo_file.shp",
+            ".shx": "/tmpe1exb9e9/foo_file.shx"
+        }
+        self.resource.save()
+
+        self.resource.refresh_from_db()
 
         actual = original_link_available(self.context, self.resource.resourcebase_ptr_id, self.url)
         self.assertTrue(actual)

geonode/proxy/views.py

@@ -18,6 +18,7 @@
 #
 #########################################################################
 
+from geonode.base.models import ResourceBase
 import io
 import os
 import re
@@ -266,7 +267,7 @@ def download(request, resourceid, sender=Layer):
                               permission='base.download_resourcebase',
                               permission_msg=_not_permitted)
 
-    if isinstance(instance, Layer):
+    if isinstance(instance, ResourceBase):
         # Create Target Folder
         dirpath = tempfile.mkdtemp(dir=settings.STATIC_ROOT)
         dir_time_suffix = get_dir_time_suffix()
@@ -276,14 +277,14 @@ def download(request, resourceid, sender=Layer):
 
         layer_files = []
         try:
-            upload_session = Upload.objects.get(layer=instance)
+            files = instance.resourcebase_ptr.files
             # Copy all Layer related files into a temporary folder
-            for lyr in upload_session.uploadfile_set.all():
-                if storage_manager.exists(lyr.file):
-                    layer_files.append(lyr)
-                    filename = os.path.basename(lyr.file)
+            for ext, file_path in files.items():
+                if storage_manager.exists(file_path):
+                    layer_files.append(file_path)
+                    filename = os.path.basename(file_path)
                     with open(f"{target_folder}/{filename}", 'wb+') as f:
-                        f.write(storage_manager.open(lyr.file).read())
+                        f.write(storage_manager.open(file_path).read())
                 else:
                     return HttpResponse(
                         loader.render_to_string(

geonode/upload/admin.py

@@ -18,7 +18,7 @@
 #
 #########################################################################
 
-from geonode.upload.models import Upload, UploadFile
+from geonode.upload.models import Upload
 
 from django.contrib import admin
 
@@ -32,23 +32,15 @@ def import_link(obj):
 
 
 class UploadAdmin(admin.ModelAdmin):
-    list_display = ('id', 'import_id', 'name', 'layer', 'user', 'date', 'state', import_link)
+    list_display = ('id', 'import_id', 'name', 'resource', 'user', 'date', 'state', import_link)
     list_display_links = ('id',)
     date_hierarchy = 'date'
-    list_filter = ('name', 'layer', 'user', 'date', 'state')
-    search_fields = ('name', 'layer', 'user', 'date', 'state')
+    list_filter = ('name', 'resource', 'user', 'date', 'state')
+    search_fields = ('name', 'resource', 'user', 'date', 'state')
 
     def delete_queryset(self, request, queryset):
         for obj in queryset:
             obj.delete()
 
 
-class UploadFileAdmin(admin.ModelAdmin):
-    list_display = ('id', 'upload', 'slug')
-    list_display_links = ('id',)
-    list_filter = ('slug', )
-    search_fields = ('slug', )
-
-
 admin.site.register(Upload, UploadAdmin)
-admin.site.register(UploadFile, UploadFileAdmin)

geonode/upload/api/serializers.py

@@ -17,13 +17,14 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 #########################################################################
+import os
+from geonode.base.models import ResourceBase
 from rest_framework import serializers
 
-from dynamic_rest.serializers import DynamicModelSerializer
 from dynamic_rest.fields.fields import DynamicRelationField, DynamicComputedField
 
 from geonode.base.utils import build_absolute_uri
-from geonode.upload.models import Upload, UploadFile
+from geonode.upload.models import Upload
 from geonode.layers.api.serializers import LayerSerializer
 from geonode.base.api.serializers import BaseDynamicModelSerializer
 
@@ -32,17 +33,26 @@
 logger = logging.getLogger(__name__)
 
 
-class UploadFileSerializer(DynamicModelSerializer):
-
+class UploadFileField(serializers.RelatedField):
     class Meta:
-        model = UploadFile
-        name = 'upload_file'
-        fields = (
-            'pk', 'name', 'slug', 'file'
-        )
-
-    name = serializers.CharField(read_only=True)
-    slug = serializers.CharField(read_only=True)
+        model = ResourceBase
+        name = 'resource-files'
+
+    def to_representation(self, obj):
+        files = []
+        for _, file in obj.files.items():
+            name, _ = os.path.splitext(os.path.basename(file))
+            files.append(
+                {
+                    "name": name,
+                    "slug": os.path.basename(file),
+                    "file": file
+                }
+            )
+        return {
+            'name': obj.title,
+            'files': files,
+        }
 
 
 class SessionSerializer(serializers.Field):
@@ -213,12 +223,12 @@ class Meta:
         fields = (
             'id', 'name', 'date', 'create_date', 'user',
             'state', 'progress', 'complete', 'import_id',
-            'uploadfile_set', 'resume_url', 'delete_url', 'import_url', 'detail_url'
+            'resume_url', 'delete_url', 'import_url', 'detail_url', "uploadfile_set"
         )
 
     progress = ProgressField(read_only=True)
     resume_url = ProgressUrlField('resume', read_only=True)
     delete_url = ProgressUrlField('delete', read_only=True)
     import_url = ProgressUrlField('import', read_only=True)
     detail_url = ProgressUrlField('detail', read_only=True)
-    uploadfile_set = DynamicRelationField(UploadFileSerializer, embed=True, many=True, read_only=True)
+    uploadfile_set = UploadFileField(source='resource', read_only=True)

geonode/upload/api/views.py

@@ -64,6 +64,7 @@ class UploadViewSet(DynamicModelViewSet):
     queryset = Upload.objects.all()
     serializer_class = UploadSerializer
     pagination_class = GeoNodeApiPagination
+    lookup_field = "resource_id"
 
     @extend_schema(methods=['put'],
                    responses={201: None},

geonode/upload/forms.py

@@ -27,19 +27,11 @@
 from ..utils import check_ogc_backend
 from ..layers.forms import JSONField
 
-from .models import UploadFile
 from .upload_validators import validate_uploaded_files
 
 logger = logging.getLogger(__name__)
 
 
-class UploadFileForm(forms.ModelForm):
-
-    class Meta:
-        model = UploadFile
-        fields = '__all__'
-
-
 class LayerUploadForm(forms.Form):
     base_file = forms.FileField()
     dbf_file = forms.FileField(required=False)