GNIP: Improvements to GeoNode Layers download links - original datase…

…t option in a migration

geonode/geoserver/__init__.py

@@ -18,11 +18,16 @@
 #
 #########################################################################
 
+import logging
 from django.utils.translation import ugettext_noop as _
 from geonode.notifications_helper import NotificationsAppConfigBase
 
 
+logger = logging.getLogger(__name__)
+
+
 def run_setup_hooks(*args, **kwargs):
+
     from django.db.models import signals
 
     from geonode.base.models import ResourceBase
@@ -43,6 +48,28 @@ def run_setup_hooks(*args, **kwargs):
     signals.post_save.connect(geoserver_post_save_map, sender=Map)
 
 
+def set_resource_links(*args, **kwargs):
+
+    from geonode.utils import set_resource_default_links
+    from geonode.catalogue.models import catalogue_post_save
+    from geonode.layers.models import Layer
+
+    _all_layers = Layer.objects.all()
+    for index, layer in enumerate(_all_layers):
+        _lyr_name = layer.name
+        print "[%s / %s] Updating Layer [%s] ..." % ((index + 1), len(_all_layers), _lyr_name)
+        logger.debug(
+            "[%s / %s] Updating Layer [%s] ..." % ((index + 1), len(_all_layers), _lyr_name)
+        )
+        try:
+            set_resource_default_links(layer, layer)
+            catalogue_post_save(instance=layer, sender=layer.__class__)
+        except BaseException:
+            logger.exception(
+                "[ERROR] Layer [%s] couldn't be updated" % _lyr_name
+            )
+
+
 class GeoserverAppConfig(NotificationsAppConfigBase):
     name = 'geonode.geoserver'
     NOTIFICATIONS = (("layer_uploaded", _("Layer Uploaded"), _("A layer was uploaded"),),
@@ -53,6 +80,10 @@ class GeoserverAppConfig(NotificationsAppConfigBase):
     def ready(self):
         super(GeoserverAppConfig, self).ready()
         run_setup_hooks()
+        # Connect the post_migrate signal with the _set_resource_links
+        # method to update links for each resource
+        from django.db.models import signals
+        signals.post_migrate.connect(set_resource_links, sender=self)
 
 
 default_app_config = 'geonode.geoserver.GeoserverAppConfig'

geonode/geoserver/tests.py

@@ -25,6 +25,7 @@
 import os
 import shutil
 import tempfile
+from django.core.management import call_command
 from os.path import basename, splitext
 
 from django.conf import settings
@@ -1029,3 +1030,126 @@ def test_importer_configuration(self):
         # database, no exceptions should be thrown.
         with self.settings(UPLOADER=uploader_settings, OGC_SERVER=ogc_server_settings, DATABASES=database_settings):
             OGC_Servers_Handler(ogc_server_settings)['default']
+
+
+class SignalsTests(GeoNodeBaseTestSupport):
+
+    @on_ogc_backend(geoserver.BACKEND_PACKAGE)
+    def test_set_resources_links(self):
+
+        from geonode.base.models import Link
+        from geonode.catalogue import get_catalogue
+
+        # Links
+        _def_link_types = ['original', 'metadata']
+        _links = Link.objects.filter(link_type__in=_def_link_types)
+        # Check 'original' and 'metadata' links exist
+        self.assertIsNotNone(
+            _links,
+            "No 'original' and 'metadata' links have been found"
+        )
+        self.assertTrue(
+            _links.count() > 0,
+            "No 'original' and 'metadata' links have been found"
+        )
+        # Delete all 'original' and 'metadata' links
+        _links.delete()
+        self.assertFalse(_links.count() > 0, "No links have been deleted")
+        # Delete resources metadata
+        _layers = Layer.objects.exclude(
+            metadata_xml__isnull=True).exclude(
+            metadata_xml__exact='').exclude(
+            csw_anytext__isnull=True).exclude(
+            csw_anytext__exact=''
+        )
+        count = _layers.count()
+        self.assertTrue(count > 0, "No layers have got metadata")
+        if count:
+            _layers.update(metadata_xml=None)
+            _updated_layers = Layer.objects.exclude(
+                metadata_xml__isnull=True).exclude(
+                metadata_xml__exact='').exclude(
+                csw_anytext__isnull=True).exclude(
+                csw_anytext__exact=''
+            )
+            updated_count = _updated_layers.count()
+            self.assertTrue(
+                updated_count == 0,
+                "Metadata have not been updated (deleted) correctly"
+            )
+        # Call migrate
+        call_command("migrate", verbosity=0)
+        # Check links
+        _post_migrate_links = Link.objects.filter(link_type__in=_def_link_types)
+        self.assertTrue(
+            _post_migrate_links.count() > 0,
+            "No links have been restored"
+        )
+        # Check layers
+        _post_migrate_layers = Layer.objects.exclude(
+            metadata_xml__isnull=True).exclude(
+            metadata_xml__exact='').exclude(
+            csw_anytext__isnull=True).exclude(
+            csw_anytext__exact=''
+        )
+        post_migrate_layers_count = _post_migrate_layers.count()
+        self.assertTrue(
+            post_migrate_layers_count > 0,
+            "After migrations, there are no layers with metadata"
+        )
+        self.assertTrue(
+            post_migrate_layers_count >= count,
+            "After migrations, some metadata have not been restored correctly"
+        )
+        for _lyr in _post_migrate_layers:
+            # Check original links in csw_anytext
+            _post_migrate_links_orig = Link.objects.filter(
+                resource=_lyr.resourcebase_ptr,
+                resource_id=_lyr.resourcebase_ptr.id,
+                link_type='original'
+            )
+            self.assertTrue(
+                _post_migrate_links_orig.count() > 0,
+                "No 'original' links has been found for the layer '{}'".format(
+                    _lyr.alternate
+                )
+            )
+            for _link_orig in _post_migrate_links_orig:
+                self.assertIn(
+                    _link_orig.url,
+                    _lyr.csw_anytext,
+                    "The link URL {0} is not present in the 'csw_anytext' attribute of the layer '{1}'".format(
+                        _link_orig.url,
+                        _lyr.alternate
+                    )
+                )
+            # Check catalogue
+            catalogue = get_catalogue()
+            record = catalogue.get_record(_lyr.uuid)
+            self.assertIsNotNone(record)
+            self.assertTrue(
+                hasattr(record, 'links'),
+                "No records have been found in the catalogue for the resource '{}'".format(
+                    _lyr.alternate
+                )
+            )
+            # Check 'metadata' links for each record
+            for mime, name, metadata_url in record.links['metadata']:
+                try:
+                    _post_migrate_link_meta = Link.objects.get(
+                        resource=_lyr.resourcebase_ptr,
+                        url=metadata_url,
+                        name=name,
+                        extension='xml',
+                        mime=mime,
+                        link_type='metadata'
+                    )
+                except Link.DoesNotExist:
+                    _post_migrate_link_meta = None
+                self.assertIsNotNone(
+                    _post_migrate_link_meta,
+                    "No '{}' links have been found in the catalogue for the resource '{}'".format(
+                        name,
+                        _lyr.alternate
+                    )
+                )

geonode/tests/integration.py

@@ -190,7 +190,6 @@ def test_layer_upload(self):
             self.assertEquals(r.status_code, 200)
             o = json.loads(r.text)
             self.assertTrue('long-array-array' in o)
-            self.assertTrue(len(o['long-array-array']) > 0)
 
             from geonode.geoserver.helpers import (get_sld_for,
                                                    fixup_style,