Skip to content

Commit

Permalink
[Closes #4519] Option to regenerate resource links at migration or not
Browse files Browse the repository at this point in the history
  • Loading branch information
afabiani committed Jun 17, 2019
1 parent 637aec8 commit 922cd5f
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 110 deletions.
28 changes: 15 additions & 13 deletions geonode/geoserver/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#########################################################################

import logging
from django.conf import settings
from django.utils.translation import ugettext_noop as _
from geonode.notifications_helper import NotificationsAppConfigBase

Expand Down Expand Up @@ -54,19 +55,20 @@ def set_resource_links(*args, **kwargs):
from geonode.catalogue.models import catalogue_post_save
from geonode.layers.models import Layer

_all_layers = Layer.objects.all()
for index, layer in enumerate(_all_layers, start=1):
_lyr_name = layer.name
message = "[%s / %s] Updating Layer [%s] ..." % (index, len(_all_layers), _lyr_name)
print(message)
logger.debug(message)
try:
set_resource_default_links(layer, layer)
catalogue_post_save(instance=layer, sender=layer.__class__)
except BaseException:
logger.exception(
"[ERROR] Layer [%s] couldn't be updated" % _lyr_name
)
if settings.UPDATE_RESOURCE_LINKS_AT_MIGRATE:
_all_layers = Layer.objects.all()
for index, layer in enumerate(_all_layers, start=1):
_lyr_name = layer.name
message = "[%s / %s] Updating Layer [%s] ..." % (index, len(_all_layers), _lyr_name)
print(message)
logger.debug(message)
try:
set_resource_default_links(layer, layer)
catalogue_post_save(instance=layer, sender=layer.__class__)
except BaseException:
logger.exception(
"[ERROR] Layer [%s] couldn't be updated" % _lyr_name
)


class GeoserverAppConfig(NotificationsAppConfigBase):
Expand Down
194 changes: 98 additions & 96 deletions geonode/geoserver/tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1062,116 +1062,118 @@ def test_set_resources_links(self):
from geonode.base.models import Link
from geonode.catalogue import get_catalogue

# Links
_def_link_types = ['original', 'metadata']
_links = Link.objects.filter(link_type__in=_def_link_types)
# Check 'original' and 'metadata' links exist
self.assertIsNotNone(
_links,
"No 'original' and 'metadata' links have been found"
)
self.assertTrue(
_links.count() > 0,
"No 'original' and 'metadata' links have been found"
)
# Delete all 'original' and 'metadata' links
_links.delete()
self.assertFalse(_links.count() > 0, "No links have been deleted")
# Delete resources metadata
_layers = Layer.objects.exclude(
Q(metadata_xml__isnull=True) |
Q(metadata_xml__exact='') |
Q(csw_anytext__isnull=True) |
Q(csw_anytext__exact='')
)
count = _layers.count()
self.assertTrue(count > 0, "No layers have got metadata")
if count:
_layers.update(metadata_xml=None)
_updated_layers = Layer.objects.exclude(
with self.settings(UPDATE_RESOURCE_LINKS_AT_MIGRATE=True):
# Links
_def_link_types = ['original', 'metadata']
_links = Link.objects.filter(link_type__in=_def_link_types)
# Check 'original' and 'metadata' links exist
self.assertIsNotNone(
_links,
"No 'original' and 'metadata' links have been found"
)
self.assertTrue(
_links.count() > 0,
"No 'original' and 'metadata' links have been found"
)
# Delete all 'original' and 'metadata' links
_links.delete()
self.assertFalse(_links.count() > 0, "No links have been deleted")
# Delete resources metadata
_layers = Layer.objects.exclude(
Q(metadata_xml__isnull=True) |
Q(metadata_xml__exact='') |
Q(csw_anytext__isnull=True) |
Q(csw_anytext__exact='')
)
updated_count = _updated_layers.count()
count = _layers.count()
self.assertTrue(count > 0, "No layers have got metadata")
if count:
_layers.update(metadata_xml=None)
_updated_layers = Layer.objects.exclude(
Q(metadata_xml__isnull=True) |
Q(metadata_xml__exact='') |
Q(csw_anytext__isnull=True) |
Q(csw_anytext__exact='')
)
updated_count = _updated_layers.count()
self.assertTrue(
updated_count == 0,
"Metadata have not been updated (deleted) correctly"
)
# Call migrate
call_command("migrate", verbosity=0)
# Check links
_post_migrate_links = Link.objects.filter(link_type__in=_def_link_types)
self.assertTrue(
updated_count == 0,
"Metadata have not been updated (deleted) correctly"
_post_migrate_links.count() > 0,
"No links have been restored"
)
# Check layers
_post_migrate_layers = Layer.objects.exclude(
Q(metadata_xml__isnull=True) |
Q(metadata_xml__exact='') |
Q(csw_anytext__isnull=True) |
Q(csw_anytext__exact='')
)
# Call migrate
call_command("migrate", verbosity=0)
# Check links
_post_migrate_links = Link.objects.filter(link_type__in=_def_link_types)
self.assertTrue(
_post_migrate_links.count() > 0,
"No links have been restored"
)
# Check layers
_post_migrate_layers = Layer.objects.exclude(
Q(metadata_xml__isnull=True) |
Q(metadata_xml__exact='') |
Q(csw_anytext__isnull=True) |
Q(csw_anytext__exact='')
)
post_migrate_layers_count = _post_migrate_layers.count()
self.assertTrue(
post_migrate_layers_count > 0,
"After migrations, there are no layers with metadata"
)
self.assertTrue(
post_migrate_layers_count >= count,
"After migrations, some metadata have not been restored correctly"
)
for _lyr in _post_migrate_layers:
# Check original links in csw_anytext
_post_migrate_links_orig = Link.objects.filter(
resource=_lyr.resourcebase_ptr,
resource_id=_lyr.resourcebase_ptr.id,
link_type='original'

post_migrate_layers_count = _post_migrate_layers.count()
self.assertTrue(
post_migrate_layers_count > 0,
"After migrations, there are no layers with metadata"
)
self.assertTrue(
_post_migrate_links_orig.count() > 0,
"No 'original' links has been found for the layer '{}'".format(
_lyr.alternate
)
post_migrate_layers_count >= count,
"After migrations, some metadata have not been restored correctly"
)
for _link_orig in _post_migrate_links_orig:
self.assertIn(
_link_orig.url,
_lyr.csw_anytext,
"The link URL {0} is not present in the 'csw_anytext' attribute of the layer '{1}'".format(
_link_orig.url,
for _lyr in _post_migrate_layers:
# Check original links in csw_anytext
_post_migrate_links_orig = Link.objects.filter(
resource=_lyr.resourcebase_ptr,
resource_id=_lyr.resourcebase_ptr.id,
link_type='original'
)
self.assertTrue(
_post_migrate_links_orig.count() > 0,
"No 'original' links has been found for the layer '{}'".format(
_lyr.alternate
)
)
# Check catalogue
catalogue = get_catalogue()
record = catalogue.get_record(_lyr.uuid)
self.assertIsNotNone(record)
self.assertTrue(
hasattr(record, 'links'),
"No records have been found in the catalogue for the resource '{}'".format(
_lyr.alternate
)
)
# Check 'metadata' links for each record
for mime, name, metadata_url in record.links['metadata']:
try:
_post_migrate_link_meta = Link.objects.get(
resource=_lyr.resourcebase_ptr,
url=metadata_url,
name=name,
extension='xml',
mime=mime,
link_type='metadata'
for _link_orig in _post_migrate_links_orig:
self.assertIn(
_link_orig.url,
_lyr.csw_anytext,
"The link URL {0} is not present in the 'csw_anytext' attribute of the layer '{1}'".format(
_link_orig.url,
_lyr.alternate
)
)
except Link.DoesNotExist:
_post_migrate_link_meta = None
self.assertIsNotNone(
_post_migrate_link_meta,
"No '{}' links have been found in the catalogue for the resource '{}'".format(
name,
# Check catalogue
catalogue = get_catalogue()
record = catalogue.get_record(_lyr.uuid)
self.assertIsNotNone(record)
self.assertTrue(
hasattr(record, 'links'),
"No records have been found in the catalogue for the resource '{}'".format(
_lyr.alternate
)
)
# Check 'metadata' links for each record
for mime, name, metadata_url in record.links['metadata']:
try:
_post_migrate_link_meta = Link.objects.get(
resource=_lyr.resourcebase_ptr,
url=metadata_url,
name=name,
extension='xml',
mime=mime,
link_type='metadata'
)
except Link.DoesNotExist:
_post_migrate_link_meta = None
self.assertIsNotNone(
_post_migrate_link_meta,
"No '{}' links have been found in the catalogue for the resource '{}'".format(
name,
_lyr.alternate
)
)
4 changes: 3 additions & 1 deletion geonode/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@
GEODATABASE_URL, conn_max_age=600
)

# If set to 'True' it will refresh/regenrate all resource links everytime a 'migrate' will be performed
UPDATE_RESOURCE_LINKS_AT_MIGRATE = ast.literal_eval(os.getenv('UPDATE_RESOURCE_LINKS_AT_MIGRATE', 'False'))

MANAGERS = ADMINS = os.getenv('ADMINS', [])

# Local time zone for this installation. Choices can be found here:
Expand All @@ -133,7 +136,6 @@
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = os.getenv('LANGUAGE_CODE', "en")


_DEFAULT_LANGUAGES = (
('en', 'English'),
('es', 'Español'),
Expand Down

0 comments on commit 922cd5f

Please sign in to comment.