Skip to content

Commit

Permalink
[Fixes #4321] GeoNode is not able to show errors properly on upload / […
Browse files Browse the repository at this point in the history 
…Closes #4241] Upload failure: saved_layer is None
  • Loading branch information
afabiani committed Apr 10, 2019
1 parent 834ef71 commit 7f88674
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 30 deletions.
88 changes: 88 additions & 0 deletions geonode/layers/tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from django.contrib.contenttypes.models import ContentType
from django.core.urlresolvers import reverse
from django.contrib.auth.models import Group
from django.conf import settings

from django.db.models import Count
from django.contrib.auth import get_user_model
Expand All @@ -59,6 +60,7 @@
from geonode.layers import LayersAppConfig
from geonode.tests.utils import NotificationsTestsHelper
from geonode.layers.populate_layers_data import create_layer_data
from geonode.base.enumerations import CHARSETS

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -1187,3 +1189,89 @@ def testLayerNotifications(self):
comment.save()

self.assertTrue(self.check_notification_out('layer_comment', self.u))


class LayersUploaderTests(GeoNodeBaseTestSupport):

GEONODE_REST_UPLOADER = {
'BACKEND': 'geonode.rest',
'OPTIONS': {
'TIME_ENABLED': True,
'MOSAIC_ENABLED': False,
'GEOGIG_ENABLED': False,
},
'SUPPORTED_CRS': [
'EPSG:4326',
'EPSG:3785',
'EPSG:3857',
'EPSG:32647',
'EPSG:32736'
],
'SUPPORTED_EXT': [
'.shp',
'.csv',
'.kml',
'.kmz',
'.json',
'.geojson',
'.tif',
'.tiff',
'.geotiff',
'.gml',
'.xml'
]
}

def setUp(self):
super(LayersUploaderTests, self).setUp()
create_layer_data()
self.user = 'admin'
self.passwd = 'admin'
self.anonymous_user = get_anonymous_user()

@override_settings(UPLOADER=GEONODE_REST_UPLOADER)
def test_geonode_rest_layer_uploader(self):
layer_upload_url = reverse('layer_upload')
self.client.login(username=self.user, password=self.passwd)
# Check upload for each charset
for charset in CHARSETS:
files = dict(
base_file=SimpleUploadedFile('foo.shp', ' '),
shx_file=SimpleUploadedFile('foo.shx', ' '),
dbf_file=SimpleUploadedFile('foo.dbf', ' '),
prj_file=SimpleUploadedFile('foo.prj', ' '))
files['permissions'] = '{}'
files['charset'] = charset[0]
files['layer_title'] = 'test layer_{}'.format(charset[0])
resp = self.client.post(layer_upload_url, data=files)
# Check response status code
self.assertEqual(resp.status_code, 200)
# Retrieve the layer from DB
data = json.loads(resp.content)
# Check success
self.assertTrue(data['success'])
_lname = data['url'].split(':')[-1]
_l = Layer.objects.get(name=_lname)
# Check the layer has been published
self.assertTrue(_l.is_published)
# Check errors
self.assertNotIn('errors', data)
self.assertNotIn('errormsgs', data)
self.assertNotIn('traceback', data)
self.assertNotIn('context', data)
self.assertNotIn('upload_session', data)
if 'info' in data:
self.assertEqual(data['info'], _l.info)
self.assertEqual(data['bbox'], _l.bbox_string)
self.assertEqual(
data['crs'],
{
'type': 'name',
'properties': _l.srid
}
)
self.assertEqual(
data['ogc_backend'],
settings.OGC_SERVER['default']['BACKEND']
)
_l.delete()
94 changes: 64 additions & 30 deletions geonode/layers/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import shutil
import base64
import traceback
from types import TracebackType
import decimal
import cPickle as pickle
from django.db.models import Q
Expand Down Expand Up @@ -172,6 +173,7 @@ def layer_upload(request, template='upload/layer_upload.html'):
tempdir = None
saved_layer = None
errormsgs = []
input_charset = None
out = {'success': False}
if form.is_valid():
title = form.cleaned_data["layer_title"]
Expand All @@ -193,6 +195,9 @@ def layer_upload(request, template='upload/layer_upload.html'):
else:
abstract = "No abstract provided."

# charset
input_charset = form.cleaned_data["charset"]

try:
# Moved this inside the try/except block because it can raise
# exceptions when unicode characters are present.
Expand All @@ -204,7 +209,7 @@ def layer_upload(request, template='upload/layer_upload.html'):
name=name,
user=request.user,
overwrite=False,
charset=form.cleaned_data["charset"],
charset=input_charset,
abstract=abstract,
title=title,
metadata_uploaded_preserve=form.cleaned_data[
Expand All @@ -222,6 +227,7 @@ def layer_upload(request, template='upload/layer_upload.html'):
exception_type, error, tb = sys.exc_info()
logger.exception(e)
out['success'] = False
out['errormsgs'] = _('Failed to upload the layer')
try:
out['errors'] = u''.join(error).encode('utf-8')
except BaseException:
Expand All @@ -240,36 +246,44 @@ def layer_upload(request, template='upload/layer_upload.html'):
user=request.user).order_by('-date')
if latest_uploads.count() > 0:
upload_session = latest_uploads[0]
upload_session.error = pickle.dumps(error).decode("utf-8", "replace")
# Ref issue #4232
if not isinstance(error, TracebackType):
upload_session.error = pickle.dumps(error).decode("utf-8", "replace")
else:
err_msg = 'The error could not be parsed'
upload_session.error = err_msg
logger.error("TypeError: can't pickle traceback objects")
upload_session.traceback = traceback.format_exc(tb)
upload_session.context = log_snippet(CONTEXT_LOG_FILE)
upload_session.save()
out['traceback'] = upload_session.traceback
out['context'] = upload_session.context
out['upload_session'] = upload_session.id
else:
out['success'] = True
if hasattr(saved_layer, 'info'):
out['info'] = saved_layer.info
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
if hasattr(saved_layer, 'bbox_string'):
out['bbox'] = saved_layer.bbox_string
if hasattr(saved_layer, 'srid'):
out['crs'] = {
'type': 'name',
'properties': saved_layer.srid
}
out['ogc_backend'] = settings.OGC_SERVER['default']['BACKEND']
upload_session = saved_layer.upload_session
if upload_session:
upload_session.processed = True
upload_session.save()
permissions = form.cleaned_data["permissions"]
if permissions is not None and len(permissions.keys()) > 0:
saved_layer.set_permissions(permissions)
saved_layer.handle_moderated_uploads()
# Prevent calls to None
if saved_layer:
out['success'] = True
if hasattr(saved_layer, 'info'):
out['info'] = saved_layer.info
out['url'] = reverse(
'layer_detail', args=[
saved_layer.service_typename])
if hasattr(saved_layer, 'bbox_string'):
out['bbox'] = saved_layer.bbox_string
if hasattr(saved_layer, 'srid'):
out['crs'] = {
'type': 'name',
'properties': saved_layer.srid
}
out['ogc_backend'] = settings.OGC_SERVER['default']['BACKEND']
upload_session = saved_layer.upload_session
if upload_session:
upload_session.processed = True
upload_session.save()
permissions = form.cleaned_data["permissions"]
if permissions is not None and len(permissions.keys()) > 0:
saved_layer.set_permissions(permissions)
saved_layer.handle_moderated_uploads()
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
Expand All @@ -283,20 +297,40 @@ def layer_upload(request, template='upload/layer_upload.html'):
else:
status_code = 400
if settings.MONITORING_ENABLED:
if saved_layer or name:
layer_name = saved_layer.alternate if hasattr(
saved_layer, 'alternate') else name
layer_name = None
if saved_layer and hasattr(saved_layer, 'alternate'):
layer_name = saved_layer.alternate
elif name:
layer_name = name
if layer_name:
request.add_resource('layer', layer_name)

# null-safe charset
layer_charset = 'UTF-8'
if saved_layer and hasattr(saved_layer, 'charset'):
layer_charset = saved_layer.charset
elif input_charset:
layer_charset = input_charset

_keys = ['info', 'errors']
for _k in _keys:
if _k in out:
if isinstance(out[_k], unicode) or isinstance(
out[_k], str):
out[_k] = out[_k].decode(saved_layer.charset).encode("utf-8")
out[_k] = out[_k].decode(layer_charset).encode("utf-8")
elif isinstance(out[_k], dict):
for key, value in out[_k].iteritems():
out[_k][key] = out[_k][key].decode(saved_layer.charset).encode("utf-8")
out[_k][key.decode(saved_layer.charset).encode("utf-8")] = out[_k].pop(key)
try:
item = out[_k][key]
# Ref issue #4241
if isinstance(item, ErrorList):
out[_k][key] = item.as_text().decode(layer_charset).encode("utf-8")
else:
out[_k][key] = item.decode(layer_charset).encode("utf-8")
out[_k][key.decode(layer_charset).encode("utf-8")] = out[_k].pop(key)
except BaseException as e:
logger.exception(e)

return HttpResponse(
json.dumps(out),
content_type='application/json',
Expand Down

0 comments on commit 7f88674

Please sign in to comment.