Fix tasks + parsers shapefile

geotrek/common/tasks.py

@@ -15,13 +15,8 @@ class GeotrekImportTask(Task):
      to be displayed on the web interface.
     '''
     def on_failure(self, exc, task_id, args, kwargs, einfo):
-        try:
-            filename, class_name, module_name, user_pk = args
-
-        except ValueError:
-            class_name, module_name, user_pk = args
-            filename = ''
-
+        filename = kwargs.get('filename', '')
+        class_name = kwargs.get('name', '')
         self.update_state(
             task_id,
             'FAILURE',
@@ -36,7 +31,12 @@ def on_failure(self, exc, task_id, args, kwargs, einfo):
 
 
 @shared_task(base=GeotrekImportTask, name='geotrek.common.import-file')
-def import_datas(class_name, filename, module_name="bulkimport.parsers", encoding='utf8', user_pk=None):
+def import_datas(**kwargs):
+    class_name = kwargs.get('name')
+    filename = kwargs.get('filename')
+    module_name = kwargs.get('module')
+    encoding = kwargs.get('encoding')
+    user_pk = kwargs.get('user', None)
     try:
         module = importlib.import_module(module_name)
         Parser = getattr(module, class_name)
@@ -77,7 +77,10 @@ def progress_cb(progress, line, eid):
 
 
 @shared_task(base=GeotrekImportTask, name='geotrek.common.import-web')
-def import_datas_from_web(class_name, module_name="bulkimport.parsers", user_pk=None):
+def import_datas_from_web(**kwargs):
+    class_name = kwargs.get('name')
+    module_name = kwargs.get('module')
+    user_pk = kwargs.get('user', None)
     try:
         module = importlib.import_module(module_name)
         Parser = getattr(module, class_name)

geotrek/common/tests/test_tasks.py

@@ -1,24 +1,52 @@
 # -*- encoding: utf-8 -*-
 
 from django.test import TestCase
-from geotrek.common.tasks import import_datas
-from geotrek.common.models import FileType
+from geotrek.common.tasks import import_datas, import_datas_from_web
+from geotrek.common.models import Organism
+from geotrek.common.parsers import ExcelParser, GlobalImportError
 
 
-class TasksTest(TestCase):
-    def setUp(self):
-        self.filetype = FileType.objects.create(type=u"Photographie")
+class OrganismParser(ExcelParser):
+    model = Organism
+    fields = {'organism': 'nOm'}
+
 
-    def test_import_exceptions(self):
-        self.assertRaises(
-            ImportError, import_datas, filename='bombadil', class_name='haricot', module_name='toto')
+class TasksTest(TestCase):
+    def test_import_datas_message_exception(self):
+        self.assertRaisesMessage(
+            ImportError,
+            "Failed to import parser class 'haricot' from module 'toto'",
+            import_datas,
+            filename='bombadil',
+            name='haricot',
+            module='toto'
+        )
 
-    def test_import_message_exception(self):
+    def test_import_datas_from_web_message_exception(self):
         self.assertRaisesMessage(
             ImportError,
             "Failed to import parser class 'haricot' from module 'toto'",
+            import_datas_from_web,
+            filename='bombadil',
+            name='haricot',
+            module='toto'
+        )
+
+    def test_import_datas_from_web_other_exception(self):
+        self.assertRaisesMessage(
+            GlobalImportError,
+            'Filename is required',
+            import_datas_from_web,
+            name='OrganismParser',
+            module='geotrek.common.tests.test_tasks'
+        )
+
+    def test_import_datas_other_exception(self):
+        self.assertRaisesMessage(
+            GlobalImportError,
+            'File does not exists at: bombadil',
             import_datas,
             filename='bombadil',
-            class_name='haricot',
-            module_name='toto'
+            name='OrganismParser',
+            module='geotrek.common.tests.test_tasks'
         )

geotrek/common/tests/test_views.py

@@ -107,7 +107,7 @@ def test_import_form_no_parser_no_superuser(self):
         self.assertEqual(response_real.status_code, 200)
         self.assertNotContains(response_real, '<form  method="post"')
 
-    def test_import_from_web(self):
+    def test_import_from_web_bad_parser(self):
         self.user.is_superuser = True
         self.user.save()
 
@@ -116,8 +116,25 @@ def test_import_from_web(self):
         response_real = self.client.post(
             url, {
                 'import-web': 'Upload',
-                'without-file-parser': '13',
+                'without-file-parser': '99',
             }
         )
         self.assertEqual(response_real.status_code, 200)
+        self.assertContains(response_real, "Select a valid choice. 99 is not one of the available choices.")
         # There is no parser available for user not superuser
+
+    def test_import_from_web_good_parser(self):
+        self.user.is_superuser = True
+        self.user.save()
+
+        url = reverse('common:import_dataset')
+        real_key = dict(self.client.get(url).context['form_without_file'].fields['parser'].choices).keys()[0]
+        response_real = self.client.post(
+            url, {
+                'import-web': 'Upload',
+                'without-file-parser': real_key,
+            }
+        )
+        self.assertEqual(response_real.status_code, 200)
+        self.assertNotContains(response_real, "Select a valid choice. {real_key} "
+                                              "is not one of the available choices.".format(real_key=real_key))

geotrek/common/utils/import_celery.py

@@ -44,7 +44,7 @@ def discover_available_parsers(user):
             if not cls.label or not cls.model:
                 continue
             codename = '{}.import_{}'.format(cls.model._meta.app_label, cls.model._meta.model_name)
-            if not user.is_superuser and not user.has_perm(codename):
+            if not user.has_perm(codename):
                 continue
             if not getattr(cls, 'url', None) and not getattr(cls, 'base_url', None):
                 choices.append((index, cls.label))

geotrek/common/views.py

@@ -193,7 +193,8 @@ def import_file(uploaded, parser, encoding, user_pk):
         for name in zfile.namelist():
             zfile.extract(name, os.path.dirname(os.path.realpath(f.name)))
             if name.endswith('shp'):
-                import_datas.delay(parser.__name__, '/'.join((destination_dir, name)), parser.__module__, encoding, user_pk)
+                import_datas.delay(name=parser.__name__, filename='/'.join((destination_dir, name)),
+                                   module=parser.__module__, encoding=encoding, user=user_pk)
 
 
 @login_required
@@ -221,9 +222,6 @@ def import_view(request):
                 uploaded = request.FILES['with-file-zipfile']
                 parser = classes[int(form['parser'].value())]
                 encoding = form.cleaned_data['encoding']
-                codename = '{}.import_{}'.format(parser.model._meta.app_label, parser.model._meta.model_name)
-                if not request.user.is_superuser and not request.user.has_perm(codename):
-                    raise PermissionDenied
                 try:
                     import_file(uploaded, parser, encoding, request.user.pk)
                 except UnicodeDecodeError:
@@ -235,11 +233,8 @@ def import_view(request):
 
             if form_without_file.is_valid():
                 parser = classes[int(form_without_file['parser'].value())]
-                codename = '{}.import_{}'.format(parser.model._meta.app_label, parser.model._meta.model_name)
-                if not request.user.is_superuser and not request.user.has_perm(codename):
-                    raise PermissionDenied
                 import_datas_from_web.delay(
-                    parser.__name__, parser.__module__, request.user.pk
+                    name=parser.__name__, module=parser.__module__, user=request.user.pk
                 )
 
     # Hide second form if parser has no web based imports.

geotrek/sensitivity/parsers.py

@@ -168,10 +168,10 @@ class RegulatorySensitiveAreaShapeParser(ShapeParser):
     fields = {
         'geom': 'geom',
         'contact': 'contact',
-        'description': 'description',
+        'description': 'descriptio',
         'species': (
             'nom',
-            'altitude'
+            'altitude',
             'periode',
             'pratiques',
             'url',