Merge 8a4c64a into fd5128c

bulkimport/__init__.py

@@ -0,0 +1,4 @@
+try:
+    from . import parsers
+except ImportError:
+    pass

conf/buildout-dev.cfg

@@ -19,6 +19,8 @@ eggs +=
     pydot
     pyparsing
 
+unzip=true
+
 [bpython]
 recipe = zc.recipe.egg:scripts
 eggs = ${buildout:eggs}

conf/buildout-prod.cfg

@@ -60,6 +60,15 @@ errorlog = ${django:deployroot}/var/log/${django:project}.log
 input = ${buildout:directory}/conf/gunicorn.conf.in
 output = ${django:deployroot}/etc/gunicorn-${django:project}_api.conf
 
+[gunicorn-celery-conf]
+recipe = collective.recipe.genshi
+# workers and timeout are set ``conf/settings-default.cfg`` and can be overidden
+bind = unix:${django:deployroot}/var/run/gunicorn-${django:project}_celery.sock
+pidfile = ${django:deployroot}/var/run/gunicorn-${django:project}_celery.pid
+errorlog = ${django:deployroot}/var/log/${django:project}.log
+input = ${buildout:directory}/conf/gunicorn.conf.in
+output = ${django:deployroot}/etc/gunicorn-${django:project}_celery.conf
+
 [nginx-conf]
 recipe = collective.recipe.genshi
 port = 80
@@ -99,6 +108,7 @@ geotrek-programs =
 # Priority means order of start/shutdown
     20 ${django:project} (stdout_logfile=${django:deployroot}/var/log/${django:project}.log stderr_logfile=${django:deployroot}/var/log/${django:project}.log) ${django:deployroot}/bin/django [run_gunicorn --config=${gunicorn-app-conf:output}] ${django:deployroot}/etc true
     20 ${django:project}_api (stdout_logfile=${django:deployroot}/var/log/${django:project}.log stderr_logfile=${django:deployroot}/var/log/${django:project}.log) ${django:deployroot}/bin/django [run_gunicorn --config=${gunicorn-api-conf:output}] ${django:deployroot}/etc true
+    20 ${django:project}_celery (stdout_logfile=${django:deployroot}/var/log/${django:project}_celery.log stderr_logfile=${django:deployroot}/var/log/${django:project}_celery.log) ${django:deployroot}/bin/django [celery worker] ${django:deployroot}/etc true
 programs =
     ${supervisor:geotrek-programs}
 

geotrek/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+
 pkg_resources = __import__('pkg_resources')
 distribution = pkg_resources.get_distribution('geotrek')
 

geotrek/celery.py

@@ -0,0 +1,22 @@
+from __future__ import absolute_import
+
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'geotrek.settings.default')
+
+from django.conf import settings
+
+app = Celery('geotrek')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))

geotrek/common/forms.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from django import forms as django_forms
 from django.db.models.fields.related import ForeignKey, ManyToManyField, FieldDoesNotExist
 
@@ -9,6 +10,10 @@
 
 from .mixins import NoDeleteMixin
 
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Layout, Div, Submit
+from crispy_forms.bootstrap import FormActions
+
 
 class CommonForm(MapEntityForm):
 
@@ -68,3 +73,39 @@ def __init__(self, *args, **kwargs):
 
         for name, field in self.fields.items():
             self.filter_related_field(name, field)
+
+
+class ImportDatasetForm(django_forms.Form):
+    parser = forms.TypedChoiceField(
+        label='Parseur',
+        widget=forms.RadioSelect,
+        required=True,
+    )
+    zipfile = forms.FileField(
+        label='Fichier',
+        required=True,
+        widget=forms.FileInput
+    )
+
+    def __init__(self, choices=None, *args, **kwargs):
+            super(ImportDatasetForm, self).__init__(*args, **kwargs)
+
+            self.fields['parser'].choices = choices
+
+            self.helper = FormHelper()
+            self.helper.layout = Layout(
+                Div(
+                    Div(
+                        'parser',
+                        'zipfile',
+                    ),
+                    FormActions(
+                        Submit('submit', u"Transférer", css_class='button white')
+                    ),
+                    css_class='file-attachment-form',
+                )
+            )
+
+    def clean_zipfile(self):
+        if self.cleaned_data['zipfile'].content_type != "application/zip":
+            raise django_forms.ValidationError("File must be of ZIP type.", code='invalid')

geotrek/common/static/common/css/import.css

@@ -0,0 +1,21 @@
+#progress-bars {
+	margin-top: 1.5em;
+}
+
+#progress-bars .close {
+	display: block;
+	float: none;
+	text-align: right;
+	width: 100%;
+}
+
+#progress-bars div.pull-left {
+	width: 35px;
+	margin-left: 5px;
+	margin-right: 5px;
+}
+
+#progress-bars div.alert-error {
+	margin-left: 45px;
+	display: none;
+}

geotrek/common/static/common/js/import.js

@@ -0,0 +1,43 @@
+function updateImportProgressBars() {
+	$.getJSON('/import-update.json', function(json) {
+		parent = document.querySelector('#progress-bars');
+		json.forEach(function(row) {
+			var local_percent = row.result.current + "%";
+			var status_class = 'pogress-info';
+
+			if (row.status == 'SUCCESS') {
+				local_percent = "100%";
+				status_class = "progress-success";
+			} else if (row.status == 'FAILURE') {
+				local_percent = "100%";
+				status_class = "progress-danger";
+			}
+
+			if (element = document.getElementById(row.id)) {
+				element.querySelector('.bar').style.width = local_percent;
+				element.querySelector('.pull-left').innerHTML = local_percent;
+			} else {
+				element = document.createElement('div');
+				element.innerHTML = document.querySelector('#import-template').innerHTML
+				element.id = row.id;
+				element.querySelector('.bar').style.width = local_percent;
+				element.querySelector('.pull-left').innerHTML = local_percent;
+
+				parent.appendChild(element);
+			}
+			if (row.result.exc_message) {
+				element.querySelector('.alert span').innerHTML = "Error message : " + row.result.exc_message;
+				element.querySelector('.alert').style.display = 'block';
+			}
+
+			if (!element.querySelector('.progress').classList.contains(status_class)) {
+				element.querySelector('.progress').classList.add(status_class);
+			}
+		});
+	});
+}
+
+$(document).ready(function() {
+	updateImportProgressBars();
+	setInterval(updateImportProgressBars, 1000);
+});

geotrek/common/tasks.py

@@ -0,0 +1,34 @@
+from __future__ import absolute_import
+
+import importlib
+import sys
+from celery import shared_task, current_task
+
+
+@shared_task
+def import_datas(filename, class_name, module_name="bulkimport.parsers"):
+    try:
+        module = importlib.import_module(module_name)
+        Parser = getattr(module, class_name)
+    except:
+        raise ImportError("Failed to import parser class '{0}' from module '{1}'".format(
+            class_name, module_name))
+
+    def progress_cb(progress):
+        current_task.update_state(
+            state='PROGRESS',
+            meta={
+                'current': int(100 * progress),
+                'total': 100,
+            }
+        )
+        sys.stdout.write(
+            "{progress:02d}%".format(progress=int(100 * progress)))
+
+    try:
+        parser = Parser(progress_cb=progress_cb)
+        parser.parse(filename)
+    except Exception as e:
+        raise e
+
+    return

geotrek/common/templates/common/common_usermenu_fragment.html

@@ -0,0 +1,17 @@
+{% load i18n %}
+
+{{appname}}
+
+{% if appname == "common" %}
+<div class="btn-group">
+    <a class="btn dropdown-toggle" data-toggle="dropdown" href="#">
+        <strong>
+        	{% trans "Imports" %}
+        </strong>
+        <span class="caret"></span>
+    </a>
+    <ul class="dropdown-menu">
+    	<li><a href="{% url 'common:import_dataset' %}">{% trans "Imports" %}</a></li>
+    </ul>
+</div>
+{% endif %}

geotrek/common/templates/common/import_dataset.html

@@ -0,0 +1,36 @@
+{% extends "mapentity/base.html" %}
+{% load static crispy_forms_tags %}
+
+{% block extrahead %}
+	<script type="text/javascript" src="{% static "common/js/import.js" %}"></script>
+	<link rel="stylesheet" href="{% static "common/css/import.css" %}" />
+{% endblock extrahead %}
+
+{% block toolbar %}
+{% endblock toolbar %}
+
+{% block mainpanel %}
+	<div class="span3 offset2">
+    {% block mainform %}
+        {% crispy form form.helper %}
+    {% endblock mainform %}
+    </div>
+    <div id="progress-bars" class="span5">
+    </div>
+
+    <script id="import-template" type="text/template">
+	    <div id="progress-tpl">
+    		<button type="button" class="close" data-dismiss="alert">&times;</button>
+			<div class="pull-left">
+				<span></span>
+			</div>
+			<div class="progress progress-striped active">
+			  <div class="bar"></div>
+			</div>
+		    <div class="alert alert-error">
+		      <span></span>
+		    </div>
+		    <hr></hr>
+		</div>
+	</script>
+{% endblock mainpanel %}

geotrek/common/tests/test_tasks.py

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+
+from django.test import TestCase
+
+from geotrek.common.tasks import import_datas
+
+
+class TasksTest(TestCase):
+
+    def test_import_exceptions(self):
+        self.assertRaises(
+            ImportError, import_datas, filename='bombadil', class_name='haricot', module_name='toto')
+
+    def test_import_message_exception(self):
+        self.assertRaisesMessage(
+            ImportError,
+            "Failed to import parser class 'haricot' from module 'toto'",
+            import_datas,
+            filename='bombadil',
+            class_name='haricot',
+            module_name='toto'
+        )

geotrek/common/tests/test_utils.py

@@ -7,9 +7,13 @@
 
 from ..utils import almostequal, sql_extent, uniquify
 from ..utils.postgresql import debug_pg_notices
+from ..utils.import_celery import create_tmp_destination, subclasses
+
+from geotrek.common.parsers import Parser
 
 
 class UtilsTest(TestCase):
+
     def test_almostequal(self):
         self.assertTrue(almostequal(0.001, 0.002))
         self.assertFalse(almostequal(0.001, 0.002, precision=3))
@@ -18,7 +22,8 @@ def test_almostequal(self):
         self.assertFalse(almostequal(1, -1))
 
     def test_sqlextent(self):
-        ext = sql_extent("SELECT ST_Extent('LINESTRING(0 0, 10 10)'::geometry)")
+        ext = sql_extent(
+            "SELECT ST_Extent('LINESTRING(0 0, 10 10)'::geometry)")
         self.assertEqual((0.0, 0.0, 10.0, 10.0), ext)
 
     def test_uniquify(self):
@@ -37,3 +42,22 @@ def raisenotice():
         with mock.patch('geotrek.common.utils.postgresql.logger') as fake_log:
             raisenotice()
             fake_log.debug.assert_called_with('hello')
+
+    def test_subclasses(self):
+        class_list = subclasses(Parser)
+        for classname in (
+                'TrekParser',
+                'TourInSoftParser',
+                'CityParser',
+                'ExcelParser',
+                'OpenSystemParser',
+                'ShapeParser',
+                'SitraParser',
+                'TourismSystemParser'):
+            self.assert_(classname not in class_list)
+
+    def test_create_tmp_directory(self):
+        self.assertTupleEqual(
+            ('/tmp/geotrek/bombadil', '/tmp/geotrek/bombadil/bombadil'),
+            create_tmp_destination('bombadil')
+        )