Add import/export functionality to the API

requirements.txt

@@ -21,7 +21,7 @@ diff-match-patch==20121119
 requests==2.4.1
 
 easy-thumbnails==2.1
-celery==3.1.12
+celery==3.1.17
 redis==2.10.3
 Unidecode==0.04.16
 raven==5.1.1

settings/common.py

@@ -201,6 +201,7 @@
     "rest_framework",
     "djmail",
     "django_jinja",
+    "django_jinja.contrib._humanize",
     "easy_thumbnails",
     "raven.contrib.django.raven_compat",
 ]
@@ -300,7 +301,8 @@
     "DEFAULT_THROTTLE_RATES": {
         "anon": None,
         "user": None,
-        "import-mode": None
+        "import-mode": None,
+        "import-dump-mode": "1/minute",
     },
     "FILTER_BACKEND": "taiga.base.filters.FilterBackend",
     "EXCEPTION_HANDLER": "taiga.base.exceptions.exception_handler",
@@ -362,6 +364,9 @@
 BITBUCKET_VALID_ORIGIN_IPS = ["131.103.20.165", "131.103.20.166"]
 GITLAB_VALID_ORIGIN_IPS = []
 
+EXPORTS_TTL = 60 * 60 * 24  # 24 hours
+CELERY_ENABLED = False
+
 # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE
 TEST_RUNNER="django.test.runner.DiscoverRunner"
 

settings/testing.py

@@ -28,5 +28,6 @@
 REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] = {
     "anon": None,
     "user": None,
-    "import-mode": None
+    "import-mode": None,
+    "import-dump-mode": None,
 }

taiga/base/management/commands/test_emails.py

@@ -14,7 +14,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
+
 from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
 
 from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail
 
@@ -76,6 +79,32 @@ def handle(self, *args, **options):
         email = mbuilder.change_email(test_email, context)
         email.send()
 
+        # Export/Import emails
+        context = {
+            "user": User.objects.all().order_by("?").first(),
+            "error_subject": "Error generating project dump",
+            "error_message": "Error generating project dump",
+        }
+        email = mbuilder.export_import_error(test_email, context)
+        email.send()
+
+        deletion_date = timezone.now() + datetime.timedelta(seconds=60*60*24)
+        context = {
+            "url": "http://dummyurl.com",
+            "user": User.objects.all().order_by("?").first(),
+            "project": Project.objects.all().order_by("?").first(),
+            "deletion_date": deletion_date,
+        }
+        email = mbuilder.dump_project(test_email, context)
+        email.send()
+
+        context = {
+            "user": User.objects.all().order_by("?").first(),
+            "project": Project.objects.all().order_by("?").first(),
+        }
+        email = mbuilder.load_dump(test_email, context)
+        email.send()
+
         # Notification emails
         notification_emails = [
             "issues/issue-change",

taiga/export_import/api.py

@@ -14,33 +14,71 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import json
+import codecs
+
 from rest_framework.exceptions import APIException
 from rest_framework.response import Response
+from rest_framework.decorators import throttle_classes
 from rest_framework import status
 
 from django.utils.decorators import method_decorator
+from django.utils.translation import ugettext_lazy as _
 from django.db.transaction import atomic
 from django.db.models import signals
+from django.conf import settings
 
 from taiga.base.api.mixins import CreateModelMixin
 from taiga.base.api.viewsets import GenericViewSet
-from taiga.base.decorators import detail_route
+from taiga.base.decorators import detail_route, list_route
+from taiga.base import exceptions as exc
 from taiga.projects.models import Project, Membership
 from taiga.projects.issues.models import Issue
 
 from . import mixins
 from . import serializers
 from . import service
 from . import permissions
+from . import tasks
+from . import dump_service
+from . import throttling
+
+from taiga.base.api.utils import get_object_or_404
 
 
 class Http400(APIException):
     status_code = 400
 
 
+class ProjectExporterViewSet(mixins.ImportThrottlingPolicyMixin, GenericViewSet):
+    model = Project
+    permission_classes = (permissions.ImportExportPermission, )
+
+    def retrieve(self, request, pk, *args, **kwargs):
+        throttle = throttling.ImportDumpModeRateThrottle()
+
+        if not throttle.allow_request(request, self):
+            self.throttled(request, throttle.wait())
+
+        project = get_object_or_404(self.get_queryset(), pk=pk)
+        self.check_permissions(request, 'export_project', project)
+
+        if settings.CELERY_ENABLED:
+            task = tasks.dump_project.delay(request.user, project)
+            tasks.delete_project_dump.apply_async((project.pk,), countdown=settings.EXPORTS_TTL)
+            return Response({"export-id": task.id}, status=status.HTTP_202_ACCEPTED)
+
+        return Response(
+            service.project_to_dict(project),
+            status=status.HTTP_200_OK,
+            headers={
+                "Content-Disposition": "attachment; filename={}.json".format(project.slug)
+            }
+        )
+
 class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixin, GenericViewSet):
     model = Project
-    permission_classes = (permissions.ImportPermission, )
+    permission_classes = (permissions.ImportExportPermission, )
 
     @method_decorator(atomic)
     def create(self, request, *args, **kwargs):
@@ -113,6 +151,39 @@ def create(self, request, *args, **kwargs):
         headers = self.get_success_headers(response_data)
         return Response(response_data, status=status.HTTP_201_CREATED, headers=headers)
 
+    @list_route(methods=["POST"])
+    @method_decorator(atomic)
+    def load_dump(self, request):
+        throttle = throttling.ImportDumpModeRateThrottle()
+
+        if not throttle.allow_request(request, self):
+            self.throttled(request, throttle.wait())
+
+        self.check_permissions(request, "load_dump", None)
+
+        dump = request.FILES.get('dump', None)
+
+        if not dump:
+            raise exc.WrongArguments(_("Needed dump file"))
+
+        reader = codecs.getreader("utf-8")
+
+        try:
+            dump = json.load(reader(dump))
+        except Exception:
+            raise exc.WrongArguments(_("Invalid dump format"))
+
+        if Project.objects.filter(slug=dump['slug']).exists():
+            del dump['slug']
+
+        if settings.CELERY_ENABLED:
+            task = tasks.load_project_dump.delay(request.user, dump)
+            return Response({"import-id": task.id}, status=status.HTTP_202_ACCEPTED)
+
+        dump_service.dict_to_project(dump, request.user.email)
+        return Response(None, status=status.HTTP_204_NO_CONTENT)
+
+
     @detail_route(methods=['post'])
     @method_decorator(atomic)
     def issue(self, request, *args, **kwargs):

taiga/export_import/dump_service.py

@@ -72,6 +72,12 @@ def store_issues(project, data):
     return issues
 
 
+def store_tags_colors(project, data):
+    project.tags_colors = data.get("tags_colors", [])
+    project.save()
+    return None
+
+
 def dict_to_project(data, owner=None):
     if owner:
         data['owner'] = owner
@@ -148,3 +154,7 @@ def dict_to_project(data, owner=None):
 
     if service.get_errors(clear=False):
         raise TaigaImportError('error importing issues')
+
+    store_tags_colors(proj, data)
+
+    return proj

taiga/export_import/permissions.py

@@ -19,6 +19,8 @@
                                         IsProjectOwner, IsAuthenticated)
 
 
-class ImportPermission(TaigaResourcePermission):
+class ImportExportPermission(TaigaResourcePermission):
     import_project_perms = IsAuthenticated()
     import_item_perms = IsProjectOwner()
+    export_project_perms = IsProjectOwner()
+    load_dump_perms = IsAuthenticated()

taiga/export_import/serializers.py

@@ -46,8 +46,10 @@ def to_native(self, obj):
         if not obj:
             return None
 
+        data = base64.b64encode(obj.read()).decode('utf-8')
+
         return OrderedDict([
-            ("data", base64.b64encode(obj.read()).decode('utf-8')),
+            ("data", data),
             ("name", os.path.basename(obj.name)),
         ])
 
@@ -120,7 +122,7 @@ def from_native(self, data):
 
 class HistoryUserField(JsonField):
     def to_native(self, obj):
-        if obj is None:
+        if obj is None or obj == {}:
             return []
         try:
             user = users_models.User.objects.get(pk=obj['pk'])
@@ -190,7 +192,7 @@ class HistoryExportSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = history_models.HistoryEntry
-        exclude = ("id", "comment_html")
+        exclude = ("id", "comment_html", "key")
 
 
 class HistoryExportSerializerMixin(serializers.ModelSerializer):

taiga/export_import/service.py

@@ -84,7 +84,7 @@ def store_choice(project, data, field, serializer):
 
 def store_choices(project, data, field, serializer):
     result = []
-    for choice_data in data[field]:
+    for choice_data in data.get(field, []):
         result.append(store_choice(project, choice_data, field, serializer))
     return result
 
@@ -102,7 +102,7 @@ def store_role(project, role):
 
 def store_roles(project, data):
     results = []
-    for role in data['roles']:
+    for role in data.get('roles', []):
         results.append(store_role(project, role))
     return results
 

taiga/export_import/tasks.py

@@ -0,0 +1,82 @@
+# Copyright (C) 2014 Andrey Antukh <niwi@niwi.be>
+# Copyright (C) 2014 Jesús Espino <jespinog@gmail.com>
+# Copyright (C) 2014 David Barragán <bameda@dbarragan.com>
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+
+from django.core.files.storage import default_storage
+from django.core.files.base import ContentFile
+from django.utils import timezone
+from django.conf import settings
+
+from djmail.template_mail import MagicMailBuilder
+
+from taiga.celery import app
+
+from .service import project_to_dict
+from .dump_service import dict_to_project
+from .renderers import ExportRenderer
+
+
+@app.task(bind=True)
+def dump_project(self, user, project):
+    mbuilder = MagicMailBuilder()
+
+    path = "exports/{}/{}.json".format(project.pk, self.request.id)
+
+    try:
+        content = ContentFile(ExportRenderer().render(project_to_dict(project), renderer_context={"indent": 4}).decode('utf-8'))
+        default_storage.save(path, content)
+        url = default_storage.url(path)
+    except Exception:
+        email = mbuilder.export_import_error(
+            user.email,
+            {
+                "user": user,
+                "error_subject": "Error generating project dump",
+                "error_message": "Error generating project dump",
+            }
+        )
+        email.send()
+        return
+
+    deletion_date = timezone.now() + datetime.timedelta(seconds=settings.EXPORTS_TTL)
+    email = mbuilder.dump_project(user.email, {"url": url, "project": project, "user": user, "deletion_date": deletion_date})
+    email.send()
+
+@app.task
+def delete_project_dump(project_id, task_id):
+    default_storage.delete("exports/{}/{}.json".format(project_id, task_id))
+
+@app.task
+def load_project_dump(user, dump):
+    mbuilder = MagicMailBuilder()
+
+    try:
+        project = dict_to_project(dump, user.email)
+    except Exception:
+        email = mbuilder.export_import_error(
+            user.email,
+            {
+                "user": user,
+                "error_subject": "Error loading project dump",
+                "error_message": "Error loading project dump",
+            }
+        )
+        email.send()
+        return
+
+    email = mbuilder.load_dump(user.email, {"user": user, "project": project})
+    email.send()

taiga/export_import/templates/emails/dump_project-body-html.jinja

@@ -0,0 +1,28 @@
+{% extends "emails/base.jinja" %}
+
+{% block body %}
+<table border="0" width="100%" cellpadding="0" cellspacing="0" class="table-body">
+    <tr>
+        <td>
+            <h1>Project dump generated</h1>
+            <p>Hello {{ user.get_full_name() }},</p>
+            <h3>Your project dump has been correctly generated.</h3>
+            <p>You can download it from here: <a style="color: #669900;" href="{{ url }}">{{ url }}</a></p>
+            <p>This file will be deleted on {{ deletion_date|date("r") }}.</p>
+            <p>The Taiga Team</p>
+        </td>
+    </tr>
+</table>
+{% endblock %}
+
+{% block footer %}
+    <em>Copyright © 2014 Taiga Agile, LLC, All rights reserved.</em>
+    <br>
+    <strong>Contact us:</strong>
+    <br>
+    <strong>Support:</strong>
+    <a href="mailto:support@taiga.io" title="Taiga Support">support@taiga.io</a>
+    <br>
+    <strong>Our mailing address is:</strong>
+    <a href="https://groups.google.com/forum/#!forum/taigaio" title="Taiga mailing list">https://groups.google.com/forum/#!forum/taigaio</a>
+{% endblock %}