Import previously exported json data (#518)

Fix #417 

* New tab upload

* Extract data from JSON

* Add users

* Black format

* Try to add bill

* Import bills

* Add french translation msg

* Black reformat missing

* Deactivated users are supported

* Test import

* Remove temp file in upload_json()

* Incomplete tests

* tests import

* Update ihatemoney/translations/fr/LC_MESSAGES/messages.po

Co-Authored-By: Rémy HUBSCHER <hubscher.remy@gmail.com>

* Remove useless variable and check json format

* Use String.IO and test for wrong json

* Remove coma

Co-authored-by: Rémy HUBSCHER <hubscher.remy@gmail.com>

.gitignore

@@ -10,4 +10,5 @@ dist
 build
 .vscode
 .env
-.pytest_cache
+.pytest_cache
+

ihatemoney/forms.py

@@ -10,6 +10,8 @@
     NumberRange,
     Optional,
 )
+from flask_wtf.file import FileField, FileAllowed, FileRequired
+
 from flask_babel import lazy_gettext as _
 from flask import request
 from werkzeug.security import generate_password_hash
@@ -110,6 +112,12 @@ def update(self, project):
         return project
 
 
+class UploadForm(FlaskForm):
+    file = FileField(
+        "JSON", validators=[FileRequired(), FileAllowed(["json", "JSON"], "JSON only!")]
+    )
+
+
 class ProjectForm(EditProjectForm):
     id = StringField(_("Project identifier"), validators=[DataRequired()])
     password = PasswordField(_("Private code"), validators=[DataRequired()])
@@ -181,6 +189,15 @@ def save(self, bill, project):
         bill.external_link = self.external_link.data
         bill.date = self.date.data
         bill.owers = [Person.query.get(ower, project) for ower in self.payed_for.data]
+        return bill
+
+    def fake_form(self, bill, project):
+        bill.payer_id = self.payer
+        bill.amount = self.amount
+        bill.what = self.what
+        bill.external_link = ""
+        bill.date = self.date
+        bill.owers = [Person.query.get(ower, project) for ower in self.payed_for]
 
         return bill
 

ihatemoney/templates/forms.html

@@ -85,6 +85,15 @@
 
 {% endmacro %}
 
+{% macro upload_json(form) %}
+    {% include "display_errors.html" %}
+    {{ form.hidden_tag() }}
+    {{ form.file }}
+    <div class="actions">
+        <button class="btn btn-primary">{{ _("Import") }}</button>
+    </div>
+{% endmacro %}
+
 {% macro add_bill(form, edit=False, title=True) %}
 
     <fieldset>

ihatemoney/templates/layout.html

@@ -42,6 +42,7 @@ <h1><a class="navbar-brand" href="{{ url_for("main.home") }}"><span>#!</span> mo
           <li class="nav-item{% if current_view == 'settle_bill' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.settle_bill") }}">{{ _("Settle") }}</a></li>
           <li class="nav-item{% if current_view == 'statistics' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.statistics") }}">{{ _("Statistics") }}</a></li>
           <li class="nav-item{% if current_view == 'edit_project' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.edit_project") }}">{{ _("Settings") }}</a></li>
+          <li class="nav-item{% if current_view == 'upload_json' %} active{% endif %}"><a class="nav-link" href="{{ url_for("main.upload_json") }}">{{ _("Import") }}</a></li>
           {% endblock %}
         {% endif %}
       </ul>

ihatemoney/templates/upload_json.html

@@ -0,0 +1,10 @@
+{% extends "layout.html" %}
+
+{% block content %}
+    <h2>{{ _("Import JSON") }}</h2>
+    <p>
+    <form class="form-horizontal" method="post" enctype="multipart/form-data">
+        {{ forms.upload_json(form) }}
+    </form>
+    </p>
+{% endblock %}

ihatemoney/tests/tests.py

@@ -1190,6 +1190,215 @@ def test_export(self):
         resp = self.client.get("/raclette/export/transactions.wrong")
         self.assertEqual(resp.status_code, 404)
 
+    def test_import_new_project(self):
+        # Import JSON in an empty project
+
+        self.post_project("raclette")
+        self.login("raclette")
+
+        project = models.Project.query.get("raclette")
+
+        json_to_import = [
+            {
+                "date": "2017-01-01",
+                "what": "refund",
+                "amount": 13.33,
+                "payer_name": "tata",
+                "payer_weight": 1.0,
+                "owers": ["fred"],
+            },
+            {
+                "date": "2016-12-31",
+                "what": "red wine",
+                "amount": 200.0,
+                "payer_name": "fred",
+                "payer_weight": 1.0,
+                "owers": ["alexis", "tata"],
+            },
+            {
+                "date": "2016-12-31",
+                "what": "fromage a raclette",
+                "amount": 10.0,
+                "payer_name": "alexis",
+                "payer_weight": 2.0,
+                "owers": ["alexis", "fred", "tata", "pepe"],
+            },
+        ]
+
+        from ihatemoney.web import import_project
+
+        file = io.StringIO()
+        json.dump(json_to_import, file)
+        file.seek(0)
+        import_project(file, project)
+
+        bills = project.get_pretty_bills()
+
+        # Check if all bills has been add
+        self.assertEqual(len(bills), len(json_to_import))
+
+        # Check if name of bills are ok
+        b = [e["what"] for e in bills]
+        b.sort()
+        ref = [e["what"] for e in json_to_import]
+        ref.sort()
+
+        self.assertEqual(b, ref)
+
+        # Check if other informations in bill are ok
+        for i in json_to_import:
+            for j in bills:
+                if j["what"] == i["what"]:
+                    self.assertEqual(j["payer_name"], i["payer_name"])
+                    self.assertEqual(j["amount"], i["amount"])
+                    self.assertEqual(j["payer_weight"], i["payer_weight"])
+                    self.assertEqual(j["date"], i["date"])
+
+                    list_project = [ower for ower in j["owers"]]
+                    list_project.sort()
+                    list_json = [ower for ower in i["owers"]]
+                    list_json.sort()
+
+                    self.assertEqual(list_project, list_json)
+
+    def test_import_partial_project(self):
+        # Import a JSON in a project with already existing data
+
+        self.post_project("raclette")
+        self.login("raclette")
+
+        project = models.Project.query.get("raclette")
+
+        self.client.post("/raclette/members/add", data={"name": "alexis", "weight": 2})
+        self.client.post("/raclette/members/add", data={"name": "fred"})
+        self.client.post("/raclette/members/add", data={"name": "tata"})
+        self.client.post(
+            "/raclette/add",
+            data={
+                "date": "2016-12-31",
+                "what": "red wine",
+                "payer": 2,
+                "payed_for": [1, 3],
+                "amount": "200",
+            },
+        )
+
+        json_to_import = [
+            {
+                "date": "2017-01-01",
+                "what": "refund",
+                "amount": 13.33,
+                "payer_name": "tata",
+                "payer_weight": 1.0,
+                "owers": ["fred"],
+            },
+            {  # This expense does not have to be present twice.
+                "date": "2016-12-31",
+                "what": "red wine",
+                "amount": 200.0,
+                "payer_name": "fred",
+                "payer_weight": 1.0,
+                "owers": ["alexis", "tata"],
+            },
+            {
+                "date": "2016-12-31",
+                "what": "fromage a raclette",
+                "amount": 10.0,
+                "payer_name": "alexis",
+                "payer_weight": 2.0,
+                "owers": ["alexis", "fred", "tata", "pepe"],
+            },
+        ]
+
+        from ihatemoney.web import import_project
+
+        file = io.StringIO()
+        json.dump(json_to_import, file)
+        file.seek(0)
+        import_project(file, project)
+
+        bills = project.get_pretty_bills()
+
+        # Check if all bills has been add
+        self.assertEqual(len(bills), len(json_to_import))
+
+        # Check if name of bills are ok
+        b = [e["what"] for e in bills]
+        b.sort()
+        ref = [e["what"] for e in json_to_import]
+        ref.sort()
+
+        self.assertEqual(b, ref)
+
+        # Check if other informations in bill are ok
+        for i in json_to_import:
+            for j in bills:
+                if j["what"] == i["what"]:
+                    self.assertEqual(j["payer_name"], i["payer_name"])
+                    self.assertEqual(j["amount"], i["amount"])
+                    self.assertEqual(j["payer_weight"], i["payer_weight"])
+                    self.assertEqual(j["date"], i["date"])
+
+                    list_project = [ower for ower in j["owers"]]
+                    list_project.sort()
+                    list_json = [ower for ower in i["owers"]]
+                    list_json.sort()
+
+                    self.assertEqual(list_project, list_json)
+
+    def test_import_wrong_json(self):
+        self.post_project("raclette")
+        self.login("raclette")
+
+        project = models.Project.query.get("raclette")
+
+        json_1 = [
+            {  # wrong keys
+                "checked": False,
+                "dimensions": {"width": 5, "height": 10},
+                "id": 1,
+                "name": "A green door",
+                "price": 12.5,
+                "tags": ["home", "green"],
+            }
+        ]
+
+        json_2 = [
+            {  # amount missing
+                "date": "2017-01-01",
+                "what": "refund",
+                "payer_name": "tata",
+                "payer_weight": 1.0,
+                "owers": ["fred"],
+            }
+        ]
+
+        from ihatemoney.web import import_project
+
+        try:
+            file = io.StringIO()
+            json.dump(json_1, file)
+            file.seek(0)
+            import_project(file, project)
+        except ValueError:
+            self.assertTrue(True)
+        except Exception:
+            self.fail("unexpected exception raised")
+        else:
+            self.fail("ExpectedException not raised")
+
+        try:
+            file = io.StringIO()
+            json.dump(json_2, file)
+            file.seek(0)
+            import_project(file, project)
+        except ValueError:
+            self.assertTrue(True)
+        except Exception:
+            self.fail("unexpected exception raised")
+        else:
+            self.fail("ExpectedException not raised")
+
 
 class APITestCase(IhatemoneyTestCase):
 

ihatemoney/translations/fr/LC_MESSAGES/messages.po

@@ -539,6 +539,16 @@ msgstr "A dépensé"
 msgid "Balance"
 msgstr "Solde"
 
+msgid "Import"
+msgstr "Importer"
+
+msgid "Project successfully uploaded"
+msgstr "Le projet a été correctement importé"
+
+msgid "Invalid JSON"
+msgstr "Le fichier JSON est invalide"
+
+
 #~ msgid ""
 #~ "The project identifier is used to "
 #~ "log in and for the URL of "

ihatemoney/utils.py

@@ -4,13 +4,15 @@
 import operator
 
 from io import BytesIO, StringIO
+
 import jinja2
 from json import dumps, JSONEncoder
 from flask import redirect, current_app
 from babel import Locale
 from werkzeug.routing import HTTPException, RoutingException
 from datetime import datetime, timedelta
 
+
 import csv
 
 
@@ -234,3 +236,24 @@ def _eval(node):
         raise ValueError("Error evaluating expression: {}".format(expr))
 
     return result
+
+
+def get_members(file):
+    members_list = list()
+    for item in file:
+        if (item["payer_name"], item["payer_weight"]) not in members_list:
+            members_list.append((item["payer_name"], item["payer_weight"]))
+    for item in file:
+        for ower in item["owers"]:
+            if ower not in [i[0] for i in members_list]:
+                members_list.append((ower, 1))
+
+    return members_list
+
+
+def same_bill(bill1, bill2):
+    attr = ["what", "payer_name", "payer_weight", "amount", "date", "owers"]
+    for a in attr:
+        if bill1[a] != bill2[a]:
+            return False
+    return True