CSV bills import (cospend compatible) (#951)

* proper import form (fix messy errors)
* csv compatible import
* cospend compatible import
* localization (best effort)
* refactoring
* revert localization (best effort)
* import return 400 on error
* fix Person.query.get_by_ids calls
* Bill explicit init parameters
* fix tests
* refacto tests with self.get_project
* separate import tests
* fix tests
* csv import test case
* fix import csv parsing
* revert DestructiveActionProjectForm renaming
* fix csv import test
* fix error redirection on import
* fix lint
* import file input type hint
* various fixes from review

Co-authored-by: Youe Graillot <youe.graillot@gmail.com>

ihatemoney/api/common.py

@@ -159,8 +159,7 @@ def get(self, project):
     def post(self, project):
         form = get_billform_for(project, True, meta={"csrf": False})
         if form.validate():
-            bill = Bill()
-            form.save(bill, project)
+            bill = form.export(project)
             db.session.add(bill)
             db.session.commit()
             return bill.id, 201

ihatemoney/forms.py

@@ -23,7 +23,7 @@
 )
 
 from ihatemoney.currency_convertor import CurrencyConverter
-from ihatemoney.models import LoggingMode, Person, Project
+from ihatemoney.models import Bill, LoggingMode, Person, Project
 from ihatemoney.utils import (
     eval_arithmetic_expression,
     render_localized_currency,
@@ -182,13 +182,15 @@ def update(self, project):
         return project
 
 
-class UploadForm(FlaskForm):
+class ImportProjectForm(FlaskForm):
     file = FileField(
-        "JSON",
-        validators=[FileRequired(), FileAllowed(["json", "JSON"], "JSON only!")],
-        description=_("Import previously exported JSON file"),
+        "File",
+        validators=[
+            FileRequired(),
+            FileAllowed(["json", "JSON", "csv", "CSV"], "Incorrect file format"),
+        ],
+        description=_("Compatible with Cospend"),
     )
-    submit = SubmitField(_("Import"))
 
 
 class ProjectForm(EditProjectForm):
@@ -319,33 +321,31 @@ class BillForm(FlaskForm):
     submit = SubmitField(_("Submit"))
     submit2 = SubmitField(_("Submit and add a new one"))
 
+    def export(self, project):
+        return Bill(
+            amount=float(self.amount.data),
+            date=self.date.data,
+            external_link=self.external_link.data,
+            original_currency=str(self.original_currency.data),
+            owers=Person.query.get_by_ids(self.payed_for.data, project),
+            payer_id=self.payer.data,
+            project_default_currency=project.default_currency,
+            what=self.what.data,
+        )
+
     def save(self, bill, project):
         bill.payer_id = self.payer.data
         bill.amount = self.amount.data
         bill.what = self.what.data
         bill.external_link = self.external_link.data
         bill.date = self.date.data
-        bill.owers = [Person.query.get(ower, project) for ower in self.payed_for.data]
+        bill.owers = Person.query.get_by_ids(self.payed_for.data, project)
         bill.original_currency = self.original_currency.data
         bill.converted_amount = self.currency_helper.exchange_currency(
             bill.amount, bill.original_currency, project.default_currency
         )
         return bill
 
-    def fake_form(self, bill, project):
-        bill.payer_id = self.payer
-        bill.amount = self.amount
-        bill.what = self.what
-        bill.external_link = ""
-        bill.date = self.date
-        bill.owers = [Person.query.get(ower, project) for ower in self.payed_for]
-        bill.original_currency = self.original_currency
-        bill.converted_amount = self.currency_helper.exchange_currency(
-            bill.amount, bill.original_currency, project.default_currency
-        )
-
-        return bill
-
     def fill(self, bill, project):
         self.payer.data = bill.payer_id
         self.amount.data = bill.amount

ihatemoney/models.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from datetime import datetime
 
+from dateutil.parser import parse
 from debts import settle
 from flask import current_app, g
 from flask_sqlalchemy import BaseQuery, SQLAlchemy
@@ -19,6 +20,7 @@
 
 from ihatemoney.currency_convertor import CurrencyConverter
 from ihatemoney.patch_sqlalchemy_continuum import PatchedBuilder
+from ihatemoney.utils import get_members, same_bill
 from ihatemoney.versioning import (
     ConditionalVersioningManager,
     LoggingMode,
@@ -320,6 +322,44 @@ def switch_currency(self, new_currency):
         db.session.add(self)
         db.session.commit()
 
+    def import_bills(self, bills: list):
+        """Import bills from a list of dictionaries"""
+        # Add members not already in the project
+        project_members = [str(m) for m in self.members]
+        new_members = [
+            m for m in get_members(bills) if str(m[0]) not in project_members
+        ]
+        for m in new_members:
+            Person(name=m[0], project=self, weight=m[1])
+        db.session.commit()
+
+        # Import bills not already in the project
+        project_bills = self.get_pretty_bills()
+        id_dict = {m.name: m.id for m in self.members}
+        for b in bills:
+            same = False
+            for p_b in project_bills:
+                if same_bill(p_b, b):
+                    same = True
+                    break
+            if not same:
+                # Create bills
+                try:
+                    new_bill = Bill(
+                        amount=b["amount"],
+                        date=parse(b["date"]),
+                        external_link="",
+                        original_currency=b["currency"],
+                        owers=Person.query.get_by_names(b["owers"], self),
+                        payer_id=id_dict[b["payer_name"]],
+                        project_default_currency=self.default_currency,
+                        what=b["what"],
+                    )
+                except Exception as e:
+                    raise ValueError(f"Unable to import csv data: {repr(e)}")
+                db.session.add(new_bill)
+        db.session.commit()
+
     def remove_member(self, member_id):
         """Remove a member from the project.
 
@@ -435,16 +475,17 @@ def create_demo_project():
             ("Alice", 20, ("Amina", "Alice"), "Beer !"),
             ("Amina", 50, ("Amina", "Alice", "Georg"), "AMAP"),
         )
-        for (payer, amount, owers, subject) in operations:
-            bill = Bill()
-            bill.payer_id = members[payer].id
-            bill.what = subject
-            bill.owers = [members[name] for name in owers]
-            bill.amount = amount
-            bill.original_currency = "XXX"
-            bill.converted_amount = amount
-
-            db.session.add(bill)
+        for (payer, amount, owers, what) in operations:
+            db.session.add(
+                Bill(
+                    amount=amount,
+                    original_currency=project.default_currency,
+                    owers=[members[name] for name in owers],
+                    payer_id=members[payer].id,
+                    project_default_currency=project.default_currency,
+                    what=what,
+                )
+            )
 
         db.session.commit()
         return project
@@ -459,6 +500,13 @@ def get_by_name(self, name, project):
                 .one_or_none()
             )
 
+        def get_by_names(self, names, project):
+            return (
+                Person.query.filter(Person.name.in_(names))
+                .filter(Person.project_id == project.id)
+                .all()
+            )
+
         def get(self, id, project=None):
             if not project:
                 project = g.project
@@ -468,6 +516,15 @@ def get(self, id, project=None):
                 .one_or_none()
             )
 
+        def get_by_ids(self, ids, project=None):
+            if not project:
+                project = g.project
+            return (
+                Person.query.filter(Person.id.in_(ids))
+                .filter(Person.project_id == project.id)
+                .all()
+            )
+
     query_class = PersonQuery
 
     # Direct SQLAlchemy-Continuum to track changes to this model
@@ -561,6 +618,31 @@ def delete(self, project, id):
 
     archive = db.Column(db.Integer, db.ForeignKey("archive.id"))
 
+    currency_helper = CurrencyConverter()
+
+    def __init__(
+        self,
+        amount: float,
+        date: datetime = None,
+        external_link: str = "",
+        original_currency: str = "",
+        owers: list = [],
+        payer_id: int = None,
+        project_default_currency: str = "",
+        what: str = "",
+    ):
+        super().__init__()
+        self.amount = amount
+        self.date = date
+        self.external_link = external_link
+        self.original_currency = original_currency
+        self.owers = owers
+        self.payer_id = payer_id
+        self.what = what
+        self.converted_amount = self.currency_helper.exchange_currency(
+            self.amount, self.original_currency, project_default_currency
+        )
+
     @property
     def _to_serialize(self):
         return {

ihatemoney/templates/edit_project.html

@@ -29,7 +29,7 @@
 <div class="container edit-project">
 
     <h2>{{ _("Edit project") }}</h2>
-    <form class="form-horizontal" method="post">
+    <form id="edit-project" class="form-horizontal" method="post">
         {{ forms.edit_project(edit_form) }}
     </form>
 
@@ -38,23 +38,10 @@ <h2>{{ _("Delete project") }}</h2>
         {{ forms.delete_project(delete_form) }}
     </form>
 
-    <h2>{{ _("Import JSON") }}</h2>
-    <form class="form-horizontal" method="post" enctype="multipart/form-data">
-        {{ import_form.hidden_tag() }}
 
-        <div class="custom-file">
-            <div class="form-group">
-                {{ import_form.file(class="custom-file-input") }}
-                <small class="form-text text-muted">
-                    {{ import_form.file.description }}
-                </small>
-            </div>
-            <label class="custom-file-label" for="customFile">{{ _('Choose file') }}</label>
-        </div>
-
-        <div class="actions">
-            {{ import_form.submit(class="btn btn-primary") }}
-        </div>
+    <h2>{{ _("Import project") }}</h2>
+    <form id="import-project" class="form-horizontal" action="{{ url_for(".import_project") }}" method="post" enctype="multipart/form-data">
+        {{ forms.import_project(import_form) }}
     </form>
 
     <h2>{{ _("Download project's data") }}</h2>

ihatemoney/templates/forms.html

@@ -118,13 +118,27 @@
 
 {% endmacro %}
 
-{% macro upload_json(form) %}
+{% macro import_project(form) %}
+
     {% include "display_errors.html" %}
     {{ form.hidden_tag() }}
-    {{ form.file }}
+
+    <p><strong>{{ _("Import previously exported project") }}</strong></p>
+
+    <div class="custom-file">
+        <div class="form-group">
+            {{ form.file(class="custom-file-input", accept=".json,.csv") }}
+            <small class="form-text text-muted">
+                {{ form.file.description }}
+            </small>
+        </div>
+        <label class="custom-file-label" for="customFile">{{ _('Choose file') }}</label>
+    </div>
+
     <div class="actions">
-        <button class="btn btn-primary">{{ _("Import") }}</button>
+        <button class="btn btn-primary">{{ _("Import project") }}</button>
     </div>
+
 {% endmacro %}
 
 {% macro delete_project_history(form) %}