Merge commit 'refs/pull/518/head' of https://github.com/oca/bank-stat…

…ement-import into 15.0-1012

account_statement_import_txt_xlsx/__manifest__.py

@@ -5,7 +5,7 @@
 {
     "name": "Bank Statement TXT/CSV/XLSX Import",
     "summary": "Import TXT/CSV or XLSX files as Bank Statements in Odoo",
-    "version": "15.0.1.0.0",
+    "version": "15.0.1.1.0",
     "category": "Accounting",
     "website": "https://github.com/OCA/bank-statement-import",
     "author": "ForgeFlow, CorporateHub, Odoo Community Association (OCA)",

account_statement_import_txt_xlsx/data/map_data.xml

@@ -7,12 +7,15 @@
 <odoo noupdate="1">
     <record id="sample_statement_map" model="account.statement.import.sheet.mapping">
         <field name="name">Sample Statement</field>
+        <field name="footer_lines_count">0</field>
+        <field name="column_labels_row">1</field>
         <field name="float_thousands_sep">comma</field>
         <field name="float_decimal_sep">dot</field>
         <field name="delimiter">comma</field>
         <field name="quotechar">"</field>
         <field name="timestamp_format">%m/%d/%Y</field>
         <field name="timestamp_column">Date</field>
+        <field name="amount_type">simple_value</field>
         <field name="amount_column">Amount</field>
         <field name="original_currency_column">Currency</field>
         <field name="original_amount_column">Amount Currency</field>

account_statement_import_txt_xlsx/migrations/15.0.1.1.0/post-migration.py

@@ -0,0 +1,25 @@
+# Copyright 2020 Akretion
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+
+from openupgradelib import openupgrade
+
+
+@openupgrade.migrate()
+def migrate(env, version):
+    openupgrade.logged_query(
+        env.cr,
+        """
+UPDATE account_statement_import_sheet_mapping
+    SET amount_type = 'absolute_value'
+    WHERE debit_credit_column IS NOT NULL;
+       """,
+    )
+
+    openupgrade.logged_query(
+        env.cr,
+        """
+UPDATE account_statement_import_sheet_mapping
+    SET amount_type = 'simple_value'
+    WHERE debit_credit_column IS NULL;
+       """,
+    )

account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py

@@ -83,10 +83,43 @@ class AccountStatementImportSheetMapping(models.Model):
             "transaction amount in original transaction currency from"
         ),
     )
+    amount_type = fields.Selection(
+        selection=[
+            ("simple_value", "Simple value"),
+            ("absolute_value", "Absolute value"),
+            ("distinct_credit_debit", "Distinct Credit/debit Column"),
+        ],
+        string="Amount type",
+        required=True,
+        default="simple_value",
+        help=(
+            "Simple value: use igned amount in ammount comlumn\n"
+            "Absolute Value: use a same comlumn for debit and credit\n"
+            "(absolute value + indicate sign)\n"
+            "Distinct Credit/debit Column: use a distinct comlumn for debit and credit"
+        ),
+    )
+    amount_column = fields.Char(
+        string="Amount column",
+        help=(
+            'Used if amount type is "Simple value" or "Absolute value"\n'
+            "Amount of transaction in journal's currency\n"
+            "Some statement formats use credit/debit columns"
+        ),
+    )
+    debit_column = fields.Char(
+        string="Debit column",
+        help='Used if amount type is "Distinct Credit/debit Column"',
+    )
+    credit_column = fields.Char(
+        string="Credit column",
+        help='Used if amount type is "Distinct Credit/debit Column"\n',
+    )
     debit_credit_column = fields.Char(
         string="Debit/credit column",
         help=(
-            "Some statement formats use absolute amount value and indicate sign"
+            'Used if amount type is "Absolute value"\n'
+            "Some statement formats use absolute amount value and indicate sign\n"
             "of the transaction by specifying if it was a debit or a credit one"
         ),
     )
@@ -111,6 +144,18 @@ class AccountStatementImportSheetMapping(models.Model):
     bank_account_column = fields.Char(
         help="Partner's bank account",
     )
+    footer_lines_count = fields.Integer(
+        string="Footer lines number",
+        help="Set the Footer lines number."
+        "Used in some csv file that integrate meta data in"
+        "last lines.",
+        default="0",
+    )
+    column_labels_row = fields.Integer(
+        string="Row number for column labels",
+        help="The number of line that contain column names.",
+        default="1",
+    )
 
     @api.onchange("float_thousands_sep")
     def onchange_thousands_separator(self):

account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py

@@ -26,21 +26,23 @@ class AccountStatementImportSheetParser(models.TransientModel):
     _description = "Bank Statement Import Sheet Parser"
 
     @api.model
-    def parse_header(self, data_file, encoding, csv_options):
+    def parse_header(self, data_file, encoding, csv_options, column_labels_row=1):
         try:
             workbook = xlrd.open_workbook(
                 file_contents=data_file,
                 encoding_override=encoding if encoding else None,
             )
             sheet = workbook.sheet_by_index(0)
-            values = sheet.row_values(0)
+            values = sheet.row_values(column_labels_row - 1)
             return [str(value) for value in values]
         except xlrd.XLRDError:
             _logger.error("Pass this method")
 
         data = StringIO(data_file.decode(encoding or "utf-8"))
         csv_data = reader(data, **csv_options)
-        return list(next(csv_data))
+        csv_data_lst = list(csv_data)
+        header = [value.strip() for value in csv_data_lst[column_labels_row - 1]]
+        return header
 
     @api.model
     def parse(self, data_file, mapping):
@@ -52,9 +54,12 @@ def parse(self, data_file, mapping):
         if not lines:
             return currency_code, account_number, [{"transactions": []}]
 
-        lines = list(sorted(lines, key=lambda line: line["timestamp"]))
-        first_line = lines[0]
-        last_line = lines[-1]
+        if lines[0]["timestamp"] > lines[-1]["timestamp"]:
+            first_line = lines[-1]
+            last_line = lines[0]
+        else:
+            first_line = lines[0]
+            last_line = lines[-1]
         data = {
             "date": first_line["timestamp"].date(),
         }
@@ -105,14 +110,34 @@ def _parse_lines(self, mapping, data_file, currency_code):
             )
 
         if isinstance(csv_or_xlsx, tuple):
-            header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
+            header = [
+                str(value).strip()
+                for value in csv_or_xlsx[1].row_values(mapping.column_labels_row - 1)
+            ]
         else:
+            for _i in range(mapping.column_labels_row - 1):
+                next(csv_or_xlsx)
             header = [value.strip() for value in next(csv_or_xlsx)]
+
         columns["timestamp_column"] = header.index(mapping.timestamp_column)
         columns["currency_column"] = (
             header.index(mapping.currency_column) if mapping.currency_column else None
         )
-        columns["amount_column"] = header.index(mapping.amount_column)
+        columns["amount_column"] = (
+            header.index(mapping.amount_column)
+            if mapping.amount_type != "distinct_credit_debit"
+            else None
+        )
+        columns["debit_column"] = (
+            header.index(mapping.debit_column)
+            if mapping.amount_type == "distinct_credit_debit"
+            else None
+        )
+        columns["credit_column"] = (
+            header.index(mapping.credit_column)
+            if mapping.amount_type == "distinct_credit_debit"
+            else None
+        )
         columns["balance_column"] = (
             header.index(mapping.balance_column) if mapping.balance_column else None
         )
@@ -160,27 +185,41 @@ def _parse_lines(self, mapping, data_file, currency_code):
             if mapping.bank_account_column
             else None
         )
-        return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns)
+        data = csv_or_xlsx, data_file
+        return self._parse_rows(mapping, currency_code, data, columns)
+
+    def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
+        csv_or_xlsx, data_file = data
+
+        # Get the numbers of rows of the file
+        if isinstance(csv_or_xlsx, tuple):
+            numrows = csv_or_xlsx[1].nrows
+        else:
+            numrows = len(str(data_file.strip()).split("\\n"))
+
+        label_line = mapping.column_labels_row
+        footer_line = numrows - mapping.footer_lines_count
 
-    def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C901
         if isinstance(csv_or_xlsx, tuple):
-            rows = range(1, csv_or_xlsx[1].nrows)
+            rows = range(mapping.column_labels_row, footer_line)
         else:
             rows = csv_or_xlsx
 
         lines = []
-        for row in rows:
+        for index, row in enumerate(rows, label_line):
             if isinstance(csv_or_xlsx, tuple):
                 book = csv_or_xlsx[0]
                 sheet = csv_or_xlsx[1]
                 values = []
-                for col_index in range(sheet.row_len(row)):
+                for col_index in range(0, sheet.row_len(row)):
                     cell_type = sheet.cell_type(row, col_index)
                     cell_value = sheet.cell_value(row, col_index)
                     if cell_type == xlrd.XL_CELL_DATE:
                         cell_value = xldate_as_datetime(cell_value, book.datemode)
                     values.append(cell_value)
             else:
+                if index >= footer_line:
+                    continue
                 values = list(row)
 
             timestamp = values[columns["timestamp_column"]]
@@ -189,7 +228,6 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C9
                 if columns["currency_column"] is not None
                 else currency_code
             )
-            amount = values[columns["amount_column"]]
             balance = (
                 values[columns["balance_column"]]
                 if columns["balance_column"] is not None
@@ -252,17 +290,25 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C9
             if isinstance(timestamp, str):
                 timestamp = datetime.strptime(timestamp, mapping.timestamp_format)
 
-            amount = self._parse_decimal(amount, mapping)
-            if balance:
-                balance = self._parse_decimal(balance, mapping)
-            else:
-                balance = None
-
-            if debit_credit:
+            amount_column = columns["amount_column"]
+            if amount_column and values[columns["amount_column"]]:
+                amount = self._parse_decimal(values[columns["amount_column"]], mapping)
+            if debit_credit is not None:
                 amount = amount.copy_abs()
                 if debit_credit == mapping.debit_value:
                     amount = -amount
 
+            debit_column = columns["debit_column"]
+            credit_column = columns["credit_column"]
+            if debit_column and credit_column:
+                debit_amount = self._parse_decimal(values[debit_column], mapping)
+                debit_amount = debit_amount.copy_abs()
+                credit_amount = self._parse_decimal(values[credit_column], mapping)
+                amount = credit_amount - debit_amount
+
+            if balance is not None:
+                balance = self._parse_decimal(balance, mapping)
+
             if original_amount:
                 original_amount = self._parse_decimal(
                     original_amount, mapping
@@ -378,6 +424,7 @@ def _parse_decimal(self, value, mapping):
             return value
         elif isinstance(value, float):
             return Decimal(value)
+        value = value or "0"
         thousands, decimal = mapping._get_float_separators()
         value = value.replace(thousands, "")
         value = value.replace(decimal, ".")

account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst

@@ -1,5 +1,6 @@
 * Alexis de Lattre <alexis.delattre@akretion.com>
 * Sebastien BEAU <sebastien.beau@akretion.com>
+* Mourad EL HADJ MIMOUNE <mourad.elhadj.mimoune@akretion.com>
 * Tecnativa (https://www.tecnativa.com)
 
   * Vicent Cubells <vicent.cubells@tecnativa.com>

account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv

@@ -0,0 +1,10 @@
+Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,,
+Account Number : 08088804068,Account Name : Account Owner,: EUR,,,
+,,,,,
+Balance at end of period,,,,"+31070,11",
+Date,Operation Number,Label,Debit,Credit,Detail
+01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1
+01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X
+02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2
+02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3
+Balance at start of period,,,,"+30000,77",