[IMP] account_bank_statement_import_txt_xlsx: CSV

[IMP] add csv meta data management
[FIX] exclude footer meta data
[IMP] import separated credit/debit column file
[FIX] make comptatible with new version of multi_step_wizard module & add migration file
[FIX] all not provided value are handled in_parse_decimal method
[REF] Remove unnecessary \n

account_bank_statement_import_txt_xlsx/data/map_data.xml

@@ -10,12 +10,16 @@
         model="account.bank.statement.import.sheet.mapping"
     >
         <field name="name">Sample Statement</field>
+        <field name="header_lines_count">1</field>
+        <field name="footer_lines_count">0</field>
+        <field name="column_names_line">1</field>
         <field name="float_thousands_sep">comma</field>
         <field name="float_decimal_sep">dot</field>
         <field name="delimiter">comma</field>
         <field name="quotechar">"</field>
         <field name="timestamp_format">%m/%d/%Y</field>
         <field name="timestamp_column">Date</field>
+        <field name="amount_type">simple_value</field>
         <field name="amount_column">Amount</field>
         <field name="original_currency_column">Currency</field>
         <field name="original_amount_column">Amount Currency</field>

account_bank_statement_import_txt_xlsx/migrations/12.0.3.0.0/post-migration.py

@@ -0,0 +1,35 @@
+# Copyright 2020 Akretion
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+
+from openupgradelib import openupgrade
+
+
+@openupgrade.migrate()
+def migrate(env, version):
+    openupgrade.logged_query(
+        env.cr,
+        """
+UPDATE account_bank_statement_import_sheet_mapping
+    SET header_lines_count = 1,
+    footer_lines_count = 0,
+    column_names_line = 1;
+       """
+    )
+
+    openupgrade.logged_query(
+        env.cr,
+        """
+UPDATE account_bank_statement_import_sheet_mapping
+    SET amount_type = 'absolute_value'
+    WHERE debit_credit_column IS NOT NULL;
+       """
+    )
+
+    openupgrade.logged_query(
+        env.cr,
+        """
+UPDATE account_bank_statement_import_sheet_mapping
+    SET amount_type = 'simple_value'
+    WHERE debit_credit_column IS NULL;
+       """
+    )

account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py

@@ -91,11 +91,43 @@ class AccountBankStatementImportSheetMapping(models.Model):
             "transaction amount in original transaction currency from"
         ),
     )
+    amount_type = fields.Selection(
+        selection=[
+            ('simple_value', 'Simple value'),
+            ('absolute_value', 'Absolute value'),
+            ('distinct_credit_debit', 'Distinct Credit/debit Column'),
+        ],
+        string='Amount type',
+        required=True,
+        default="simple_value",
+        help=(
+            'Simple value: use igned amount in ammount comlumn\n'
+            'Absolute Value: use a same comlumn for debit and credit\n'
+            '(absolute value + indicate sign)\n'
+            'Distinct Credit/debit Column: use a distinct comlumn for debit and credit'
+        ),
+    )
+    amount_column = fields.Char(
+        string='Amount column',
+        help=(
+            'Used if amount type is "Simple value" or "Absolute value"\n'
+            'Amount of transaction in journal\'s currency\n'
+            'Some statement formats use credit/debit columns'),
+    )
+    debit_column = fields.Char(
+        string='Debit column',
+        help='Used if amount type is "Distinct Credit/debit Column"',
+    )
+    credit_column = fields.Char(
+        string='Credit column',
+        help='Used if amount type is "Distinct Credit/debit Column"\n',
+    )
     debit_credit_column = fields.Char(
         string="Debit/credit column",
         help=(
-            "Some statement formats use absolute amount value and indicate sign"
-            "of the transaction by specifying if it was a debit or a credit one"
+            'Used if amount type is "Absolute value"\n'
+            'Some statement formats use absolute amount value and indicate sign\n'
+            'of the transaction by specifying if it was a debit or a credit one'
         ),
     )
     debit_value = fields.Char(
@@ -117,6 +149,30 @@ class AccountBankStatementImportSheetMapping(models.Model):
     bank_account_column = fields.Char(
         string="Bank Account column", help="Partner's bank account",
     )
+    with_metadata = fields.Boolean(
+        string='File wWth Metadata ',
+        help='Check if file containt meta data in first lines',
+    )
+    header_lines_count = fields.Integer(
+        string='Header lines number',
+        help='Set the Header lines number.'
+             'Used in some csv file that integrate meta data in'
+             'first lines. This number contain the number of'
+             'the all meta data lines including columns names',
+        default="1",
+    )
+    footer_lines_count = fields.Integer(
+        string='Footer lines number',
+        help='Set the Footer lines number.'
+             'Used in some csv file that integrate meta data in'
+             'last lines.',
+        default="0",
+    )
+    column_names_line = fields.Integer(
+        string='The number of line that contan column names',
+        help='The number of line that contan column names.',
+        default="1",
+    )
 
     @api.onchange("float_thousands_sep")
     def onchange_thousands_separator(self):

account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py

@@ -35,21 +35,25 @@ class AccountBankStatementImportSheetParser(models.TransientModel):
     _description = "Account Bank Statement Import Sheet Parser"
 
     @api.model
-    def parse_header(self, data_file, encoding, csv_options):
+    def parse_header(
+            self, data_file, encoding, csv_options, column_names_line=1):
         try:
             workbook = xlrd.open_workbook(
                 file_contents=data_file,
                 encoding_override=encoding if encoding else None,
             )
             sheet = workbook.sheet_by_index(0)
-            values = sheet.row_values(0)
+            values = sheet.row_values(column_names_line - 1)
             return [str(value) for value in values]
         except xlrd.XLRDError:
             pass
 
         data = StringIO(data_file.decode(encoding or "utf-8"))
         csv_data = reader(data, **csv_options)
-        return list(next(csv_data))
+        csv_data_lst = list(csv_data)
+        header = [value.strip()
+                  for value in csv_data_lst[column_names_line - 1]]
+        return header
 
     @api.model
     def parse(self, data_file, mapping, filename):
@@ -189,21 +193,29 @@ def _get_values_from_column(self, values, columns, column_name):
 
     def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C901
         if isinstance(csv_or_xlsx, tuple):
-            rows = range(1, csv_or_xlsx[1].nrows)
+            rows = range(
+                mapping.header_lines_count,
+                csv_or_xlsx[1].nrows - mapping.footer_lines_count)
         else:
-            rows = csv_or_xlsx
+            stat_first_index = mapping.header_lines_count
+            stat_last_index = - mapping.footer_lines_count
+            if stat_last_index:
+                rows = csv_or_xlsx_lst[stat_first_index: stat_last_index]
+            else:
+                rows = csv_or_xlsx_lst[stat_first_index:]
 
         lines = []
         for row in rows:
             if isinstance(csv_or_xlsx, tuple):
                 book = csv_or_xlsx[0]
                 sheet = csv_or_xlsx[1]
                 values = []
-                for col_index in range(sheet.row_len(row)):
+                for col_index in range(0, sheet.row_len(row)):
                     cell_type = sheet.cell_type(row, col_index)
                     cell_value = sheet.cell_value(row, col_index)
                     if cell_type == xlrd.XL_CELL_DATE:
-                        cell_value = xldate_as_datetime(cell_value, book.datemode)
+                        cell_value = xldate_as_datetime(
+                            cell_value, book.datemode)
                     values.append(cell_value)
             else:
                 values = list(row)
@@ -281,16 +293,23 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C9
             if isinstance(timestamp, str):
                 timestamp = datetime.strptime(timestamp, mapping.timestamp_format)
 
-            amount = self._parse_decimal(amount, mapping)
-            if balance:
-                balance = self._parse_decimal(balance, mapping)
-            else:
-                balance = None
-
-            if debit_credit:
+            if amount_column:
+                amount = self._parse_decimal(
+                    values[amount_column], mapping)
+            if debit_credit is not None:
                 amount = amount.copy_abs()
                 if debit_credit == mapping.debit_value:
                     amount = -amount
+            if debit_column and credit_column:
+                debit_amount = self._parse_decimal(
+                    values[debit_column], mapping)
+                debit_amount = debit_amount.copy_abs()
+                credit_amount = self._parse_decimal(
+                    values[credit_column], mapping)
+                amount = credit_amount - debit_amount
+
+            if balance is not None:
+                balance = self._parse_decimal(balance, mapping)
 
             if not original_currency:
                 original_currency = currency
@@ -401,6 +420,7 @@ def _parse_decimal(self, value, mapping):
             return value
         elif isinstance(value, float):
             return Decimal(value)
+        value = value or "0"
         thousands, decimal = mapping._get_float_separators()
         value = value.replace(thousands, "")
         value = value.replace(decimal, ".")

account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst

@@ -1,5 +1,6 @@
 * Alexis de Lattre <alexis.delattre@akretion.com>
 * Sebastien BEAU <sebastien.beau@akretion.com>
+* Mourad EL HADJ MIMOUNE <mourad.elhadj.mimoune@akretion.com>
 * Tecnativa (https://www.tecnativa.com)
 
   * Vicent Cubells <vicent.cubells@tecnativa.com>
@@ -13,3 +14,4 @@
 * `CorporateHub <https://corporatehub.eu/>`__
 
   * Alexey Pelykh <alexey.pelykh@corphub.eu>
+

account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv

@@ -0,0 +1,10 @@
+Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,,
+Account Number : 08088804068,Account Name : Account Owner,: EUR,,,
+,,,,,
+Balance at end of period,,,,"+31070,11",
+Date,Operation Number,Label,Debit,Credit,Detail
+01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1
+01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X
+02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2
+02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3
+Balance at start of period,,,,"+30000,77",

account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py

@@ -422,3 +422,50 @@ def test_debit_credit(self):
         self.assertEqual(statement.balance_start, 10.0)
         self.assertEqual(statement.balance_end_real, 1510.0)
         self.assertEqual(statement.balance_end, 1510.0)
+
+    def test_metadata_separated_debit_credit(self):
+        journal = self.AccountJournal.create({
+            'name': 'Bank',
+            'type': 'bank',
+            'code': 'BANK',
+            'currency_id': self.currency_usd.id,
+        })
+        statement_map = self.sample_statement_map.copy({
+            'header_lines_count': 5,
+            'footer_lines_count': 1,
+            'column_names_line': 5,
+            'amount_column': None,
+            'partner_name_column': None,
+            'bank_account_column': None,
+            'float_thousands_sep': 'none',
+            'float_decimal_sep': 'comma',
+            'timestamp_format': '%m/%d/%y',
+            'original_currency_column': None,
+            'original_amount_column': None,
+            'amount_type': 'distinct_credit_debit',
+            'debit_column': 'Debit',
+            'credit_column': 'Credit',
+        })
+        wizard = self.AccountBankStatementImport.with_context({
+            'journal_id': journal.id,
+        }).create({
+            'filename': 'fixtures/meta_data_separated_credit_debit.csv',
+            'data_file': self._data_file(
+                'fixtures/meta_data_separated_credit_debit.csv',
+                'utf-8'
+            ),
+            'sheet_mapping_id': statement_map.id,
+        })
+        wizard.with_context({
+            'journal_id': journal.id,
+            'account_bank_statement_import_txt_xlsx_test': True,
+        }).import_file()
+        statement = self.AccountBankStatement.search([
+            ('journal_id', '=', journal.id),
+        ])
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 4)
+        line1 = statement.line_ids.filtered(lambda x: x.name == 'LABEL 1')
+        line4 = statement.line_ids.filtered(lambda x: x.name == 'LABEL 4')
+        self.assertEqual(line1.amount, -50)
+        self.assertEqual(line4.amount, 1300)

account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml

@@ -63,6 +63,14 @@
                                 attrs="{'required': [('debit_credit_column', '!=', False)]}"
                             />
                         </group>
+                        <group>
+                            <field name="with_metadata"/>
+                            <group>
+                                <field name="header_lines_count"/>
+                                <field name="footer_lines_count"/>
+                                <field name="column_names_line"/>
+                            </group>
+                        </group>
                     </group>
                     <group string="Columns">
                         <group colspan="4" col="2">
@@ -77,11 +85,30 @@
                         <group>
                             <field name="timestamp_column" />
                             <field name="currency_column" />
-                            <field name="amount_column" />
+                            <field name="amount_type" />
+                            <field name="amount_column"
+                                attrs="{
+                                'required': [('amount_type', '!=', 'distinct_credit_debit'),],
+                                'invisible': [('amount_type', '=', 'distinct_credit_debit')],
+                                }"/>
+                            <field name="debit_column"
+                                attrs="{
+                                'required': [('amount_type', '=', 'distinct_credit_debit')],
+                                'invisible': [('amount_type', '!=', 'distinct_credit_debit')],
+                                }"/>
+                            <field name="credit_column"
+                                attrs="{
+                                'required': [('amount_type', '=', 'distinct_credit_debit')],
+                                'invisible': [('amount_type', '!=', 'distinct_credit_debit')],
+                                }"/>
+                            <field name="debit_credit_column"
+                                attrs="{
+                                'required': [('amount_type', '=', 'absolute_value')],
+                                'invisible': [('amount_type', '!=', 'absolute_value')],
+                                }"/>
                             <field name="balance_column" />
                             <field name="original_currency_column" />
                             <field name="original_amount_column" />
-                            <field name="debit_credit_column" />
                             <field name="transaction_id_column" />
                             <field name="description_column" />
                             <field name="notes_column" />