From e714ff9f994679f3adc2ad986dda4d7fc4874e81 Mon Sep 17 00:00:00 2001 From: Mourad Date: Wed, 3 Jun 2020 13:10:21 +0200 Subject: [PATCH] [IMP] account_bank_statement_import_txt_xlsx: CSV [IMP] add csv meta data management [FIX] exclude footer meta data [IMP] import separated credit/debit column file [FIX] make comptatible with new version of multi_step_wizard module & add migration file [FIX] all not provided value are handled in_parse_decimal method [REF] Remove unnecessary \n --- .../data/map_data.xml | 4 ++ .../migrations/12.0.3.0.0/post-migration.py | 35 +++++++++++ ...unt_bank_statement_import_sheet_mapping.py | 60 ++++++++++++++++++- ...ount_bank_statement_import_sheet_parser.py | 48 ++++++++++----- .../readme/CONTRIBUTORS.rst | 2 + .../meta_data_separated_credit_debit.csv | 10 ++++ ..._account_bank_statement_import_txt_xlsx.py | 47 +++++++++++++++ ...nt_bank_statement_import_sheet_mapping.xml | 31 +++++++++- ...k_statement_import_sheet_mapping_wizard.py | 36 ++++++++++- ..._statement_import_sheet_mapping_wizard.xml | 41 +++++++++++-- 10 files changed, 289 insertions(+), 25 deletions(-) create mode 100644 account_bank_statement_import_txt_xlsx/migrations/12.0.3.0.0/post-migration.py create mode 100644 account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv diff --git a/account_bank_statement_import_txt_xlsx/data/map_data.xml b/account_bank_statement_import_txt_xlsx/data/map_data.xml index 4f56e64eb..30d4edc9a 100644 --- a/account_bank_statement_import_txt_xlsx/data/map_data.xml +++ b/account_bank_statement_import_txt_xlsx/data/map_data.xml @@ -10,12 +10,16 @@ model="account.bank.statement.import.sheet.mapping" > Sample Statement + 1 + 0 + 1 comma dot comma " %m/%d/%Y Date + simple_value Amount Currency Amount Currency diff --git a/account_bank_statement_import_txt_xlsx/migrations/12.0.3.0.0/post-migration.py b/account_bank_statement_import_txt_xlsx/migrations/12.0.3.0.0/post-migration.py new file mode 100644 index 000000000..a1a7c67dc --- /dev/null +++ b/account_bank_statement_import_txt_xlsx/migrations/12.0.3.0.0/post-migration.py @@ -0,0 +1,35 @@ +# Copyright 2020 Akretion +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). + +from openupgradelib import openupgrade + + +@openupgrade.migrate() +def migrate(env, version): + openupgrade.logged_query( + env.cr, + """ +UPDATE account_bank_statement_import_sheet_mapping + SET header_lines_count = 1, + footer_lines_count = 0, + column_names_line = 1; + """ + ) + + openupgrade.logged_query( + env.cr, + """ +UPDATE account_bank_statement_import_sheet_mapping + SET amount_type = 'absolute_value' + WHERE debit_credit_column IS NOT NULL; + """ + ) + + openupgrade.logged_query( + env.cr, + """ +UPDATE account_bank_statement_import_sheet_mapping + SET amount_type = 'simple_value' + WHERE debit_credit_column IS NULL; + """ + ) diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py index 5a927b1eb..981fa6df5 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py @@ -91,11 +91,43 @@ class AccountBankStatementImportSheetMapping(models.Model): "transaction amount in original transaction currency from" ), ) + amount_type = fields.Selection( + selection=[ + ('simple_value', 'Simple value'), + ('absolute_value', 'Absolute value'), + ('distinct_credit_debit', 'Distinct Credit/debit Column'), + ], + string='Amount type', + required=True, + default="simple_value", + help=( + 'Simple value: use igned amount in ammount comlumn\n' + 'Absolute Value: use a same comlumn for debit and credit\n' + '(absolute value + indicate sign)\n' + 'Distinct Credit/debit Column: use a distinct comlumn for debit and credit' + ), + ) + amount_column = fields.Char( + string='Amount column', + help=( + 'Used if amount type is "Simple value" or "Absolute value"\n' + 'Amount of transaction in journal\'s currency\n' + 'Some statement formats use credit/debit columns'), + ) + debit_column = fields.Char( + string='Debit column', + help='Used if amount type is "Distinct Credit/debit Column"', + ) + credit_column = fields.Char( + string='Credit column', + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) debit_credit_column = fields.Char( string="Debit/credit column", help=( - "Some statement formats use absolute amount value and indicate sign" - "of the transaction by specifying if it was a debit or a credit one" + 'Used if amount type is "Absolute value"\n' + 'Some statement formats use absolute amount value and indicate sign\n' + 'of the transaction by specifying if it was a debit or a credit one' ), ) debit_value = fields.Char( @@ -117,6 +149,30 @@ class AccountBankStatementImportSheetMapping(models.Model): bank_account_column = fields.Char( string="Bank Account column", help="Partner's bank account", ) + with_metadata = fields.Boolean( + string='File wWth Metadata ', + help='Check if file containt meta data in first lines', + ) + header_lines_count = fields.Integer( + string='Header lines number', + help='Set the Header lines number.' + 'Used in some csv file that integrate meta data in' + 'first lines. This number contain the number of' + 'the all meta data lines including columns names', + default="1", + ) + footer_lines_count = fields.Integer( + string='Footer lines number', + help='Set the Footer lines number.' + 'Used in some csv file that integrate meta data in' + 'last lines.', + default="0", + ) + column_names_line = fields.Integer( + string='The number of line that contan column names', + help='The number of line that contan column names.', + default="1", + ) @api.onchange("float_thousands_sep") def onchange_thousands_separator(self): diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py index 4b42d0f9a..ebbc8be0a 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py @@ -35,21 +35,25 @@ class AccountBankStatementImportSheetParser(models.TransientModel): _description = "Account Bank Statement Import Sheet Parser" @api.model - def parse_header(self, data_file, encoding, csv_options): + def parse_header( + self, data_file, encoding, csv_options, column_names_line=1): try: workbook = xlrd.open_workbook( file_contents=data_file, encoding_override=encoding if encoding else None, ) sheet = workbook.sheet_by_index(0) - values = sheet.row_values(0) + values = sheet.row_values(column_names_line - 1) return [str(value) for value in values] except xlrd.XLRDError: pass data = StringIO(data_file.decode(encoding or "utf-8")) csv_data = reader(data, **csv_options) - return list(next(csv_data)) + csv_data_lst = list(csv_data) + header = [value.strip() + for value in csv_data_lst[column_names_line - 1]] + return header @api.model def parse(self, data_file, mapping, filename): @@ -189,9 +193,16 @@ def _get_values_from_column(self, values, columns, column_name): def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 if isinstance(csv_or_xlsx, tuple): - rows = range(1, csv_or_xlsx[1].nrows) + rows = range( + mapping.header_lines_count, + csv_or_xlsx[1].nrows - mapping.footer_lines_count) else: - rows = csv_or_xlsx + stat_first_index = mapping.header_lines_count + stat_last_index = - mapping.footer_lines_count + if stat_last_index: + rows = csv_or_xlsx_lst[stat_first_index: stat_last_index] + else: + rows = csv_or_xlsx_lst[stat_first_index:] lines = [] for row in rows: @@ -199,11 +210,12 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C9 book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(sheet.row_len(row)): + for col_index in range(0, sheet.row_len(row)): cell_type = sheet.cell_type(row, col_index) cell_value = sheet.cell_value(row, col_index) if cell_type == xlrd.XL_CELL_DATE: - cell_value = xldate_as_datetime(cell_value, book.datemode) + cell_value = xldate_as_datetime( + cell_value, book.datemode) values.append(cell_value) else: values = list(row) @@ -281,16 +293,23 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C9 if isinstance(timestamp, str): timestamp = datetime.strptime(timestamp, mapping.timestamp_format) - amount = self._parse_decimal(amount, mapping) - if balance: - balance = self._parse_decimal(balance, mapping) - else: - balance = None - - if debit_credit: + if amount_column: + amount = self._parse_decimal( + values[amount_column], mapping) + if debit_credit is not None: amount = amount.copy_abs() if debit_credit == mapping.debit_value: amount = -amount + if debit_column and credit_column: + debit_amount = self._parse_decimal( + values[debit_column], mapping) + debit_amount = debit_amount.copy_abs() + credit_amount = self._parse_decimal( + values[credit_column], mapping) + amount = credit_amount - debit_amount + + if balance is not None: + balance = self._parse_decimal(balance, mapping) if not original_currency: original_currency = currency @@ -401,6 +420,7 @@ def _parse_decimal(self, value, mapping): return value elif isinstance(value, float): return Decimal(value) + value = value or "0" thousands, decimal = mapping._get_float_separators() value = value.replace(thousands, "") value = value.replace(decimal, ".") diff --git a/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst b/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst index b5e09af68..576415a36 100644 --- a/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst +++ b/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst @@ -1,5 +1,6 @@ * Alexis de Lattre * Sebastien BEAU +* Mourad EL HADJ MIMOUNE * Tecnativa (https://www.tecnativa.com) * Vicent Cubells @@ -13,3 +14,4 @@ * `CorporateHub `__ * Alexey Pelykh + diff --git a/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv b/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv new file mode 100644 index 000000000..0b3862128 --- /dev/null +++ b/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv @@ -0,0 +1,10 @@ +Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,, +Account Number : 08088804068,Account Name : Account Owner,: EUR,,, +,,,,, +Balance at end of period,,,,"+31070,11", +Date,Operation Number,Label,Debit,Credit,Detail +01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1 +01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X +02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2 +02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3 +Balance at start of period,,,,"+30000,77", diff --git a/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py b/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py index a474ff69a..0987c95d0 100644 --- a/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py +++ b/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py @@ -422,3 +422,50 @@ def test_debit_credit(self): self.assertEqual(statement.balance_start, 10.0) self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) + + def test_metadata_separated_debit_credit(self): + journal = self.AccountJournal.create({ + 'name': 'Bank', + 'type': 'bank', + 'code': 'BANK', + 'currency_id': self.currency_usd.id, + }) + statement_map = self.sample_statement_map.copy({ + 'header_lines_count': 5, + 'footer_lines_count': 1, + 'column_names_line': 5, + 'amount_column': None, + 'partner_name_column': None, + 'bank_account_column': None, + 'float_thousands_sep': 'none', + 'float_decimal_sep': 'comma', + 'timestamp_format': '%m/%d/%y', + 'original_currency_column': None, + 'original_amount_column': None, + 'amount_type': 'distinct_credit_debit', + 'debit_column': 'Debit', + 'credit_column': 'Credit', + }) + wizard = self.AccountBankStatementImport.with_context({ + 'journal_id': journal.id, + }).create({ + 'filename': 'fixtures/meta_data_separated_credit_debit.csv', + 'data_file': self._data_file( + 'fixtures/meta_data_separated_credit_debit.csv', + 'utf-8' + ), + 'sheet_mapping_id': statement_map.id, + }) + wizard.with_context({ + 'journal_id': journal.id, + 'account_bank_statement_import_txt_xlsx_test': True, + }).import_file() + statement = self.AccountBankStatement.search([ + ('journal_id', '=', journal.id), + ]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.name == 'LABEL 1') + line4 = statement.line_ids.filtered(lambda x: x.name == 'LABEL 4') + self.assertEqual(line1.amount, -50) + self.assertEqual(line4.amount, 1300) diff --git a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml index 299eac72b..fb6cc76e4 100644 --- a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml +++ b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml @@ -63,6 +63,14 @@ attrs="{'required': [('debit_credit_column', '!=', False)]}" /> + + + + + + + + @@ -77,11 +85,30 @@ - + + + + + - diff --git a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py index 430deefaf..7e3522823 100644 --- a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py +++ b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py @@ -20,6 +20,13 @@ class AccountBankStatementImportSheetMappingWizard(models.TransientModel): required=True, relation="account_bank_statement_import_sheet_mapping_wiz_attachment_rel", ) + column_names_line = fields.Integer( + string='Header line', + help='The number of line that contan column names.\n' + 'Used if csv/xls files contain\n' + 'meta data in first lines\n ', + default="1", + ) header = fields.Char() file_encoding = fields.Selection( string="Encoding", selection=lambda self: self._selection_file_encoding(), @@ -36,9 +43,33 @@ class AccountBankStatementImportSheetMappingWizard(models.TransientModel): "transaction from" ), ) + amount_type = fields.Selection( + selection=[ + ('simple_value', 'Simple value'), + ('absolute_value', 'Absolute value'), + ('distinct_credit_debit', 'Distinct Credit/debit Column'), + ], + string='Amount type', + required=True, + default="simple_value", + help=( + 'Simple value: use igned amount in ammount comlumn\n' + 'Absolute Value: use a same comlumn for debit and credit\n' + '(absolute value + indicate sign)\n' + 'Distinct Credit/debit Column: use a distinct comlumn for debit and credit' + ), + ) amount_column = fields.Char( string="Amount column", help="Amount of transaction in journal's currency", ) + debit_column = fields.Char( + string='Debit column', + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) + credit_column = fields.Char( + string='Credit column', + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) balance_column = fields.Char( string="Balance column", help="Balance after transaction in journal's currency", ) @@ -117,7 +148,7 @@ def _onchange_attachment_ids(self): header = [] for data_file in self.attachment_ids: header += Parser.parse_header( - b64decode(data_file.datas), self.file_encoding, csv_options + b64decode(data_file.datas), self.file_encoding, csv_options, self.column_names_line ) header = list(set(header)) self.header = json.dumps(header) @@ -143,6 +174,9 @@ def _get_mapping_values(self): "timestamp_format": "%d/%m/%Y", "timestamp_column": self.timestamp_column, "currency_column": self.currency_column, + "amount_type": self.amount_type, + "debit_column": self.debit_column, + "credit_column": self.credit_column, "amount_column": self.amount_column, "balance_column": self.balance_column, "original_currency_column": self.original_currency_column, diff --git a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml index 01880e3fa..835c80326 100644 --- a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml +++ b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml @@ -29,10 +29,15 @@ /> + + + + + @@ -50,33 +55,57 @@ values="statement_columns" context="{'header': header}" /> + + + + attrs="{ + 'required': [('amount_type', '=', 'absolute_value')], + 'invisible': [('amount_type', '!=', 'absolute_value')], + }"/>