diff --git a/account_statement_import_txt_xlsx/__manifest__.py b/account_statement_import_txt_xlsx/__manifest__.py index 3dbd1590a..0d71a2805 100644 --- a/account_statement_import_txt_xlsx/__manifest__.py +++ b/account_statement_import_txt_xlsx/__manifest__.py @@ -5,7 +5,7 @@ { "name": "Bank Statement TXT/CSV/XLSX Import", "summary": "Import TXT/CSV or XLSX files as Bank Statements in Odoo", - "version": "15.0.1.0.0", + "version": "15.0.1.1.0", "category": "Accounting", "website": "https://github.com/OCA/bank-statement-import", "author": "ForgeFlow, CorporateHub, Odoo Community Association (OCA)", diff --git a/account_statement_import_txt_xlsx/data/map_data.xml b/account_statement_import_txt_xlsx/data/map_data.xml index f87466015..29f48cdd8 100644 --- a/account_statement_import_txt_xlsx/data/map_data.xml +++ b/account_statement_import_txt_xlsx/data/map_data.xml @@ -7,12 +7,15 @@ Sample Statement + 0 + 1 comma dot comma " %m/%d/%Y Date + simple_value Amount Currency Amount Currency diff --git a/account_statement_import_txt_xlsx/migrations/15.0.1.1.0/post-migration.py b/account_statement_import_txt_xlsx/migrations/15.0.1.1.0/post-migration.py new file mode 100644 index 000000000..498ef8435 --- /dev/null +++ b/account_statement_import_txt_xlsx/migrations/15.0.1.1.0/post-migration.py @@ -0,0 +1,25 @@ +# Copyright 2020 Akretion +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html). + +from openupgradelib import openupgrade + + +@openupgrade.migrate() +def migrate(env, version): + openupgrade.logged_query( + env.cr, + """ +UPDATE account_statement_import_sheet_mapping + SET amount_type = 'absolute_value' + WHERE debit_credit_column IS NOT NULL; + """, + ) + + openupgrade.logged_query( + env.cr, + """ +UPDATE account_statement_import_sheet_mapping + SET amount_type = 'simple_value' + WHERE debit_credit_column IS NULL; + """, + ) diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index 70d11b5d3..48a198478 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -83,10 +83,43 @@ class AccountStatementImportSheetMapping(models.Model): "transaction amount in original transaction currency from" ), ) + amount_type = fields.Selection( + selection=[ + ("simple_value", "Simple value"), + ("absolute_value", "Absolute value"), + ("distinct_credit_debit", "Distinct Credit/debit Column"), + ], + string="Amount type", + required=True, + default="simple_value", + help=( + "Simple value: use igned amount in ammount comlumn\n" + "Absolute Value: use a same comlumn for debit and credit\n" + "(absolute value + indicate sign)\n" + "Distinct Credit/debit Column: use a distinct comlumn for debit and credit" + ), + ) + amount_column = fields.Char( + string="Amount column", + help=( + 'Used if amount type is "Simple value" or "Absolute value"\n' + "Amount of transaction in journal's currency\n" + "Some statement formats use credit/debit columns" + ), + ) + debit_column = fields.Char( + string="Debit column", + help='Used if amount type is "Distinct Credit/debit Column"', + ) + credit_column = fields.Char( + string="Credit column", + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) debit_credit_column = fields.Char( string="Debit/credit column", help=( - "Some statement formats use absolute amount value and indicate sign" + 'Used if amount type is "Absolute value"\n' + "Some statement formats use absolute amount value and indicate sign\n" "of the transaction by specifying if it was a debit or a credit one" ), ) @@ -111,6 +144,18 @@ class AccountStatementImportSheetMapping(models.Model): bank_account_column = fields.Char( help="Partner's bank account", ) + footer_lines_count = fields.Integer( + string="Footer lines number", + help="Set the Footer lines number." + "Used in some csv file that integrate meta data in" + "last lines.", + default="0", + ) + column_labels_row = fields.Integer( + string="Row number for column labels", + help="The number of line that contain column names.", + default="1", + ) @api.onchange("float_thousands_sep") def onchange_thousands_separator(self): diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 615715456..04851902c 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -26,21 +26,23 @@ class AccountStatementImportSheetParser(models.TransientModel): _description = "Bank Statement Import Sheet Parser" @api.model - def parse_header(self, data_file, encoding, csv_options): + def parse_header(self, data_file, encoding, csv_options, column_labels_row=1): try: workbook = xlrd.open_workbook( file_contents=data_file, encoding_override=encoding if encoding else None, ) sheet = workbook.sheet_by_index(0) - values = sheet.row_values(0) + values = sheet.row_values(column_labels_row - 1) return [str(value) for value in values] except xlrd.XLRDError: _logger.error("Pass this method") data = StringIO(data_file.decode(encoding or "utf-8")) csv_data = reader(data, **csv_options) - return list(next(csv_data)) + csv_data_lst = list(csv_data) + header = [value.strip() for value in csv_data_lst[column_labels_row - 1]] + return header @api.model def parse(self, data_file, mapping): @@ -52,9 +54,12 @@ def parse(self, data_file, mapping): if not lines: return currency_code, account_number, [{"transactions": []}] - lines = list(sorted(lines, key=lambda line: line["timestamp"])) - first_line = lines[0] - last_line = lines[-1] + if lines[0]["timestamp"] > lines[-1]["timestamp"]: + first_line = lines[-1] + last_line = lines[0] + else: + first_line = lines[0] + last_line = lines[-1] data = { "date": first_line["timestamp"].date(), } @@ -105,14 +110,34 @@ def _parse_lines(self, mapping, data_file, currency_code): ) if isinstance(csv_or_xlsx, tuple): - header = [str(value) for value in csv_or_xlsx[1].row_values(0)] + header = [ + str(value).strip() + for value in csv_or_xlsx[1].row_values(mapping.column_labels_row - 1) + ] else: + for _i in range(mapping.column_labels_row - 1): + next(csv_or_xlsx) header = [value.strip() for value in next(csv_or_xlsx)] + columns["timestamp_column"] = header.index(mapping.timestamp_column) columns["currency_column"] = ( header.index(mapping.currency_column) if mapping.currency_column else None ) - columns["amount_column"] = header.index(mapping.amount_column) + columns["amount_column"] = ( + header.index(mapping.amount_column) + if mapping.amount_type != "distinct_credit_debit" + else None + ) + columns["debit_column"] = ( + header.index(mapping.debit_column) + if mapping.amount_type == "distinct_credit_debit" + else None + ) + columns["credit_column"] = ( + header.index(mapping.credit_column) + if mapping.amount_type == "distinct_credit_debit" + else None + ) columns["balance_column"] = ( header.index(mapping.balance_column) if mapping.balance_column else None ) @@ -160,27 +185,41 @@ def _parse_lines(self, mapping, data_file, currency_code): if mapping.bank_account_column else None ) - return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + data = csv_or_xlsx, data_file + return self._parse_rows(mapping, currency_code, data, columns) + + def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901 + csv_or_xlsx, data_file = data + + # Get the numbers of rows of the file + if isinstance(csv_or_xlsx, tuple): + numrows = csv_or_xlsx[1].nrows + else: + numrows = len(str(data_file.strip()).split("\\n")) + + label_line = mapping.column_labels_row + footer_line = numrows - mapping.footer_lines_count - def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 if isinstance(csv_or_xlsx, tuple): - rows = range(1, csv_or_xlsx[1].nrows) + rows = range(mapping.column_labels_row, footer_line) else: rows = csv_or_xlsx lines = [] - for row in rows: + for index, row in enumerate(rows, label_line): if isinstance(csv_or_xlsx, tuple): book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(sheet.row_len(row)): + for col_index in range(0, sheet.row_len(row)): cell_type = sheet.cell_type(row, col_index) cell_value = sheet.cell_value(row, col_index) if cell_type == xlrd.XL_CELL_DATE: cell_value = xldate_as_datetime(cell_value, book.datemode) values.append(cell_value) else: + if index >= footer_line: + continue values = list(row) timestamp = values[columns["timestamp_column"]] @@ -189,7 +228,6 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C9 if columns["currency_column"] is not None else currency_code ) - amount = values[columns["amount_column"]] balance = ( values[columns["balance_column"]] if columns["balance_column"] is not None @@ -252,17 +290,25 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C9 if isinstance(timestamp, str): timestamp = datetime.strptime(timestamp, mapping.timestamp_format) - amount = self._parse_decimal(amount, mapping) - if balance: - balance = self._parse_decimal(balance, mapping) - else: - balance = None - - if debit_credit: + amount_column = columns["amount_column"] + if amount_column and values[columns["amount_column"]]: + amount = self._parse_decimal(values[columns["amount_column"]], mapping) + if debit_credit is not None: amount = amount.copy_abs() if debit_credit == mapping.debit_value: amount = -amount + debit_column = columns["debit_column"] + credit_column = columns["credit_column"] + if debit_column and credit_column: + debit_amount = self._parse_decimal(values[debit_column], mapping) + debit_amount = debit_amount.copy_abs() + credit_amount = self._parse_decimal(values[credit_column], mapping) + amount = credit_amount - debit_amount + + if balance is not None: + balance = self._parse_decimal(balance, mapping) + if original_amount: original_amount = self._parse_decimal( original_amount, mapping @@ -378,6 +424,7 @@ def _parse_decimal(self, value, mapping): return value elif isinstance(value, float): return Decimal(value) + value = value or "0" thousands, decimal = mapping._get_float_separators() value = value.replace(thousands, "") value = value.replace(decimal, ".") diff --git a/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst b/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst index b5e09af68..6b9085d24 100644 --- a/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst +++ b/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst @@ -1,5 +1,6 @@ * Alexis de Lattre * Sebastien BEAU +* Mourad EL HADJ MIMOUNE * Tecnativa (https://www.tecnativa.com) * Vicent Cubells diff --git a/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv new file mode 100644 index 000000000..0b3862128 --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv @@ -0,0 +1,10 @@ +Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,, +Account Number : 08088804068,Account Name : Account Owner,: EUR,,, +,,,,, +Balance at end of period,,,,"+31070,11", +Date,Operation Number,Label,Debit,Credit,Detail +01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1 +01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X +02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2 +02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3 +Balance at start of period,,,,"+30000,77", diff --git a/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx new file mode 100644 index 000000000..c7ae2b92e Binary files /dev/null and b/account_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx differ diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py index 0ffecd930..5226582db 100644 --- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py @@ -345,3 +345,95 @@ def test_debit_credit(self): self.assertEqual(statement.balance_start, 10.0) self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) + + def test_metadata_separated_debit_credit_csv(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_count": 1, + "column_labels_row": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "debit_column": "Debit", + "credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, -50) + self.assertEqual(line4.amount, 1300) + + def test_metadata_separated_debit_credit_xlsx(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "footer_lines_count": 1, + "column_labels_row": 5, + "amount_column": None, + "partner_name_column": None, + "bank_account_column": None, + "float_thousands_sep": "none", + "float_decimal_sep": "comma", + "timestamp_format": "%m/%d/%y", + "original_currency_column": None, + "original_amount_column": None, + "amount_type": "distinct_credit_debit", + "debit_column": "Debit", + "credit_column": "Credit", + } + ) + data = self._data_file("fixtures/meta_data_separated_credit_debit.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.xlsx", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 4) + line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") + line4 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 4") + self.assertEqual(line1.amount, -50) + self.assertEqual(line4.amount, 1300) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 8f9718b5a..09b762b0c 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -51,15 +51,46 @@ attrs="{'required': [('debit_credit_column', '!=', False)]}" /> + + + + - + + + + + - diff --git a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py index 47f4eeab2..33892a65a 100644 --- a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py +++ b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py @@ -20,6 +20,13 @@ class AccountStatementImportSheetMappingWizard(models.TransientModel): required=True, relation="account_statement_import_sheet_mapping_wiz_attachment_rel", ) + column_labels_row = fields.Integer( + string="Header line", + help="The number of line that contan column names.\n" + "Used if csv/xls files contain\n" + "meta data in first lines\n ", + default="1", + ) header = fields.Char() file_encoding = fields.Selection( string="Encoding", @@ -39,9 +46,33 @@ class AccountStatementImportSheetMappingWizard(models.TransientModel): "transaction from" ), ) + amount_type = fields.Selection( + selection=[ + ("simple_value", "Simple value"), + ("absolute_value", "Absolute value"), + ("distinct_credit_debit", "Distinct Credit/debit Column"), + ], + string="Amount type", + required=True, + default="simple_value", + help=( + "Simple value: use igned amount in ammount comlumn\n" + "Absolute Value: use a same comlumn for debit and credit\n" + "(absolute value + indicate sign)\n" + "Distinct Credit/debit Column: use a distinct comlumn for debit and credit" + ), + ) amount_column = fields.Char( help="Amount of transaction in journal's currency", ) + debit_column = fields.Char( + string="Debit column", + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) + credit_column = fields.Char( + string="Credit column", + help='Used if amount type is "Distinct Credit/debit Column"\n', + ) balance_column = fields.Char( help="Balance after transaction in journal's currency", ) @@ -120,7 +151,10 @@ def _onchange_attachment_ids(self): header = [] for data_file in self.attachment_ids: header += Parser.parse_header( - b64decode(data_file.datas), self.file_encoding, csv_options + b64decode(data_file.datas), + self.file_encoding, + csv_options, + self.column_labels_row, ) header = list(set(header)) self.header = json.dumps(header) @@ -146,6 +180,9 @@ def _get_mapping_values(self): "timestamp_format": "%d/%m/%Y", "timestamp_column": self.timestamp_column, "currency_column": self.currency_column, + "amount_type": self.amount_type, + "debit_column": self.debit_column, + "credit_column": self.credit_column, "amount_column": self.amount_column, "balance_column": self.balance_column, "original_currency_column": self.original_currency_column, diff --git a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml index 51097816f..7d657a33b 100644 --- a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml +++ b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml @@ -24,6 +24,7 @@ /> + @@ -45,33 +46,63 @@ values="statement_columns" context="{'header': header}" /> + + +