diff --git a/account_bank_statement_import_txt_xlsx/data/map_data.xml b/account_bank_statement_import_txt_xlsx/data/map_data.xml
index 4f56e64eb..708f4eebd 100644
--- a/account_bank_statement_import_txt_xlsx/data/map_data.xml
+++ b/account_bank_statement_import_txt_xlsx/data/map_data.xml
@@ -10,12 +10,15 @@
model="account.bank.statement.import.sheet.mapping"
>
Sample Statement
+ 0
+ 1
comma
dot
comma
"
%m/%d/%Y
Date
+ simple_value
Amount
Currency
Amount Currency
diff --git a/account_bank_statement_import_txt_xlsx/migrations/13.0.1.0.5/post-migration.py b/account_bank_statement_import_txt_xlsx/migrations/13.0.1.0.5/post-migration.py
new file mode 100644
index 000000000..800c608c1
--- /dev/null
+++ b/account_bank_statement_import_txt_xlsx/migrations/13.0.1.0.5/post-migration.py
@@ -0,0 +1,15 @@
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+
+from openupgradelib import openupgrade
+
+
+@openupgrade.migrate()
+def migrate(env, version):
+ openupgrade.logged_query(
+ env.cr,
+ """
+UPDATE account_bank_statement_import_sheet_mapping
+ SET amount_type = 'absolute_value'
+ WHERE debit_credit_column IS NOT NULL;
+ """,
+ )
diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py
index 5a927b1eb..c7fcefa36 100644
--- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py
+++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py
@@ -69,8 +69,11 @@ class AccountBankStatementImportSheetMapping(models.Model):
)
amount_column = fields.Char(
string="Amount column",
- required=True,
- help="Amount of transaction in journal's currency",
+ help=(
+ 'Used if amount type is "Simple value" or "Absolute value"\n'
+ "Amount of transaction in journal's currency\n"
+ "Some statement formats use credit/debit columns"
+ ),
)
balance_column = fields.Char(
string="Balance column", help="Balance after transaction in journal's currency",
@@ -91,10 +94,35 @@ class AccountBankStatementImportSheetMapping(models.Model):
"transaction amount in original transaction currency from"
),
)
+ amount_type = fields.Selection(
+ selection=[
+ ("simple_value", "Simple value"),
+ ("absolute_value", "Absolute value"),
+ ("distinct_credit_debit", "Distinct Credit/debit Column"),
+ ],
+ string="Amount type",
+ required=True,
+ default="simple_value",
+ help=(
+ "Simple value: use igned amount in ammount comlumn\n"
+ "Absolute Value: use a same comlumn for debit and credit\n"
+ "(absolute value + indicate sign)\n"
+ "Distinct Credit/debit Column: use a distinct comlumn for debit and credit"
+ ),
+ )
+ debit_column = fields.Char(
+ string="Debit column",
+ help='Used if amount type is "Distinct Credit/debit Column"',
+ )
+ credit_column = fields.Char(
+ string="Credit column",
+ help='Used if amount type is "Distinct Credit/debit Column"\n',
+ )
debit_credit_column = fields.Char(
string="Debit/credit column",
help=(
- "Some statement formats use absolute amount value and indicate sign"
+ 'Used if amount type is "Absolute value"\n'
+ "Some statement formats use absolute amount value and indicate sign\n"
"of the transaction by specifying if it was a debit or a credit one"
),
)
@@ -117,6 +145,18 @@ class AccountBankStatementImportSheetMapping(models.Model):
bank_account_column = fields.Char(
string="Bank Account column", help="Partner's bank account",
)
+ footer_lines_count = fields.Integer(
+ string="Footer lines number",
+ help="Set the Footer lines number."
+ "Used in some csv file that integrate meta data in"
+ "last lines.",
+ default="0",
+ )
+ column_labels_row = fields.Integer(
+ string="Row number for column labels",
+ help="The number of line that contain column names.",
+ default="1",
+ )
@api.onchange("float_thousands_sep")
def onchange_thousands_separator(self):
diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py
index 4b42d0f9a..1630c81da 100644
--- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py
+++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py
@@ -35,21 +35,23 @@ class AccountBankStatementImportSheetParser(models.TransientModel):
_description = "Account Bank Statement Import Sheet Parser"
@api.model
- def parse_header(self, data_file, encoding, csv_options):
+ def parse_header(self, data_file, encoding, csv_options, column_labels_row=1):
try:
workbook = xlrd.open_workbook(
file_contents=data_file,
encoding_override=encoding if encoding else None,
)
sheet = workbook.sheet_by_index(0)
- values = sheet.row_values(0)
+ values = sheet.row_values(column_labels_row - 1)
return [str(value) for value in values]
except xlrd.XLRDError:
pass
data = StringIO(data_file.decode(encoding or "utf-8"))
csv_data = reader(data, **csv_options)
- return list(next(csv_data))
+ csv_data_lst = list(csv_data)
+ header = [value.strip() for value in csv_data_lst[column_labels_row - 1]]
+ return header
@api.model
def parse(self, data_file, mapping, filename):
@@ -61,9 +63,12 @@ def parse(self, data_file, mapping, filename):
if not lines:
return currency_code, account_number, [{"transactions": []}]
- lines = list(sorted(lines, key=lambda line: line["timestamp"]))
- first_line = lines[0]
- last_line = lines[-1]
+ if lines[0]["timestamp"] > lines[-1]["timestamp"]:
+ first_line = lines[-1]
+ last_line = lines[0]
+ else:
+ first_line = lines[0]
+ last_line = lines[-1]
data = {
"date": first_line["timestamp"].date(),
"name": _("%s: %s") % (journal.code, path.basename(filename),),
@@ -127,6 +132,8 @@ def _get_column_names(self):
"partner_name_column",
"bank_name_column",
"bank_account_column",
+ "debit_column",
+ "credit_column",
]
def _parse_lines(self, mapping, data_file, currency_code):
@@ -163,14 +170,23 @@ def _parse_lines(self, mapping, data_file, currency_code):
header = False
if not mapping.no_header:
if isinstance(csv_or_xlsx, tuple):
- header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
+ header = [
+ str(value).strip()
+ for value in csv_or_xlsx[1].row_values(
+ mapping.column_labels_row - 1
+ )
+ ]
else:
+ for _i in range(mapping.column_labels_row - 1):
+ next(csv_or_xlsx)
header = [value.strip() for value in next(csv_or_xlsx)]
+
for column_name in self._get_column_names():
columns[column_name] = self._get_column_indexes(
header, column_name, mapping
)
- return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns)
+ data = csv_or_xlsx, data_file
+ return self._parse_rows(mapping, currency_code, data, columns)
def _get_values_from_column(self, values, columns, column_name):
indexes = columns[column_name]
@@ -187,25 +203,38 @@ def _get_values_from_column(self, values, columns, column_name):
return " ".join(content_l)
return content_l[0]
- def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901
+ def _parse_rows(self, mapping, currency_code, data, columns): # noqa: C901
+ csv_or_xlsx, data_file = data
+
+ # Get the numbers of rows of the file
+ if isinstance(csv_or_xlsx, tuple):
+ numrows = csv_or_xlsx[1].nrows
+ else:
+ numrows = len(str(data_file.strip()).split("\\n"))
+
+ label_line = mapping.column_labels_row
+ footer_line = numrows - mapping.footer_lines_count
+
if isinstance(csv_or_xlsx, tuple):
- rows = range(1, csv_or_xlsx[1].nrows)
+ rows = range(mapping.column_labels_row, footer_line)
else:
rows = csv_or_xlsx
lines = []
- for row in rows:
+ for index, row in enumerate(rows, label_line):
if isinstance(csv_or_xlsx, tuple):
book = csv_or_xlsx[0]
sheet = csv_or_xlsx[1]
values = []
- for col_index in range(sheet.row_len(row)):
+ for col_index in range(0, sheet.row_len(row)):
cell_type = sheet.cell_type(row, col_index)
cell_value = sheet.cell_value(row, col_index)
if cell_type == xlrd.XL_CELL_DATE:
cell_value = xldate_as_datetime(cell_value, book.datemode)
values.append(cell_value)
else:
+ if index >= footer_line:
+ continue
values = list(row)
timestamp = self._get_values_from_column(
@@ -281,17 +310,27 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C9
if isinstance(timestamp, str):
timestamp = datetime.strptime(timestamp, mapping.timestamp_format)
- amount = self._parse_decimal(amount, mapping)
- if balance:
- balance = self._parse_decimal(balance, mapping)
+ if mapping.amount_type == "distinct_credit_debit":
+ debit_column = columns["debit_column"][0]
+ credit_column = columns["credit_column"][0]
+ if debit_column and credit_column:
+ debit_amount = self._parse_decimal(values[debit_column], mapping)
+ debit_amount = debit_amount.copy_abs()
+ credit_amount = self._parse_decimal(values[credit_column], mapping)
+ amount = credit_amount - debit_amount
else:
- balance = None
+ amount_column = columns["amount_column"][0]
+ if amount_column and values[amount_column]:
+ amount = self._parse_decimal(values[amount_column], mapping)
- if debit_credit:
+ if debit_credit is not None:
amount = amount.copy_abs()
if debit_credit == mapping.debit_value:
amount = -amount
+ if balance is not None:
+ balance = self._parse_decimal(balance, mapping)
+
if not original_currency:
original_currency = currency
original_amount = amount
@@ -401,6 +440,7 @@ def _parse_decimal(self, value, mapping):
return value
elif isinstance(value, float):
return Decimal(value)
+ value = value or "0"
thousands, decimal = mapping._get_float_separators()
value = value.replace(thousands, "")
value = value.replace(decimal, ".")
diff --git a/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst b/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst
index b5e09af68..05ec13a85 100644
--- a/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst
+++ b/account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst
@@ -1,5 +1,6 @@
* Alexis de Lattre
* Sebastien BEAU
+* Mourad EL HADJ MIMOUNE
* Tecnativa (https://www.tecnativa.com)
* Vicent Cubells
@@ -13,3 +14,8 @@
* `CorporateHub `__
* Alexey Pelykh
+
+* ADHOC (https://www.adhoc.com.ar/)
+
+ * Juan Jose Scarafia
+ * Katherine Zaoral
diff --git a/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv b/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv
new file mode 100644
index 000000000..0b3862128
--- /dev/null
+++ b/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.csv
@@ -0,0 +1,10 @@
+Bank code : 1001010101,Agency Code : 10000,Download start date : 01/04/2020,Download end date : 02/04/2020,,
+Account Number : 08088804068,Account Name : Account Owner,: EUR,,,
+,,,,,
+Balance at end of period,,,,"+31070,11",
+Date,Operation Number,Label,Debit,Credit,Detail
+01/04/20,UNIQUE OP 1,LABEL 1,"-50,00",,DETAILS 1
+01/04/20,UNIQUE OP 2,LABEL 2,"-100,00",,CLIENTS X
+02/04/20,UNIQUE OP 3,LABEL 3,"-80,68",,DETAILS 2
+02/04/20,UNIQUE OP 4,LABEL 4,,"1300,00",DETAILS 3
+Balance at start of period,,,,"+30000,77",
diff --git a/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx b/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx
new file mode 100644
index 000000000..c7ae2b92e
Binary files /dev/null and b/account_bank_statement_import_txt_xlsx/tests/fixtures/meta_data_separated_credit_debit.xlsx differ
diff --git a/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py b/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py
index a474ff69a..ba3d3f86c 100644
--- a/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py
+++ b/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py
@@ -422,3 +422,119 @@ def test_debit_credit(self):
self.assertEqual(statement.balance_start, 10.0)
self.assertEqual(statement.balance_end_real, 1510.0)
self.assertEqual(statement.balance_end, 1510.0)
+
+ def test_metadata_separated_debit_credit_csv(self):
+ journal = self.AccountJournal.create(
+ {
+ "name": "Bank",
+ "type": "bank",
+ "code": "BANK",
+ "currency_id": self.currency_usd.id,
+ }
+ )
+ statement_map = self.sample_statement_map.copy(
+ {
+ "footer_lines_count": 1,
+ "column_labels_row": 5,
+ "amount_column": None,
+ "partner_name_column": None,
+ "bank_account_column": None,
+ "float_thousands_sep": "none",
+ "float_decimal_sep": "comma",
+ "timestamp_format": "%m/%d/%y",
+ "original_currency_column": None,
+ "original_amount_column": None,
+ "amount_type": "distinct_credit_debit",
+ "debit_column": "Debit",
+ "credit_column": "Credit",
+ }
+ )
+ data = self._data_file("fixtures/meta_data_separated_credit_debit.csv", "utf-8")
+ wizard = self.AccountBankStatementImport.with_context(
+ {"journal_id": journal.id}
+ ).create(
+ {
+ "attachment_ids": [
+ (
+ 0,
+ 0,
+ {
+ "name": "fixtures/meta_data_separated_credit_debit.csv",
+ "datas": data,
+ },
+ )
+ ],
+ "sheet_mapping_id": statement_map.id,
+ }
+ )
+ wizard.with_context(
+ {
+ "journal_id": journal.id,
+ "account_bank_statement_import_txt_xlsx_test": True,
+ }
+ ).import_file()
+ statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+ self.assertEqual(len(statement), 1)
+ self.assertEqual(len(statement.line_ids), 4)
+ line1 = statement.line_ids.filtered(lambda x: x.name == "LABEL 1")
+ line4 = statement.line_ids.filtered(lambda x: x.name == "LABEL 4")
+ self.assertEqual(line1.amount, -50)
+ self.assertEqual(line4.amount, 1300)
+
+ def test_metadata_separated_debit_credit_xlsx(self):
+ journal = self.AccountJournal.create(
+ {
+ "name": "Bank",
+ "type": "bank",
+ "code": "BANK",
+ "currency_id": self.currency_usd.id,
+ }
+ )
+ statement_map = self.sample_statement_map.copy(
+ {
+ "footer_lines_count": 1,
+ "column_labels_row": 5,
+ "amount_column": None,
+ "partner_name_column": None,
+ "bank_account_column": None,
+ "float_thousands_sep": "none",
+ "float_decimal_sep": "comma",
+ "timestamp_format": "%m/%d/%y",
+ "original_currency_column": None,
+ "original_amount_column": None,
+ "amount_type": "distinct_credit_debit",
+ "debit_column": "Debit",
+ "credit_column": "Credit",
+ }
+ )
+ data = self._data_file("fixtures/meta_data_separated_credit_debit.xlsx")
+ wizard = self.AccountBankStatementImport.with_context(
+ journal_id=journal.id
+ ).create(
+ {
+ "attachment_ids": [
+ (
+ 0,
+ 0,
+ {
+ "name": "fixtures/meta_data_separated_credit_debit.xlsx",
+ "datas": data,
+ },
+ )
+ ],
+ "sheet_mapping_id": statement_map.id,
+ }
+ )
+ wizard.with_context(
+ {
+ "journal_id": journal.id,
+ "account_bank_statement_import_txt_xlsx_test": True,
+ }
+ ).import_file()
+ statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+ self.assertEqual(len(statement), 1)
+ self.assertEqual(len(statement.line_ids), 4)
+ line1 = statement.line_ids.filtered(lambda x: x.name == "LABEL 1")
+ line4 = statement.line_ids.filtered(lambda x: x.name == "LABEL 4")
+ self.assertEqual(line1.amount, -50)
+ self.assertEqual(line4.amount, 1300)
diff --git a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml
index 299eac72b..1f1d18277 100644
--- a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml
+++ b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml
@@ -63,6 +63,10 @@
attrs="{'required': [('debit_credit_column', '!=', False)]}"
/>
+
+
+
+
@@ -77,11 +81,38 @@
-
+
+
+
+
+
-
diff --git a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py
index 430deefaf..fbe35a012 100644
--- a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py
+++ b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.py
@@ -20,6 +20,13 @@ class AccountBankStatementImportSheetMappingWizard(models.TransientModel):
required=True,
relation="account_bank_statement_import_sheet_mapping_wiz_attachment_rel",
)
+ column_labels_row = fields.Integer(
+ string="Header line",
+ help="The number of line that contan column names.\n"
+ "Used if csv/xls files contain\n"
+ "meta data in first lines\n ",
+ default="1",
+ )
header = fields.Char()
file_encoding = fields.Selection(
string="Encoding", selection=lambda self: self._selection_file_encoding(),
@@ -36,9 +43,33 @@ class AccountBankStatementImportSheetMappingWizard(models.TransientModel):
"transaction from"
),
)
+ amount_type = fields.Selection(
+ selection=[
+ ("simple_value", "Simple value"),
+ ("absolute_value", "Absolute value"),
+ ("distinct_credit_debit", "Distinct Credit/debit Column"),
+ ],
+ string="Amount type",
+ required=True,
+ default="simple_value",
+ help=(
+ "Simple value: use igned amount in ammount comlumn\n"
+ "Absolute Value: use a same comlumn for debit and credit\n"
+ "(absolute value + indicate sign)\n"
+ "Distinct Credit/debit Column: use a distinct comlumn for debit and credit"
+ ),
+ )
amount_column = fields.Char(
string="Amount column", help="Amount of transaction in journal's currency",
)
+ debit_column = fields.Char(
+ string="Debit column",
+ help='Used if amount type is "Distinct Credit/debit Column"\n',
+ )
+ credit_column = fields.Char(
+ string="Credit column",
+ help='Used if amount type is "Distinct Credit/debit Column"\n',
+ )
balance_column = fields.Char(
string="Balance column", help="Balance after transaction in journal's currency",
)
@@ -117,7 +148,10 @@ def _onchange_attachment_ids(self):
header = []
for data_file in self.attachment_ids:
header += Parser.parse_header(
- b64decode(data_file.datas), self.file_encoding, csv_options
+ b64decode(data_file.datas),
+ self.file_encoding,
+ csv_options,
+ self.column_labels_row,
)
header = list(set(header))
self.header = json.dumps(header)
@@ -143,6 +177,9 @@ def _get_mapping_values(self):
"timestamp_format": "%d/%m/%Y",
"timestamp_column": self.timestamp_column,
"currency_column": self.currency_column,
+ "amount_type": self.amount_type,
+ "debit_column": self.debit_column,
+ "credit_column": self.credit_column,
"amount_column": self.amount_column,
"balance_column": self.balance_column,
"original_currency_column": self.original_currency_column,
diff --git a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml
index 01880e3fa..60443b97e 100644
--- a/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml
+++ b/account_bank_statement_import_txt_xlsx/wizards/account_bank_statement_import_sheet_mapping_wizard.xml
@@ -29,6 +29,7 @@
/>
+
@@ -50,33 +51,63 @@
values="statement_columns"
context="{'header': header}"
/>
+
+
+