[IMP] account_bank_statement_import_txt_xlsx: header/footer trimming,…

… separate debit/credit

account_bank_statement_import_txt_xlsx/data/map_data.xml

@@ -10,9 +10,8 @@
         model="account.bank.statement.import.sheet.mapping"
     >
         <field name="name">Sample Statement</field>
-        <field name="header_lines_count">1</field>
         <field name="footer_lines_count">0</field>
-        <field name="column_names_line">1</field>
+        <field name="column_labels_row">1</field>
         <field name="float_thousands_sep">comma</field>
         <field name="float_decimal_sep">dot</field>
         <field name="delimiter">comma</field>

account_bank_statement_import_txt_xlsx/migrations/13.0.1.0.5/post-migration.py

@@ -0,0 +1,15 @@
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+
+from openupgradelib import openupgrade
+
+
+@openupgrade.migrate()
+def migrate(env, version):
+    openupgrade.logged_query(
+        env.cr,
+        """
+UPDATE account_bank_statement_import_sheet_mapping
+    SET amount_type = 'absolute_value'
+    WHERE debit_credit_column IS NOT NULL;
+       """,
+    )

account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py

@@ -69,8 +69,11 @@ class AccountBankStatementImportSheetMapping(models.Model):
     )
     amount_column = fields.Char(
         string="Amount column",
-        required=True,
-        help="Amount of transaction in journal's currency",
+        help=(
+            'Used if amount type is "Simple value" or "Absolute value"\n'
+            "Amount of transaction in journal's currency\n"
+            "Some statement formats use credit/debit columns"
+        ),
     )
     balance_column = fields.Char(
         string="Balance column", help="Balance after transaction in journal's currency",
@@ -93,41 +96,34 @@ class AccountBankStatementImportSheetMapping(models.Model):
     )
     amount_type = fields.Selection(
         selection=[
-            ('simple_value', 'Simple value'),
-            ('absolute_value', 'Absolute value'),
-            ('distinct_credit_debit', 'Distinct Credit/debit Column'),
+            ("simple_value", "Simple value"),
+            ("absolute_value", "Absolute value"),
+            ("distinct_credit_debit", "Distinct Credit/debit Column"),
         ],
-        string='Amount type',
+        string="Amount type",
         required=True,
         default="simple_value",
         help=(
-            'Simple value: use igned amount in ammount comlumn\n'
-            'Absolute Value: use a same comlumn for debit and credit\n'
-            '(absolute value + indicate sign)\n'
-            'Distinct Credit/debit Column: use a distinct comlumn for debit and credit'
+            "Simple value: use igned amount in ammount comlumn\n"
+            "Absolute Value: use a same comlumn for debit and credit\n"
+            "(absolute value + indicate sign)\n"
+            "Distinct Credit/debit Column: use a distinct comlumn for debit and credit"
         ),
     )
-    amount_column = fields.Char(
-        string='Amount column',
-        help=(
-            'Used if amount type is "Simple value" or "Absolute value"\n'
-            'Amount of transaction in journal\'s currency\n'
-            'Some statement formats use credit/debit columns'),
-    )
     debit_column = fields.Char(
-        string='Debit column',
+        string="Debit column",
         help='Used if amount type is "Distinct Credit/debit Column"',
     )
     credit_column = fields.Char(
-        string='Credit column',
+        string="Credit column",
         help='Used if amount type is "Distinct Credit/debit Column"\n',
     )
     debit_credit_column = fields.Char(
         string="Debit/credit column",
         help=(
             'Used if amount type is "Absolute value"\n'
-            'Some statement formats use absolute amount value and indicate sign\n'
-            'of the transaction by specifying if it was a debit or a credit one'
+            "Some statement formats use absolute amount value and indicate sign\n"
+            "of the transaction by specifying if it was a debit or a credit one"
         ),
     )
     debit_value = fields.Char(
@@ -149,28 +145,16 @@ class AccountBankStatementImportSheetMapping(models.Model):
     bank_account_column = fields.Char(
         string="Bank Account column", help="Partner's bank account",
     )
-    with_metadata = fields.Boolean(
-        string='File wWth Metadata ',
-        help='Check if file containt meta data in first lines',
-    )
-    header_lines_count = fields.Integer(
-        string='Header lines number',
-        help='Set the Header lines number.'
-             'Used in some csv file that integrate meta data in'
-             'first lines. This number contain the number of'
-             'the all meta data lines including columns names',
-        default="1",
-    )
     footer_lines_count = fields.Integer(
-        string='Footer lines number',
-        help='Set the Footer lines number.'
-             'Used in some csv file that integrate meta data in'
-             'last lines.',
+        string="Footer lines number",
+        help="Set the Footer lines number."
+        "Used in some csv file that integrate meta data in"
+        "last lines.",
         default="0",
     )
-    column_names_line = fields.Integer(
-        string='The number of line that contan column names',
-        help='The number of line that contan column names.',
+    column_labels_row = fields.Integer(
+        string="Row number for column labels",
+        help="The number of line that contain column names.",
         default="1",
     )
 

account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py

@@ -35,24 +35,22 @@ class AccountBankStatementImportSheetParser(models.TransientModel):
     _description = "Account Bank Statement Import Sheet Parser"
 
     @api.model
-    def parse_header(
-            self, data_file, encoding, csv_options, column_names_line=1):
+    def parse_header(self, data_file, encoding, csv_options, column_labels_row=1):
         try:
             workbook = xlrd.open_workbook(
                 file_contents=data_file,
                 encoding_override=encoding if encoding else None,
             )
             sheet = workbook.sheet_by_index(0)
-            values = sheet.row_values(column_names_line - 1)
+            values = sheet.row_values(column_labels_row - 1)
             return [str(value) for value in values]
         except xlrd.XLRDError:
             pass
 
         data = StringIO(data_file.decode(encoding or "utf-8"))
         csv_data = reader(data, **csv_options)
         csv_data_lst = list(csv_data)
-        header = [value.strip()
-                  for value in csv_data_lst[column_names_line - 1]]
+        header = [value.strip() for value in csv_data_lst[column_labels_row - 1]]
         return header
 
     @api.model
@@ -65,9 +63,12 @@ def parse(self, data_file, mapping, filename):
         if not lines:
             return currency_code, account_number, [{"transactions": []}]
 
-        lines = list(sorted(lines, key=lambda line: line["timestamp"]))
-        first_line = lines[0]
-        last_line = lines[-1]
+        if lines[0]["timestamp"] > lines[-1]["timestamp"]:
+            first_line = lines[-1]
+            last_line = lines[0]
+        else:
+            first_line = lines[0]
+            last_line = lines[-1]
         data = {
             "date": first_line["timestamp"].date(),
             "name": _("%s: %s") % (journal.code, path.basename(filename),),
@@ -131,6 +132,8 @@ def _get_column_names(self):
             "partner_name_column",
             "bank_name_column",
             "bank_account_column",
+            "debit_column",
+            "credit_column",
         ]
 
     def _parse_lines(self, mapping, data_file, currency_code):
@@ -167,14 +170,23 @@ def _parse_lines(self, mapping, data_file, currency_code):
         header = False
         if not mapping.no_header:
             if isinstance(csv_or_xlsx, tuple):
-                header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
+                header = [
+                    str(value).strip()
+                    for value in csv_or_xlsx[1].row_values(
+                        mapping.column_labels_row - 1
+                    )
+                ]
             else:
+                for _i in range(mapping.column_labels_row - 1):
+                    next(csv_or_xlsx)
                 header = [value.strip() for value in next(csv_or_xlsx)]
+
         for column_name in self._get_column_names():
             columns[column_name] = self._get_column_indexes(
                 header, column_name, mapping
             )
-        return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns)
+        data = csv_or_xlsx, data_file
+        return self._parse_rows(mapping, currency_code, data, columns)
 
     def _get_values_from_column(self, values, columns, column_name):
         indexes = columns[column_name]
@@ -191,21 +203,25 @@ def _get_values_from_column(self, values, columns, column_name):
             return " ".join(content_l)
         return content_l[0]
 
-    def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C901
+    def _parse_rows(self, mapping, currency_code, data, columns):  # noqa: C901
+        csv_or_xlsx, data_file = data
+
+        # Get the numbers of rows of the file
         if isinstance(csv_or_xlsx, tuple):
-            rows = range(
-                mapping.header_lines_count,
-                csv_or_xlsx[1].nrows - mapping.footer_lines_count)
+            numrows = csv_or_xlsx[1].nrows
         else:
-            stat_first_index = mapping.header_lines_count
-            stat_last_index = - mapping.footer_lines_count
-            if stat_last_index:
-                rows = csv_or_xlsx_lst[stat_first_index: stat_last_index]
-            else:
-                rows = csv_or_xlsx_lst[stat_first_index:]
+            numrows = len(str(data_file.strip()).split("\\n"))
+
+        label_line = mapping.column_labels_row
+        footer_line = numrows - mapping.footer_lines_count
+
+        if isinstance(csv_or_xlsx, tuple):
+            rows = range(mapping.column_labels_row, footer_line)
+        else:
+            rows = csv_or_xlsx
 
         lines = []
-        for row in rows:
+        for index, row in enumerate(rows, label_line):
             if isinstance(csv_or_xlsx, tuple):
                 book = csv_or_xlsx[0]
                 sheet = csv_or_xlsx[1]
@@ -214,10 +230,11 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C9
                     cell_type = sheet.cell_type(row, col_index)
                     cell_value = sheet.cell_value(row, col_index)
                     if cell_type == xlrd.XL_CELL_DATE:
-                        cell_value = xldate_as_datetime(
-                            cell_value, book.datemode)
+                        cell_value = xldate_as_datetime(cell_value, book.datemode)
                     values.append(cell_value)
             else:
+                if index >= footer_line:
+                    continue
                 values = list(row)
 
             timestamp = self._get_values_from_column(
@@ -293,20 +310,23 @@ def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C9
             if isinstance(timestamp, str):
                 timestamp = datetime.strptime(timestamp, mapping.timestamp_format)
 
-            if amount_column:
-                amount = self._parse_decimal(
-                    values[amount_column], mapping)
+            if mapping.amount_type == "distinct_credit_debit":
+                debit_column = columns["debit_column"][0]
+                credit_column = columns["credit_column"][0]
+                if debit_column and credit_column:
+                    debit_amount = self._parse_decimal(values[debit_column], mapping)
+                    debit_amount = debit_amount.copy_abs()
+                    credit_amount = self._parse_decimal(values[credit_column], mapping)
+                    amount = credit_amount - debit_amount
+            else:
+                amount_column = columns["amount_column"][0]
+                if amount_column and values[amount_column]:
+                    amount = self._parse_decimal(values[amount_column], mapping)
+
             if debit_credit is not None:
                 amount = amount.copy_abs()
                 if debit_credit == mapping.debit_value:
                     amount = -amount
-            if debit_column and credit_column:
-                debit_amount = self._parse_decimal(
-                    values[debit_column], mapping)
-                debit_amount = debit_amount.copy_abs()
-                credit_amount = self._parse_decimal(
-                    values[credit_column], mapping)
-                amount = credit_amount - debit_amount
 
             if balance is not None:
                 balance = self._parse_decimal(balance, mapping)

account_bank_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst

@@ -15,3 +15,7 @@
 
   * Alexey Pelykh <alexey.pelykh@corphub.eu>
 
+* ADHOC (https://www.adhoc.com.ar/)
+
+  * Juan Jose Scarafia <jjs@adhoc.com.ar>
+  * Katherine Zaoral <kz@adhoc.com.ar>