rcv: Improve CSV parsing preprocessing and logging

src/cl_sii/rcv/parse_csv.py

@@ -518,6 +518,22 @@ def validate_schema(self, data: dict, original_data: dict, **kwargs: Any) -> Non
     def to_detalle_entry(self, data: dict) -> RcvDetalleEntry:
         raise NotImplementedError
 
+    @marshmallow.pre_load
+    def preprocess(self, in_data: dict, **kwargs: Any) -> dict:
+        # Get required field names from the schema
+        required_fields = {
+            field.data_key
+            for name, field in self.fields.items()
+            if field.required and field.allow_none is False
+        }
+        # Remove only required fields that are None or empty string
+        for field in required_fields:
+            if field in in_data.keys() and (
+                in_data[field] is None or str(in_data[field]).strip() == ''
+            ):
+                del in_data[field]
+        return in_data
+
 
 class RcvVentaCsvRowSchema(_RcvCsvRowSchemaBase):
     FIELD_FECHA_RECEPCION_DT_TZ = SII_OFFICIAL_TZ
@@ -586,6 +602,7 @@ class RcvVentaCsvRowSchema(_RcvCsvRowSchemaBase):
 
     @marshmallow.pre_load
     def preprocess(self, in_data: dict, **kwargs: Any) -> dict:
+        in_data = super().preprocess(in_data, **kwargs)
         # note: required fields checks are run later on automatically thus we may not assume that
         #   values of required fields (`required=True`) exist.
 
@@ -725,6 +742,7 @@ class RcvCompraRegistroCsvRowSchema(_RcvCsvRowSchemaBase):
 
     @marshmallow.pre_load
     def preprocess(self, in_data: dict, **kwargs: Any) -> dict:
+        in_data = super().preprocess(in_data, **kwargs)
         # note: required fields checks are run later on automatically thus we may not assume that
         #   values of required fields (`required=True`) exist.
 
@@ -891,6 +909,7 @@ class RcvCompraReclamadoCsvRowSchema(_RcvCsvRowSchemaBase):
 
     @marshmallow.pre_load
     def preprocess(self, in_data: dict, **kwargs: Any) -> dict:
+        in_data = super().preprocess(in_data, **kwargs)
         # note: required fields checks are run later on automatically thus we may not assume that
         #   values of required fields (`required=True`) exist.
 
@@ -1018,6 +1037,7 @@ class RcvCompraPendienteCsvRowSchema(_RcvCsvRowSchemaBase):
 
     @marshmallow.pre_load
     def preprocess(self, in_data: dict, **kwargs: Any) -> dict:
+        in_data = super().preprocess(in_data, **kwargs)
         # note: required fields checks are run later on automatically thus we may not assume that
         #   values of required fields (`required=True`) exist.
 
@@ -1168,14 +1188,16 @@ def _parse_rcv_csv_file(
                 except Exception as exc:
                     conversion_error = str(exc)
                     logger.exception(
-                        "Deserialized data to data model instance conversion failed "
-                        "(probably a programming error)."
+                        "Deserialized row data conversion failed for row %d: %s",
+                        row_ix,
+                        conversion_error,
+                        extra={'deserialized_row_data': deserialized_row_data},
                     )
 
             # Instead of empty dicts, lists, str, etc, we want to have None.
             if validation_errors:
                 row_errors['validation'] = validation_errors
             if conversion_error:
-                row_errors['other'] = conversion_error
+                row_errors['conversion_errors'] = conversion_error
 
             yield entry, row_ix, row_data, row_errors

src/tests/test_data/sii-rcv/RCV-venta-missing-required-fields.csv

@@ -0,0 +1,3 @@
+Nro;Tipo Doc;Tipo Venta;Rut cliente;Razon Social;Folio;Fecha Docto;Fecha Recepcion;Fecha Acuse Recibo;Fecha Reclamo;Monto Exento;Monto Neto;Monto IVA;Monto total;IVA Retenido Total;IVA Retenido Parcial;IVA no retenido;IVA propio;IVA Terceros;RUT Emisor Liquid. Factura;Neto Comision Liquid. Factura;Exento Comision Liquid. Factura;IVA Comision Liquid. Factura;IVA fuera de plazo;Tipo Docto. Referencia;Folio Docto. Referencia;Num. Ident. Receptor Extranjero;Nacionalidad Receptor Extranjero;Credito empresa constructora;Impto. Zona Franca (Ley 18211);Garantia Dep. Envases;Indicador Venta sin Costo;Indicador Servicio Periodico;Monto No facturable;Total Monto Periodo;Venta Pasajes Transporte Nacional;Venta Pasajes Transporte Internacional;Numero Interno;Codigo Sucursal;NCE o NDE sobre Fact. de Compra;Codigo Otro Imp.;Valor Otro Imp.;Tasa Otro Imp.
+1;"";Del Giro;12345678-5;Fake Company S.A. ;506;04/06/2019;"";;;0;1750181;332534;2082715;0;0;0;0;0;-;0;0;0;0;;;;;0;;0;2;0;0;0;;;;0;;;;;
+23;33;Del Giro;12345678-5;  Fake Company S.A.;508;28/06/2019;01/07/2019 13:49:42;;;0;2209597;419823;2629420;0;0;0;0;0;-;0;0;0;0;0;;;;0;;0;2;0;0;0;;;;0;;;;;

src/tests/test_rcv_parse_csv.py

@@ -1,5 +1,6 @@
 import unittest
 from typing import Callable
+from unittest import mock
 
 from cl_sii.rcv.parse_csv import (  # noqa: F401
     RcvCompraNoIncluirCsvRowSchema,
@@ -72,6 +73,29 @@ def test_parse_rcv_venta_csv_file_receptor_rz_leading_trailing_whitespace(self)
         self.assertEqual(entry_struct.receptor_razon_social, 'Fake Company S.A.')
         self.assertEqual(len(row_parsing_errors), 0)
 
+    def test_parse_rcv_venta_csv_file_missing_required_fields(self):
+        # This CSV should have a required field (e.g., 'Folio') as an empty string
+        rcv_file_path = get_test_file_path(
+            'test_data/sii-rcv/RCV-venta-missing-required-fields.csv',
+        )
+
+        items = parse_rcv_venta_csv_file(
+            rut=Rut('1-9'),
+            input_file_path=rcv_file_path,
+            n_rows_offset=0,
+            max_n_rows=None,
+        )
+
+        entry_struct, row_ix, row_data, row_parsing_errors = next(items)
+        self.assertIsNone(entry_struct)
+        self.assertIn('validation', row_parsing_errors)
+        self.assertIn('Fecha Recepcion', row_parsing_errors['validation'])
+        self.assertIn('Tipo Doc', row_parsing_errors['validation'])
+        self.assertEqual(
+            row_parsing_errors['validation']['Fecha Recepcion'],
+            ['Missing data for required field.'],
+        )
+
     def _test_parse_rcv_compra_csv_file_emisor_rz_leading_trailing_whitespace(
         self,
         parse_rcv_compra_csv_file_function: Callable,
@@ -98,6 +122,38 @@ def _test_parse_rcv_compra_csv_file_emisor_rz_leading_trailing_whitespace(
         self.assertEqual(entry_struct.emisor_razon_social, 'Fake Company S.A.')
         self.assertEqual(len(row_parsing_errors), 0)
 
+    def test_parse_rcv_venta_csv_file_conversion_error(self):
+        rcv_file_path = get_test_file_path(
+            'test_data/sii-rcv/RCV-venta-rz_leading_trailing_whitespace.csv',
+        )
+
+        # Patch the to_detalle_entry method to raise an exception
+        with (
+            self.assertLogs('cl_sii.rcv', level='ERROR') as assert_logs_cm,
+            mock.patch.object(
+                RcvVentaCsvRowSchema,
+                'to_detalle_entry',
+                side_effect=ValueError('Mocked conversion error'),
+            ),
+        ):
+            items = parse_rcv_venta_csv_file(
+                rut=Rut('1-9'),
+                input_file_path=rcv_file_path,
+                n_rows_offset=0,
+                max_n_rows=1,
+            )
+            entry_struct, row_ix, row_data, row_parsing_errors = next(items)
+            self.assertIsNone(entry_struct)
+            self.assertIn('conversion_errors', row_parsing_errors)
+            self.assertIn('Mocked conversion error', row_parsing_errors['conversion_errors'])
+            self.assertRegex(
+                assert_logs_cm.output[0],
+                (
+                    'ERROR:cl_sii.rcv.parse_csv:'
+                    'Deserialized row data conversion failed for row 1: Mocked conversion error'
+                ),
+            )
+
     def test_parse_rcv_compra_registro_csv_file(self) -> None:
         # TODO: implement for 'parse_rcv_compra_registro_csv_file'.
         pass