diff --git a/src/cl_sii/rcv/parse_csv.py b/src/cl_sii/rcv/parse_csv.py index 8a5d6d08..1fbfef53 100644 --- a/src/cl_sii/rcv/parse_csv.py +++ b/src/cl_sii/rcv/parse_csv.py @@ -518,6 +518,22 @@ def validate_schema(self, data: dict, original_data: dict, **kwargs: Any) -> Non def to_detalle_entry(self, data: dict) -> RcvDetalleEntry: raise NotImplementedError + @marshmallow.pre_load + def preprocess(self, in_data: dict, **kwargs: Any) -> dict: + # Get required field names from the schema + required_fields = { + field.data_key + for name, field in self.fields.items() + if field.required and field.allow_none is False + } + # Remove only required fields that are None or empty string + for field in required_fields: + if field in in_data.keys() and ( + in_data[field] is None or str(in_data[field]).strip() == '' + ): + del in_data[field] + return in_data + class RcvVentaCsvRowSchema(_RcvCsvRowSchemaBase): FIELD_FECHA_RECEPCION_DT_TZ = SII_OFFICIAL_TZ @@ -586,6 +602,7 @@ class RcvVentaCsvRowSchema(_RcvCsvRowSchemaBase): @marshmallow.pre_load def preprocess(self, in_data: dict, **kwargs: Any) -> dict: + in_data = super().preprocess(in_data, **kwargs) # note: required fields checks are run later on automatically thus we may not assume that # values of required fields (`required=True`) exist. @@ -725,6 +742,7 @@ class RcvCompraRegistroCsvRowSchema(_RcvCsvRowSchemaBase): @marshmallow.pre_load def preprocess(self, in_data: dict, **kwargs: Any) -> dict: + in_data = super().preprocess(in_data, **kwargs) # note: required fields checks are run later on automatically thus we may not assume that # values of required fields (`required=True`) exist. @@ -891,6 +909,7 @@ class RcvCompraReclamadoCsvRowSchema(_RcvCsvRowSchemaBase): @marshmallow.pre_load def preprocess(self, in_data: dict, **kwargs: Any) -> dict: + in_data = super().preprocess(in_data, **kwargs) # note: required fields checks are run later on automatically thus we may not assume that # values of required fields (`required=True`) exist. @@ -1018,6 +1037,7 @@ class RcvCompraPendienteCsvRowSchema(_RcvCsvRowSchemaBase): @marshmallow.pre_load def preprocess(self, in_data: dict, **kwargs: Any) -> dict: + in_data = super().preprocess(in_data, **kwargs) # note: required fields checks are run later on automatically thus we may not assume that # values of required fields (`required=True`) exist. @@ -1168,14 +1188,16 @@ def _parse_rcv_csv_file( except Exception as exc: conversion_error = str(exc) logger.exception( - "Deserialized data to data model instance conversion failed " - "(probably a programming error)." + "Deserialized row data conversion failed for row %d: %s", + row_ix, + conversion_error, + extra={'deserialized_row_data': deserialized_row_data}, ) # Instead of empty dicts, lists, str, etc, we want to have None. if validation_errors: row_errors['validation'] = validation_errors if conversion_error: - row_errors['other'] = conversion_error + row_errors['conversion_errors'] = conversion_error yield entry, row_ix, row_data, row_errors diff --git a/src/tests/test_data/sii-rcv/RCV-venta-missing-required-fields.csv b/src/tests/test_data/sii-rcv/RCV-venta-missing-required-fields.csv new file mode 100644 index 00000000..60718859 --- /dev/null +++ b/src/tests/test_data/sii-rcv/RCV-venta-missing-required-fields.csv @@ -0,0 +1,3 @@ +Nro;Tipo Doc;Tipo Venta;Rut cliente;Razon Social;Folio;Fecha Docto;Fecha Recepcion;Fecha Acuse Recibo;Fecha Reclamo;Monto Exento;Monto Neto;Monto IVA;Monto total;IVA Retenido Total;IVA Retenido Parcial;IVA no retenido;IVA propio;IVA Terceros;RUT Emisor Liquid. Factura;Neto Comision Liquid. Factura;Exento Comision Liquid. Factura;IVA Comision Liquid. Factura;IVA fuera de plazo;Tipo Docto. Referencia;Folio Docto. Referencia;Num. Ident. Receptor Extranjero;Nacionalidad Receptor Extranjero;Credito empresa constructora;Impto. Zona Franca (Ley 18211);Garantia Dep. Envases;Indicador Venta sin Costo;Indicador Servicio Periodico;Monto No facturable;Total Monto Periodo;Venta Pasajes Transporte Nacional;Venta Pasajes Transporte Internacional;Numero Interno;Codigo Sucursal;NCE o NDE sobre Fact. de Compra;Codigo Otro Imp.;Valor Otro Imp.;Tasa Otro Imp. +1;"";Del Giro;12345678-5;Fake Company S.A. ;506;04/06/2019;"";;;0;1750181;332534;2082715;0;0;0;0;0;-;0;0;0;0;;;;;0;;0;2;0;0;0;;;;0;;;;; +23;33;Del Giro;12345678-5; Fake Company S.A.;508;28/06/2019;01/07/2019 13:49:42;;;0;2209597;419823;2629420;0;0;0;0;0;-;0;0;0;0;0;;;;0;;0;2;0;0;0;;;;0;;;;; diff --git a/src/tests/test_rcv_parse_csv.py b/src/tests/test_rcv_parse_csv.py index d0665595..459dec54 100644 --- a/src/tests/test_rcv_parse_csv.py +++ b/src/tests/test_rcv_parse_csv.py @@ -1,5 +1,6 @@ import unittest from typing import Callable +from unittest import mock from cl_sii.rcv.parse_csv import ( # noqa: F401 RcvCompraNoIncluirCsvRowSchema, @@ -72,6 +73,29 @@ def test_parse_rcv_venta_csv_file_receptor_rz_leading_trailing_whitespace(self) self.assertEqual(entry_struct.receptor_razon_social, 'Fake Company S.A.') self.assertEqual(len(row_parsing_errors), 0) + def test_parse_rcv_venta_csv_file_missing_required_fields(self): + # This CSV should have a required field (e.g., 'Folio') as an empty string + rcv_file_path = get_test_file_path( + 'test_data/sii-rcv/RCV-venta-missing-required-fields.csv', + ) + + items = parse_rcv_venta_csv_file( + rut=Rut('1-9'), + input_file_path=rcv_file_path, + n_rows_offset=0, + max_n_rows=None, + ) + + entry_struct, row_ix, row_data, row_parsing_errors = next(items) + self.assertIsNone(entry_struct) + self.assertIn('validation', row_parsing_errors) + self.assertIn('Fecha Recepcion', row_parsing_errors['validation']) + self.assertIn('Tipo Doc', row_parsing_errors['validation']) + self.assertEqual( + row_parsing_errors['validation']['Fecha Recepcion'], + ['Missing data for required field.'], + ) + def _test_parse_rcv_compra_csv_file_emisor_rz_leading_trailing_whitespace( self, parse_rcv_compra_csv_file_function: Callable, @@ -98,6 +122,38 @@ def _test_parse_rcv_compra_csv_file_emisor_rz_leading_trailing_whitespace( self.assertEqual(entry_struct.emisor_razon_social, 'Fake Company S.A.') self.assertEqual(len(row_parsing_errors), 0) + def test_parse_rcv_venta_csv_file_conversion_error(self): + rcv_file_path = get_test_file_path( + 'test_data/sii-rcv/RCV-venta-rz_leading_trailing_whitespace.csv', + ) + + # Patch the to_detalle_entry method to raise an exception + with ( + self.assertLogs('cl_sii.rcv', level='ERROR') as assert_logs_cm, + mock.patch.object( + RcvVentaCsvRowSchema, + 'to_detalle_entry', + side_effect=ValueError('Mocked conversion error'), + ), + ): + items = parse_rcv_venta_csv_file( + rut=Rut('1-9'), + input_file_path=rcv_file_path, + n_rows_offset=0, + max_n_rows=1, + ) + entry_struct, row_ix, row_data, row_parsing_errors = next(items) + self.assertIsNone(entry_struct) + self.assertIn('conversion_errors', row_parsing_errors) + self.assertIn('Mocked conversion error', row_parsing_errors['conversion_errors']) + self.assertRegex( + assert_logs_cm.output[0], + ( + 'ERROR:cl_sii.rcv.parse_csv:' + 'Deserialized row data conversion failed for row 1: Mocked conversion error' + ), + ) + def test_parse_rcv_compra_registro_csv_file(self) -> None: # TODO: implement for 'parse_rcv_compra_registro_csv_file'. pass